diff --git a/lib/logstash/outputs/elasticsearch/common_configs.rb b/lib/logstash/outputs/elasticsearch/common_configs.rb index dc2efea56..f86a1c87b 100644 --- a/lib/logstash/outputs/elasticsearch/common_configs.rb +++ b/lib/logstash/outputs/elasticsearch/common_configs.rb @@ -50,6 +50,12 @@ def self.included(mod) # If not set, the included template will be used. mod.config :template, :validate => :path + # The legacy_template option will use the old /_template (ELK >7.7) + # path for creating index templates. + # It will also construct the ilm configurations for the template in a manner + # which is compatible with legacy templates + mod.config :legacy_template, :validate => :boolean, :default => true + # The template_overwrite option will always overwrite the indicated template # in Elasticsearch with either the one indicated by template or the included one. # This option is set to false by default. If you always want to stay up to date diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index 32a37e82a..7c9a11aa8 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -252,6 +252,10 @@ def http_compression client_settings.fetch(:http_compression, false) end + def legacy_template + client_settings.fetch(:legacy) + end + def build_adapter(options) timeout = options[:timeout] || 0 @@ -343,11 +347,19 @@ def exists?(path, use_get=false) end def template_exists?(name) - exists?("/_template/#{name}") + if legacy_template + exists?("/_template/#{name}") + else + exists?("/_index_template/#{name}") + end end def template_put(name, template) - path = "_template/#{name}" + if legacy_template + path = "_template/#{name}" + else + path = "_index_template/#{name}" + end logger.info("Installing elasticsearch template to #{path}") @pool.put(path, nil, LogStash::Json.dump(template)) end diff --git a/lib/logstash/outputs/elasticsearch/http_client_builder.rb b/lib/logstash/outputs/elasticsearch/http_client_builder.rb index fd8827f71..bbc57fa66 100644 --- a/lib/logstash/outputs/elasticsearch/http_client_builder.rb +++ b/lib/logstash/outputs/elasticsearch/http_client_builder.rb @@ -9,7 +9,8 @@ def self.build(logger, hosts, params) :pool_max_per_route => params["pool_max_per_route"], :check_connection_timeout => params["validate_after_inactivity"], :http_compression => params["http_compression"], - :headers => params["custom_headers"] || {} + :headers => params["custom_headers"] || {}, + :legacy => params["legacy_template"] } client_settings[:proxy] = params["proxy"] if params["proxy"] diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 20f2cbd2c..0c139a6f8 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -32,14 +32,63 @@ def self.install(client, template_name, template, template_overwrite) end def self.add_ilm_settings_to_template(plugin, template) - # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern - # definition - remove any existing definition of 'template' - template.delete('template') if template.include?('template') - template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" - if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias']) - plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled.") + if plugin.original_params.key?('ilm_rollover_alias') + # If no index patterns in the 'index_pattern' array would include the 'ilm_rollover_alias' add the 'ilm_rollover_alias' to the 'index_pattern' array + if template.key?('index_patterns') && template['index_patterns'].kind_of?(Array) && !template['index_patterns'].any? { |idx_pattern| "#{plugin.ilm_rollover_alias}-".start_with?(idx_pattern.tr('*','')) } + plugin.logger.info("Adding index pattern name for rollover alias as ILM is enabled.") + template['index_patterns'].append("#{plugin.ilm_rollover_alias}-*") + # If 'index_pattern' is not an array and doesn't include the 'ilm_rollover_alias' set the 'index_pattern' to the 'ilm_rollover_alias' + elsif template.key?('index_patterns') && !template['index_patterns'].kind_of?(Array) && !"#{plugin.ilm_rollover_alias}-".start_with?(template['index_patterns'].tr('*','')) + plugin.logger.info("Overwriting index pattern name for rollover alias as ILM is enabled.") + template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" + # If 'index_pattern' doesn't exist, set it to the 'ilm_rollover_alias' + elsif !template.key?('index_patterns') + plugin.logger.info("Setting index pattern name for rollover alias as ILM is enabled.") + template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" + end + end + if plugin.legacy_template + # definition - remove any existing definition of 'template' + template.delete('template') if template.include?('template') + # Create settings hash if not in the template, but ilm is enabled + if !template.key?('settings') + template['settings'] = { } + end + if plugin.original_params.key?('ilm_rollover_alias') + if template['settings'] + # Overwrite the rollover_alias, sense it was defined in the output plugin + plugin.logger.info("Overwriting index rollover alias with plugin defined alias.") + template['settings'].update({ 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) + end + end + if plugin.original_params.key?('ilm_policy') + if template['settings'] + # Overwrite the ilm_policy, sense it was defined in the output plugin + plugin.logger.info("Overwriting index lifecycle name with plugin defined ilm policy.") + template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy}) + end + end + else + #plugin.logger.warn(plugin.original_params.key?('ilm_rollover_alias')) + # Create settings hash if not in the template, but ilm is enabled + if !template['template'].key?('settings') + template['template']['settings'] = { } + end + if plugin.original_params.key?('ilm_rollover_alias') + if template['template']['settings'] + # Overwrite the rollover_alias, sense it was defined in the output plugin + plugin.logger.info("Overwriting index rollover alias with plugin defined alias.") + template['template']['settings'].update({ 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) + end + end + if plugin.original_params.key?('ilm_policy') + if template['template']['settings'] + # Overwrite the ilm_policy, sense it was defined in the output plugin + plugin.logger.info("Overwriting index lifecycle name with plugin defined ilm policy.") + template['template']['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy}) + end + end end - template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) end # Template name - if template_name set, use it