[Backport][main] NDJSON headers

elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb

@@ -187,9 +187,7 @@ def bulk(arguments = {})
                     body
                   end
 
-        headers.merge!({
-                         'Content-Type' => 'application/vnd.elasticsearch+x-ndjson; compatible-with=9'
-                       })
+        Utils.update_ndjson_headers!(headers, transport.options.dig(:transport_options, :headers))
         Elasticsearch::API::Response.new(
           perform_request(method, path, params, payload, headers, request_opts)
         )

elasticsearch-api/lib/elasticsearch/api/actions/fleet/msearch.rb

@@ -108,9 +108,7 @@ def msearch(arguments = {})
             payload = body
           end
 
-          headers.merge!({
-                           'Content-Type' => 'application/vnd.elasticsearch+x-ndjson; compatible-with=9'
-                         })
+          Utils.update_ndjson_headers!(headers, client.transport.options.dig(:transport_options, :headers))
           Elasticsearch::API::Response.new(
             perform_request(method, path, params, payload, headers, request_opts)
           )

elasticsearch-api/lib/elasticsearch/api/actions/msearch.rb

@@ -120,9 +120,7 @@ def msearch(arguments = {})
           payload = body
         end
 
-        headers.merge!({
-                         'Content-Type' => 'application/vnd.elasticsearch+x-ndjson; compatible-with=9'
-                       })
+        Utils.update_ndjson_headers!(headers, transport.options.dig(:transport_options, :headers))
         Elasticsearch::API::Response.new(
           perform_request(method, path, params, payload, headers, request_opts)
         )

elasticsearch-api/lib/elasticsearch/api/actions/msearch_template.rb

@@ -93,9 +93,7 @@ def msearch_template(arguments = {})
           payload = body
         end
 
-        headers.merge!({
-                         'Content-Type' => 'application/vnd.elasticsearch+x-ndjson; compatible-with=9'
-                       })
+        Utils.update_ndjson_headers!(headers, transport.options.dig(:transport_options, :headers))
         Elasticsearch::API::Response.new(
           perform_request(method, path, params, payload, headers, request_opts)
         )

elasticsearch-api/lib/elasticsearch/api/actions/text_structure/find_structure.rb

@@ -136,9 +136,7 @@ def find_structure(arguments = {})
                       body
                     end
 
-          headers.merge!({
-                           'Content-Type' => 'application/vnd.elasticsearch+x-ndjson; compatible-with=9'
-                         })
+          Utils.update_ndjson_headers!(headers, client.transport.options.dig(:transport_options, :headers))
           Elasticsearch::API::Response.new(
             perform_request(method, path, params, payload, headers, request_opts)
           )

elasticsearch-api/lib/elasticsearch/api/utils.rb

@@ -177,6 +177,21 @@ def rescue_from_not_found(&block)
         end
       end
 
+      # Updates ndjson headers for msearch, bulk, and others
+      #
+      def update_ndjson_headers!(headers, client_headers)
+        current_content = client_headers.keys.find { |c| c.match?(/content-?_?type/i) } || 'content-type'
+        current_accept = client_headers.keys.find { |c| c.match?(/accept/i) } || 'accept'
+        version = client_headers[current_content].match(/compatible-with=([0-9]+)/)[1] || 9
+
+        headers.merge!(
+          {
+            current_content => "application/vnd.elasticsearch+x-ndjson; compatible-with=#{version}",
+            current_accept => "application/vnd.elasticsearch+x-ndjson; compatible-with=#{version}"
+          }
+        )
+      end
+
       extend self
     end
   end

elasticsearch-api/spec/spec_helper.rb

@@ -57,6 +57,13 @@ def self.included(context)
   end
 end
 
+Transport ||= Struct.new('Transport', :options)
+
+def dummy_ndjson_headers
+  allow(client_double).to receive(:transport).and_return Transport.new({ transport_options: { headers: {} } })
+  allow(Elasticsearch::API::Utils).to receive(:update_ndjson_headers!).and_return({})
+end
+
 RSpec.configure do |config|
   config.include(HelperModule)
   config.filter_run_excluding skip: true

elasticsearch-api/spec/unit/actions/bulk_spec.rb

@@ -29,11 +29,12 @@
     ]
   end
 
-  let(:headers) {
-    {
-      'Content-Type' => 'application/vnd.elasticsearch+x-ndjson; compatible-with=9'
-    }
-  }
+  let(:headers) { {} }
+
+  # This test only cares about the body, there's another test for the ndjson headers.
+  before do
+    dummy_ndjson_headers
+  end
 
   let(:params) { {} }
   let(:url) { '_bulk' }
@@ -53,12 +54,16 @@
     end
 
     it 'performs the request' do
-      expect(client_double.bulk(:body => [
-          { :index =>  { :_index => 'myindexA', :_id => '1', :data => { :title => 'Test' } } },
-          { :update => { :_index => 'myindexB', :_id => '2', :data => { :doc => { :title => 'Update' } } } },
-          { :delete => { :_index => 'myindexC', :_id => '3' } },
-          { :index =>  { :_index => 'myindexD', :_id => '1', :data => { :data => 'MYDATA' } } },
-      ])).to be_a Elasticsearch::API::Response
+      expect(
+        client_double.bulk(
+          body: [
+            { index:  { _index: 'myindexA', _id: '1', data: { title: 'Test' } } },
+            { update: { _index: 'myindexB', _id: '2', data: { doc: { title: 'Update' } } } },
+            { delete: { _index: 'myindexC', _id: '3' } },
+            { index:  { _index: 'myindexD', _id: '1', data: { data: 'MYDATA' } } }
+          ]
+        )
+      ).to be_a Elasticsearch::API::Response
     end
   end
 
@@ -72,7 +77,7 @@
         params,
         body,
         headers,
-        { defined_params: { index: 'myindex' }, :endpoint=>"bulk"}
+        { defined_params: { index: 'myindex' }, endpoint: 'bulk' }
       ]
     end
 
@@ -90,8 +95,14 @@
     end
 
     it 'performs the request' do
-      expect(client_double.bulk(body:[ { :update => { :_index => 'myindex', :_id => '1' } },
-                                       { :doc => { :data => { :title => 'Update' } } } ])).to be_a Elasticsearch::API::Response
+      expect(
+        client_double.bulk(
+          body: [
+            { update: { _index: 'myindex', _id: '1' } },
+            { doc: { data: { title: 'Update' } } }
+          ]
+        )
+      ).to be_a Elasticsearch::API::Response
     end
   end
 
@@ -137,7 +148,7 @@
         params,
         body,
         headers,
-        { defined_params: { index: 'foo^bar' }, :endpoint=>"bulk"}
+        { defined_params: { index: 'foo^bar' }, endpoint: 'bulk'}
       ]
     end
 

elasticsearch-api/spec/unit/actions/fleet/msearch_spec.rb

@@ -24,28 +24,17 @@
       '_fleet/_fleet_msearch',
       {},
       {},
-      headers,
+      {},
       { endpoint: 'fleet.msearch' }
     ]
   end
 
-  let(:headers) {
-    {
-      'Content-Type' => 'application/vnd.elasticsearch+x-ndjson; compatible-with=9'
-    }
-  }
+  # This test only cares about the body, there's another test for the ndjson headers.
+  before do
+    dummy_ndjson_headers
+  end
 
   it 'performs the request' do
     expect(client_double.fleet.msearch(body: {})).to be_a Elasticsearch::API::Response
   end
-
-  let(:client) do
-    Class.new { include Elasticsearch::API }.new
-  end
-
-  it 'requires the :body argument' do
-    expect {
-      client.fleet.msearch
-    }.to raise_exception(ArgumentError)
-  end
 end

elasticsearch-api/spec/unit/actions/msearch_spec.rb

@@ -41,24 +41,16 @@
     {}
   end
 
-  let(:headers) {
-    {
-      'Content-Type' => 'application/vnd.elasticsearch+x-ndjson; compatible-with=9'
-    }
-  }
-
-  let(:client) do
-    Class.new { include Elasticsearch::API }.new
+  let(:headers) do
+    {}
   end
 
-  it 'requires the :body argument' do
-    expect {
-      client.msearch
-    }.to raise_exception(ArgumentError)
+  # This test only cares about the body, there's another test for the ndjson headers.
+  before do
+    dummy_ndjson_headers
   end
 
   context 'when the body is an object' do
-
     let(:body) do
       <<-PAYLOAD.gsub(/^\s+/, '')
             {"index":"foo"}
@@ -72,15 +64,14 @@
 
     it 'performs the request' do
       expect(client_double.msearch body: [
-          { index: 'foo', search: { query: { match_all: {}  } } },
+          { index: 'foo', search: { query: { match_all: {} } } },
           { index: 'bar', search: { query: { match: { foo: 'bar' } } } },
           { search_type: 'count', search: { facets: { tags: {} } } }
       ])
     end
   end
 
   context 'when the body is a string' do
-
     let(:body) do
       %Q|{"foo":"bar"}\n{"moo":"lam"}|
     end
@@ -141,7 +132,6 @@
   end
 
   context 'when the request needs to be URL-escaped' do
-
     let(:url) do
       'foo%5Ebar/_msearch'
     end
@@ -167,7 +157,6 @@
   end
 
   context 'when the URL params need to be URL-encoded' do
-
     let(:url) do
       '_msearch'
     end

elasticsearch-api/spec/unit/actions/msearch_template_spec.rb

@@ -37,11 +37,12 @@
     {}
   end
 
-  let(:headers) {
-    {
-      'Content-Type' => 'application/vnd.elasticsearch+x-ndjson; compatible-with=9'
-    }
-  }
+  let(:headers) {{}}
+
+  # This test only cares about the body, there's another test for the ndjson headers.
+  before do
+    dummy_ndjson_headers
+  end
 
   let(:url) do
     '_msearch/template'

elasticsearch-api/spec/unit/actions/text_structure/find_structure_spec.rb

@@ -17,23 +17,23 @@
 
 require 'spec_helper'
 
-describe 'client.text_structure#test_grok_pattern' do
+describe 'client.text_structure#find_structure' do
   let(:expected_args) do
     [
       'POST',
       '_text_structure/find_structure',
       {},
       {},
-      headers,
+      {},
       { endpoint: 'text_structure.find_structure' }
     ]
   end
 
-  let(:headers) {
-    {
-      'Content-Type' => 'application/vnd.elasticsearch+x-ndjson; compatible-with=9'
-    }
-  }
+  # This test only cares about the body, there's another test for the ndjson headers.
+  before do
+    dummy_ndjson_headers
+    allow(client_double.text_structure.client).to receive(:transport).and_return Transport.new({ transport_options: { headers: {} } })
+  end
 
   it 'performs the request' do
     expect(client_double.text_structure.find_structure(body: {})).to be_a Elasticsearch::API::Response