Skip to content

Commit 6efbe8c

Browse files
picandocodigopicandocodigo
committed
[Gem] Adds ES|QL Helper - Backports #2328 
1 parent 6e0d376 commit 6efbe8c

File tree

7 files changed

+292
-23
lines changed

7 files changed

+292
-23
lines changed

‎docs/helpers.asciidoc

+95
Original file line numberDiff line numberDiff line change
@@ -160,3 +160,98 @@ end
160160
scroll_helper.clear
161161
----
162162
--
163+
164+
[discrete]
165+
[[esql-helper]]
166+
=== ES|QL Helper
167+
168+
This functionality is Experimental and may be changed or removed completely in a future release. If you have any feedback on this helper, please https://github.com/elastic/elasticsearch-ruby/issues/new/choose[let us know].
169+
170+
The helper provides an object response from the ESQL `query` API instead of the default JSON value.
171+
172+
To use the ES|QL helper, require it in your code:
173+
174+
[source,ruby]
175+
----
176+
require 'elasticsearch/helpers/esql_helper'
177+
----
178+
179+
By default, the `query` API returns a Hash response with `columns` and `values` like so:
180+
181+
[source,ruby]
182+
----
183+
query = <<ESQL
184+
FROM sample_data
185+
| EVAL duration_ms = ROUND(event.duration / 1000000.0, 1)
186+
ESQL
187+
188+
response = client.esql.query(body: { query: query})
189+
puts response
190+
191+
{"columns"=>[
192+
{"name"=>"@timestamp", "type"=>"date"},
193+
{"name"=>"client.ip", "type"=>"ip"},
194+
{"name"=>"event.duration", "type"=>"long"},
195+
{"name"=>"message", "type"=>"keyword"},
196+
{"name"=>"duration_ms", "type"=>"double"}
197+
],
198+
"values"=>[
199+
["2023-10-23T12:15:03.360Z", "172.21.2.162", 3450233, "Connected to 10.1.0.3", 3.5],
200+
["2023-10-23T12:27:28.948Z", "172.21.2.113", 2764889, "Connected to 10.1.0.2", 2.8],
201+
["2023-10-23T13:33:34.937Z", "172.21.0.5", 1232382, "Disconnected", 1.2],
202+
["2023-10-23T13:51:54.732Z", "172.21.3.15", 725448, "Connection error", 0.7],
203+
["2023-10-23T13:52:55.015Z", "172.21.3.15", 8268153, "Connection error", 8.3],
204+
["2023-10-23T13:53:55.832Z", "172.21.3.15", 5033755, "Connection error", 5.0],
205+
["2023-10-23T13:55:01.543Z", "172.21.3.15", 1756467, "Connected to 10.1.0.1", 1.8]
206+
]}
207+
----
208+
209+
The helper returns an array of hashes with the columns as keys and the respective values. So for the previous example, it would return the following:
210+
211+
[source,ruby]
212+
----
213+
response = Elasticsearch::Helpers::ESQLHelper.query(client, query)
214+
215+
puts response
216+
217+
{"duration_ms"=>3.5, "message"=>"Connected to 10.1.0.3", "event.duration"=>3450233, "client.ip"=>"172.21.2.162", "@timestamp"=>"2023-10-23T12:15:03.360Z"}
218+
{"duration_ms"=>2.8, "message"=>"Connected to 10.1.0.2", "event.duration"=>2764889, "client.ip"=>"172.21.2.113", "@timestamp"=>"2023-10-23T12:27:28.948Z"}
219+
{"duration_ms"=>1.2, "message"=>"Disconnected", "event.duration"=>1232382, "client.ip"=>"172.21.0.5", "@timestamp"=>"2023-10-23T13:33:34.937Z"}
220+
{"duration_ms"=>0.7, "message"=>"Connection error", "event.duration"=>725448, "client.ip"=>"172.21.3.15", "@timestamp"=>"2023-10-23T13:51:54.732Z"}
221+
{"duration_ms"=>8.3, "message"=>"Connection error", "event.duration"=>8268153, "client.ip"=>"172.21.3.15", "@timestamp"=>"2023-10-23T13:52:55.015Z"}
222+
----
223+
224+
Additionally, you can transform the data in the response by passing in a Hash of `column => Proc` values. You could use this for example to convert '@timestamp' into a DateTime object. Pass in a Hash to `query` as a `parser` defining a `Proc` for each value you'd like to parse:
225+
226+
[source,ruby]
227+
----
228+
require 'elasticsearch/helpers/esql_helper'
229+
230+
parser = {
231+
'@timestamp' => Proc.new { |t| DateTime.parse(t) }
232+
}
233+
response = Elasticsearch::Helpers::ESQLHelper.query(client, query, parser: parser)
234+
response.first['@timestamp']
235+
# <DateTime: 2023-10-23T12:15:03+00:00 ((2460241j,44103s,360000000n),+0s,2299161j)>
236+
----
237+
238+
You can pass in as many Procs as there are columns in the response. For example:
239+
240+
[source,ruby]
241+
----
242+
parser = {
243+
'@timestamp' => Proc.new { |t| DateTime.parse(t) },
244+
'client.ip' => Proc.new { |i| IPAddr.new(i) },
245+
'event.duration' => Proc.new { |d| d.to_s }
246+
}
247+
248+
response = Elasticsearch::Helpers::ESQLHelper.query(client, query, parser: parser)
249+
250+
puts response
251+
252+
{"duration_ms"=>3.5, "message"=>"Connected to 10.1.0.3", "event.duration"=>"3450233", "client.ip"=>#<IPAddr: IPv4:172.21.2.162/255.255.255.255>, "@timestamp"=>#<DateTime: 2023-10-23T12:15:03+00:00 ((2460241j,44103s,360000000n),+0s,2299161j)>}
253+
{"duration_ms"=>2.8, "message"=>"Connected to 10.1.0.2", "event.duration"=>"2764889", "client.ip"=>#<IPAddr: IPv4:172.21.2.113/255.255.255.255>, "@timestamp"=>#<DateTime: 2023-10-23T12:27:28+00:00 ((2460241j,44848s,948000000n),+0s,2299161j)>}
254+
{"duration_ms"=>1.2, "message"=>"Disconnected", "event.duration"=>"1232382", "client.ip"=>#<IPAddr: IPv4:172.21.0.5/255.255.255.255>, "@timestamp"=>#<DateTime: 2023-10-23T13:33:34+00:00 ((2460241j,48814s,937000000n),+0s,2299161j)>}
255+
{"duration_ms"=>0.7, "message"=>"Connection error", "event.duration"=>"725448", "client.ip"=>#<IPAddr: IPv4:172.21.3.15/255.255.255.255>, "@timestamp"=>#<DateTime: 2023-10-23T13:51:54+00:00 ((2460241j,49914s,732000000n),+0s,2299161j)>}
256+
{"duration_ms"=>8.3, "message"=>"Connection error", "event.duration"=>"8268153", "client.ip"=>#<IPAddr: IPv4:172.21.3.15/255.255.255.255>, "@timestamp"=>#<DateTime: 2023-10-23T13:52:55+00:00 ((2460241j,49975s,15000000n),+0s,2299161j)>}
257+
----

‎elasticsearch/lib/elasticsearch/helpers/esql_helper.rb

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Licensed to Elasticsearch B.V. under one or more contributor
2+
# license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright
4+
# ownership. Elasticsearch B.V. licenses this file to you under
5+
# the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
module Elasticsearch
19+
module Helpers
20+
# Elasticsearch Client Helper for the ES|QL API
21+
#
22+
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-query-api.html
23+
#
24+
module ESQLHelper
25+
# Query helper for ES|QL
26+
#
27+
# By default, the `esql.query` API returns a Hash response with the following keys:
28+
#
29+
# * `columns` with the value being an Array of `{ name: type }` Hashes for each column.
30+
#
31+
# * `values` with the value being an Array of Arrays with the values for each row.
32+
#
33+
# This helper function returns an Array of hashes with the columns as keys and the respective
34+
# values: `{ column['name'] => value }`.
35+
#
36+
# @param client [Elasticsearch::Client] an instance of the Client to use for the query.
37+
# @param query [Hash, String] The query to be passed to the ES|QL query API.
38+
# @param params [Hash] options to pass to the ES|QL query API.
39+
# @param parser [Hash] Hash of column name keys and Proc values to transform the value of
40+
# a given column.
41+
# @example Using the ES|QL helper
42+
# require 'elasticsearch/helpers/esql_helper'
43+
# query = <<~ESQL
44+
# FROM sample_data
45+
# | EVAL duration_ms = ROUND(event.duration / 1000000.0, 1)
46+
# ESQL
47+
# response = Elasticsearch::Helpers::ESQLHelper.query(client, query)
48+
#
49+
# @example Using the ES|QL helper with a parser
50+
# response = Elasticsearch::Helpers::ESQLHelper.query(
51+
# client,
52+
# query,
53+
# parser: { '@timestamp' => Proc.new { |t| DateTime.parse(t) } }
54+
# )
55+
#
56+
# @see https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/Helpers.html#_esql_helper
57+
#
58+
def self.query(client, query, params = {}, parser: {})
59+
response = client.esql.query({ body: { query: query }, format: 'json' }.merge(params))
60+
columns = response['columns']
61+
response['values'].map do |value|
62+
(value.length - 1).downto(0).map do |index|
63+
key = columns[index]['name']
64+
value[index] = parser[key].call value[index] if parser[key]
65+
{ key => value[index] }
66+
end.reduce({}, :merge)
67+
end
68+
end
69+
end
70+
end
71+
end

‎elasticsearch/spec/integration/client_integration_spec.rb

+1-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
end
5555

5656
context 'Reports the right meta header' do
57-
it 'Reports es service name and gem versio' do
57+
it 'Reports es service name and gem version' do
5858
headers = client.transport.connections.first.connection.headers
5959
version = Class.new.extend(Elastic::Transport::MetaHeader).send(:client_meta_version, Elasticsearch::VERSION)
6060
expect(headers['x-elastic-client-meta']).to match /^es=#{version}/

‎elasticsearch/spec/integration/helpers/bulk_helper_spec.rb

+1-11
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,12 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17-
ELASTICSEARCH_URL = ENV['TEST_ES_SERVER'] || "http://localhost:#{(ENV['PORT'] || 9200)}"
18-
raise URI::InvalidURIError unless ELASTICSEARCH_URL =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/
19-
17+
require_relative 'helpers_spec_helper'
2018
require 'elasticsearch/helpers/bulk_helper'
21-
require 'spec_helper'
2219
require 'tempfile'
2320

2421
context 'Elasticsearch client helpers' do
2522
context 'Bulk helper' do
26-
let(:client) do
27-
Elasticsearch::Client.new(
28-
host: ELASTICSEARCH_URL,
29-
user: 'elastic',
30-
password: 'changeme'
31-
)
32-
end
3323
let(:index) { 'bulk_animals' }
3424
let(:params) { { refresh: 'wait_for' } }
3525
let(:bulk_helper) { Elasticsearch::Helpers::BulkHelper.new(client, index, params) }

‎elasticsearch/spec/integration/helpers/esql_helper_spec.rb

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
# Licensed to Elasticsearch B.V. under one or more contributor
2+
# license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright
4+
# ownership. Elasticsearch B.V. licenses this file to you under
5+
# the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
require_relative 'helpers_spec_helper'
18+
require 'elasticsearch/helpers/esql_helper'
19+
20+
context 'Elasticsearch client helpers' do
21+
let(:index) { 'esql_helper_test' }
22+
let(:body) { { size: 12, query: { match_all: {} } } }
23+
let(:esql_helper) { Elasticsearch::Helpers::ESQLHelper }
24+
let(:query) do
25+
<<~ESQL
26+
FROM #{index}
27+
| EVAL duration_ms = ROUND(event.duration / 1000000.0, 1)
28+
ESQL
29+
end
30+
31+
before do
32+
client.indices.create(
33+
index: index,
34+
body: {
35+
mappings: {
36+
properties: { 'client.ip' => { type: 'ip' }, message: { type: 'keyword' } }
37+
}
38+
}
39+
)
40+
client.bulk(
41+
index: index,
42+
body: [
43+
{'index': {}},
44+
{'@timestamp' => '2023-10-23T12:15:03.360Z', 'client.ip' => '172.21.2.162', message: 'Connected to 10.1.0.3', 'event.duration' => 3450233},
45+
{'index': {}},
46+
{'@timestamp' => '2023-10-23T12:27:28.948Z', 'client.ip' => '172.21.2.113', message: 'Connected to 10.1.0.2', 'event.duration' => 2764889},
47+
{'index': {}},
48+
{'@timestamp' => '2023-10-23T13:33:34.937Z', 'client.ip' => '172.21.0.5', message: 'Disconnected', 'event.duration' => 1232382},
49+
{'index': {}},
50+
{'@timestamp' => '2023-10-23T13:51:54.732Z', 'client.ip' => '172.21.3.15', message: 'Connection error', 'event.duration' => 725448},
51+
{'index': {}},
52+
{'@timestamp' => '2023-10-23T13:52:55.015Z', 'client.ip' => '172.21.3.15', message: 'Connection error', 'event.duration' => 8268153},
53+
{'index': {}},
54+
{'@timestamp' => '2023-10-23T13:53:55.832Z', 'client.ip' => '172.21.3.15', message: 'Connection error', 'event.duration' => 5033755},
55+
{'index': {}},
56+
{'@timestamp' => '2023-10-23T13:55:01.543Z', 'client.ip' => '172.21.3.15', message: 'Connected to 10.1.0.1', 'event.duration' => 1756467}
57+
],
58+
refresh: true
59+
)
60+
end
61+
62+
after do
63+
client.indices.delete(index: index)
64+
end
65+
66+
it 'returns an ESQL response as a relational key/value object' do
67+
response = esql_helper.query(client, query)
68+
expect(response.count).to eq 7
69+
expect(response.first.keys).to eq ['duration_ms', 'message', 'event.duration', 'client.ip', '@timestamp']
70+
response.each do |r|
71+
expect(r['@timestamp']).to be_a String
72+
expect(r['client.ip']).to be_a String
73+
expect(r['message']).to be_a String
74+
expect(r['event.duration']).to be_a Integer
75+
end
76+
end
77+
78+
it 'parses iterated objects when procs are passed in' do
79+
require 'ipaddr'
80+
81+
parser = {
82+
'@timestamp' => Proc.new { |t| DateTime.parse(t) },
83+
'client.ip' => Proc.new { |i| IPAddr.new(i) },
84+
'event.duration' => Proc.new { |d| d.to_s }
85+
}
86+
response = esql_helper.query(client, query, parser: parser)
87+
response.each do |r|
88+
expect(r['@timestamp']).to be_a DateTime
89+
expect(r['client.ip']).to be_a IPAddr
90+
expect(r['message']).to be_a String
91+
expect(r['event.duration']).to be_a String
92+
end
93+
end
94+
end

‎elasticsearch/spec/integration/helpers/helpers_spec_helper.rb

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Licensed to Elasticsearch B.V. under one or more contributor
2+
# license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright
4+
# ownership. Elasticsearch B.V. licenses this file to you under
5+
# the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
require 'spec_helper'
19+
20+
ELASTICSEARCH_URL = ENV['TEST_ES_SERVER'] || "http://localhost:#{(ENV['PORT'] || 9200)}"
21+
raise URI::InvalidURIError unless ELASTICSEARCH_URL =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/
22+
23+
def client
24+
@client ||= Elasticsearch::Client.new(
25+
host: ELASTICSEARCH_URL,
26+
user: 'elastic',
27+
password: 'changeme'
28+
)
29+
end

‎elasticsearch/spec/integration/helpers/scroll_helper_spec.rb

+1-11
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,10 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17-
ELASTICSEARCH_URL = ENV['TEST_ES_SERVER'] || "http://localhost:#{(ENV['PORT'] || 9200)}"
18-
raise URI::InvalidURIError unless ELASTICSEARCH_URL =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/
19-
20-
require 'spec_helper'
17+
require_relative 'helpers_spec_helper'
2118
require 'elasticsearch/helpers/scroll_helper'
2219

2320
context 'Elasticsearch client helpers' do
24-
let(:client) do
25-
Elasticsearch::Client.new(
26-
host: ELASTICSEARCH_URL,
27-
user: 'elastic',
28-
password: 'changeme'
29-
)
30-
end
3121
let(:index) { 'books' }
3222
let(:body) { { size: 12, query: { match_all: {} } } }
3323
let(:scroll_helper) { Elasticsearch::Helpers::ScrollHelper.new(client, index, body) }

0 commit comments

Comments
 (0)