Skip to content

Commit bb48e4e

Browse files
authored
Merge branch 'master' into zstd_comp_support
2 parents cdb963f + 29189a1 commit bb48e4e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+2364
-513
lines changed

.github/workflows/scorecards.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828

2929
steps:
3030
- name: "Checkout code"
31-
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
31+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
3232
with:
3333
persist-credentials: false
3434

@@ -55,7 +55,7 @@ jobs:
5555
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
5656
# format to the repository Actions tab.
5757
- name: "Upload artifact"
58-
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
58+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
5959
with:
6060
name: SARIF file
6161
path: results.sarif
@@ -64,6 +64,6 @@ jobs:
6464
# Upload the results to GitHub's code scanning dashboard (optional).
6565
# Commenting out will disable upload of results to your repo's Code Scanning dashboard
6666
- name: "Upload to code-scanning"
67-
uses: github/codeql-action/upload-sarif@2c779ab0d087cd7fe7b826087247c2c81f27bfa6 # v3.26.5
67+
uses: github/codeql-action/upload-sarif@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
6868
with:
6969
sarif_file: results.sarif

.github/workflows/test-ruby-head.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ jobs:
1919

2020
name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
2121
steps:
22-
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
22+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
2323
- name: Set up Ruby
24-
uses: ruby/setup-ruby@a6e6f86333f0a2523ece813039b8b4be04560854 # v1.190.0
24+
uses: ruby/setup-ruby@28c4deda893d5a96a6b2d958c5b47fc18d65c9d3 # v1.213.0
2525
with:
2626
ruby-version: ${{ matrix.ruby-version }}
2727
- name: Install addons

.github/workflows/test.yml

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ on:
1212
- '*.md'
1313
- 'lib/fluent/version.rb'
1414

15+
concurrency:
16+
group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
17+
cancel-in-progress: true
18+
1519
permissions: read-all
1620

1721
jobs:
@@ -22,15 +26,16 @@ jobs:
2226
fail-fast: false
2327
matrix:
2428
os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
25-
ruby-version: ['3.3', '3.2', '3.1', '3.0', '2.7']
29+
ruby-version: ['3.4', '3.3', '3.2']
2630

2731
name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
2832
steps:
29-
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
33+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
3034
- name: Set up Ruby
31-
uses: ruby/setup-ruby@a6e6f86333f0a2523ece813039b8b4be04560854 # v1.190.0
35+
uses: ruby/setup-ruby@28c4deda893d5a96a6b2d958c5b47fc18d65c9d3 # v1.213.0
3236
with:
3337
ruby-version: ${{ matrix.ruby-version }}
38+
rubygems: latest
3439
- name: Install addons
3540
if: ${{ matrix.os == 'ubuntu-latest' }}
3641
run: sudo apt-get install libgmp3-dev libcap-ng-dev

CHANGELOG.md

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,57 @@
1+
# v1.18
2+
3+
## Release v1.18.0 - 2024/11/29
4+
5+
### Enhancement
6+
7+
* Add zero-downtime-restart feature for non-Windows
8+
https://github.com/fluent/fluentd/pull/4624
9+
* Add with-source-only feature
10+
https://github.com/fluent/fluentd/pull/4661
11+
* `fluentd` command: Add `--with-source-only` option
12+
* System configuration: Add `with_source_only` option
13+
* Embedded plugin: Add `out_buffer` plugin, which can be used for buffering and relabeling events
14+
https://github.com/fluent/fluentd/pull/4661
15+
* Config File Syntax: Extend Embedded Ruby Code support for Hashes and Arrays
16+
https://github.com/fluent/fluentd/pull/4580
17+
* Example: `key {"foo":"#{1 + 1}"} => key {"foo":"2"}`
18+
* Please note that this is not backward compatible, although we assume that this will never affect to actual existing configs.
19+
* In case the behavior changes unintentionally, you can disable this feature by surrounding the entire value with single quotes.
20+
* `key '{"foo":"#{1 + 1}"}' => key {"foo":"#{1 + 1}"}`
21+
* transport tls: Use SSL_VERIFY_NONE by default
22+
https://github.com/fluent/fluentd/pull/4718
23+
* transport tls: Add ensure_fips option to ensure FIPS compliant mode
24+
https://github.com/fluent/fluentd/pull/4720
25+
* plugin_helper/server: Add receive_buffer_size parameter in transport section
26+
https://github.com/fluent/fluentd/pull/4649
27+
* filter_parser: Now able to handle multiple parsed results
28+
https://github.com/fluent/fluentd/pull/4620
29+
* in_http: add `add_tag_prefix` option
30+
https://github.com/fluent/fluentd/pull/4655
31+
* System configuration: add `path` option in `log` section
32+
https://github.com/fluent/fluentd/pull/4604
33+
34+
### Bug Fix
35+
36+
* command: fix NoMethodError of --daemon under Windows
37+
https://github.com/fluent/fluentd/pull/4716
38+
* `fluentd` command: fix `--plugin` (`-p`) option not to overwrite default value
39+
https://github.com/fluent/fluentd/pull/4605
40+
41+
### Misc
42+
43+
* http_server: Ready to support Async 2.0 gem
44+
https://github.com/fluent/fluentd/pull/4619
45+
* Minor code refactoring
46+
* https://github.com/fluent/fluentd/pull/4641
47+
* CI fixes
48+
* https://github.com/fluent/fluentd/pull/4638
49+
* https://github.com/fluent/fluentd/pull/4644
50+
* https://github.com/fluent/fluentd/pull/4675
51+
* https://github.com/fluent/fluentd/pull/4676
52+
* https://github.com/fluent/fluentd/pull/4677
53+
* https://github.com/fluent/fluentd/pull/4686
54+
155
# v1.17
256

357
## Release v1.17.1 - 2024/08/19

README.md

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ Fluentd: Open-Source Log Collector
3030

3131
### Prerequisites
3232

33-
- Ruby 2.7 or later
33+
- Ruby 3.2 or later
3434
- git
3535

3636
`git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.
@@ -72,5 +72,3 @@ See [SECURITY](SECURITY.md) to contact us about vulnerability.
7272
## Contributors:
7373

7474
Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).
75-
76-
[<img src="https://ga-beacon.appspot.com/UA-24890265-6/fluent/fluentd" />](https://github.com/fluent/fluentd)

SECURITY.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44

55
| Version | Supported |
66
| ------- | ------------------ |
7-
| 1.17.x | :white_check_mark: |
7+
| 1.18.x | :white_check_mark: |
8+
| 1.17.x | :x: |
89
| 1.16.x | :white_check_mark: |
910
| <= 1.15.x | :x: |
1011

fluentd.gemspec

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
2020
gem.require_paths = ["lib"]
2121
gem.license = "Apache-2.0"
2222

23-
gem.required_ruby_version = '>= 2.7'
23+
gem.required_ruby_version = '>= 3.2'
2424

2525
gem.add_runtime_dependency("bundler")
2626
gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
@@ -41,7 +41,9 @@ Gem::Specification.new do |gem|
4141
gem.add_runtime_dependency("drb", ["~> 2.2"])
4242

4343
# gems that aren't default gems as of Ruby 3.5
44-
gem.add_runtime_dependency("logger", ["~> 1.6"])
44+
# logger 1.6.3 or later cause bug on windows,
45+
# hold on 1.6.2 for a while. see https://github.com/ruby/logger/issues/107
46+
gem.add_runtime_dependency("logger", ["1.6.2"])
4547

4648
# build gem for a certain platform. see also Rakefile
4749
fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
@@ -64,8 +66,7 @@ Gem::Specification.new do |gem|
6466
gem.add_development_dependency("test-unit", ["~> 3.3"])
6567
gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
6668
gem.add_development_dependency("oj", [">= 2.14", "< 4"])
67-
gem.add_development_dependency("async", "~> 1.23")
68-
gem.add_development_dependency("async-http", ">= 0.50.0")
69+
gem.add_development_dependency("async-http", "~> 0.86")
6970
gem.add_development_dependency("aws-sigv4", ["~> 1.8"])
7071
gem.add_development_dependency("aws-sdk-core", ["~> 3.191"])
7172
gem.add_development_dependency("rexml", ["~> 3.2"])

lib/fluent/command/fluentd.rb

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,12 @@
127127
cmd_opts[:without_source] = b
128128
}
129129

130+
unless Fluent.windows?
131+
op.on('--with-source-only', "Invoke a fluentd only with input plugins. The data is stored in a temporary buffer. Send SIGWINCH to cancel this mode and process the data (Not supported on Windows).", TrueClass) {|b|
132+
cmd_opts[:with_source_only] = b
133+
}
134+
end
135+
130136
op.on('--config-file-type VALU', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|
131137
if (s == 'yaml') || (s == 'yml')
132138
cmd_opts[:config_file_type] = s.to_sym

lib/fluent/config/literal_parser.rb

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -254,8 +254,15 @@ def scan_json(is_array)
254254
buffer << line_buffer + "\n"
255255
line_buffer = ""
256256
else
257-
# '#' is a char in json string
258-
line_buffer << char
257+
if @ss.exist?(/^\{[^}]+\}/)
258+
# if it's interpolated string
259+
skip(/\{/)
260+
line_buffer << eval_embedded_code(scan_embedded_code)
261+
skip(/\}/)
262+
else
263+
# '#' is a char in json string
264+
line_buffer << char
265+
end
259266
end
260267

261268
next # This char '#' MUST NOT terminate json object.

lib/fluent/engine.rb

Lines changed: 49 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,15 @@ def initialize
4343
@system_config = SystemConfig.new
4444

4545
@supervisor_mode = false
46+
47+
@root_agent_mutex = Mutex.new
4648
end
4749

4850
MAINLOOP_SLEEP_INTERVAL = 0.3
4951

5052
attr_reader :root_agent, :system_config, :supervisor_mode
5153

52-
def init(system_config, supervisor_mode: false)
54+
def init(system_config, supervisor_mode: false, start_in_parallel: false)
5355
@system_config = system_config
5456
@supervisor_mode = supervisor_mode
5557

@@ -58,7 +60,7 @@ def init(system_config, supervisor_mode: false)
5860

5961
@log_event_verbose = system_config.log_event_verbose unless system_config.log_event_verbose.nil?
6062

61-
@root_agent = RootAgent.new(log: log, system_config: @system_config)
63+
@root_agent = RootAgent.new(log: log, system_config: @system_config, start_in_parallel: start_in_parallel)
6264

6365
self
6466
end
@@ -133,7 +135,15 @@ def emit_stream(tag, es)
133135
end
134136

135137
def flush!
136-
@root_agent.flush!
138+
@root_agent_mutex.synchronize do
139+
@root_agent.flush!
140+
end
141+
end
142+
143+
def cancel_source_only!
144+
@root_agent_mutex.synchronize do
145+
@root_agent.cancel_source_only!
146+
end
137147
end
138148

139149
def now
@@ -144,7 +154,9 @@ def now
144154
def run
145155
begin
146156
$log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
147-
start
157+
@root_agent_mutex.synchronize do
158+
start
159+
end
148160

149161
@fluent_log_event_router.start
150162

@@ -158,47 +170,51 @@ def run
158170
raise
159171
end
160172

161-
stop_phase(@root_agent)
173+
@root_agent_mutex.synchronize do
174+
stop_phase(@root_agent)
175+
end
162176
end
163177

164178
# @param conf [Fluent::Config]
165179
# @param supervisor [Bool]
166180
# @reutrn nil
167181
def reload_config(conf, supervisor: false)
168-
# configure first to reduce down time while restarting
169-
new_agent = RootAgent.new(log: log, system_config: @system_config)
170-
ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
171-
172-
ret.all_plugins.each do |plugin|
173-
if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
174-
raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
182+
@root_agent_mutex.synchronize do
183+
# configure first to reduce down time while restarting
184+
new_agent = RootAgent.new(log: log, system_config: @system_config)
185+
ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
186+
187+
ret.all_plugins.each do |plugin|
188+
if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
189+
raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
190+
end
175191
end
176-
end
177192

178-
# Assign @root_agent to new root_agent
179-
# for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
180-
old_agent, @root_agent = @root_agent, new_agent
181-
begin
182-
@root_agent.configure(conf)
183-
rescue
184-
@root_agent = old_agent
185-
raise
186-
end
193+
# Assign @root_agent to new root_agent
194+
# for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
195+
old_agent, @root_agent = @root_agent, new_agent
196+
begin
197+
@root_agent.configure(conf)
198+
rescue
199+
@root_agent = old_agent
200+
raise
201+
end
187202

188-
unless @suppress_config_dump
189-
$log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
190-
end
203+
unless @suppress_config_dump
204+
$log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
205+
end
191206

192-
# supervisor doesn't handle actual data. so the following code is unnecessary.
193-
if supervisor
194-
old_agent.shutdown # to close thread created in #configure
195-
return
196-
end
207+
# supervisor doesn't handle actual data. so the following code is unnecessary.
208+
if supervisor
209+
old_agent.shutdown # to close thread created in #configure
210+
return
211+
end
197212

198-
stop_phase(old_agent)
213+
stop_phase(old_agent)
199214

200-
$log.info 'restart fluentd worker', worker: worker_id
201-
start_phase(new_agent)
215+
$log.info 'restart fluentd worker', worker: worker_id
216+
start_phase(new_agent)
217+
end
202218
end
203219

204220
def stop

0 commit comments

Comments
 (0)