From f290af14ff466595a3071bd9a2141fa21be0bc8f Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Tue, 25 Feb 2025 01:40:07 -0700 Subject: [PATCH 01/14] Add capability to discard duplicate jobs with concurrency configuration --- .../solid_queue/job/concurrency_controls.rb | 10 +++- app/models/solid_queue/job/executable.rb | 3 +- app/models/solid_queue/semaphore.rb | 12 +++++ lib/active_job/concurrency_controls.rb | 4 +- test/models/solid_queue/job_test.rb | 50 ++++++++++++++++++- 5 files changed, 74 insertions(+), 5 deletions(-) diff --git a/app/models/solid_queue/job/concurrency_controls.rb b/app/models/solid_queue/job/concurrency_controls.rb index 6ae12e28..87b723a0 100644 --- a/app/models/solid_queue/job/concurrency_controls.rb +++ b/app/models/solid_queue/job/concurrency_controls.rb @@ -8,7 +8,7 @@ module ConcurrencyControls included do has_one :blocked_execution - delegate :concurrency_limit, :concurrency_duration, to: :job_class + delegate :concurrency_limit, :concurrency_on_duplicate, :concurrency_duration, to: :job_class before_destroy :unblock_next_blocked_job, if: -> { concurrency_limited? && ready? } end @@ -34,6 +34,14 @@ def blocked? end private + def duplicate? + Semaphore.at_limit?(self) + end + + def discard_on_duplicate? + concurrency_on_duplicate == :discard && duplicate? + end + def acquire_concurrency_lock return true unless concurrency_limited? diff --git a/app/models/solid_queue/job/executable.rb b/app/models/solid_queue/job/executable.rb index e2146a67..ba01132f 100644 --- a/app/models/solid_queue/job/executable.rb +++ b/app/models/solid_queue/job/executable.rb @@ -65,7 +65,8 @@ def prepare_for_execution end def dispatch - if acquire_concurrency_lock then ready + if discard_on_duplicate? then discard + elsif acquire_concurrency_lock then ready else block end diff --git a/app/models/solid_queue/semaphore.rb b/app/models/solid_queue/semaphore.rb index 62eeb035..96cce74a 100644 --- a/app/models/solid_queue/semaphore.rb +++ b/app/models/solid_queue/semaphore.rb @@ -10,6 +10,10 @@ def wait(job) Proxy.new(job).wait end + def at_limit?(job) + Proxy.new(job).at_limit? + end + def signal(job) Proxy.new(job).signal end @@ -39,6 +43,14 @@ def initialize(job) @job = job end + def at_limit? + if semaphore = Semaphore.find_by(key: key) + semaphore.value.zero? + else + false + end + end + def wait if semaphore = Semaphore.find_by(key: key) semaphore.value > 0 && attempt_decrement diff --git a/lib/active_job/concurrency_controls.rb b/lib/active_job/concurrency_controls.rb index 0ea290f6..6b0f08e8 100644 --- a/lib/active_job/concurrency_controls.rb +++ b/lib/active_job/concurrency_controls.rb @@ -11,15 +11,17 @@ module ConcurrencyControls class_attribute :concurrency_group, default: DEFAULT_CONCURRENCY_GROUP, instance_accessor: false class_attribute :concurrency_limit + class_attribute :concurrency_on_duplicate class_attribute :concurrency_duration, default: SolidQueue.default_concurrency_control_period end class_methods do - def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period) + def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period, on_duplicate: :block) self.concurrency_key = key self.concurrency_limit = to self.concurrency_group = group self.concurrency_duration = duration + self.concurrency_on_duplicate = on_duplicate end end diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index 17a658d7..eb1154b6 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -10,6 +10,14 @@ def perform(job_result) end end + class DiscardedNonOverlappingJob < NonOverlappingJob + limits_concurrency key: ->(job_result, **) { job_result }, on_duplicate: :discard + end + + class DiscardedOverlappingJob < NonOverlappingJob + limits_concurrency to: 2, key: ->(job_result, **) { job_result }, on_duplicate: :discard + end + class NonOverlappingGroupedJob1 < NonOverlappingJob limits_concurrency key: ->(job_result, **) { job_result }, group: "MyGroup" end @@ -98,6 +106,40 @@ class NonOverlappingGroupedJob2 < NonOverlappingJob assert_equal active_job.concurrency_key, job.concurrency_key end + test "enqueue jobs with discarding concurrency controls" do + assert_ready do + active_job = DiscardedNonOverlappingJob.perform_later(@result, name: "A") + assert_equal 1, active_job.concurrency_limit + assert_equal "SolidQueue::JobTest::DiscardedNonOverlappingJob/JobResult/#{@result.id}", active_job.concurrency_key + end + + assert_discarded do + active_job = DiscardedNonOverlappingJob.perform_later(@result, name: "A") + assert_equal 1, active_job.concurrency_limit + assert_equal "SolidQueue::JobTest::DiscardedNonOverlappingJob/JobResult/#{@result.id}", active_job.concurrency_key + end + end + + test "enqueue jobs with discarding concurrency controls when below limit" do + assert_ready do + active_job = DiscardedOverlappingJob.perform_later(@result, name: "A") + assert_equal 2, active_job.concurrency_limit + assert_equal "SolidQueue::JobTest::DiscardedOverlappingJob/JobResult/#{@result.id}", active_job.concurrency_key + end + + assert_ready do + active_job = DiscardedOverlappingJob.perform_later(@result, name: "A") + assert_equal 2, active_job.concurrency_limit + assert_equal "SolidQueue::JobTest::DiscardedOverlappingJob/JobResult/#{@result.id}", active_job.concurrency_key + end + + assert_discarded do + active_job = DiscardedOverlappingJob.perform_later(@result, name: "A") + assert_equal 2, active_job.concurrency_limit + assert_equal "SolidQueue::JobTest::DiscardedOverlappingJob/JobResult/#{@result.id}", active_job.concurrency_key + end + end + test "enqueue jobs with concurrency controls in the same concurrency group" do assert_ready do active_job = NonOverlappingGroupedJob1.perform_later(@result, name: "A") @@ -289,8 +331,12 @@ def assert_blocked(&block) assert SolidQueue::Job.last.blocked? end - def assert_job_counts(ready: 0, scheduled: 0, blocked: 0, &block) - assert_difference -> { SolidQueue::Job.count }, +(ready + scheduled + blocked) do + def assert_discarded(&block) + assert_job_counts(discarded: 1, &block) + end + + def assert_job_counts(ready: 0, scheduled: 0, blocked: 0, discarded: 0, &block) + assert_difference -> { SolidQueue::Job.count }, +(ready + scheduled + blocked + discarded) do assert_difference -> { SolidQueue::ReadyExecution.count }, +ready do assert_difference -> { SolidQueue::ScheduledExecution.count }, +scheduled do assert_difference -> { SolidQueue::BlockedExecution.count }, +blocked, &block From e561356edcf8f0684b9c9fd5da4f5296d0e203f5 Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Tue, 25 Feb 2025 02:02:21 -0700 Subject: [PATCH 02/14] Remove 'duplicate' verbiage and use concurrency limits instead, simplify control flow --- app/models/solid_queue/job/concurrency_controls.rb | 11 ++++------- app/models/solid_queue/job/executable.rb | 5 +++-- lib/active_job/concurrency_controls.rb | 6 +++--- test/models/solid_queue/job_test.rb | 4 ++-- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/app/models/solid_queue/job/concurrency_controls.rb b/app/models/solid_queue/job/concurrency_controls.rb index 87b723a0..230f8811 100644 --- a/app/models/solid_queue/job/concurrency_controls.rb +++ b/app/models/solid_queue/job/concurrency_controls.rb @@ -8,7 +8,7 @@ module ConcurrencyControls included do has_one :blocked_execution - delegate :concurrency_limit, :concurrency_on_duplicate, :concurrency_duration, to: :job_class + delegate :concurrency_limit, :concurrency_at_limit, :concurrency_duration, to: :job_class before_destroy :unblock_next_blocked_job, if: -> { concurrency_limited? && ready? } end @@ -34,16 +34,13 @@ def blocked? end private - def duplicate? - Semaphore.at_limit?(self) - end - - def discard_on_duplicate? - concurrency_on_duplicate == :discard && duplicate? + def discard_concurrent? + concurrency_at_limit == :discard end def acquire_concurrency_lock return true unless concurrency_limited? + return false if Semaphore.at_limit?(self) && discard_concurrent? Semaphore.wait(self) end diff --git a/app/models/solid_queue/job/executable.rb b/app/models/solid_queue/job/executable.rb index ba01132f..b5e30499 100644 --- a/app/models/solid_queue/job/executable.rb +++ b/app/models/solid_queue/job/executable.rb @@ -65,8 +65,9 @@ def prepare_for_execution end def dispatch - if discard_on_duplicate? then discard - elsif acquire_concurrency_lock then ready + if acquire_concurrency_lock then ready + elsif discard_concurrent? + discard else block end diff --git a/lib/active_job/concurrency_controls.rb b/lib/active_job/concurrency_controls.rb index 6b0f08e8..76d75dfa 100644 --- a/lib/active_job/concurrency_controls.rb +++ b/lib/active_job/concurrency_controls.rb @@ -11,17 +11,17 @@ module ConcurrencyControls class_attribute :concurrency_group, default: DEFAULT_CONCURRENCY_GROUP, instance_accessor: false class_attribute :concurrency_limit - class_attribute :concurrency_on_duplicate + class_attribute :concurrency_at_limit class_attribute :concurrency_duration, default: SolidQueue.default_concurrency_control_period end class_methods do - def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period, on_duplicate: :block) + def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period, at_limit: :block) self.concurrency_key = key self.concurrency_limit = to self.concurrency_group = group self.concurrency_duration = duration - self.concurrency_on_duplicate = on_duplicate + self.concurrency_at_limit = at_limit end end diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index eb1154b6..f6fddcb0 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -11,11 +11,11 @@ def perform(job_result) end class DiscardedNonOverlappingJob < NonOverlappingJob - limits_concurrency key: ->(job_result, **) { job_result }, on_duplicate: :discard + limits_concurrency key: ->(job_result, **) { job_result }, at_limit: :discard end class DiscardedOverlappingJob < NonOverlappingJob - limits_concurrency to: 2, key: ->(job_result, **) { job_result }, on_duplicate: :discard + limits_concurrency to: 2, key: ->(job_result, **) { job_result }, at_limit: :discard end class NonOverlappingGroupedJob1 < NonOverlappingJob From 90793ad7bb2127a167276a83d066d565eff3959d Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Fri, 7 Mar 2025 13:33:12 -0700 Subject: [PATCH 03/14] Fix race condition vulnerability by changing logic to enqueue --- app/models/solid_queue/job/concurrency_controls.rb | 1 - app/models/solid_queue/job/executable.rb | 3 +-- app/models/solid_queue/semaphore.rb | 12 ------------ 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/app/models/solid_queue/job/concurrency_controls.rb b/app/models/solid_queue/job/concurrency_controls.rb index 230f8811..8cb7b8e5 100644 --- a/app/models/solid_queue/job/concurrency_controls.rb +++ b/app/models/solid_queue/job/concurrency_controls.rb @@ -40,7 +40,6 @@ def discard_concurrent? def acquire_concurrency_lock return true unless concurrency_limited? - return false if Semaphore.at_limit?(self) && discard_concurrent? Semaphore.wait(self) end diff --git a/app/models/solid_queue/job/executable.rb b/app/models/solid_queue/job/executable.rb index b5e30499..33849247 100644 --- a/app/models/solid_queue/job/executable.rb +++ b/app/models/solid_queue/job/executable.rb @@ -66,8 +66,7 @@ def prepare_for_execution def dispatch if acquire_concurrency_lock then ready - elsif discard_concurrent? - discard + elsif discard_concurrent? then discard else block end diff --git a/app/models/solid_queue/semaphore.rb b/app/models/solid_queue/semaphore.rb index 96cce74a..62eeb035 100644 --- a/app/models/solid_queue/semaphore.rb +++ b/app/models/solid_queue/semaphore.rb @@ -10,10 +10,6 @@ def wait(job) Proxy.new(job).wait end - def at_limit?(job) - Proxy.new(job).at_limit? - end - def signal(job) Proxy.new(job).signal end @@ -43,14 +39,6 @@ def initialize(job) @job = job end - def at_limit? - if semaphore = Semaphore.find_by(key: key) - semaphore.value.zero? - else - false - end - end - def wait if semaphore = Semaphore.find_by(key: key) semaphore.value > 0 && attempt_decrement From b8dae8e5d510864214c285f10887580119e983d9 Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Fri, 7 Mar 2025 13:40:02 -0700 Subject: [PATCH 04/14] Add assertions when bulk enqueuing jobs with concurrency controls --- test/models/solid_queue/job_test.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index f6fddcb0..5e2f8920 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -120,6 +120,16 @@ class NonOverlappingGroupedJob2 < NonOverlappingJob end end + test "enqueuing multiple jobs with enqueue_all and concurrency controls" do + jobs = [ + DiscardedNonOverlappingJob.new(@result, name: "A"), + DiscardedNonOverlappingJob.new(@result, name: "A") + ] + + enqueued_jobs_count = SolidQueue::Job.enqueue_all(jobs) + assert_equal enqueued_jobs_count, 1 + end + test "enqueue jobs with discarding concurrency controls when below limit" do assert_ready do active_job = DiscardedOverlappingJob.perform_later(@result, name: "A") From ddb513f1b3b3ecc70034c3c00242cd1b2a8338ca Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Fri, 7 Mar 2025 17:27:43 -0700 Subject: [PATCH 05/14] Dispatch jobs in the order they were enqueued --- app/models/solid_queue/job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/solid_queue/job.rb b/app/models/solid_queue/job.rb index 8574c1ec..137b2d7c 100644 --- a/app/models/solid_queue/job.rb +++ b/app/models/solid_queue/job.rb @@ -49,7 +49,7 @@ def create_from_active_job(active_job) def create_all_from_active_jobs(active_jobs) job_rows = active_jobs.map { |job| attributes_from_active_job(job) } insert_all(job_rows) - where(active_job_id: active_jobs.map(&:job_id)) + where(active_job_id: active_jobs.map(&:job_id)).order(id: :asc) end def attributes_from_active_job(active_job) From a3a70493e7696dc9f022e304ccd30a93bed63d6e Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Sat, 8 Mar 2025 13:22:59 -0700 Subject: [PATCH 06/14] Set ActiveJob successfully_enqueued for both enqueued/blocked and discarded jobs --- app/models/solid_queue/job.rb | 17 ++++++++++------- test/models/solid_queue/job_test.rb | 7 +++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/app/models/solid_queue/job.rb b/app/models/solid_queue/job.rb index 137b2d7c..e83fdeeb 100644 --- a/app/models/solid_queue/job.rb +++ b/app/models/solid_queue/job.rb @@ -10,19 +10,22 @@ class EnqueueError < StandardError; end class << self def enqueue_all(active_jobs) - active_jobs_by_job_id = active_jobs.index_by(&:job_id) + enqueued_jobs_count = 0 transaction do jobs = create_all_from_active_jobs(active_jobs) - prepare_all_for_execution(jobs).tap do |enqueued_jobs| - enqueued_jobs.each do |enqueued_job| - active_jobs_by_job_id[enqueued_job.active_job_id].provider_job_id = enqueued_job.id - active_jobs_by_job_id[enqueued_job.active_job_id].successfully_enqueued = true - end + enqueued_jobs_by_active_job_id = prepare_all_for_execution(jobs).index_by(&:active_job_id) + + active_jobs.each do |active_job| + job = enqueued_jobs_by_active_job_id[active_job.job_id] + active_job.provider_job_id = job&.id + active_job.successfully_enqueued = job.present? end + + enqueued_jobs_count = enqueued_jobs_by_active_job_id.count end - active_jobs.count(&:successfully_enqueued?) + enqueued_jobs_count end def enqueue(active_job, scheduled_at: Time.current) diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index 5e2f8920..d5d3942c 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -122,12 +122,15 @@ class NonOverlappingGroupedJob2 < NonOverlappingJob test "enqueuing multiple jobs with enqueue_all and concurrency controls" do jobs = [ - DiscardedNonOverlappingJob.new(@result, name: "A"), - DiscardedNonOverlappingJob.new(@result, name: "A") + job_1 = DiscardedNonOverlappingJob.new(@result, name: "A"), + job_2 = DiscardedNonOverlappingJob.new(@result, name: "B") ] enqueued_jobs_count = SolidQueue::Job.enqueue_all(jobs) assert_equal enqueued_jobs_count, 1 + + assert job_1.successfully_enqueued? + assert_not job_2.successfully_enqueued? end test "enqueue jobs with discarding concurrency controls when below limit" do From f7a445069cbdd78842ca519e5507decb0d957fa8 Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Sun, 22 Jun 2025 13:59:33 -0600 Subject: [PATCH 07/14] Change concurrency 'at_limit' -> 'on_conflict' --- app/models/solid_queue/job/concurrency_controls.rb | 4 ++-- lib/active_job/concurrency_controls.rb | 6 +++--- test/models/solid_queue/job_test.rb | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/solid_queue/job/concurrency_controls.rb b/app/models/solid_queue/job/concurrency_controls.rb index 8cb7b8e5..6c37cab4 100644 --- a/app/models/solid_queue/job/concurrency_controls.rb +++ b/app/models/solid_queue/job/concurrency_controls.rb @@ -8,7 +8,7 @@ module ConcurrencyControls included do has_one :blocked_execution - delegate :concurrency_limit, :concurrency_at_limit, :concurrency_duration, to: :job_class + delegate :concurrency_limit, :concurrency_on_conflict, :concurrency_duration, to: :job_class before_destroy :unblock_next_blocked_job, if: -> { concurrency_limited? && ready? } end @@ -35,7 +35,7 @@ def blocked? private def discard_concurrent? - concurrency_at_limit == :discard + concurrency_on_conflict == :discard end def acquire_concurrency_lock diff --git a/lib/active_job/concurrency_controls.rb b/lib/active_job/concurrency_controls.rb index 76d75dfa..2ec0b682 100644 --- a/lib/active_job/concurrency_controls.rb +++ b/lib/active_job/concurrency_controls.rb @@ -11,17 +11,17 @@ module ConcurrencyControls class_attribute :concurrency_group, default: DEFAULT_CONCURRENCY_GROUP, instance_accessor: false class_attribute :concurrency_limit - class_attribute :concurrency_at_limit + class_attribute :concurrency_on_conflict class_attribute :concurrency_duration, default: SolidQueue.default_concurrency_control_period end class_methods do - def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period, at_limit: :block) + def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period, on_conflict: :discard) self.concurrency_key = key self.concurrency_limit = to self.concurrency_group = group self.concurrency_duration = duration - self.concurrency_at_limit = at_limit + self.concurrency_on_conflict = on_conflict end end diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index d5d3942c..b5351177 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -11,11 +11,11 @@ def perform(job_result) end class DiscardedNonOverlappingJob < NonOverlappingJob - limits_concurrency key: ->(job_result, **) { job_result }, at_limit: :discard + limits_concurrency key: ->(job_result, **) { job_result }, on_conflict: :discard end class DiscardedOverlappingJob < NonOverlappingJob - limits_concurrency to: 2, key: ->(job_result, **) { job_result }, at_limit: :discard + limits_concurrency to: 2, key: ->(job_result, **) { job_result }, on_conflict: :discard end class NonOverlappingGroupedJob1 < NonOverlappingJob From 88e9ef6f0b375395ff704b55586dbddf15293993 Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Mon, 23 Jun 2025 00:35:38 -0600 Subject: [PATCH 08/14] Update discard logic to trigger an ActiveRecord rollback when attempting dispatch to prevent discarded job creation --- app/models/solid_queue/job.rb | 14 ++-- app/models/solid_queue/job/executable.rb | 14 +++- test/models/solid_queue/job_test.rb | 81 +++++++++++++----------- 3 files changed, 64 insertions(+), 45 deletions(-) diff --git a/app/models/solid_queue/job.rb b/app/models/solid_queue/job.rb index e83fdeeb..df4ab405 100644 --- a/app/models/solid_queue/job.rb +++ b/app/models/solid_queue/job.rb @@ -2,7 +2,7 @@ module SolidQueue class Job < Record - class EnqueueError < StandardError; end + class EnqueueError < ActiveJob::EnqueueError; end include Executable, Clearable, Recurrable @@ -14,15 +14,17 @@ def enqueue_all(active_jobs) transaction do jobs = create_all_from_active_jobs(active_jobs) - enqueued_jobs_by_active_job_id = prepare_all_for_execution(jobs).index_by(&:active_job_id) + prepare_all_for_execution(jobs) + jobs_by_active_job_id = jobs.index_by(&:active_job_id) active_jobs.each do |active_job| - job = enqueued_jobs_by_active_job_id[active_job.job_id] + job = jobs_by_active_job_id[active_job.job_id] + active_job.provider_job_id = job&.id - active_job.successfully_enqueued = job.present? + active_job.enqueue_error = job&.enqueue_error + active_job.successfully_enqueued = job.present? && job.enqueue_error.nil? + enqueued_jobs_count += 1 if active_job.successfully_enqueued? end - - enqueued_jobs_count = enqueued_jobs_by_active_job_id.count end enqueued_jobs_count diff --git a/app/models/solid_queue/job/executable.rb b/app/models/solid_queue/job/executable.rb index 33849247..28c1ac35 100644 --- a/app/models/solid_queue/job/executable.rb +++ b/app/models/solid_queue/job/executable.rb @@ -13,6 +13,8 @@ module Executable after_create :prepare_for_execution + attr_accessor :enqueue_error + scope :finished, -> { where.not(finished_at: nil) } end @@ -37,7 +39,13 @@ def dispatch_all_at_once(jobs) end def dispatch_all_one_by_one(jobs) - jobs.each(&:dispatch) + jobs.each do |job| + begin + job.dispatch + rescue EnqueueError => e + job.enqueue_error = e + end + end end def successfully_dispatched(jobs) @@ -66,7 +74,9 @@ def prepare_for_execution def dispatch if acquire_concurrency_lock then ready - elsif discard_concurrent? then discard + elsif discard_concurrent? + discard + raise EnqueueError.new("Dispatched job discarded due to concurrent configuration.") else block end diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index b5351177..668db0ee 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -28,6 +28,9 @@ class NonOverlappingGroupedJob2 < NonOverlappingJob setup do @result = JobResult.create!(queue_name: "default") + @discarded_concurrent_error = SolidQueue::Job::EnqueueError.new( + "Dispatched job discarded due to concurrent configuration." + ) end test "enqueue active job to be executed right away" do @@ -109,61 +112,64 @@ class NonOverlappingGroupedJob2 < NonOverlappingJob test "enqueue jobs with discarding concurrency controls" do assert_ready do active_job = DiscardedNonOverlappingJob.perform_later(@result, name: "A") - assert_equal 1, active_job.concurrency_limit - assert_equal "SolidQueue::JobTest::DiscardedNonOverlappingJob/JobResult/#{@result.id}", active_job.concurrency_key - end + assert active_job.successfully_enqueued? - assert_discarded do - active_job = DiscardedNonOverlappingJob.perform_later(@result, name: "A") - assert_equal 1, active_job.concurrency_limit - assert_equal "SolidQueue::JobTest::DiscardedNonOverlappingJob/JobResult/#{@result.id}", active_job.concurrency_key + assert_not DiscardedNonOverlappingJob.perform_later(@result, name: "B") do |overlapping_active_job| + assert_not overlapping_active_job.successfully_enqueued? + assert_equal @discarded_concurrent_error, overlapping_active_job.enqueue_error + end end end - test "enqueuing multiple jobs with enqueue_all and concurrency controls" do + test "enqueues jobs in bulk with discarding concurrency controls" do jobs = [ job_1 = DiscardedNonOverlappingJob.new(@result, name: "A"), job_2 = DiscardedNonOverlappingJob.new(@result, name: "B") ] - enqueued_jobs_count = SolidQueue::Job.enqueue_all(jobs) - assert_equal enqueued_jobs_count, 1 + assert_job_counts(ready: 1, discarded: 1) do + enqueued_jobs_count = SolidQueue::Job.enqueue_all(jobs) + assert_equal enqueued_jobs_count, 1 + end assert job_1.successfully_enqueued? assert_not job_2.successfully_enqueued? + assert_equal SolidQueue::Job::EnqueueError, job_2.enqueue_error.class + assert_equal @discarded_concurrent_error.message, job_2.enqueue_error.message end test "enqueue jobs with discarding concurrency controls when below limit" do - assert_ready do - active_job = DiscardedOverlappingJob.perform_later(@result, name: "A") - assert_equal 2, active_job.concurrency_limit - assert_equal "SolidQueue::JobTest::DiscardedOverlappingJob/JobResult/#{@result.id}", active_job.concurrency_key - end + assert_job_counts(ready: 2) do + assert_ready do + active_job = DiscardedOverlappingJob.perform_later(@result, name: "A") + assert active_job.successfully_enqueued? + end - assert_ready do - active_job = DiscardedOverlappingJob.perform_later(@result, name: "A") - assert_equal 2, active_job.concurrency_limit - assert_equal "SolidQueue::JobTest::DiscardedOverlappingJob/JobResult/#{@result.id}", active_job.concurrency_key - end + assert_ready do + active_job = DiscardedOverlappingJob.perform_later(@result, name: "B") + assert active_job.successfully_enqueued? + end - assert_discarded do - active_job = DiscardedOverlappingJob.perform_later(@result, name: "A") - assert_equal 2, active_job.concurrency_limit - assert_equal "SolidQueue::JobTest::DiscardedOverlappingJob/JobResult/#{@result.id}", active_job.concurrency_key + assert_not DiscardedOverlappingJob.perform_later(@result, name: "C") do |overlapping_active_job| + assert_not overlapping_active_job.successfully_enqueued? + assert_equal @discarded_concurrent_error, overlapping_active_job.enqueue_error + end end end test "enqueue jobs with concurrency controls in the same concurrency group" do - assert_ready do - active_job = NonOverlappingGroupedJob1.perform_later(@result, name: "A") - assert_equal 1, active_job.concurrency_limit - assert_equal "MyGroup/JobResult/#{@result.id}", active_job.concurrency_key - end + assert_job_counts(ready: 1) do + assert_ready do + active_job = NonOverlappingGroupedJob1.perform_later(@result, name: "A") + assert_equal 1, active_job.concurrency_limit + assert_equal "MyGroup/JobResult/#{@result.id}", active_job.concurrency_key + end - assert_blocked do - active_job = NonOverlappingGroupedJob2.perform_later(@result, name: "B") - assert_equal 1, active_job.concurrency_limit - assert_equal "MyGroup/JobResult/#{@result.id}", active_job.concurrency_key + assert_not NonOverlappingGroupedJob2.perform_later(@result, name: "B") do |blocked_active_job| + assert_not blocked_active_job.successfully_enqueued? + assert_equal 1, blocked_active_job.concurrency_limit + assert_equal "MyGroup/JobResult/#{@result.id}", blocked_active_job.concurrency_key + end end end @@ -252,10 +258,11 @@ class NonOverlappingGroupedJob2 < NonOverlappingJob end test "release blocked locks when discarding a ready job" do - NonOverlappingJob.perform_later(@result, name: "ready") - NonOverlappingJob.perform_later(@result, name: "blocked") - ready_job, blocked_job = SolidQueue::Job.last(2) - semaphore = SolidQueue::Semaphore.last + ready_job = NonOverlappingJob.perform_later(@result, name: "ready") + blocked_job = NonOverlappingJob.perform_later(@result, name: "blocked") + assert_equal({}, ready_job) + assert_equal({}, blocked_job) + # semaphore = SolidQueue::Semaphore.last assert ready_job.ready? assert blocked_job.blocked? From 58fe247a604d74a473872c76a17a61ca9f321368 Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Mon, 23 Jun 2025 00:46:06 -0600 Subject: [PATCH 09/14] Change default on_conflict concurrency option to old behaviour (blocking execution) --- lib/active_job/concurrency_controls.rb | 2 +- test/models/solid_queue/job_test.rb | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/active_job/concurrency_controls.rb b/lib/active_job/concurrency_controls.rb index 2ec0b682..ceb03e7e 100644 --- a/lib/active_job/concurrency_controls.rb +++ b/lib/active_job/concurrency_controls.rb @@ -16,7 +16,7 @@ module ConcurrencyControls end class_methods do - def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period, on_conflict: :discard) + def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period, on_conflict: :block) self.concurrency_key = key self.concurrency_limit = to self.concurrency_group = group diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index fcf8e43a..1dff8657 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -258,11 +258,10 @@ class NonOverlappingGroupedJob2 < NonOverlappingJob end test "release blocked locks when discarding a ready job" do - ready_job = NonOverlappingJob.perform_later(@result, name: "ready") - blocked_job = NonOverlappingJob.perform_later(@result, name: "blocked") - assert_equal({}, ready_job) - assert_equal({}, blocked_job) - # semaphore = SolidQueue::Semaphore.last + NonOverlappingJob.perform_later(@result, name: "ready") + NonOverlappingJob.perform_later(@result, name: "blocked") + ready_job, blocked_job = SolidQueue::Job.last(2) + semaphore = SolidQueue::Semaphore.last assert ready_job.ready? assert blocked_job.blocked? From d1dc3fb5012ba69fd60fb21254d897a7f383f501 Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Mon, 23 Jun 2025 01:07:18 -0600 Subject: [PATCH 10/14] Add concurrent on_conflict documentation to README --- README.md | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 40cea400..a197d896 100644 --- a/README.md +++ b/README.md @@ -426,11 +426,11 @@ In the case of recurring tasks, if such error is raised when enqueuing the job c ## Concurrency controls -Solid Queue extends Active Job with concurrency controls, that allows you to limit how many jobs of a certain type or with certain arguments can run at the same time. When limited in this way, jobs will be blocked from running, and they'll stay blocked until another job finishes and unblocks them, or after the set expiry time (concurrency limit's _duration_) elapses. Jobs are never discarded or lost, only blocked. +Solid Queue extends Active Job with concurrency controls, that allows you to limit how many jobs of a certain type or with certain arguments can run at the same time. When limited in this way, jobs will be blocked from running, and they'll stay blocked until another job finishes and unblocks them, or after the set expiry time (concurrency limit's _duration_) elapses. Jobs can can be configured to either be discarded or blocked. ```ruby class MyJob < ApplicationJob - limits_concurrency to: max_concurrent_executions, key: ->(arg1, arg2, **) { ... }, duration: max_interval_to_guarantee_concurrency_limit, group: concurrency_group + limits_concurrency to: max_concurrent_executions, key: ->(arg1, arg2, **) { ... }, duration: max_interval_to_guarantee_concurrency_limit, group: concurrency_group, on_conflict: conflict_behaviour # ... ``` @@ -438,6 +438,9 @@ class MyJob < ApplicationJob - `to` is `1` by default. - `duration` is set to `SolidQueue.default_concurrency_control_period` by default, which itself defaults to `3 minutes`, but that you can configure as well. - `group` is used to control the concurrency of different job classes together. It defaults to the job class name. +- `on_conflict` controls behaviour when enqueuing a job which is above the max concurrent executions for your configuration. + - (default) `:block`; the job is blocked and is dispatched until another job completes and unblocks it + - `:discard`; the job is discarded When a job includes these controls, we'll ensure that, at most, the number of jobs (indicated as `to`) that yield the same `key` will be performed concurrently, and this guarantee will last for `duration` for each job enqueued. Note that there's no guarantee about _the order of execution_, only about jobs being performed at the same time (overlapping). @@ -480,6 +483,31 @@ Jobs are unblocked in order of priority but queue order is not taken into accoun Finally, failed jobs that are automatically or manually retried work in the same way as new jobs that get enqueued: they get in the queue for getting an open semaphore, and whenever they get it, they'll be run. It doesn't matter if they had already gotten an open semaphore in the past. +### Discarding conflicting jobs + +When configuring `on_conflict` with `:discard`, jobs enqueued above the concurrent execution limit are discarded and failed to be enqueued. + +```ruby +class ConcurrentJob < ApplicationJob + limits_concurrency key: ->(record) { record }, on_conflict: :discard + + def perform(user); end +end + +enqueued_job = ConcurrentJob.perform_later(record) +# => instance of ConcurrentJob +enqueued_job.successfully_enqueued? +# => true + +second_enqueued_job = ConcurrentJob.perform_later(record) do |job| + job.successfully_enqueued? + # => false +end + +second_enqueued_job +# => false +``` + ### Performance considerations Concurrency controls introduce significant overhead (blocked executions need to be created and promoted to ready, semaphores need to be created and updated) so you should consider carefully whether you need them. For throttling purposes, where you plan to have `limit` significantly larger than 1, I'd encourage relying on a limited number of workers per queue instead. For example: @@ -503,6 +531,10 @@ production: Or something similar to that depending on your setup. You can also assign a different queue to a job on the moment of enqueuing so you can decide whether to enqueue a job in the throttled queue or another queue depending on the arguments, or pass a block to `queue_as` as explained [here](https://guides.rubyonrails.org/active_job_basics.html#queues). +### Discarding concurrent jobs + + + ## Failed jobs and retries Solid Queue doesn't include any automatic retry mechanism, it [relies on Active Job for this](https://edgeguides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs). Jobs that fail will be kept in the system, and a _failed execution_ (a record in the `solid_queue_failed_executions` table) will be created for these. The job will stay there until manually discarded or re-enqueued. You can do this in a console as: From de9bcf52f7d5c6b97388b948afaca1d374134845 Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Mon, 23 Jun 2025 01:14:12 -0600 Subject: [PATCH 11/14] Add test for discarding grouped concurrent jobs --- test/models/solid_queue/job_test.rb | 31 +++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index 1dff8657..23ec008c 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -26,6 +26,14 @@ class NonOverlappingGroupedJob2 < NonOverlappingJob limits_concurrency key: ->(job_result, **) { job_result }, group: "MyGroup" end + class DiscardedNonOverlappingGroupedJob1 < NonOverlappingJob + limits_concurrency key: ->(job_result, **) { job_result }, group: "DiscardingGroup", on_conflict: :discard + end + + class DiscardedNonOverlappingGroupedJob2 < NonOverlappingJob + limits_concurrency key: ->(job_result, **) { job_result }, group: "DiscardingGroup", on_conflict: :discard + end + setup do @result = JobResult.create!(queue_name: "default") @discarded_concurrent_error = SolidQueue::Job::EnqueueError.new( @@ -158,17 +166,32 @@ class NonOverlappingGroupedJob2 < NonOverlappingJob end test "enqueue jobs with concurrency controls in the same concurrency group" do + assert_ready do + active_job = NonOverlappingGroupedJob1.perform_later(@result, name: "A") + assert_equal 1, active_job.concurrency_limit + assert_equal "MyGroup/JobResult/#{@result.id}", active_job.concurrency_key + end + + assert_blocked do + active_job = NonOverlappingGroupedJob2.perform_later(@result, name: "B") + assert_equal 1, active_job.concurrency_limit + assert_equal "MyGroup/JobResult/#{@result.id}", active_job.concurrency_key + end + end + + test "enqueue jobs with discarding concurrency controls in the same concurrency group" do assert_job_counts(ready: 1) do assert_ready do - active_job = NonOverlappingGroupedJob1.perform_later(@result, name: "A") + active_job = DiscardedNonOverlappingGroupedJob1.perform_later(@result, name: "A") + assert active_job.successfully_enqueued? assert_equal 1, active_job.concurrency_limit - assert_equal "MyGroup/JobResult/#{@result.id}", active_job.concurrency_key + assert_equal "DiscardingGroup/JobResult/#{@result.id}", active_job.concurrency_key end - assert_not NonOverlappingGroupedJob2.perform_later(@result, name: "B") do |blocked_active_job| + assert_not DiscardedNonOverlappingGroupedJob2.perform_later(@result, name: "B") do |blocked_active_job| assert_not blocked_active_job.successfully_enqueued? assert_equal 1, blocked_active_job.concurrency_limit - assert_equal "MyGroup/JobResult/#{@result.id}", blocked_active_job.concurrency_key + assert_equal "DiscardingGroup/JobResult/#{@result.id}", blocked_active_job.concurrency_key end end end From af166502b84dfabbccabc8612273bfdbfe6b6eb8 Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Mon, 23 Jun 2025 01:28:04 -0600 Subject: [PATCH 12/14] Fix tests which expect raising enqueue errors --- test/models/solid_queue/job_test.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index 23ec008c..52085112 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -333,13 +333,15 @@ class DiscardedNonOverlappingGroupedJob2 < NonOverlappingJob test "raise EnqueueError when there's an ActiveRecordError" do SolidQueue::Job.stubs(:create!).raises(ActiveRecord::Deadlocked) - active_job = AddToBufferJob.new(1).set(priority: 8, queue: "test") assert_raises SolidQueue::Job::EnqueueError do + active_job = AddToBufferJob.new(1).set(priority: 8, queue: "test") SolidQueue::Job.enqueue(active_job) end - assert_raises SolidQueue::Job::EnqueueError do - AddToBufferJob.perform_later(1) + # #perform_later doesn't raise ActiveJob::EnqueueError, and instead set's successfully_enqueued? to false + assert_not AddToBufferJob.perform_later(1) do |active_job| + assert_not active_job.successfully_enqueued? + assert_equal SolidQueue::Job::EnqueueError, active_job.enqueue_error.class end end From 09152880e680d1d2110c2dba6c3e1d5796116ec6 Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Mon, 23 Jun 2025 02:13:54 -0600 Subject: [PATCH 13/14] Add test to confirm scheduled jobs are also discarded --- test/models/solid_queue/job_test.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index 52085112..cb2fb377 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -129,6 +129,30 @@ class DiscardedNonOverlappingGroupedJob2 < NonOverlappingJob end end + test "enqueue scheduled job with discarding concurrency controls" do + assert_ready do + active_job = DiscardedNonOverlappingJob.perform_later(@result, name: "A") + assert active_job.successfully_enqueued? + end + + scheduled_job_id = nil + + assert_scheduled do + scheduled_active_job = DiscardedNonOverlappingJob.set(wait: 0.5.seconds).perform_later(@result, name: "B") + assert scheduled_active_job.successfully_enqueued? + assert_nil scheduled_active_job.enqueue_error + + scheduled_job_id = scheduled_active_job.provider_job_id + end + + scheduled_job = SolidQueue::Job.find(scheduled_job_id) + wait_for { scheduled_job.due? } + + dispatched = SolidQueue::ScheduledExecution.dispatch_next_batch(10) + assert_equal 0, dispatched + assert_raises(ActiveRecord::RecordNotFound) { scheduled_job.reload } + end + test "enqueues jobs in bulk with discarding concurrency controls" do jobs = [ job_1 = DiscardedNonOverlappingJob.new(@result, name: "A"), From 6af96be8580d37770a2d7c08a3564ab492b0cbdd Mon Sep 17 00:00:00 2001 From: Joel Warrington Date: Mon, 23 Jun 2025 07:39:44 -0600 Subject: [PATCH 14/14] Fix typo Co-authored-by: Philippe Tring --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a197d896..1da184ca 100644 --- a/README.md +++ b/README.md @@ -426,7 +426,7 @@ In the case of recurring tasks, if such error is raised when enqueuing the job c ## Concurrency controls -Solid Queue extends Active Job with concurrency controls, that allows you to limit how many jobs of a certain type or with certain arguments can run at the same time. When limited in this way, jobs will be blocked from running, and they'll stay blocked until another job finishes and unblocks them, or after the set expiry time (concurrency limit's _duration_) elapses. Jobs can can be configured to either be discarded or blocked. +Solid Queue extends Active Job with concurrency controls, that allows you to limit how many jobs of a certain type or with certain arguments can run at the same time. When limited in this way, jobs will be blocked from running, and they'll stay blocked until another job finishes and unblocks them, or after the set expiry time (concurrency limit's _duration_) elapses. Jobs can be configured to either be discarded or blocked. ```ruby class MyJob < ApplicationJob