9b62f88a2f
Sidekiq has a useful optimisation called `push_bulk` that enqueues many jobs at once, eliminating the repeated Redis roundtrips. However, this feature is not exposed through Active Job, so it only works for `Sidekiq::Worker` jobs. This adds a barrier to Active Job adoption for apps that rely on this feature. It also makes it harder for other queue adapters to implement similar functionality, as they then have to take care of serialization, callbacks, etc. themselves. This commit adds `ActiveJob.perform_all_later(<job1>, <job2>)`, backed by Sidekiq's `push_bulk` and with a fallback to enqueuing serially if the queue adapter does not support bulk enqueue. The performance benefit for 1000 jobs can be more than an order of magnitude: | Enqueue type | Serial time (ms) | Bulk time (ms) | Speedup | | ------------------ | ---------------- | -------------- | ------- | | Raw Sidekiq | 2661 | 119 | 22x | | Active Job Sidekiq | 2853 | 208 | 14x | (Measured in a simple test app in our production environment.) Instrumentation for perform_all_later uses a new event `enqueue_all.active_job`
74 lines
2.4 KiB
Ruby
74 lines
2.4 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require "helper"
|
|
require "jobs/hello_job"
|
|
require "jobs/enqueue_error_job"
|
|
require "jobs/multiple_kwargs_job"
|
|
require "active_support/core_ext/numeric/time"
|
|
|
|
class QueuingTest < ActiveSupport::TestCase
|
|
setup do
|
|
JobBuffer.clear
|
|
end
|
|
|
|
test "run queued job" do
|
|
HelloJob.perform_later
|
|
assert_equal "David says hello", JobBuffer.last_value
|
|
end
|
|
|
|
test "run queued job with arguments" do
|
|
HelloJob.perform_later "Jamie"
|
|
assert_equal "Jamie says hello", JobBuffer.last_value
|
|
end
|
|
|
|
test "run queued job later" do
|
|
result = HelloJob.set(wait_until: 1.second.ago).perform_later "Jamie"
|
|
assert result
|
|
rescue NotImplementedError
|
|
skip
|
|
end
|
|
|
|
test "job returned by enqueue has the arguments available" do
|
|
job = HelloJob.perform_later "Jamie"
|
|
assert_equal [ "Jamie" ], job.arguments
|
|
end
|
|
|
|
test "job returned by perform_at has the timestamp available" do
|
|
job = HelloJob.set(wait_until: Time.utc(2014, 1, 1)).perform_later
|
|
assert_equal Time.utc(2014, 1, 1).to_f, job.scheduled_at
|
|
rescue NotImplementedError
|
|
skip
|
|
end
|
|
|
|
test "job is yielded to block after enqueue with successfully_enqueued property set" do
|
|
HelloJob.perform_later "John" do |job|
|
|
assert_equal "John says hello", JobBuffer.last_value
|
|
assert_equal [ "John" ], job.arguments
|
|
assert_equal true, job.successfully_enqueued?
|
|
assert_nil job.enqueue_error
|
|
end
|
|
end
|
|
|
|
test "when enqueuing raises an EnqueueError job is yielded to block with error set on job" do
|
|
EnqueueErrorJob.perform_later do |job|
|
|
assert_equal false, job.successfully_enqueued?
|
|
assert_equal ActiveJob::EnqueueError, job.enqueue_error.class
|
|
end
|
|
end
|
|
|
|
test "run multiple queued jobs" do
|
|
ActiveJob.perform_all_later(HelloJob.new("Jamie"), HelloJob.new("John"))
|
|
assert_equal ["Jamie says hello", "John says hello"], JobBuffer.values.sort
|
|
end
|
|
|
|
test "run multiple queued jobs passed as array" do
|
|
ActiveJob.perform_all_later([HelloJob.new("Jamie"), HelloJob.new("John")])
|
|
assert_equal ["Jamie says hello", "John says hello"], JobBuffer.values.sort
|
|
end
|
|
|
|
test "run multiple queued jobs of different classes" do
|
|
ActiveJob.perform_all_later([HelloJob.new("Jamie"), MultipleKwargsJob.new(argument1: "John", argument2: 42)])
|
|
assert_equal ["Jamie says hello", "Job with argument1: John, argument2: 42"], JobBuffer.values.sort
|
|
end
|
|
end
|