diff --git a/test/integration/async_processes_lifecycle_test.rb b/test/integration/async_processes_lifecycle_test.rb index 7f2e7007..65a17ad6 100644 --- a/test/integration/async_processes_lifecycle_test.rb +++ b/test/integration/async_processes_lifecycle_test.rb @@ -128,7 +128,7 @@ class AsyncProcessesLifecycleTest < ActiveSupport::TestCase test "term supervisor exceeding timeout while there are jobs in-flight" do no_pause = enqueue_store_result_job("no pause") - pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 10.second) + pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 30.seconds) # Wait for the "no pause" job to complete and the pause job to start. # A claimed execution alone is not enough here because the worker may have @@ -153,8 +153,15 @@ class AsyncProcessesLifecycleTest < ActiveSupport::TestCase assert_completed_job_results("no pause") assert_job_status(no_pause, :finished) - # The pause job should not have completed - assert_not_equal "completed", skip_active_record_query_cache { JobResult.find_by(value: "pause")&.status } + # The pause job should not have completed. Its pause is far longer than this + # test's entire timeline, so a completed result can only come from outside + # the test's own flow — include enough state to tell where it came from. + skip_active_record_query_cache do + assert_not_equal "completed", JobResult.find_by(value: "pause")&.status, + "Expected the pause job not to complete. " \ + "Job results: #{JobResult.all.map { |r| { id: r.id, queue: r.queue_name, status: r.status, value: r.value, updated_at: r.updated_at } }.inspect}; " \ + "registered processes: #{SolidQueue::Process.all.map { |p| { id: p.id, kind: p.kind, pid: p.pid, last_heartbeat_at: p.last_heartbeat_at } }.inspect}" + end # After shutdown, the pause job may be either: # - claimed (exit! called, no cleanup) OR diff --git a/test/integration/concurrency_controls_test.rb b/test/integration/concurrency_controls_test.rb index 9cdb3d65..0f20034c 100644 --- a/test/integration/concurrency_controls_test.rb +++ b/test/integration/concurrency_controls_test.rb @@ -174,7 +174,7 @@ class ConcurrencyControlsTest < ActiveSupport::TestCase end test "don't block claimed executions that get released" do - NonOverlappingUpdateResultJob.perform_later(@result, name: "I'll be released to ready", pause: SolidQueue.shutdown_timeout + 10.seconds) + NonOverlappingUpdateResultJob.perform_later(@result, name: "I'll be released to ready", pause: SolidQueue.shutdown_timeout + 30.seconds) job = SolidQueue::Job.last wait_for(timeout: 2.seconds) { job.reload.claimed? } diff --git a/test/integration/forked_processes_lifecycle_test.rb b/test/integration/forked_processes_lifecycle_test.rb index deb196d3..036ba239 100644 --- a/test/integration/forked_processes_lifecycle_test.rb +++ b/test/integration/forked_processes_lifecycle_test.rb @@ -124,7 +124,7 @@ class ForkedProcessesLifecycleTest < ActiveSupport::TestCase test "term supervisor exceeding timeout while there are jobs in-flight" do no_pause = enqueue_store_result_job("no pause") - pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 10.seconds) + pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 30.seconds) wait_while_with_timeout(5.seconds) { SolidQueue::ReadyExecution.joins(:job).exists?(solid_queue_jobs: { active_job_id: pause.job_id }) @@ -238,12 +238,15 @@ class ForkedProcessesLifecycleTest < ActiveSupport::TestCase end test "kill worker individually" do - killed_pause = enqueue_store_result_job("killed_pause", pause: 2.seconds) + killed_pause = enqueue_store_result_job("killed_pause", pause: 5.seconds) enqueue_store_result_job("pause", :default, pause: 0.5.seconds) wait_for_jobs_to_finish_for(1.second, except: [ killed_pause ]) - # Ensure the long job has written its "started" row before we SIGKILL the worker. - wait_while_with_timeout(2.seconds) do + # Ensure the long job has written its "started" row before we SIGKILL the + # worker. Raise if it never does: proceeding would kill the worker before + # it starts the job, so no "started" row would ever exist and the test + # would fail later, in a way much harder to trace back here. + wait_while_with_timeout!(10.seconds) do JobResult.where(status: "started", value: "killed_pause").none? end