Give in-flight jobs enough pause to survive slow shutdowns in tests

test/integration/async_processes_lifecycle_test.rb

@@ -128,7 +128,7 @@ class AsyncProcessesLifecycleTest < ActiveSupport::TestCase
 
   test "term supervisor exceeding timeout while there are jobs in-flight" do
     no_pause = enqueue_store_result_job("no pause")
-    pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 10.second)
+    pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 30.seconds)
 
     # Wait for the "no pause" job to complete and the pause job to start.
     # A claimed execution alone is not enough here because the worker may have
@@ -153,8 +153,15 @@ class AsyncProcessesLifecycleTest < ActiveSupport::TestCase
     assert_completed_job_results("no pause")
     assert_job_status(no_pause, :finished)
 
-    # The pause job should not have completed
-    assert_not_equal "completed", skip_active_record_query_cache { JobResult.find_by(value: "pause")&.status }
+    # The pause job should not have completed. Its pause is far longer than this
+    # test's entire timeline, so a completed result can only come from outside
+    # the test's own flow — include enough state to tell where it came from.
+    skip_active_record_query_cache do
+      assert_not_equal "completed", JobResult.find_by(value: "pause")&.status,
+        "Expected the pause job not to complete. " \
+        "Job results: #{JobResult.all.map { |r| { id: r.id, queue: r.queue_name, status: r.status, value: r.value, updated_at: r.updated_at } }.inspect}; " \
+        "registered processes: #{SolidQueue::Process.all.map { |p| { id: p.id, kind: p.kind, pid: p.pid, last_heartbeat_at: p.last_heartbeat_at } }.inspect}"
+    end
 
     # After shutdown, the pause job may be either:
     # - claimed (exit! called, no cleanup) OR

test/integration/concurrency_controls_test.rb

@@ -174,7 +174,7 @@ class ConcurrencyControlsTest < ActiveSupport::TestCase
   end
 
   test "don't block claimed executions that get released" do
-    NonOverlappingUpdateResultJob.perform_later(@result, name: "I'll be released to ready", pause: SolidQueue.shutdown_timeout + 10.seconds)
+    NonOverlappingUpdateResultJob.perform_later(@result, name: "I'll be released to ready", pause: SolidQueue.shutdown_timeout + 30.seconds)
     job = SolidQueue::Job.last
 
     wait_for(timeout: 2.seconds) { job.reload.claimed? }

test/integration/forked_processes_lifecycle_test.rb

@@ -124,7 +124,7 @@ class ForkedProcessesLifecycleTest < ActiveSupport::TestCase
 
   test "term supervisor exceeding timeout while there are jobs in-flight" do
     no_pause = enqueue_store_result_job("no pause")
-    pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 10.seconds)
+    pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 30.seconds)
 
     wait_while_with_timeout(5.seconds) {
       SolidQueue::ReadyExecution.joins(:job).exists?(solid_queue_jobs: { active_job_id: pause.job_id })
@@ -238,12 +238,15 @@ class ForkedProcessesLifecycleTest < ActiveSupport::TestCase
   end
 
   test "kill worker individually" do
-    killed_pause = enqueue_store_result_job("killed_pause", pause: 2.seconds)
+    killed_pause = enqueue_store_result_job("killed_pause", pause: 5.seconds)
     enqueue_store_result_job("pause", :default, pause: 0.5.seconds)
 
     wait_for_jobs_to_finish_for(1.second, except: [ killed_pause ])
-    # Ensure the long job has written its "started" row before we SIGKILL the worker.
-    wait_while_with_timeout(2.seconds) do
+    # Ensure the long job has written its "started" row before we SIGKILL the
+    # worker. Raise if it never does: proceeding would kill the worker before
+    # it starts the job, so no "started" row would ever exist and the test
+    # would fail later, in a way much harder to trace back here.
+    wait_while_with_timeout!(10.seconds) do
       JobResult.where(status: "started", value: "killed_pause").none?
     end