diff --git a/ruby/README.md b/ruby/README.md index 92ba5b9..7b3fc7f 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -38,6 +38,19 @@ minitest-queue --queue redis://example.com run -Itest test/**/*_test.rb Additionally you can configure the requeue settings (see main README) with `--max-requeues` and `--requeue-tolerance`. +To recover artifacts lost with a retried distributed worker, replay every test +reserved by that worker before resuming the shared queue: + +```bash +minitest-queue --queue redis://example.com \ + --retry-mode worker-history \ + --recovery-manifest log/ci-queue-recovery.json \ + run -Itest test/**/*_test.rb +``` + +This mode requires the retry to retain its worker ID and queue build ID. It +fails when the worker's reservation history or suite chunk metadata is missing. + If you'd like to centralize the error reporting you can do so with: diff --git a/ruby/lib/ci/queue.rb b/ruby/lib/ci/queue.rb index a38aaf6..31236d8 100644 --- a/ruby/lib/ci/queue.rb +++ b/ruby/lib/ci/queue.rb @@ -10,6 +10,7 @@ require 'ci/queue/common' require 'ci/queue/build_record' require 'ci/queue/static' +require 'ci/queue/worker_history_recovery' require 'ci/queue/file' require 'ci/queue/grind' require 'ci/queue/bisect' diff --git a/ruby/lib/ci/queue/configuration.rb b/ruby/lib/ci/queue/configuration.rb index 9e9c133..3985908 100644 --- a/ruby/lib/ci/queue/configuration.rb +++ b/ruby/lib/ci/queue/configuration.rb @@ -15,6 +15,7 @@ class Configuration attr_accessor :timing_redis_url attr_accessor :write_duration_averages attr_accessor :heartbeat_grace_period, :heartbeat_interval + attr_accessor :retry_mode, :recovery_manifest, :retry_count attr_reader :circuit_breakers attr_writer :seed, :build_id attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout @@ -30,6 +31,7 @@ def from_env(env) redis_ttl: env['CI_QUEUE_REDIS_TTL']&.to_i || 8 * 60 * 60, known_flaky_tests: load_known_flaky_tests(env['CI_QUEUE_KNOWN_FLAKY_TESTS']), branch: env['BUILDKITE_BRANCH'], + retry_count: env['BUILDKITE_RETRY_COUNT']&.to_i || 0, ) end @@ -66,7 +68,10 @@ def initialize( branch: nil, timing_redis_url: nil, heartbeat_grace_period: 30, - heartbeat_interval: 10 + heartbeat_interval: 10, + retry_mode: :failures, + recovery_manifest: nil, + retry_count: 0 ) @build_id = build_id @circuit_breakers = [CircuitBreaker::Disabled] @@ -105,6 +110,9 @@ def initialize( @write_duration_averages = false @heartbeat_grace_period = heartbeat_grace_period @heartbeat_interval = heartbeat_interval + @retry_mode = retry_mode + @recovery_manifest = recovery_manifest + @retry_count = retry_count end def queue_init_timeout diff --git a/ruby/lib/ci/queue/redis.rb b/ruby/lib/ci/queue/redis.rb index c3876ef..6602e14 100644 --- a/ruby/lib/ci/queue/redis.rb +++ b/ruby/lib/ci/queue/redis.rb @@ -18,6 +18,7 @@ module Queue module Redis Error = Class.new(StandardError) LostMaster = Class.new(Error) + WorkerHistoryError = Class.new(Error) class << self diff --git a/ruby/lib/ci/queue/redis/worker.rb b/ruby/lib/ci/queue/redis/worker.rb index 7de1f17..6f2cd9d 100644 --- a/ruby/lib/ci/queue/redis/worker.rb +++ b/ruby/lib/ci/queue/redis/worker.rb @@ -16,6 +16,15 @@ class << self self.max_sleep_time = 2 class Worker < Base + class WorkerHistory + attr_reader :history_items, :test_ids + + def initialize(history_items:, test_ids:) + @history_items = history_items + @test_ids = test_ids.freeze + end + end + DEFAULT_SLEEP_SECONDS = 0.5 attr_reader :total @@ -154,6 +163,25 @@ def retry_queue Retry.new(log, config, redis: redis) end + def worker_history + reservations = redis.lrange(key('worker', worker_id, 'queue'), 0, -1) + if reservations.empty? + raise WorkerHistoryError, "Reservation history is missing for worker #{worker_id}" + end + + seen = {} + test_ids = reservations.reverse_each.each_with_object([]) do |reservation_id, ids| + expand_reservation(reservation_id).each do |test_id| + next if seen[test_id] + + seen[test_id] = true + ids << test_id + end + end + + WorkerHistory.new(history_items: reservations.size, test_ids: test_ids) + end + def supervisor Supervisor.new(redis_url, config) end @@ -525,6 +553,24 @@ def chunk_id?(id) id.include?(':chunk_') end + def expand_reservation(id) + return [id] unless chunk_id?(id) + + chunk_json = redis.get(key('chunk', id)) + unless chunk_json + raise WorkerHistoryError, "Chunk metadata is missing for #{id}" + end + + test_ids = CI::Queue::TestChunk.from_json(id, chunk_json).test_ids + if test_ids.empty? + raise WorkerHistoryError, "Chunk metadata contains no tests for #{id}" + end + + test_ids + rescue JSON::ParserError => error + raise WorkerHistoryError, "Chunk metadata is invalid for #{id}: #{error.message}" + end + def resolve_executable(id) # Detect chunk by ID pattern if chunk_id?(id) diff --git a/ruby/lib/ci/queue/worker_history_recovery.rb b/ruby/lib/ci/queue/worker_history_recovery.rb new file mode 100644 index 0000000..eac374d --- /dev/null +++ b/ruby/lib/ci/queue/worker_history_recovery.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +module CI + module Queue + class WorkerHistoryRecovery + attr_reader :config, :history_items, :replayed_tests + + def initialize(shared_queue, history) + @shared_queue = shared_queue + @config = shared_queue.config + @history_items = history.history_items + @replay_ids = history.test_ids.dup + @replayed_tests = history.test_ids.size + @replay_completed = false + @resumed_shared_queue = false + @replay_failures = 0 + @shutdown_required = false + @phase = :replay + end + + def distributed? + true + end + + def populate(tests, random: Random.new) + @index = tests.map { |test| [test.id, test] }.to_h + shared_queue.populate(tests, random: random) + self + end + + def populated? + defined?(@index) && shared_queue.populated? + end + + def poll(&block) + while replaying? && replay_allowed? && (id = @replay_ids.shift) + block.call(index.fetch(id)) + end + + return unless @replay_ids.empty? + return unless replay_allowed? + + @replay_completed = true + @resumed_shared_queue = true + @phase = :shared + shared_queue.poll(&block) + end + + def replay_completed? + @replay_completed + end + + def resumed_shared_queue? + @resumed_shared_queue + end + + def acknowledge(test) + return shared_queue.acknowledge(test) unless replaying? + + true + end + + def requeue(test, **options) + return false if replaying? + + if options.empty? + shared_queue.requeue(test) + else + shared_queue.requeue(test, **options) + end + end + + def increment_test_failed + if replaying? + @replay_failures += 1 + else + shared_queue.increment_test_failed + end + end + + def test_failed + replaying? ? @replay_failures : shared_queue.test_failed + end + + def max_test_failed? + return false if config.max_test_failed.nil? + + test_failed >= config.max_test_failed + end + + def exhausted? + @replay_ids.empty? && shared_queue.exhausted? + end + + def size + @replay_ids.size + shared_queue.size + end + + def total + shared_queue.total + end + + def progress + shared_queue.progress + end + + def to_a + @replay_ids.map { |id| index.fetch(id) } + shared_queue.to_a + end + + def build + shared_queue.build + end + + def supervisor + shared_queue.supervisor + end + + def retrying? + true + end + + def retry_queue + self + end + + def expired? + shared_queue.expired? + end + + def created_at=(timestamp) + shared_queue.created_at = timestamp + end + + def release! + shared_queue.release! + end + + def shutdown! + @shutdown_required = true + shared_queue.shutdown! + end + + def flaky?(test) + shared_queue.flaky?(test) + end + + def report_failure! + shared_queue.report_failure! + end + + def report_success! + shared_queue.report_success! + end + + def rescue_connection_errors(handler = ->(_error) { nil }, &block) + shared_queue.rescue_connection_errors(handler, &block) + end + + private + + attr_reader :index, :shared_queue + + def replaying? + @phase == :replay + end + + def replay_allowed? + !@shutdown_required && config.circuit_breakers.none?(&:open?) && !max_test_failed? + end + end + end +end diff --git a/ruby/lib/minitest/queue.rb b/ruby/lib/minitest/queue.rb index 9b0770c..59a5487 100644 --- a/ruby/lib/minitest/queue.rb +++ b/ruby/lib/minitest/queue.rb @@ -15,6 +15,7 @@ require 'minitest/queue/grind_reporter' require 'minitest/queue/test_time_recorder' require 'minitest/queue/test_time_reporter' +require 'minitest/queue/recovery_reporter' module Minitest class Requeue < Skip diff --git a/ruby/lib/minitest/queue/recovery_reporter.rb b/ruby/lib/minitest/queue/recovery_reporter.rb new file mode 100644 index 0000000..7ab9fd3 --- /dev/null +++ b/ruby/lib/minitest/queue/recovery_reporter.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'json' +require 'minitest/reporters' + +module Minitest + module Queue + class RecoveryReporter < Minitest::Reporters::BaseReporter + def initialize(path:, queue:, config:) + super({}) + @path = path + @queue = queue + @config = config + end + + def report + super + return unless queue.exhausted? + return if recovery? && !queue.replay_completed? + + FileUtils.mkdir_p(File.dirname(path)) + File.write(path, JSON.pretty_generate(manifest)) + end + + private + + attr_reader :config, :path, :queue + + def recovery? + queue.is_a?(CI::Queue::WorkerHistoryRecovery) + end + + def manifest + { + schema_version: 1, + worker_id: config.worker_id.to_s, + retry_count: config.retry_count, + history_items: recovery? ? queue.history_items : 0, + replayed_tests: recovery? ? queue.replayed_tests : 0, + resumed_shared_queue: recovery? && queue.resumed_shared_queue?, + replay_completed: !recovery? || queue.replay_completed? + } + end + end + end +end diff --git a/ruby/lib/minitest/queue/runner.rb b/ruby/lib/minitest/queue/runner.rb index cc7c204..70359d6 100644 --- a/ruby/lib/minitest/queue/runner.rb +++ b/ruby/lib/minitest/queue/runner.rb @@ -54,12 +54,16 @@ def run_command abort! "The test run is too old and can't be retried" end reset_counters - retry_queue = queue.retry_queue - if retry_queue.exhausted? - puts "The retry queue does not contain any failure, we'll process the main queue instead." + if worker_history_retry? + recover_worker_history else - puts "Retrying failed tests." - self.queue = retry_queue + retry_queue = queue.retry_queue + if retry_queue.exhausted? + puts "The retry queue does not contain any failure, we'll process the main queue instead." + else + puts "Retrying failed tests." + self.queue = retry_queue + end end end @@ -81,6 +85,13 @@ def run_command if queue_config.statsd_endpoint reporters << Minitest::Reporters::StatsdReporter.new(statsd_endpoint: queue_config.statsd_endpoint) end + if queue_config.recovery_manifest + reporters << RecoveryReporter.new( + path: queue_config.recovery_manifest, + queue: queue, + config: queue_config + ) + end Minitest.queue_reporters = reporters trap('TERM') { Minitest.queue.shutdown! } @@ -324,6 +335,22 @@ def reset_counters queue.build.reset_stats(BuildStatusRecorder::COUNTERS) end + def recover_worker_history + unless queue.distributed? && queue.respond_to?(:worker_history) + abort! 'Worker history recovery requires a distributed Redis queue' + end + + history = queue.worker_history + puts "Replaying #{history.test_ids.size} tests from #{history.history_items} worker reservations." + self.queue = CI::Queue::WorkerHistoryRecovery.new(queue, history) + rescue CI::Queue::Redis::WorkerHistoryError => error + abort! error.message + end + + def worker_history_retry? + queue_config.retry_mode == :worker_history + end + def populate_queue Minitest.queue.populate(Minitest.loaded_tests, random: ordering_seed, &:id) end @@ -494,6 +521,22 @@ def parser queue_config.worker_id = worker_id end + help = <<~EOS + Retry behavior: failures (default) or worker-history. + EOS + opts.separator "" + opts.on('--retry-mode MODE', %w[failures worker-history], help) do |mode| + queue_config.retry_mode = mode.tr('-', '_').to_sym + end + + help = <<~EOS + Write worker-history recovery details to a JSON file after a completed run. + EOS + opts.separator "" + opts.on('--recovery-manifest PATH', help) do |path| + queue_config.recovery_manifest = path + end + help = <<~EOS Defines how many time a single test can be requeued. Defaults to 0. diff --git a/ruby/test/ci/queue/configuration_test.rb b/ruby/test/ci/queue/configuration_test.rb index 64d7410..35a7ccd 100644 --- a/ruby/test/ci/queue/configuration_test.rb +++ b/ruby/test/ci/queue/configuration_test.rb @@ -33,10 +33,20 @@ def test_buildkite_defaults 'BUILDKITE_BUILD_ID' => '9e08ef3c-d6e6-4a86-91dd-577ce5205b8e', 'BUILDKITE_PARALLEL_JOB' => '12', 'BUILDKITE_COMMIT' => 'faa647bbb8168a77cf338e7488c3f8445c3e6554', + 'BUILDKITE_RETRY_COUNT' => '2', ) assert_equal '9e08ef3c-d6e6-4a86-91dd-577ce5205b8e', config.build_id assert_equal '12', config.worker_id assert_equal 'faa647bbb8168a77cf338e7488c3f8445c3e6554', config.seed + assert_equal 2, config.retry_count + end + + def test_retry_defaults + config = Configuration.new + + assert_equal :failures, config.retry_mode + assert_equal 0, config.retry_count + assert_nil config.recovery_manifest end def test_travis_defaults diff --git a/ruby/test/ci/queue/redis/worker_chunk_test.rb b/ruby/test/ci/queue/redis/worker_chunk_test.rb index e0b2592..8e2f974 100644 --- a/ruby/test/ci/queue/redis/worker_chunk_test.rb +++ b/ruby/test/ci/queue/redis/worker_chunk_test.rb @@ -243,6 +243,49 @@ def test_populate_with_many_chunks_uses_batching end end + def test_worker_history_expands_chunks_and_deduplicates_tests + tests = create_mock_tests(['TestA#test_1', 'TestA#test_2', 'TestB#test_1']) + chunk = CI::Queue::TestChunk.new( + 'TestA:chunk_0', + 'TestA', + ['TestA#test_1', 'TestA#test_2'], + 2000.0 + ) + + @worker.stub(:reorder_tests, [chunk, tests.last]) do + @worker.populate(tests) + end + + history_key = 'build:42:worker:1:queue' + @redis.del(history_key) + [chunk.id, tests.last.id, chunk.id, tests[1].id].each do |id| + @redis.lpush(history_key, id) + end + + history = @worker.worker_history + + assert_equal 4, history.history_items + assert_equal ['TestA#test_1', 'TestA#test_2', 'TestB#test_1'], history.test_ids + end + + def test_worker_history_requires_reservations + error = assert_raises(CI::Queue::Redis::WorkerHistoryError) do + @worker.worker_history + end + + assert_equal 'Reservation history is missing for worker 1', error.message + end + + def test_worker_history_requires_chunk_metadata + @redis.lpush('build:42:worker:1:queue', 'TestA:chunk_0') + + error = assert_raises(CI::Queue::Redis::WorkerHistoryError) do + @worker.worker_history + end + + assert_equal 'Chunk metadata is missing for TestA:chunk_0', error.message + end + private def create_mock_tests(test_ids) diff --git a/ruby/test/ci/queue/worker_history_recovery_test.rb b/ruby/test/ci/queue/worker_history_recovery_test.rb new file mode 100644 index 0000000..050f391 --- /dev/null +++ b/ruby/test/ci/queue/worker_history_recovery_test.rb @@ -0,0 +1,201 @@ +# frozen_string_literal: true + +require 'test_helper' + +class CI::Queue::WorkerHistoryRecoveryTest < Minitest::Test + Test = Struct.new(:id) + History = Struct.new(:history_items, :test_ids) + + def test_replays_history_then_resumes_shared_queue + shared_queue = SharedQueue.new(['TestC#test_1']) + history = History.new(3, ['TestA#test_1', 'TestB#test_1']) + queue = CI::Queue::WorkerHistoryRecovery.new(shared_queue, history) + queue.populate(tests('TestA#test_1', 'TestB#test_1', 'TestC#test_1')) + + order = [] + queue.poll do |test| + order << test.id + queue.acknowledge(test) + end + + assert_equal ['TestA#test_1', 'TestB#test_1', 'TestC#test_1'], order + assert_equal ['TestC#test_1'], shared_queue.acknowledged + assert queue.replay_completed? + assert queue.resumed_shared_queue? + assert_equal 3, queue.history_items + assert_equal 2, queue.replayed_tests + end + + def test_replay_requeue_does_not_mutate_shared_queue + shared_queue = SharedQueue.new(['TestB#test_1']) + history = History.new(1, ['TestA#test_1']) + queue = CI::Queue::WorkerHistoryRecovery.new(shared_queue, history) + queue.populate(tests('TestA#test_1', 'TestB#test_1')) + + queue.poll do |test| + queue.requeue(test) if test.id == 'TestA#test_1' + queue.acknowledge(test) + end + + assert_equal [], shared_queue.requeued + assert_equal ['TestB#test_1'], shared_queue.acknowledged + end + + def test_failed_replay_does_not_resume_shared_queue + config = CI::Queue::Configuration.new(max_consecutive_failures: 1) + shared_queue = SharedQueue.new(['TestC#test_1'], config: config) + history = History.new(2, ['TestA#test_1', 'TestB#test_1']) + queue = CI::Queue::WorkerHistoryRecovery.new(shared_queue, history) + queue.populate(tests('TestA#test_1', 'TestB#test_1', 'TestC#test_1')) + + order = [] + queue.poll do |test| + order << test.id + queue.report_failure! + queue.acknowledge(test) + end + + assert_equal ['TestA#test_1'], order + refute queue.replay_completed? + refute queue.resumed_shared_queue? + assert_equal [], shared_queue.acknowledged + end + + def test_circuit_breaker_on_last_replay_does_not_resume_shared_queue + config = CI::Queue::Configuration.new(max_consecutive_failures: 1) + shared_queue = SharedQueue.new(['TestB#test_1'], config: config) + history = History.new(1, ['TestA#test_1']) + queue = CI::Queue::WorkerHistoryRecovery.new(shared_queue, history) + queue.populate(tests('TestA#test_1', 'TestB#test_1')) + + order = [] + queue.poll do |test| + order << test.id + queue.report_failure! + queue.acknowledge(test) + end + + assert_equal ['TestA#test_1'], order + refute queue.replay_completed? + refute queue.resumed_shared_queue? + assert_equal [], shared_queue.acknowledged + end + + def test_max_failures_on_last_replay_does_not_resume_shared_queue + config = CI::Queue::Configuration.new(max_test_failed: 1) + shared_queue = SharedQueue.new(['TestB#test_1'], config: config) + history = History.new(1, ['TestA#test_1']) + queue = CI::Queue::WorkerHistoryRecovery.new(shared_queue, history) + queue.populate(tests('TestA#test_1', 'TestB#test_1')) + + order = [] + queue.poll do |test| + order << test.id + queue.increment_test_failed + queue.acknowledge(test) + end + + assert_equal ['TestA#test_1'], order + assert_equal 1, queue.test_failed + refute queue.replay_completed? + refute queue.resumed_shared_queue? + assert_equal [], shared_queue.acknowledged + end + + def test_missing_replay_test_fails_before_resuming_shared_queue + shared_queue = SharedQueue.new(['TestB#test_1']) + history = History.new(1, ['MissingTest#test_1']) + queue = CI::Queue::WorkerHistoryRecovery.new(shared_queue, history) + queue.populate(tests('TestB#test_1')) + + assert_raises(KeyError) do + queue.poll { |test| queue.acknowledge(test) } + end + + refute queue.replay_completed? + refute queue.resumed_shared_queue? + assert_equal [], shared_queue.acknowledged + end + + private + + def tests(*ids) + ids.map { |id| Test.new(id) } + end + + class SharedQueue + attr_reader :acknowledged, :config, :requeued, :total + + def initialize(ids, config: CI::Queue::Configuration.new) + @ids = ids + @config = config + @acknowledged = [] + @requeued = [] + @total = ids.size + end + + def populate(tests, **_options) + @index = tests.map { |test| [test.id, test] }.to_h + self + end + + def populated? + defined?(@index) + end + + def poll + @ids.each { |id| yield @index.fetch(id) } + @ids.clear + end + + def acknowledge(test) + @acknowledged << test.id + true + end + + def requeue(test, **) + @requeued << test.id + true + end + + def increment_test_failed + @test_failed = test_failed + 1 + end + + def test_failed + @test_failed ||= 0 + end + + def exhausted? + @ids.empty? + end + + def size + @ids.size + end + + def progress + total - size + end + + def to_a + @ids.map { |id| @index.fetch(id) } + end + + def build + @build ||= CI::Queue::BuildRecord.new(self) + end + + def report_failure! + config.circuit_breakers.each(&:report_failure!) + end + + def report_success! + config.circuit_breakers.each(&:report_success!) + end + + def flaky?(_test) + false + end + end +end diff --git a/ruby/test/integration/minitest_redis_test.rb b/ruby/test/integration/minitest_redis_test.rb index ab18673..70fe629 100644 --- a/ruby/test/integration/minitest_redis_test.rb +++ b/ruby/test/integration/minitest_redis_test.rb @@ -220,6 +220,50 @@ def test_retry_success assert_equal 'All tests were ran already', output end + def test_worker_history_retry_writes_recovery_manifest + Dir.mktmpdir do |directory| + manifest_path = File.join(directory, 'recovery.json') + run_worker_history_worker(manifest_path, retry_count: 0) + + first_manifest = JSON.parse(File.read(manifest_path)) + assert_equal 0, first_manifest['retry_count'] + assert_equal 0, first_manifest['history_items'] + assert_equal 0, first_manifest['replayed_tests'] + refute first_manifest['resumed_shared_queue'] + assert first_manifest['replay_completed'] + + out, = run_worker_history_worker(manifest_path, retry_count: 1) + + assert_includes out, 'Replaying 100 tests from 100 worker reservations.' + retry_manifest = JSON.parse(File.read(manifest_path)) + assert_equal 1, retry_manifest['schema_version'] + assert_equal '1', retry_manifest['worker_id'] + assert_equal 1, retry_manifest['retry_count'] + assert_equal 100, retry_manifest['history_items'] + assert_equal 100, retry_manifest['replayed_tests'] + assert retry_manifest['resumed_shared_queue'] + assert retry_manifest['replay_completed'] + end + end + + def test_worker_history_retry_fails_without_worker_reservations + run_worker_history_worker(nil, retry_count: 0, build_id: 'missing-history', worker_id: '1') + + Dir.mktmpdir do |directory| + manifest_path = File.join(directory, 'recovery.json') + out, = run_worker_history_worker( + manifest_path, + retry_count: 1, + build_id: 'missing-history', + worker_id: '2' + ) + + refute_predicate $?, :success? + assert_includes out, 'Reservation history is missing for worker 2' + refute_path_exists manifest_path + end + end + def test_retry_fails_when_test_run_is_expired out, err = capture_subprocess_io do system( @@ -788,6 +832,28 @@ def test_utf8_tests_and_marshal private + def run_worker_history_worker(manifest_path, retry_count:, build_id: 'worker-history', worker_id: '1') + args = [ + @exe, 'run', + '--queue', @redis_url, + '--seed', 'foobar', + '--build', build_id, + '--worker', worker_id, + '--timeout', '1', + '--retry-mode', 'worker-history' + ] + args.push('--recovery-manifest', manifest_path) if manifest_path + args.push('-Itest', 'test/passing_test.rb') + + capture_subprocess_io do + system( + { 'BUILDKITE_RETRY_COUNT' => retry_count.to_s }, + *args, + chdir: 'test/fixtures/' + ) + end + end + def normalize_xml(output) freeze_xml_timing(rewrite_paths(output)) end diff --git a/ruby/test/minitest/queue/recovery_reporter_test.rb b/ruby/test/minitest/queue/recovery_reporter_test.rb new file mode 100644 index 0000000..ff72ab3 --- /dev/null +++ b/ruby/test/minitest/queue/recovery_reporter_test.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'test_helper' + +module Minitest + module Queue + class RecoveryReporterTest < Minitest::Test + def test_does_not_write_manifest_for_incomplete_run + Dir.mktmpdir do |directory| + path = File.join(directory, 'recovery.json') + queue = Struct.new(:exhausted?).new(false) + config = CI::Queue::Configuration.new(worker_id: '17') + reporter = RecoveryReporter.new(path: path, queue: queue, config: config) + + reporter.start + reporter.report + + refute_path_exists path + end + end + end + end +end