Skip to content

Commit

Permalink
Avoid worker to wait until setup timeout on asset errors
Browse files Browse the repository at this point in the history
* Abort the job if the asset caching has already run into an
  error
* Tested locally by enabling the cache service, cloning a job
  from o3 and patching the worker so it would cache the asset
  from o3 and not the local web UI
    * After the caching task had exhausted all retries the Minion
      job was marked as failed (but not "processed") and then
      the openQA worker also marked the openQA job as incomplete
      with the reason `Reason: cache failure: Failed to
      download …` instead of letting it run into the setup
      timeout.
* See https://progress.opensuse.org/issues/132434
  • Loading branch information
Martchus committed Jul 11, 2023
1 parent 6634b1c commit 18e231d
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 3 deletions.
2 changes: 1 addition & 1 deletion lib/OpenQA/Worker/Engines/isotovideo.pm
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ sub _poll_cache_service ($job, $cache_client, $request, $delay, $callback) {
my $status = $cache_client->status($request);
return Mojo::IOLoop->singleton->timer(
$delay => sub { _poll_cache_service($job, $cache_client, $request, $delay, $callback) })
unless $status->is_processed;
if !$status->is_processed && !$status->has_error;
return $callback->({error => 'Job has been cancelled'}, undef) if $job->is_stopped_or_stopping;
return $callback->({error => $status->error}, undef) if $status->has_error;
return $callback->(undef, $status);
Expand Down
13 changes: 11 additions & 2 deletions t/24-worker-engine.t
Original file line number Diff line number Diff line change
Expand Up @@ -185,12 +185,13 @@ subtest 'asset caching' => sub {
is $error, $test_dir, 'Cache directory updated';
};

sub _mock_cache_service_client ($status_data) {
sub _mock_cache_service_client ($status_data, $error = undef) {
my $cache_client_mock = Test::MockModule->new('OpenQA::CacheService::Client');
$cache_client_mock->redefine(enqueue => 'some enqueue error');
$cache_client_mock->redefine(
info => OpenQA::CacheService::Response::Info->new(data => {active_workers => 1}, error => undef));
$cache_client_mock->redefine(status => OpenQA::CacheService::Response::Status->new(data => $status_data));
$cache_client_mock->redefine(
status => OpenQA::CacheService::Response::Status->new(data => $status_data, error => $error));
return $cache_client_mock;
}

Expand All @@ -217,6 +218,14 @@ subtest 'problems when caching assets' => sub {
Mojo::IOLoop->start;
is $error->{error}, 'Failed to download FOO to some/path', 'asset not found';
is $error->{category}, WORKER_EC_ASSET_FAILURE, 'category set so problem is treated as asset failure';

$cache_client_mock = _mock_cache_service_client {}, 'some severe error';
$cache_client_mock->redefine(asset_request => Test::FakeRequest->new);
$cache_client_mock->redefine(enqueue => 0);
@assets = ('ISO_1');
OpenQA::Worker::Engines::isotovideo::cache_assets(OpenQA::CacheService::Client->new, @args);
Mojo::IOLoop->start;
is $error->{error}, 'some severe error', 'job not "processed" due to some error';
};

subtest '_handle_asset_processed' => sub {
Expand Down

0 comments on commit 18e231d

Please sign in to comment.