diff --git a/dependencies.yaml b/dependencies.yaml index 1753ec92916e..dfcaec10d582 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -74,6 +74,7 @@ devel_no_selenium_requires: curl: rsync: postgresql-devel: + openssh-common: sudo: tar: xorg-x11-fonts: diff --git a/dist/rpm/openQA.spec b/dist/rpm/openQA.spec index 0e76bb28d1cc..6a93c6cb83b4 100644 --- a/dist/rpm/openQA.spec +++ b/dist/rpm/openQA.spec @@ -73,7 +73,7 @@ # The following line is generated from dependencies.yaml %define cover_requires perl(Devel::Cover) perl(Devel::Cover::Report::Codecovbash) # The following line is generated from dependencies.yaml -%define devel_no_selenium_requires %build_requires %cover_requires %qemu %test_requires curl perl(Perl::Tidy) postgresql-devel rsync sudo tar xorg-x11-fonts +%define devel_no_selenium_requires %build_requires %cover_requires %qemu %test_requires curl openssh-common perl(Perl::Tidy) postgresql-devel rsync sudo tar xorg-x11-fonts # The following line is generated from dependencies.yaml %define devel_requires %devel_no_selenium_requires chromedriver @@ -199,6 +199,8 @@ Recommends: qemu Recommends: rsync # Optionally enabled with USE_PNGQUANT=1 Recommends: pngquant +# for Build Service Authentication +Recommends: openssh-common %if 0%{?suse_version} >= 1330 Requires(pre): group(nogroup) %endif diff --git a/etc/openqa/openqa.ini b/etc/openqa/openqa.ini index 69d23008b9b9..7b52a90f1b46 100644 --- a/etc/openqa/openqa.ini +++ b/etc/openqa/openqa.ini @@ -296,3 +296,14 @@ concurrent = 0 # lookup_depth = 10 # Specify at how many state changes the search will be aborted (state = combination of failed/softfailed/skipped modules): # state_changes_limit = 3 + +# Configuration for the OBS rsync plugin +[obs_rsync] +# project_status_url = %obs_instance%/build/%%PROJECT/_result; +# concurrency = 2 +# queue_limit = 200 +# retry_interval = 60 +# retry_max_count = 2 +# home = +# username = openqa-user +# ssh_key_file = ~/.ssh/id_rsa diff --git a/lib/OpenQA/Setup.pm b/lib/OpenQA/Setup.pm index 41bff36dfe5c..8dfb4f6071bf 100644 --- a/lib/OpenQA/Setup.pm +++ b/lib/OpenQA/Setup.pm @@ -132,6 +132,8 @@ sub read_config ($app) { queue_limit => 200, concurrency => 2, project_status_url => '', + username => '', + ssh_key_file => '' }, cleanup => { concurrent => 0, diff --git a/lib/OpenQA/WebAPI/Plugin/ObsRsync.pm b/lib/OpenQA/WebAPI/Plugin/ObsRsync.pm index 6ac8edd5317d..86d099c20820 100644 --- a/lib/OpenQA/WebAPI/Plugin/ObsRsync.pm +++ b/lib/OpenQA/WebAPI/Plugin/ObsRsync.pm @@ -2,13 +2,13 @@ # SPDX-License-Identifier: GPL-2.0-or-later package OpenQA::WebAPI::Plugin::ObsRsync; -use Mojo::Base 'Mojolicious::Plugin'; +use Mojo::Base 'Mojolicious::Plugin', -signatures; use Mojo::File; use Mojo::URL; use Mojo::UserAgent; use POSIX 'strftime'; - +use File::Which qw(which); use OpenQA::Log qw(log_error); my $dirty_status_filename = '.dirty_status'; @@ -40,11 +40,16 @@ sub register { my $plugin_r = $app->routes->find('ensure_operator'); my $plugin_api_r = $app->routes->find('api_ensure_operator'); + # ssh-keygen is needed for the Build Service authentication. + die("ssh-keygen is not availabe. Aborting.\n") unless which('ssh-keygen'); + if (!$plugin_r) { $app->log->error('Routes not configured, plugin ObsRsync will be disabled') unless $plugin_r; } else { $app->helper('obs_rsync.home' => sub { shift->app->config->{obs_rsync}->{home} }); + $app->helper('obs_rsync.username' => sub { shift->app->config->{obs_rsync}->{username} }); + $app->helper('obs_rsync.ssh_key_file' => sub { shift->app->config->{obs_rsync}->{ssh_key_file} }); $app->helper('obs_rsync.concurrency' => sub { shift->app->config->{obs_rsync}->{concurrency} }); $app->helper('obs_rsync.retry_interval' => sub { shift->app->config->{obs_rsync}->{retry_interval} }); $app->helper('obs_rsync.retry_max_count' => sub { shift->app->config->{obs_rsync}->{retry_max_count} }); @@ -58,7 +63,7 @@ sub register { my $repo = $helper->get_api_repo($alias); my $url = $helper->get_api_dirty_status_url($project); return undef unless $url; - my @res = $self->_is_obs_project_status_dirty($url, $project, $repo); + my @res = $self->_is_obs_project_status_dirty($url, $project, $repo, $helper); if ($trace && scalar @res > 1 && $res[1]) { # ignore potential errors because we use this only for cosmetic rendering open(my $fh, '>', Mojo::File->new($c->obs_rsync->home, $project, $dirty_status_filename)) @@ -165,11 +170,24 @@ sub register { # try to determine whether project is dirty # undef means status is unknown sub _is_obs_project_status_dirty { - my ($self, $url, $project, $repo) = @_; + my ($self, $url, $project, $repo, $helper) = @_; return undef unless $url; + # Use only one UserAgent my $ua = $self->{ua} ||= Mojo::UserAgent->new; - my $res = $ua->get($url)->result; + my $tx = $ua->get($url); + my $res = $tx->result; + # Retry if authentication is required + if ($res->code == 401) { + my $username = $helper->username; + my $ssh_key_file = $helper->ssh_key_file; + my $auth_header = _bs_ssh_auth($res->headers->www_authenticate, $username, $ssh_key_file); + + # Reassign the results + $tx = $ua->get($url, {Authorization => $auth_header}); + $res = $tx->result; + } + return undef unless $res->is_success; return _parse_obs_response_dirty($res, $repo); } @@ -515,4 +533,24 @@ sub _for_every_batch { return @ret; } +# Based on https://www.suse.com/c/multi-factor-authentication-on-suses-build-service/ +sub _bs_ssh_sign ($key, $realm, $value) { + die "SSH Key File not found at $key" unless (-e $key || -z $key); + # This needs to be a bit portable for CI testing + my $tmp = Mojo::File::tempfile->spew($value); + my @lines = split "\n", qx/ssh-keygen -Y sign -f "$key" -q -n "$realm" < $tmp/; + shift @lines; + pop @lines; + return join '', @lines; +} + +sub _bs_ssh_auth ($challenge, $user, $key) { + die "Unexpected OBS challenge: $challenge" unless $challenge =~ /realm="([^"]+)".*headers="\(created\)"/; + my $realm = $1; + + my $now = time(); + my $signature = _bs_ssh_sign($key, $realm, "(created): $now"); + return qq{Signature keyId="$user",algorithm="ssh",signature="$signature",headers="(created)",created="$now"}; +} + 1; diff --git a/t/config.t b/t/config.t index 15d3f9dab6c9..a81c2cdaa5c2 100644 --- a/t/config.t +++ b/t/config.t @@ -115,6 +115,8 @@ subtest 'Test configuration default modes' => sub { queue_limit => 200, concurrency => 2, project_status_url => '', + username => '', + ssh_key_file => '', }, cleanup => { concurrent => 0, diff --git a/t/ui/27-plugin_obs_rsync_obs_status.t b/t/ui/27-plugin_obs_rsync_obs_status.t index d9e47c076b1b..f8ff031ee7f0 100644 --- a/t/ui/27-plugin_obs_rsync_obs_status.t +++ b/t/ui/27-plugin_obs_rsync_obs_status.t @@ -15,13 +15,24 @@ use Mojo::Server::Daemon; use Mojo::IOLoop::Server; use Mojo::IOLoop::ReadWriteProcess 'process'; use Mojo::IOLoop::ReadWriteProcess::Session 'session'; +use Test::MockModule; +use Mojo::File qw(path tempfile); + +my $mocked_time = 0; + +BEGIN { + *CORE::GLOBAL::time = sub { + return $mocked_time if $mocked_time; + return time(); + }; +} $SIG{INT} = sub { session->clean }; END { session->clean } my $port = Mojo::IOLoop::Server->generate_port; my $host = "http://127.0.0.1:$port"; -my $url = "$host/public/build/%%PROJECT/_result"; +my $url = "$host/build/%%PROJECT/_result"; my %fake_response_by_project = ( Proj3 => ' @@ -71,28 +82,87 @@ my %fake_response_by_project = ( Proj0 => 'invalid XML', ); +my $auth_header_exact + = qq(Signature keyId="dummy-username",algorithm="ssh",) + . qq(signature="U1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgSKpcECPm8Vjo9UznZS+) + . qq(M/QLjmXXmLzoBxkIbZ8Z/oPkAAAAaVXNlIHlvdXIgZGV2ZWxvcGVyIGFjY291bnQAAAAAAAAABn) + . qq(NoYTUxMgAAAFMAAAALc3NoLWVkMjU1MTkAAABA8cmvTy1PgpW2XhHWxQ1yw/wPGAfT2M3CGRJ3II) + . qq(7uT5Orqn1a0bWlo/lEV0WiqP+pPcQdajQ4a2YGJvpfzT1uBA==",) + . qq (headers="(created)",created="1664187470"); + note 'Starting fake API server'; -my $server_instance = process sub { - my $mock = Mojolicious->new; - $mock->mode('test'); - for my $project (sort keys %fake_response_by_project) { - $mock->routes->get( - "/public/build/$project/_result" => sub { - my $c = shift; - my $pkg = $c->param('package'); - return $c->render(status => 404) if !$pkg and $project ne 'Proj1'; - return $c->render(status => 200, text => $fake_response_by_project{$project}); - }); + +my $server_process = sub { + my $mock = Mojolicious->new; # uncoverable statement + $mock->mode('test'); # uncoverable statement + + my $www_authenticate = qq(Signature realm="Use your developer account",headers="(created)"); # uncoverable statement + $mock->routes->get( + '/build/ProjWithAuth/_result' => sub { + my $c = shift; # uncoverable statement + + if ($c->req->headers->authorization) { # uncoverable statement + return $c->render(status => 200, text => $fake_response_by_project{Proj1}); # uncoverable statement + } + + $c->res->headers->www_authenticate($www_authenticate); # uncoverable statement + ; # uncoverable statement + return $c->render(status => 401, text => 'login'); # uncoverable statement + }); # uncoverable statement + + $mock->routes->get( + '/build/ProjTestingSignature/_result' => sub { # uncoverable statement + my $c = shift; # uncoverable statement + + my $client_auth_header = $c->req->headers->authorization // ''; # uncoverable statement + + if ($auth_header_exact eq $client_auth_header) { # uncoverable statement + + return $c->render(status => 200, text => $fake_response_by_project{Proj1}); # uncoverable statement + } # uncoverable statement + + $c->res->headers->www_authenticate($www_authenticate); # uncoverable statement + return $c->render(status => 401, text => 'login'); # uncoverable statement + }); # uncoverable statement + + for my $project (sort keys %fake_response_by_project) { # uncoverable statement + $mock->routes->get( # uncoverable statement + "/build/$project/_result" => sub { # uncoverable statement + my $c = shift; # uncoverable statement + my $pkg = $c->param('package'); # uncoverable statement + return $c->render(status => 404) if !$pkg and $project ne 'Proj1'; # uncoverable statement + return $c->render(status => 200, text => $fake_response_by_project{$project}); # uncoverable statement + }); # uncoverable statement } - my $daemon = Mojo::Server::Daemon->new(app => $mock, listen => [$host]); - $daemon->run; - note 'Fake API server stopped'; - _exit(0); + my $daemon = Mojo::Server::Daemon->new(app => $mock, listen => [$host]); # uncoverable statement + $daemon->run; # uncoverable statement + note 'Fake API server stopped'; # uncoverable statement + _exit(0); # uncoverable statement }; + +my $server_instance = process($server_process); $server_instance->set_pipes(0)->start; wait_for_or_bail_out { IO::Socket::INET->new(PeerAddr => '127.0.0.1', PeerPort => $port) } 'API'; -my ($t, $tempdir, $home, $params) = setup_obs_rsync_test(url => $url); +my $ssh_keyfile = tempfile(); +# using the key from [0] to have a reproduceable output. +$ssh_keyfile->spew(< $url, + config => { + username => 'dummy-username', + ssh_key_file => path($ssh_keyfile), + }); my $app = $t->app; my $helper = $app->obs_rsync; @@ -110,9 +180,9 @@ subtest 'test api package helper' => sub { }; subtest 'test api url helper' => sub { - is($helper->get_api_dirty_status_url('Proj1'), "$host/public/build/Proj1/_result"); - is($helper->get_api_dirty_status_url('Proj2'), "$host/public/build/Proj2/_result?package=0product"); - is($helper->get_api_dirty_status_url('BatchedProj'), "$host/public/build/BatchedProj/_result?package=000product"); + is($helper->get_api_dirty_status_url('Proj1'), "$host/build/Proj1/_result"); + is($helper->get_api_dirty_status_url('Proj2'), "$host/build/Proj2/_result?package=0product"); + is($helper->get_api_dirty_status_url('BatchedProj'), "$host/build/BatchedProj/_result?package=000product"); }; subtest 'test builds_text helper' => sub { @@ -153,5 +223,35 @@ $t->get_ok('/admin/obs_rsync/queue')->status_is(200, 'jobs list')->content_like( $t->get_ok('/admin/obs_rsync/')->status_is(200, 'project list')->content_like(qr/published/)->content_like(qr/dirty/) ->content_like(qr/publishing/); +subtest 'build service ssh authentication' => sub { + is($helper->is_status_dirty('ProjWithAuth'), 1, 're-authenticate with ssh auth'); +}; + +subtest 'build service authentication: signature generation' => sub { + $mocked_time = 1664187470; + note "time right now: " . time(); + + is(time(), $mocked_time, 'Time is not frozen!'); + + is($helper->is_status_dirty('ProjTestingSignature'), 1, 'signature matches fixture'); + $mocked_time = undef; +}; + +subtest 'build service authentication: error handling' => sub { + $ssh_keyfile->remove(); + throws_ok { + $helper->is_status_dirty('ProjTestingSignature') + } + qr/SSH Key File not found at/, 'Key detection logic failed (not existing key file)'; + + $ssh_keyfile->touch(); + throws_ok { + $helper->is_status_dirty('ProjTestingSignature') + } + qr/SSH Key File not found at/, 'Key detection logic failed (empty key file)'; +}; + $server_instance->stop; done_testing(); + +# [0]: https://www.suse.com/c/multi-factor-authentication-on-suses-build-service/ diff --git a/tools/ci/ci-packages.txt b/tools/ci/ci-packages.txt index a28b449eb1d3..44c8a6f5a3a4 100644 --- a/tools/ci/ci-packages.txt +++ b/tools/ci/ci-packages.txt @@ -58,6 +58,7 @@ libXt6-1.1.5 libXvnc1-1.12.0 lsof-4.91 luit-20150706 +openssh-common-8.4p1 optipng-0.7.7 pciutils-3.5.6 perl-Algorithm-C3-0.11