ObsRsync Plugin support for HTTP authentication in Build Service

Heavily inspired by: - https://github.com/openSUSE/obs-build/blob/master/PBuild/SigAuth.pm - openSUSE/cavil@583009d - https://www.suse.com/c/multi-factor-authentication-on-suses-build-service/ Addresses poo#139073 - Add openssh to devel dependencies for OBSRsync. Co-authored-by: Martchus <[email protected]>
os-autoinst · Nov 15, 2023 · da55360 · da55360
1 parent e6799a9
commit da55360
Show file tree

Hide file tree

Showing 8 changed files with 183 additions and 26 deletions.
diff --git a/dependencies.yaml b/dependencies.yaml
@@ -74,6 +74,7 @@ devel_no_selenium_requires:
   curl:
   rsync:
   postgresql-devel:
+  openssh-common:
   sudo:
   tar:
   xorg-x11-fonts:

diff --git a/dist/rpm/openQA.spec b/dist/rpm/openQA.spec
@@ -73,7 +73,7 @@
 # The following line is generated from dependencies.yaml
 %define cover_requires perl(Devel::Cover) perl(Devel::Cover::Report::Codecovbash)
 # The following line is generated from dependencies.yaml
-%define devel_no_selenium_requires %build_requires %cover_requires %qemu %test_requires curl perl(Perl::Tidy) postgresql-devel rsync sudo tar xorg-x11-fonts
+%define devel_no_selenium_requires %build_requires %cover_requires %qemu %test_requires curl openssh-common perl(Perl::Tidy) postgresql-devel rsync sudo tar xorg-x11-fonts
 # The following line is generated from dependencies.yaml
 %define devel_requires %devel_no_selenium_requires chromedriver
 
@@ -199,6 +199,8 @@ Recommends:     qemu
 Recommends:     rsync
 # Optionally enabled with USE_PNGQUANT=1
 Recommends:     pngquant
+# for Build Service Authentication
+Recommends:     openssh-common
 %if 0%{?suse_version} >= 1330
 Requires(pre):  group(nogroup)
 %endif

diff --git a/etc/openqa/openqa.ini b/etc/openqa/openqa.ini
@@ -296,3 +296,14 @@ concurrent = 0
 # lookup_depth = 10
 # Specify at how many state changes the search will be aborted (state = combination of failed/softfailed/skipped modules):
 # state_changes_limit = 3
+
+# Configuration for the OBS rsync plugin
+[obs_rsync]
+# project_status_url = %obs_instance%/build/%%PROJECT/_result;
+# concurrency = 2
+# queue_limit = 200
+# retry_interval = 60
+# retry_max_count = 2
+# home =
+# username = openqa-user
+# ssh_key_file = ~/.ssh/id_rsa
diff --git a/lib/OpenQA/Setup.pm b/lib/OpenQA/Setup.pm
@@ -132,6 +132,8 @@ sub read_config ($app) {
             queue_limit => 200,
             concurrency => 2,
             project_status_url => '',
+            username => '',
+            ssh_key_file => ''
         },
         cleanup => {
             concurrent => 0,

diff --git a/lib/OpenQA/WebAPI/Plugin/ObsRsync.pm b/lib/OpenQA/WebAPI/Plugin/ObsRsync.pm
@@ -2,13 +2,13 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 package OpenQA::WebAPI::Plugin::ObsRsync;
-use Mojo::Base 'Mojolicious::Plugin';
+use Mojo::Base 'Mojolicious::Plugin', -signatures;
 
 use Mojo::File;
 use Mojo::URL;
 use Mojo::UserAgent;
 use POSIX 'strftime';
-
+use File::Which qw(which);
 use OpenQA::Log qw(log_error);
 
 my $dirty_status_filename = '.dirty_status';
@@ -40,11 +40,16 @@ sub register {
     my $plugin_r = $app->routes->find('ensure_operator');
     my $plugin_api_r = $app->routes->find('api_ensure_operator');
 
+    # ssh-keygen is needed for the Build Service authentication.
+    die("ssh-keygen is not availabe. Aborting.\n") unless which('ssh-keygen');
+
     if (!$plugin_r) {
         $app->log->error('Routes not configured, plugin ObsRsync will be disabled') unless $plugin_r;
     }
     else {
         $app->helper('obs_rsync.home' => sub { shift->app->config->{obs_rsync}->{home} });
+        $app->helper('obs_rsync.username' => sub { shift->app->config->{obs_rsync}->{username} });
+        $app->helper('obs_rsync.ssh_key_file' => sub { shift->app->config->{obs_rsync}->{ssh_key_file} });
         $app->helper('obs_rsync.concurrency' => sub { shift->app->config->{obs_rsync}->{concurrency} });
         $app->helper('obs_rsync.retry_interval' => sub { shift->app->config->{obs_rsync}->{retry_interval} });
         $app->helper('obs_rsync.retry_max_count' => sub { shift->app->config->{obs_rsync}->{retry_max_count} });
@@ -58,7 +63,7 @@ sub register {
                 my $repo = $helper->get_api_repo($alias);
                 my $url = $helper->get_api_dirty_status_url($project);
                 return undef unless $url;
-                my @res = $self->_is_obs_project_status_dirty($url, $project, $repo);
+                my @res = $self->_is_obs_project_status_dirty($url, $project, $repo, $helper);
                 if ($trace && scalar @res > 1 && $res[1]) {
                     # ignore potential errors because we use this only for cosmetic rendering
                     open(my $fh, '>', Mojo::File->new($c->obs_rsync->home, $project, $dirty_status_filename))
@@ -165,11 +170,24 @@ sub register {
 # try to determine whether project is dirty
 # undef means status is unknown
 sub _is_obs_project_status_dirty {
-    my ($self, $url, $project, $repo) = @_;
+    my ($self, $url, $project, $repo, $helper) = @_;
     return undef unless $url;
 
+    # Use only one UserAgent
     my $ua = $self->{ua} ||= Mojo::UserAgent->new;
-    my $res = $ua->get($url)->result;
+    my $tx = $ua->get($url);
+    my $res = $tx->result;
+    # Retry if authentication is required
+    if ($res->code == 401) {
+        my $username = $helper->username;
+        my $ssh_key_file = $helper->ssh_key_file;
+        my $auth_header = _bs_ssh_auth($res->headers->www_authenticate, $username, $ssh_key_file);
+
+        # Reassign the results
+        $tx = $ua->get($url, {Authorization => $auth_header});
+        $res = $tx->result;
+    }
+
     return undef unless $res->is_success;
     return _parse_obs_response_dirty($res, $repo);
 }
@@ -515,4 +533,24 @@ sub _for_every_batch {
     return @ret;
 }
 
+# Based on https://www.suse.com/c/multi-factor-authentication-on-suses-build-service/
+sub _bs_ssh_sign ($key, $realm, $value) {
+    die "SSH Key File not found at $key" unless (-e $key || -z $key);
+    # This needs to be a bit portable for CI testing
+    my $tmp = Mojo::File::tempfile->spew($value);
+    my @lines = split "\n", qx/ssh-keygen -Y sign -f "$key" -q -n "$realm" < $tmp/;
+    shift @lines;
+    pop @lines;
+    return join '', @lines;
+}
+
+sub _bs_ssh_auth ($challenge, $user, $key) {
+    die "Unexpected OBS challenge: $challenge" unless $challenge =~ /realm="([^"]+)".*headers="\(created\)"/;
+    my $realm = $1;
+
+    my $now = time();
+    my $signature = _bs_ssh_sign($key, $realm, "(created): $now");
+    return qq{Signature keyId="$user",algorithm="ssh",signature="$signature",headers="(created)",created="$now"};
+}
+
 1;
diff --git a/t/config.t b/t/config.t
@@ -115,6 +115,8 @@ subtest 'Test configuration default modes' => sub {
             queue_limit => 200,
             concurrency => 2,
             project_status_url => '',
+            username => '',
+            ssh_key_file => '',
         },
         cleanup => {
             concurrent => 0,

diff --git a/t/ui/27-plugin_obs_rsync_obs_status.t b/t/ui/27-plugin_obs_rsync_obs_status.t
@@ -15,13 +15,24 @@ use Mojo::Server::Daemon;
 use Mojo::IOLoop::Server;
 use Mojo::IOLoop::ReadWriteProcess 'process';
 use Mojo::IOLoop::ReadWriteProcess::Session 'session';
+use Test::MockModule;
+use Mojo::File qw(path tempfile);
+
+my $mocked_time = 0;
+
+BEGIN {
+    *CORE::GLOBAL::time = sub {
+        return $mocked_time if $mocked_time;
+        return time();
+    };
+}
 
 $SIG{INT} = sub { session->clean };
 END { session->clean }
 
 my $port = Mojo::IOLoop::Server->generate_port;
 my $host = "http://127.0.0.1:$port";
-my $url = "$host/public/build/%%PROJECT/_result";
+my $url = "$host/build/%%PROJECT/_result";
 my %fake_response_by_project = (
     Proj3 => '
 <!-- This project is published. -->
@@ -71,28 +82,87 @@ my %fake_response_by_project = (
     Proj0 => 'invalid XML',
 );
 
+my $auth_header_exact
+  = qq(Signature keyId="dummy-username",algorithm="ssh",)
+  . qq(signature="U1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgSKpcECPm8Vjo9UznZS+)
+  . qq(M/QLjmXXmLzoBxkIbZ8Z/oPkAAAAaVXNlIHlvdXIgZGV2ZWxvcGVyIGFjY291bnQAAAAAAAAABn)
+  . qq(NoYTUxMgAAAFMAAAALc3NoLWVkMjU1MTkAAABA8cmvTy1PgpW2XhHWxQ1yw/wPGAfT2M3CGRJ3II)
+  . qq(7uT5Orqn1a0bWlo/lEV0WiqP+pPcQdajQ4a2YGJvpfzT1uBA==",)
+  . qq (headers="(created)",created="1664187470");
+
 note 'Starting fake API server';
-my $server_instance = process sub {
-    my $mock = Mojolicious->new;
-    $mock->mode('test');
-    for my $project (sort keys %fake_response_by_project) {
-        $mock->routes->get(
-            "/public/build/$project/_result" => sub {
-                my $c = shift;
-                my $pkg = $c->param('package');
-                return $c->render(status => 404) if !$pkg and $project ne 'Proj1';
-                return $c->render(status => 200, text => $fake_response_by_project{$project});
-            });
+
+my $server_process = sub {
+    my $mock = Mojolicious->new;    # uncoverable statement
+    $mock->mode('test');    # uncoverable statement
+
+    my $www_authenticate = qq(Signature realm="Use your developer account",headers="(created)"); # uncoverable statement
+    $mock->routes->get(
+        '/build/ProjWithAuth/_result' => sub {
+            my $c = shift;    # uncoverable statement
+
+            if ($c->req->headers->authorization) {    # uncoverable statement
+                return $c->render(status => 200, text => $fake_response_by_project{Proj1});    # uncoverable statement
+            }
+
+            $c->res->headers->www_authenticate($www_authenticate);    # uncoverable statement
+            ;    # uncoverable statement
+            return $c->render(status => 401, text => 'login');    # uncoverable statement
+        });    # uncoverable statement
+
+    $mock->routes->get(
+        '/build/ProjTestingSignature/_result' => sub {    # uncoverable statement
+            my $c = shift;    # uncoverable statement
+
+            my $client_auth_header = $c->req->headers->authorization // '';    # uncoverable statement
+
+            if ($auth_header_exact eq $client_auth_header) {    # uncoverable statement
+
+                return $c->render(status => 200, text => $fake_response_by_project{Proj1});    # uncoverable statement
+            }    # uncoverable statement
+
+            $c->res->headers->www_authenticate($www_authenticate);    # uncoverable statement
+            return $c->render(status => 401, text => 'login');    # uncoverable statement
+        });    # uncoverable statement
+
+    for my $project (sort keys %fake_response_by_project) {    # uncoverable statement
+        $mock->routes->get(    # uncoverable statement
+            "/build/$project/_result" => sub {    # uncoverable statement
+                my $c = shift;    # uncoverable statement
+                my $pkg = $c->param('package');    # uncoverable statement
+                return $c->render(status => 404) if !$pkg and $project ne 'Proj1';    # uncoverable statement
+                return $c->render(status => 200, text => $fake_response_by_project{$project});   # uncoverable statement
+            });    # uncoverable statement
     }
-    my $daemon = Mojo::Server::Daemon->new(app => $mock, listen => [$host]);
-    $daemon->run;
-    note 'Fake API server stopped';
-    _exit(0);
+    my $daemon = Mojo::Server::Daemon->new(app => $mock, listen => [$host]);    # uncoverable statement
+    $daemon->run;    # uncoverable statement
+    note 'Fake API server stopped';    # uncoverable statement
+    _exit(0);    # uncoverable statement
 };
+
+my $server_instance = process($server_process);
 $server_instance->set_pipes(0)->start;
 wait_for_or_bail_out { IO::Socket::INET->new(PeerAddr => '127.0.0.1', PeerPort => $port) } 'API';
 
-my ($t, $tempdir, $home, $params) = setup_obs_rsync_test(url => $url);
+my $ssh_keyfile = tempfile();
+# using the key from [0] to have a reproduceable output.
+$ssh_keyfile->spew(<<EOF);
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACBIqlwQI+bxWOj1TOdlL4z9AuOZdeYvOgHGQhtnxn+g+QAAAJiRS1EekUtR
+HgAAAAtzc2gtZWQyNTUxOQAAACBIqlwQI+bxWOj1TOdlL4z9AuOZdeYvOgHGQhtnxn+g+Q
+AAAECrZDKH46WiRLiazilOn4+BlnESdV8CNReMvlm2Pr6Yr0iqXBAj5vFY6PVM52UvjP0C
+45l15i86AcZCG2fGf6D5AAAAE3NhbXBsZS1tZmEtZmxvd0BpYnMBAg==
+-----END OPENSSH PRIVATE KEY-----
+EOF
+
+
+my ($t, $tempdir, $home, $params) = setup_obs_rsync_test(
+    url => $url,
+    config => {
+        username => 'dummy-username',
+        ssh_key_file => path($ssh_keyfile),
+    });
 my $app = $t->app;
 my $helper = $app->obs_rsync;
 
@@ -110,9 +180,9 @@ subtest 'test api package helper' => sub {
 };
 
 subtest 'test api url helper' => sub {
-    is($helper->get_api_dirty_status_url('Proj1'), "$host/public/build/Proj1/_result");
-    is($helper->get_api_dirty_status_url('Proj2'), "$host/public/build/Proj2/_result?package=0product");
-    is($helper->get_api_dirty_status_url('BatchedProj'), "$host/public/build/BatchedProj/_result?package=000product");
+    is($helper->get_api_dirty_status_url('Proj1'), "$host/build/Proj1/_result");
+    is($helper->get_api_dirty_status_url('Proj2'), "$host/build/Proj2/_result?package=0product");
+    is($helper->get_api_dirty_status_url('BatchedProj'), "$host/build/BatchedProj/_result?package=000product");
 };
 
 subtest 'test builds_text helper' => sub {
@@ -153,5 +223,35 @@ $t->get_ok('/admin/obs_rsync/queue')->status_is(200, 'jobs list')->content_like(
 $t->get_ok('/admin/obs_rsync/')->status_is(200, 'project list')->content_like(qr/published/)->content_like(qr/dirty/)
   ->content_like(qr/publishing/);
 
+subtest 'build service ssh authentication' => sub {
+    is($helper->is_status_dirty('ProjWithAuth'), 1, 're-authenticate with ssh auth');
+};
+
+subtest 'build service authentication: signature generation' => sub {
+    $mocked_time = 1664187470;
+    note "time right now: " . time();
+
+    is(time(), $mocked_time, 'Time is not frozen!');
+
+    is($helper->is_status_dirty('ProjTestingSignature'), 1, 'signature matches fixture');
+    $mocked_time = undef;
+};
+
+subtest 'build service authentication: error handling' => sub {
+    $ssh_keyfile->remove();
+    throws_ok {
+        $helper->is_status_dirty('ProjTestingSignature')
+    }
+    qr/SSH Key File not found at/, 'Key detection logic failed (not existing key file)';
+
+    $ssh_keyfile->touch();
+    throws_ok {
+        $helper->is_status_dirty('ProjTestingSignature')
+    }
+    qr/SSH Key File not found at/, 'Key detection logic failed (empty key file)';
+};
+
 $server_instance->stop;
 done_testing();
+
+# [0]: https://www.suse.com/c/multi-factor-authentication-on-suses-build-service/
diff --git a/tools/ci/ci-packages.txt b/tools/ci/ci-packages.txt
@@ -58,6 +58,7 @@ libXt6-1.1.5
 libXvnc1-1.12.0
 lsof-4.91
 luit-20150706
+openssh-common-8.4p1
 optipng-0.7.7
 pciutils-3.5.6
 perl-Algorithm-C3-0.11