Allow tests to be run without a PR

For validating proposed Fedora updates we want to run the tests every
night without opening a PR and with a special nightly test scenario. If
the tests fail we open an issue as there is no PR attached where we can
comment or make failing tests visible.

task/test-tests-scan

@@ -51,6 +51,12 @@ GITHUB_DATA = {
         "statuses": [],
         "sha": "abcdef",
     },
+    # HACK: we can't change the test map dynamically when invoked via test-scan
+    "/repos/cockpit-project/cockpit/commits/abcdef/status?page=1&per_page=100": {
+        "state": "pending",
+        "statuses": [],
+        "sha": "abcdef",
+    },
     "/users/user/repos": [{"full_name": "project/repo"}]
 }
 
@@ -172,6 +178,34 @@ class TestTestsScan(unittest.TestCase):
         self.assertEqual(output.strip(), expected_output)
         self.assertIsNone(stderr)
 
+    def test_no_pull_request(self):
+        repo = "cockpit-project/cockpit"
+        args = ["--dry", "--sha", self.revision, "--repo", repo,
+                "--context", self.context]
+        proc, output, stderr = self.run_tests_scan(args)
+        expected_output = (f"./s3-streamer --repo {repo} --test-name pull-\\d+-\\d+-\\d+"
+                           f" --github-context {self.context} --revision {self.revision} -- /bin/sh -c"
+                           f" \"PRIORITY=0006 ./make-checkout --verbose --repo={repo} --rebase=main {self.revision}"
+                           f" && cd make-checkout-workdir && TEST_OS=fedora BASE_BRANCH=main"
+                           " COCKPIT_BOTS_REF=main TEST_SCENARIO=nightly ../tests-invoke"
+                           f" --revision {self.revision} --repo {repo}\"")
+
+        self.assertEqual(proc.returncode, 0)
+        self.assertRegex(output.strip(), expected_output)
+        self.assertIsNone(stderr)
+
+    def test_no_pull_request_human(self):
+        repo = "cockpit-project/cockpit"
+        args = ["--dry", "--sha", self.revision, "--repo", repo,
+                "--context", self.context, "-v"]
+        proc, output, stderr = self.run_tests_scan(args)
+        expected_output = (f"pull-0      {self.context}            {self.revision}"
+                           f"     6.0  ({repo}) [bots@main]   {{main}}")
+
+        self.assertEqual(proc.returncode, 0)
+        self.assertEqual(output.strip(), expected_output)
+        self.assertIsNone(stderr)
+
 
 if __name__ == '__main__':
     unittest.main()

tests-invoke

@@ -33,8 +33,7 @@ sys.dont_write_bytecode = True
 def main():
     parser = argparse.ArgumentParser(description='Run integration tests')
     parser.add_argument('--repo', help="The repository in which the tested PR is opened", default=None)
-    parser.add_argument('--pull-number', help="The number of the pull request to test",
-                        required=True)
+    parser.add_argument('--pull-number', help="The number of the pull request to test")
     parser.add_argument('--revision', help="Revision of the PR head", required=True)
     opts = parser.parse_args()
 
@@ -94,6 +93,14 @@ def main():
                 time.sleep(60)
         return_code = p.returncode
         sys.stderr.write("Test run finished, return code: {0}\n".format(return_code))
+        if not opts.pull_number and return_code != 0:
+            api = github.GitHub(repo=opts.repo)
+            data = {
+                "title": "Nightly tests did not succeed",
+                "body": f"Tests failed on {opts.revision}",
+                "labels": ["nightly"]
+            }
+            api.post("issues", data)
         return return_code
     except RuntimeError as ex:
         ret = str(ex)

tests-scan

@@ -72,7 +72,7 @@ def main():
     parser.add_argument('--pull-data', default=None,
                         help='pull_request event GitHub JSON data to evaluate; mutualy exclusive with -p and -s')
     parser.add_argument('-s', '--sha', default=None,
-                        help='SHA belonging to pull request to scan for tasks')
+                        help='SHA to scan for tasks')
     parser.add_argument('--amqp', default=None,
                         help='The host:port of the AMQP server to publish to (format host:port)')
 
@@ -143,7 +143,11 @@ def tests_invoke(priority, name, number, revision, ref, context, base,
     (image, _, scenario) = context.partition("/")
 
     checkout = "PRIORITY={priority:04d} ./make-checkout --verbose --repo={repo}"
-    invoke = "../tests-invoke --pull-number {pull_number} --revision {revision} --repo {github_base}"
+    # Special case for when running tests without a PR
+    if number == 0:
+        invoke = "../tests-invoke --revision {revision} --repo {github_base}"
+    else:
+        invoke = "../tests-invoke --pull-number {pull_number} --revision {revision} --repo {github_base}"
     test_env = "TEST_OS={image} BASE_BRANCH={base}"
     wrapper = "./s3-streamer --repo {github_base} --test-name {name}-{current} " \
               "--github-context {github_context} --revision {revision}"
@@ -287,6 +291,21 @@ def cockpit_tasks(api, update, contexts, repo, pull_data, pull_number, sha, amqp
         else:
             logging.error("Can't find pull request %s", pull_number)
             return 1
+    elif sha:
+        pulls.append({
+            "title": f"{sha}",
+            "number": 0,
+            "head": {
+                "sha": sha,
+                "user": {
+                    "login": "cockpit-project"
+                }
+            },
+            "base": {
+                "ref": testmap.get_default_branch(repo)
+            },
+            "labels": [],
+        })
     else:
         pulls = api.pulls()