diff --git a/task/github.py b/task/github.py
index 759ea39390..a7e8cbdfd8 100644
--- a/task/github.py
+++ b/task/github.py
@@ -376,6 +376,12 @@ def issues(self, labels=["bot"], state="open", since=None):
             result = result + issues
         return result
 
+    def has_open_prs(self, sha):
+        pulls = self.get(f"commits/{sha}/pulls")
+        if pulls:
+            return any((pull['state'] == 'open' for pull in pulls))
+        return True
+
     def getHead(self, pr):
         pull = self.get("pulls/{0}".format(pr))
         if pull:
diff --git a/task/test-tests-scan b/task/test-tests-scan
index 599ac8846e..d0783b96b3 100755
--- a/task/test-tests-scan
+++ b/task/test-tests-scan
@@ -58,7 +58,7 @@ GITHUB_DATA = {
         "statuses": [],
         "sha": "abcdef",
     },
-    "/users/user/repos": [{"full_name": "project/repo"}]
+    "/users/user/repos": [{"full_name": "project/repo"}],
 }
 
 
@@ -70,8 +70,12 @@ class Handler(MockHandler):
             self.replyJson([])
         elif self.path.startswith('/repos/project/repo/pulls'):
             self.replyJson([self.server.data['/repos/project/repo/pulls/1']])
+        elif self.path == "/repos/cockpit-project/cockpit/commits/abcdef/pulls":
+            self.replyJson([{"state": "closed"}])
+        elif self.path == "/repos/cockpit-project/cockpit/commits/abcdef2/pulls":
+            self.replyJson([{"state": "open"}])
         elif self.path.endswith("/issues"):
-            issues = self.server.data['issues']
+            issues = self.server.data.get('issues', [])
             self.replyJson(issues)
         else:
             self.send_error(404, 'Mock Not Found: ' + self.path)
@@ -241,6 +245,52 @@ class TestTestsScan(unittest.TestCase):
             self.assertEqual(issues[0]['labels'], ["nightly"])
             self.assertIsNone(stderr)
 
+    def disabled_test_tests_invoke_no_issue(self):
+        repo = "cockpit-project/cockpit"
+        revision = "abcdef1"
+        args = ["--revision", revision, "--repo", repo]
+        script = os.path.join(BOTS_DIR, "tests-invoke")
+        with tempfile.TemporaryDirectory() as tempdir:
+            testdir = f"{tempdir}/.cockpit-ci"
+            os.makedirs(testdir)
+            with open(f"{testdir}/run", "w") as fp:
+                fp.write("#!/bin/bash\nexit 1")
+            os.system(f"chmod +x {testdir}/run")
+            proc = subprocess.Popen([script, *args], stdout=subprocess.PIPE, universal_newlines=True,
+                                    env={"GITHUB_API": f"http://{ADDRESS[0]}:{ADDRESS[1]}",
+                                         "TEST_INVOKE_SLEEP": "1"},
+                                    cwd=tempdir)
+            output, stderr = proc.communicate()
+            api = github.GitHub(f"http://{ADDRESS[0]}:{ADDRESS[1]}/")
+            issues = api.get("issues")
+            self.assertEqual(output, "")
+            self.assertEqual(proc.returncode, 1)
+            self.assertEqual(len(issues), 0)
+            self.assertIsNone(stderr)
+
+    def disabled_test_tests_invoke_open_pr(self):
+        repo = "cockpit-project/cockpit"
+        revision = "abcdef2"
+        args = ["--revision", revision, "--repo", repo]
+        script = os.path.join(BOTS_DIR, "tests-invoke")
+        with tempfile.TemporaryDirectory() as tempdir:
+            testdir = f"{tempdir}/.cockpit-ci"
+            os.makedirs(testdir)
+            with open(f"{testdir}/run", "w") as fp:
+                fp.write("#!/bin/bash\nexit 1")
+            os.system(f"chmod +x {testdir}/run")
+            proc = subprocess.Popen([script, *args], stdout=subprocess.PIPE, universal_newlines=True,
+                                    env={"GITHUB_API": f"http://{ADDRESS[0]}:{ADDRESS[1]}",
+                                         "TEST_INVOKE_SLEEP": "1"},
+                                    cwd=tempdir)
+            output, stderr = proc.communicate()
+            api = github.GitHub(f"http://{ADDRESS[0]}:{ADDRESS[1]}/")
+            issues = api.get("issues")
+            self.assertEqual(output, "")
+            self.assertEqual(proc.returncode, 1)
+            self.assertEqual(len(issues), 0)
+            self.assertIsNone(stderr)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests-invoke b/tests-invoke
index 55b09b2b0f..b41bbd1107 100755
--- a/tests-invoke
+++ b/tests-invoke
@@ -39,6 +39,7 @@ def main():
 
     # keep track of the most recent pull.updated_at attribute
     last_updated_at = "0"
+    api = github.GitHub(repo=opts.repo)
 
     def detect_collisions(opts):
         nonlocal last_updated_at
@@ -93,8 +94,7 @@ def main():
                 time.sleep(int(os.getenv("TEST_INVOKE_SLEEP", "60")))
         return_code = p.returncode
         sys.stderr.write("Test run finished, return code: {0}\n".format(return_code))
-        if not opts.pull_number and return_code != 0:
-            api = github.GitHub(repo=opts.repo)
+        if not api.has_open_prs(opts.revision) and return_code != 0:
             data = {
                 "title": "Nightly tests did not succeed",
                 "body": f"Tests failed on {opts.revision}",
diff --git a/tests-scan b/tests-scan
index ded60bbf51..449c07d178 100755
--- a/tests-scan
+++ b/tests-scan
@@ -291,7 +291,8 @@ def cockpit_tasks(api, update, contexts, repo, pull_data, pull_number, sha, amqp
         else:
             logging.error("Can't find pull request %s", pull_number)
             return 1
-    elif sha:
+    elif sha and not api.has_open_prs(sha):
+        logging.info("Processing revision %s without pull request", sha)
         pulls.append({
             "title": f"{sha}",
             "number": 0,