Introduce ability to limit host-tags a project may use

jobserv/api/project.py

@@ -1,6 +1,7 @@
 # Copyright (C) 2017 Linaro Limited
 # Author: Andy Doan <[email protected]>
 
+import json
 from flask import Blueprint, request, url_for
 
 from jobserv.flask import permissions
@@ -31,15 +32,43 @@ def project_create():
     if not proj:
         raise ApiError(401, 'Missing required parameter: "name"')
     sync = d.get("synchronous-builds", False)
+    allowed_host_tags = d.get("allowed-host-tags")
+    if allowed_host_tags and not isinstance(allowed_host_tags, list):
+        raise ApiError(400, 'Invalid type for "allowed-host-tags", must be "list"')
 
     permissions.assert_create_project(proj)
-    db.session.add(Project(proj, sync))
+    db.session.add(Project(proj, sync, allowed_host_tags))
     db.session.commit()
 
     url = url_for("api_project.project_get", proj=proj, _external=True)
     return jsendify({"url": url}, 201)
 
 
+@blueprint.route("/<project:proj>/", methods=("PATCH",))
+def project_update(proj):
+    permissions.assert_create_project(proj)
+    p = get_or_404(Project.query.filter_by(name=proj))
+
+    d = request.get_json() or {}
+    if "allowed-host-tags" in d:
+        allowed = d.get("allowed-host-tags")
+        if allowed:
+            if not isinstance(allowed, list):
+                raise ApiError(
+                    400, 'Invalid type for "allowed-host-tags", must be "list"'
+                )
+            p.allowed_host_tags_str = json.dumps(allowed)
+        else:
+            p.allowed_hosts_tags_str = ""
+        db.session.commit()
+    else:
+        raise ApiError(
+            400,
+            '"allowed-host-tags" is the only project attribute that can be modified',
+        )
+    return jsendify({}, 200)
+
+
 @blueprint.route("/<project:proj>/", methods=("DELETE",))
 def project_delete(proj):
     permissions.assert_can_delete(proj)

jobserv/app.py

@@ -337,6 +337,8 @@ def run_status(project, build, run, status=None):
                 db.session.delete(t)
         run.set_status(status)
         db.session.commit()
+        if run.status in (BuildStatus.FAILED, BuildStatus.PASSED):
+            Storage().copy_log(run)
         click.echo("Run is now: %r" % run)
 
 

jobserv/models.py

@@ -93,20 +93,32 @@ class Project(db.Model):
     name = db.Column(db.String(80), unique=True)
 
     synchronous_builds = db.Column(db.Boolean, default=False)
+    allowed_host_tags_str = db.Column(db.Text())
 
     builds = db.relationship("Build", order_by="-Build.id")
     triggers = db.relationship("ProjectTrigger")
 
-    def __init__(self, name=None, synchronous_builds=False):
+    def __init__(self, name=None, synchronous_builds=False, allowed_host_tags=None):
         self.name = name
         self.synchronous_builds = synchronous_builds
+        if allowed_host_tags:
+            self.allowed_host_tags_str = json.dumps(allowed_host_tags)
+
+    @property
+    def allowed_host_tags(self):
+        if self.allowed_host_tags_str:
+            return json.loads(self.allowed_host_tags_str)
+        return None
 
     def as_json(self, detailed=False):
         data = {
             "name": self.name,
             "synchronous-builds": self.synchronous_builds,
             "url": url_for("api_project.project_get", proj=self.name, _external=True),
         }
+        allowed_host_tags = self.allowed_host_tags
+        if allowed_host_tags:
+            data["allowed-host-tags"] = allowed_host_tags
         if detailed:
             data["builds_url"] = url_for(
                 "api_build.build_list", proj=self.name, _external=True

jobserv/trigger.py

@@ -36,6 +36,7 @@ def _check_for_trigger_upgrade(rundef, trigger_type, parent_trigger_type):
 def trigger_runs(
     storage, projdef, build, trigger, params, secrets, parent_type, queue_priority=0
 ):
+    allowed_host_tags = build.project.allowed_host_tags
     name_fmt = trigger.get("run-names")
     added = []
     try:
@@ -54,6 +55,11 @@ def trigger_runs(
             db.session.flush()
             added.append(r)
             rundef = projdef.get_run_definition(r, run, trigger, params, secrets)
+            if allowed_host_tags and r.host_tag not in allowed_host_tags:
+                error = f"Run requested a host-tag that is not configured for this project: {r.host_tag}"
+                logging.error(error)
+                _fail_run(r, error)
+                build.status = BuildStatus.FAILED
             _check_for_trigger_upgrade(rundef, trigger["type"], parent_type)
             storage.set_run_definition(r, rundef)
     except ApiError:
@@ -70,6 +76,14 @@ def trigger_runs(
         raise ApiError(500, str(e) + "\n" + traceback.format_exc())
 
 
+def _fail_run(run, reason):
+    run.set_status(BuildStatus.FAILED)
+    storage = Storage()
+    with storage.console_logfd(run, "a") as f:
+        f.write(reason)
+    storage.copy_log(run)
+
+
 def _fail_unexpected(build, exception):
     r = Run(build, "build-failure")
     db.session.add(r)

migrations/versions/bfe4b68df66e_.py

@@ -0,0 +1,32 @@
+"""empty message
+
+Revision ID: bfe4b68df66e
+Revises: 8c2f916d3b24
+Create Date: 2024-09-20 10:00:10.309091
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'bfe4b68df66e'
+down_revision = '8c2f916d3b24'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('projects', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('allowed_host_tags_str', sa.Text(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('projects', schema=None) as batch_op:
+        batch_op.drop_column('allowed_host_tags_str')
+
+    # ### end Alembic commands ###

tests/test_api_build.py

@@ -146,6 +146,44 @@ def test_build_unexpected(self, storage):
         self._post(self.urlbase, json.dumps(data), headers, 500)
         self.assertEqual([BuildStatus.FAILED], [x.status for x in Run.query])
 
+    @patch("jobserv.trigger.Storage")
+    def test_build_trigger_bad_tag(self, storage):
+        """Assert we can trigger a minimal build."""
+
+        self.project.allowed_host_tags_str = json.dumps(["amd64"])
+        db.session.commit()
+
+        headers = {"Content-type": "application/json"}
+        _sign("http://localhost/projects/proj-1/builds/", headers, "POST")
+
+        data = {
+            "trigger-name": "bad-host-tag",
+            "project-definition": {
+                "timeout": 1,
+                "triggers": [
+                    {
+                        "name": "bad-host-tag",
+                        "type": "simple",
+                        "runs": [
+                            {
+                                "name": "r1",
+                                "container": "alpine",
+                                "script": "compile",
+                                "host-tag": "dont_allow_this",
+                            },
+                        ],
+                    }
+                ],
+                "scripts": {
+                    "compile": "does not matter",
+                },
+            },
+        }
+        self._post(self.urlbase, json.dumps(data), headers, 201)
+        build = self.project.builds[0]
+        self.assertEqual(BuildStatus.FAILED, build.runs[0].status)
+        self.assertEqual(BuildStatus.FAILED, build.status)
+
     @patch("jobserv.api.build.trigger_build")
     def test_build_trigger_simple(self, trigger_build):
         """Assert we can trigger a minimal build."""

tests/test_api_project.py

@@ -48,6 +48,17 @@ def test_project_create_denied(self):
         )
         self.assertEqual(401, r.status_code)
 
+    def test_project_create_bad_allowed_hosts(self):
+        url = "http://localhost/projects/"
+        headers = {"Content-type": "application/json"}
+        _sign(url, headers, "POST")
+        r = self.client.post(
+            "/projects/",
+            headers=headers,
+            data=json.dumps({"name": "foo", "allowed-host-tags": "12"}),
+        )
+        self.assertEqual(400, r.status_code, r.text)
+
     def test_project_create(self):
         url = "http://localhost/projects/"
         headers = {"Content-type": "application/json"}
@@ -65,6 +76,34 @@ def test_project_create(self):
         p = Project.query.filter(Project.name == "foo2").one()
         self.assertTrue(p.synchronous_builds)
 
+        url = "http://localhost/projects/"
+        headers = {"Content-type": "application/json"}
+        _sign(url, headers, "POST")
+        r = self.client.post(
+            url,
+            headers=headers,
+            data=json.dumps({"name": "bar", "allowed-host-tags": ["1", "2"]}),
+        )
+        self.assertEqual(201, r.status_code, r.data)
+        p = Project.query.filter(Project.name == "bar").one()
+        self.assertEqual(["1", "2"], p.allowed_host_tags)
+
+    def test_project_patch(self):
+        self.create_projects("job-1")
+
+        url = "http://localhost/projects/job-1/"
+        headers = {"Content-type": "application/json"}
+        _sign(url, headers, "PATCH")
+        r = self.client.patch(url, headers=headers, data=json.dumps({"name": "foo"}))
+        self.assertEqual(400, r.status_code, r.data)
+
+        r = self.client.patch(
+            url, headers=headers, data=json.dumps({"allowed-host-tags": ["1", "2"]})
+        )
+        self.assertEqual(200, r.status_code, r.data)
+        p = Project.query.filter(Project.name == "job-1").one()
+        self.assertEqual(["1", "2"], p.allowed_host_tags)
+
     def test_project_delete_denied(self):
         self.create_projects("proj-1")
         url = "http://localhost/projects/proj-1/"