Merge pull request #351 from kbst/test-matrix

Run variants in parallel

.github/actions/builder/builder/main.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import logging
 from os import environ, listdir, mkdir
 from os.path import isdir, join
 from shutil import copytree, ignore_patterns, make_archive, rmtree
@@ -15,7 +16,7 @@ def create_archive(name, version):
     copytree(src, module_dist, ignore=ignore_patterns('_*'))
 
     make_archive(module, 'zip', module_dist)
-    print(f"[INFO] created `{module}.zip`")
+    logging.info(f"[INFO] created `{module}.zip`")
 
 
 def get_build_targets(ref):

.github/actions/get-matrix/Dockerfile

@@ -0,0 +1,5 @@
+FROM python:3
+
+COPY src/ /opt/get-matrix/
+
+CMD ["python", "/opt/get-matrix/main.py"]

.github/actions/get-matrix/action.yaml

@@ -0,0 +1,5 @@
+name: "Get Matrix for Catalog Build"
+description: "Get test matrix based on build action output."
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/get-matrix/src/main.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+from json import dumps
+from os import environ, listdir
+from os.path import isdir, isfile, join
+from shutil import unpack_archive
+from sys import exit
+from tempfile import TemporaryDirectory
+
+DISTDIR = "_dist"
+
+
+if __name__ == '__main__':
+    output = {
+        "include": []
+    }
+
+    for name in listdir(DISTDIR):
+        if not isfile(name) and (not name.startswith('module-') or not name.endswith('.zip')):
+            continue
+
+        with TemporaryDirectory() as root:
+            mut = join(root, "mut")
+            archive = join(DISTDIR, name)
+            unpack_archive(archive, mut, "zip")
+            for variant in listdir(mut):
+                variant_path = join(mut, variant)
+                if not isdir(variant_path):
+                    continue
+
+                output["include"].append({
+                    "variant": variant,
+                    "name": name
+                })
+
+    outputsFile = environ.get('GITHUB_OUTPUT')
+    outputData = dumps(output)
+
+    if outputsFile:
+        with open(outputsFile, 'w') as f:
+            f.write(f'matrix={outputData}')
+        exit(0)
+
+    print(outputData)
+    exit(0)

.github/workflows/main.yml

@@ -1,4 +1,5 @@
 name: Build
+run-name: 'Build & test :: ${{ github.ref }}'
 
 on:
   push:
@@ -18,15 +19,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3.0.0
+    - uses: actions/checkout@v4.0.0
 
     # Run builder
     - name: Run builder
       uses: ./.github/actions/builder
 
     # Upload artifacts
     - name: 'Upload artifacts'
-      uses: actions/upload-artifact@v3.0.0
+      uses: actions/upload-artifact@v3.1.3
       with:
         name: _dist
         path: _dist/*.zip
@@ -40,7 +41,7 @@ jobs:
 
     steps:
     # Checkout
-    - uses: actions/checkout@v3.0.0
+    - uses: actions/checkout@v4.0.0
 
     # Setup Terraform
     - name: Setup Terraform
@@ -57,13 +58,36 @@ jobs:
   #
   #
   # Test deploy to k3d
-  test-k3d:
+  get-matrix:
     runs-on: ubuntu-latest
     needs: build
 
+    outputs:
+      matrix: ${{ steps.get-matrix.outputs.matrix }}
+
+    steps:
+      - uses: actions/[email protected]
+
+      - name: 'Download build-artifacts'
+        uses: actions/[email protected]
+        with:
+          name: _dist
+          path: _dist
+
+      - id: get-matrix
+        uses: ./.github/actions/get-matrix
+
+  test-k3d:
+    runs-on: ubuntu-latest
+    needs: get-matrix
+
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.get-matrix.outputs.matrix) }}
+
     steps:
     # Checkout
-    - uses: actions/checkout@v3.0.0
+    - uses: actions/checkout@v4.0.0
 
     # Setup k3d
     - name: Setup k3d
@@ -76,18 +100,14 @@ jobs:
 
     # Download build artifacts
     - name: 'Download build-artifacts'
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v3.0.2
       with:
         name: _dist
         path: _dist
 
-    # Build test image
-    - name: Build test image
-      run: docker build -t test-k3d test/k3d/
-
     # Deploy to cluster
     - name: Run test container
-      run: docker run --network host --rm -v `pwd`/_dist:/_dist -v $HOME/.kube/config:/opt/test/.kubeconfig test-k3d
+      run: make test-k3d test-name=${{ matrix.name }} test-variant=${{ matrix.variant}}
 
 
   #
@@ -100,15 +120,17 @@ jobs:
     steps:
     # Download build artifacts
     - name: 'Download build-artifacts'
-      uses: actions/[email protected].0
+      uses: actions/[email protected].2
       with:
         name: _dist
         path: _dist
 
     # Upload archive
-    - uses: google-github-actions/[email protected].1
+    - uses: google-github-actions/[email protected].1
       with:
-        service_account_key: ${{ secrets.GCLOUD_AUTH }}
+        credentials_json: '${{ secrets.GCLOUD_AUTH }}'
+
+    - uses: google-github-actions/[email protected]
 
     - run: gsutil -m cp _dist/*.zip gs://dev.catalog.kubestack.com
 

.github/workflows/promote.yml

@@ -1,4 +1,5 @@
 name: Promote
+run-name: 'Promote :: ${{ github.ref }}'
 
 on: workflow_dispatch
 
@@ -15,9 +16,11 @@ jobs:
 
     steps:
     # Setup gcloud CLI
-    - uses: google-github-actions/[email protected].1
+    - uses: google-github-actions/[email protected].1
       with:
-        service_account_key: ${{ secrets.GCLOUD_AUTH }}
+        credentials_json: '${{ secrets.GCLOUD_AUTH }}'
+
+    - uses: google-github-actions/[email protected]
 
     # Promote archive
     - run: |

Makefile

@@ -10,16 +10,16 @@ dist:
 	docker build -t catalog:dist-${GITHUB_SHA} .github/actions/builder
 	docker run --rm -v `pwd`:/workdir:z --workdir=/workdir -e GITHUB_REF=${GITHUB_REF} -e GITHUB_SHA=${GITHUB_SHA} catalog:dist-${GITHUB_SHA}
 
-test-kustomize: dist
-	docker build -t catalog:test-kustomize-${GITHUB_SHA} test/kustomize/
-	docker run --rm -v `pwd`/_dist:/_dist:z catalog:test-kustomize-${GITHUB_SHA}
-
 k3d:
 	k3d cluster delete catalog-tests
 	k3d cluster create catalog-tests -s 1 -a 3 --no-lb --k3s-arg "--disable=traefik@server:*" --k3s-arg "--disable=servicelb@server:*" --k3s-arg="--node-label=ingress-ready=true@agent:*"
 	kubectl config use-context k3d-catalog-tests
 	kubectl cluster-info
 
+get-matrix: dist
+	docker build -t catalog:get-matrix-${GITHUB_SHA} .github/actions/get-matrix
+	docker run --network host --rm -v `pwd`/_dist:/_dist:z catalog:get-matrix-${GITHUB_SHA}
+
 test-k3d: dist
 	docker build -t catalog:test-k3d-${GITHUB_SHA} test/k3d/
-	docker run --network host --rm -v `pwd`/_dist:/_dist:z -v ${HOME}/.kube/config:/opt/test/.kubeconfig:z catalog:test-k3d-${GITHUB_SHA}
+	docker run --network host --rm -v `pwd`/_dist:/_dist:z -v ${HOME}/.kube/config:/opt/test/.kubeconfig:z catalog:test-k3d-${GITHUB_SHA} $(test-name) $(test-variant)

test/k3d/Dockerfile

@@ -1,16 +1,16 @@
-FROM kubestack/framework:v0.18.0-beta.0-kind
+FROM kubestack/framework:v0.19.1-beta.0-kind
 
 COPY Pipfile Pipfile.lock /opt/
 WORKDIR /opt
 
 RUN pip install --no-cache-dir pipenv &&\
     PIPENV_VENV_IN_PROJECT=true pipenv install
 
-COPY main.tf.tpl test.py /opt/test/
+COPY main.tf.tpl main.py /opt/test/
 
 ENV PATH=/opt/.venv/bin:$PATH \
     KUBECONFIG=/opt/test/.kubeconfig \
     KUBECONFIG_PATH=/opt/test/.kubeconfig
 
 WORKDIR /opt/test
-CMD ["nosetests", "-s", "--logging-level", "WARNING", "test.py"]
+ENTRYPOINT ["python", "main.py"]

test/k3d/Pipfile

@@ -6,7 +6,6 @@ verify_ssl = true
 [dev-packages]
 
 [packages]
-nose = "*"
 kubernetes = "*"
 jinja2 = "*"
 

test/k3d/test.py → test/k3d/main.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python3
 
-import time
 import logging
-from os import listdir
-from os.path import isdir, isfile, join, abspath
-from subprocess import Popen, PIPE
+import sys
+import time
+
+from os.path import isdir, join
+from subprocess import Popen
 from tempfile import TemporaryDirectory
 from shutil import unpack_archive
+
 from kubernetes import client, config
 from jinja2 import Environment, FileSystemLoader
 
@@ -103,30 +105,33 @@ def run_steps(name, path):
             wait_retries(name, TIMEOUT)
 
 
-def test_variants():
-    for name in listdir(DISTDIR):
-        if not isfile(name) and (not name.startswith('module-') or not name.endswith('.zip')):
-            continue
-
-        with TemporaryDirectory() as root:
-            mut = join(root, "mut")
-            archive = join(DISTDIR, name)
-            unpack_archive(archive, mut, "zip")
-            for variant in listdir(mut):
-                variant_path = join(mut, variant)
-                if not isdir(variant_path):
-                    continue
-
-                # write main.tf into root_module
-                jinja = Environment(loader=FileSystemLoader("."))
-                template = jinja.get_template('main.tf.tpl')
-                data = template.render(
-                    {'entry_module': mut, 'variant': variant})
-
-                with open(f'{root}/main.tf', 'w') as f:
-                    f.write(data)
-                    # always include newline at end of file
-                    f.write('\n')
-
-                # yield instructs nose to treat each variant as a separate test
-                yield run_steps, f"{name} - {variant}", root
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        logging.fatal(f"missing 2 args: name and variant")
+        sys.exit(1)
+
+    name = sys.argv[1]
+    variant = sys.argv[2]
+
+    with TemporaryDirectory() as root:
+        mut = join(root, "mut")
+        archive = join(DISTDIR, name)
+        unpack_archive(archive, mut, "zip")
+
+        variant_path = join(mut, variant)
+        if not isdir(variant_path):
+            logging.fatal(f"path not found: {variant_path}")
+            sys.exit(1)
+
+        # write main.tf into root_module
+        jinja = Environment(loader=FileSystemLoader("."))
+        template = jinja.get_template('main.tf.tpl')
+        data = template.render(
+            {'entry_module': mut, 'variant': variant})
+
+        with open(f'{root}/main.tf', 'w') as f:
+            f.write(data)
+            # always include newline at end of file
+            f.write('\n')
+
+        run_steps(f"{name} - {variant}", root)