add native apps (#236)

* add native binaries * sync * fix integration tests * fix script * fix tag
eclipse-velocitas · Apr 29, 2024 · 4e79f94 · 4e79f94
1 parent 53f7d7a
commit 4e79f94
Show file tree

Hide file tree

Showing 9 changed files with 126 additions and 53 deletions.
diff --git a/.github/scripts/deploy_image_from_artifact.sh b/.github/scripts/deploy_image_from_artifact.sh
@@ -24,7 +24,7 @@ APP_REGISTRY="localhost:12345"
 local_tag="$APP_REGISTRY/$APP_NAME_LOWERCASE:local"
 echo "Local URL: $local_tag"
 
-docker load -i "$APP_ARTIFACT_NAME.tar" | sed -n 's/^Loaded image: \([0-9a-f]*\).*/\1/p' | xargs -i docker tag {} $local_tag
+docker load -i "$APP_ARTIFACT_NAME.tar" | cut -d ':' -f 3 | xargs -i docker tag {} $local_tag
 docker push $local_tag
 
 cd $ROOT_DIRECTORY

diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml
@@ -30,6 +30,8 @@ jobs:
     name: "Building image (${{ inputs.app_name }})"
     runs-on: ubuntu-22.04
     container: ghcr.io/eclipse-velocitas/devcontainer-base-images/python:v0.3
+    outputs:
+      archs: ${{ steps.set_args.outputs.archs_matrix }}
     env:
       APP_NAME: ${{ inputs.app_name }}
 
@@ -74,15 +76,19 @@ jobs:
       - name: Set Arguments for next step
         id: set_args
         run: |
+          archs=""
           if [ ${{ inputs.platform }} = "multiarch" ]; then
             echo "Build Multiarch"
             echo "platforms=linux/amd64, linux/arm64" >> $GITHUB_OUTPUT
-            echo "type=oci,dest=./${{ env.APP_NAME }}.tar" >> $GITHUB_OUTPUT
+            archs=$(echo "linux/amd64, linux/arm64" | tr -d "linux\/,")
           else
             echo "Build ${{inputs.platform}}"
             echo "platforms=linux/${{ inputs.platform }}" >> $GITHUB_OUTPUT
-            echo "type=docker,dest=./${{ env.APP_NAME }}.tar" >> $GITHUB_OUTPUT
+            archs=${{ inputs.platform }}
           fi
+          echo "archs=$archs" >> $GITHUB_OUTPUT
+          json_array=$(echo "$archs" | jq -R 'sub("^ "; "") | split(" ")' )
+          echo "archs_matrix=$(jq -cn --argjson archs "$json_array" '{arch: $archs}')" >> $GITHUB_OUTPUT
         shell: bash
 
       - name: "${{ env.APP_NAME }} -- Build image"
@@ -93,7 +99,7 @@ jobs:
           pull: true
           push: false
           outputs: |
-            type=${{ steps.set_args.outputs.type }}
+            type=oci,dest=./${{ env.APP_NAME }}-oci-${{inputs.platform}}.tar
           file: ./app/Dockerfile
           context: .
           platforms: ${{ steps.set_args.outputs.platforms }}
@@ -108,16 +114,88 @@ jobs:
           sudo apt-get update
           sudo apt-get -y install skopeo
 
-      - name: "${{ env.APP_NAME }} -- Inspect tar image with skopeo"
+      - name: "${{ env.APP_NAME }} -- Inspect image with skopeo and create docker archives"
+        id: inspect_tar
         run: |
-          skopeo inspect --raw oci-archive:${{ env.APP_NAME }}.tar | jq
-          skopeo inspect oci-archive:${{ env.APP_NAME }}.tar
-          skopeo copy oci-archive:${{ env.APP_NAME }}.tar docker-archive:${{ env.APP_NAME }}-arch.tar
+          skopeo inspect --raw oci-archive:${{ env.APP_NAME }}-oci-${{inputs.platform}}.tar | jq
+          skopeo inspect oci-archive:${{ env.APP_NAME }}-oci-${{inputs.platform}}.tar
+          for arch in ${{ steps.set_args.outputs.archs }}; do
+            skopeo copy --override-arch $arch oci-archive:${{ env.APP_NAME }}-oci-${{inputs.platform}}.tar docker-archive:${{ env.APP_NAME }}-docker-$arch.tar
+          done
+
+      - name: "${{ env.APP_NAME }} -- Get Native Binaries from image"
+        run: |
+          for arch in ${{ steps.set_args.outputs.archs }}; do
+            image=$(docker load -i ${{ env.APP_NAME }}-docker-$arch.tar | cut -d ':' -f 3)
+            id=$(docker create $image --platform linux/$arch)
+            mkdir -p ./out
+            app_name=$(echo ${{ env.APP_NAME }}_$arch | tr '[:upper:]' '[:lower:]')
+            docker cp $id:/app ./out/$app_name
+          done
+
+      - name: "${{ env.APP_NAME }} -- Upload native binaries to artifacts"
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries
+          path: |
+            out/*
+
+      - name: "${{ env.APP_NAME }} -- Upload oci compliant image to artifacts"
+        if: ${{ steps.image_build.outcome == 'success' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.APP_NAME }}-${{ inputs.platform }}-oci-archive
+          path: ./${{ env.APP_NAME }}-oci*.tar
+          if-no-files-found: error
+
+      - name: "${{ env.APP_NAME }} -- Upload docker image to artifacts"
+        if: ${{ steps.image_build.outcome == 'success' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.APP_NAME }}-${{ inputs.platform }}-docker-archive
+          path: ./${{ env.APP_NAME }}-docker*.tar
+          if-no-files-found: error
+
+      - name: "${{ env.APP_NAME }} -- Upload AppManifest.json to artifacts"
+        if: ${{ steps.image_build.outcome == 'success' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: AppManifest
+          path: ./app/AppManifest.json
+          if-no-files-found: error
+
+  scan-image:
+    name: "Scan image (${{ inputs.app_name }}-${{ matrix.arch }})"
+    runs-on: ubuntu-22.04
+    needs: build-image
+    strategy:
+      matrix: ${{fromJSON(needs.build-image.outputs.archs)}}
+    env:
+      APP_NAME: ${{ inputs.app_name }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+
+      - name: Clone Release Documentation Action repository
+        uses: actions/checkout@v4
+        with:
+          repository: eclipse-velocitas/release-documentation-action
+          path: "./.github/actions"
+
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: .
+          pattern: ${{ env.APP_NAME }}*-docker-archive
+          merge-multiple: true
 
       - name: "${{ env.APP_NAME }} -- Scan docker image for vulnerabilities"
-        uses: aquasecurity/trivy-action@0.11.2
+        uses: aquasecurity/trivy-action@0.19.0
         with:
-          input: ${{ env.APP_NAME }}-arch.tar
+          input: ${{ env.APP_NAME }}-docker-${{ matrix.arch }}.tar
           exit-code: "0"
           ignore-unfixed: true
           severity: "CRITICAL,HIGH"
@@ -139,37 +217,17 @@ jobs:
           packagePath: results/Documentation/renderer
 
       - name: "${{ env.APP_NAME }} -- Upload trivy report as artifacts"
-        if: ${{ always() && steps.image_build.outcome == 'success' }}
         uses: actions/upload-artifact@v4
         with:
-          name: test-results-trivy
+          name: test-results-trivy-${{ matrix.arch }}
           path: |
             results/Documentation/renderer/*
 
       - name: "${{ env.APP_NAME }} -- Publish Trivy Scan Results"
         uses: mikepenz/action-junit-report@v4
-        if: ${{ always() && steps.image_build.outcome == 'success' }}
         with:
           check_name: Trivy Scan Results (${{ env.APP_NAME }})
           report_paths: ./junit.xml
           summary: true
           update_check: true
           annotate_only: true
-
-      - name: "${{ env.APP_NAME }} -- Upload image to artifacts"
-        if: ${{ steps.image_build.outcome == 'success' }}
-        uses: actions/upload-artifact@v4
-        env:
-          VAPP_IMAGE: ${{ env.APP_NAME }}-${{ inputs.platform }}
-        with:
-          name: ${{ env.VAPP_IMAGE }}
-          path: ./${{ env.APP_NAME }}.tar
-          if-no-files-found: error
-
-      - name: "${{ env.APP_NAME }} -- Upload AppManifest.json to artifacts"
-        if: ${{ steps.image_build.outcome == 'success' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: AppManifest
-          path: ./app/AppManifest.json
-          if-no-files-found: error
diff --git a/.github/workflows/build-multiarch-image.yml b/.github/workflows/build-multiarch-image.yml
@@ -45,3 +45,15 @@ jobs:
     with:
       platform: multiarch
       app_name: ${{ needs.get-app-name.outputs.app_name }}
+
+  merge-test-results:
+    runs-on: ubuntu-22.04
+    name: Merge Trivy results
+    needs: build-image-multiarch
+    steps:
+      - name: Merge Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          delete-merged: true
+          name: test-results-trivy
+          pattern: test-results-*
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -190,15 +190,15 @@ jobs:
 
       - name: Download stored image from artifacts
         uses: actions/download-artifact@v4
-        env:
-          VAPP_IMAGE: ${{ env.APP_NAME }}-amd64
         with:
-          name: ${{ env.VAPP_IMAGE }}
+          name: ${{ env.APP_NAME }}-amd64-docker-archive
           path: ./.github/scripts/
 
       - name: Deploy image
         working-directory: ./.github/scripts
-        run: ./deploy_image_from_artifact.sh
+        run: |
+          mv ${{ env.APP_NAME }}-docker-amd64.tar ${{ env.APP_NAME }}.tar
+          ./deploy_image_from_artifact.sh
 
       - name: Run Python integration tests
         shell: bash
@@ -242,11 +242,12 @@ jobs:
             results/Documentation/renderer/*
 
   merge-test-results:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: run-integration-tests
     steps:
       - name: Merge Artifacts
         uses: actions/upload-artifact/merge@v4
         with:
+          delete-merged: true
           name: test-results
           pattern: test-results-*
diff --git a/.github/workflows/gen-desired-state.yml b/.github/workflows/gen-desired-state.yml
@@ -13,7 +13,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-name: Build Docker Image for single arch
+name: Generate desired state
 
 on:
   workflow_call:
@@ -26,6 +26,7 @@ jobs:
   gen-desired-state:
     name: "Generate desired state for ${{ inputs.app_name }}"
     runs-on: ubuntu-22.04
+    container: ghcr.io/eclipse-velocitas/devcontainer-base-images/python:v0.3
 
     steps:
       - name: Checkout repository
@@ -59,7 +60,7 @@ jobs:
           velocitas exec pantaris-integration generate-desired-state -s $(echo $REGISTRY/${{ inputs.app_name }}:$VAPP_VERSION | tr '[:upper:]' '[:lower:]')
 
       - name: Upload desired state manifest
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         if: startsWith(github.ref, 'refs/tags/')
         with:
           files: |

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -73,9 +73,9 @@ jobs:
           string: ${{ github.repository }}
 
       - name: Wait for Multi-Arch build to succeed
-        uses: fountainhead/action-wait-for-check@v1.1.0
+        uses: fountainhead/action-wait-for-check@v1.2.0
         with:
-          checkName: build-image-multiarch / Building image (${{ env.APP_NAME }})
+          checkName: Merge Trivy results
           token: ${{ secrets.GITHUB_TOKEN }}
           timeoutSeconds: 1800
           intervalSeconds: 20
@@ -92,7 +92,7 @@ jobs:
       - name: "${{ env.APP_NAME }} -- Publish release image to GHCR"
         working-directory: ${{github.workspace}}
         env:
-          VAPP_IMAGE: ${{ env.APP_NAME }}-multiarch/${{ env.APP_NAME }}.tar
+          VAPP_IMAGE: ${{ env.APP_NAME }}-multiarch-oci-archive/${{ env.APP_NAME }}-oci-multiarch.tar
           VAPP_NAME: ${{ env.APP_NAME }}
           VAPP_VERSION: ${{ steps.get_version.outputs.version-without-v }}
           REGISTRY: "ghcr.io/${{steps.github-repository-name-case-adjusted.outputs.lowercase}}"
@@ -102,11 +102,13 @@ jobs:
           skopeo copy --all  oci-archive:$VAPP_IMAGE "$tag"
 
       - name: ${{ env.APP_NAME }} -- Upload assets
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         if: startsWith(github.ref, 'refs/tags/')
         with:
           files: |
             ${{github.workspace}}/AppManifest/AppManifest.json
+            ${{github.workspace}}/binaries/*
+            ${{github.workspace}}/${{ env.APP_NAME }}-multiarch-docker-archive/*
 
   release-documentation:
     name: Generate release documentation
@@ -138,7 +140,7 @@ jobs:
           if_false: ${{ github.sha }}
 
       - name: Wait for CI workflow to succeed
-        uses: fountainhead/action-wait-for-check@v1.1.0
+        uses: fountainhead/action-wait-for-check@v1.2.0
         with:
           checkName: Run Integration Tests (${{ env.APP_NAME }})
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -232,7 +234,7 @@ jobs:
           zip -r .vehicleApp/Documentation/release-documentation.zip ${{github.workspace}}/hugo/public
 
       - name: Upload assets
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         if: startsWith(github.ref, 'refs/tags/')
         with:
           files: |

diff --git a/.velocitas.json b/.velocitas.json
@@ -6,7 +6,7 @@
         },
         {
             "repo": "devenv-github-workflows",
-            "version": "v5.0.0"
+            "version": "v6.0.0"
         },
         {
             "repo": "devenv-github-templates",

diff --git a/NOTICE-3RD-PARTY-CONTENT.md b/NOTICE-3RD-PARTY-CONTENT.md
@@ -57,7 +57,7 @@
 |actions/setup-node|v4|MIT License|
 |actions/setup-python|v5|MIT License|
 |actions/upload-artifact|v4|MIT License|
-|aquasecurity/trivy-action|0.11.2|Apache License 2.0|
+|aquasecurity/trivy-action|0.19.0|Apache License 2.0|
 |ASzc/change-string-case-action|v6|ISC License|
 |battila7/get-version-action|v2|MIT License|
 |dawidd6/action-download-artifact|v3|MIT License|
@@ -67,11 +67,11 @@
 |docker/login-action|v3|Apache License 2.0|
 |docker/setup-buildx-action|v3|Apache License 2.0|
 |docker/setup-qemu-action|v3|Apache License 2.0|
-|fountainhead/action-wait-for-check|v1.1.0|MIT License|
+|fountainhead/action-wait-for-check|v1.2.0|MIT License|
 |haya14busa/action-cond|v1|MIT License|
 |irongut/CodeCoverageSummary|v1.3.0|MIT License|
 |mikepenz/action-junit-report|v4|Apache License 2.0|
 |peaceiris/actions-gh-pages|v3|MIT License|
 |peaceiris/actions-hugo|v2|MIT License|
 |pre-commit/action|v3.0.0|MIT License|
-|softprops/action-gh-release|v1|MIT License|
+|softprops/action-gh-release|v2|MIT License|
diff --git a/app/Dockerfile b/app/Dockerfile
@@ -51,15 +51,14 @@ WORKDIR /workspace/app/dist
 RUN staticx main run-exe
 
 # Runner stage, to copy the executable
-FROM scratch
+FROM scratch as runner
 
-COPY --from=builder ./workspace/app/dist/run-exe /dist/
+COPY --from=builder ./workspace/app/dist/run-exe /app
 
 WORKDIR /tmp
-WORKDIR /dist
 
-ENV PATH="/dist:$PATH"
+ENV PATH="/:$PATH"
 
 LABEL org.opencontainers.image.source="https://github.com/eclipse-velocitas/vehicle-app-python-template"
 
-CMD ["./run-exe"]
+CMD ["/app"]