241 consolidate data and app services (#326)

NOAA-GSL · May 9, 2023 · 7258fe2 · 7258fe2
2 parents 9315274 + 17bccb6
commit 7258fe2
Show file tree

Hide file tree

Showing 67 changed files with 377 additions and 2,378 deletions.
diff --git a/services/data/.coveragerc → .coveragerc b/services/data/.coveragerc → .coveragerc
diff --git a/services/data/.dockerignore → .dockerignore b/services/data/.dockerignore → .dockerignore
@@ -1,4 +1,5 @@
 **/__pycache__/
-tmp/
+node_modules/
+/tmp
 
 *.pyc
diff --git a/.github/scripts/extract_git_ref.py b/.github/scripts/extract_git_ref.py
@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 
-from typing import Optional
 import os
 import re
+from typing import Optional
 
 
 def get_branch(gh_event: str, gh_ref: str, gh_head_ref: Optional[str]) -> str:

diff --git a/.github/scripts/extract_git_ref_test.py b/.github/scripts/extract_git_ref_test.py
@@ -1,5 +1,4 @@
 import pytest
-
 from extract_git_ref import get_branch
 
 

diff --git a/.github/workflows/api-cleanup-pr-images.yaml b/.github/workflows/api-cleanup-pr-images.yaml
@@ -1,12 +1,13 @@
-name: Cleanup API PR images
+name: Cleanup Application PR images
 
 on:
   pull_request:
     types: [closed]
     paths:
-      - 'services/api/**'
-      - '.github/workflows/**'
-      - '.github/scripts/**'
+      - "src/**/*.py"
+      - "src/**/*.html"
+      - ".github/workflows/**"
+      - ".github/scripts/**"
 jobs:
   purge-ghcr-images:
     name: Cleanup PR images from ghcr.io
@@ -19,6 +20,13 @@ jobs:
           name: unified-graphics/api
           token: ${{ secrets.GHCR_CLEANUP_PAT }}
           tag: ${{ github.head_ref }}
+      - name: Cleanup images
+        uses: bots-house/[email protected]
+        with:
+          owner: noaa-gsl
+          name: unified-graphics/data
+          token: ${{ secrets.GHCR_CLEANUP_PAT }}
+          tag: ${{ github.head_ref }}
   purge-ecr-images:
     name: Cleanup PR images from ECR
     runs-on: ubuntu-latest
@@ -37,3 +45,8 @@ jobs:
           aws ecr batch-delete-image \
             --repository-name rtma-vis/api \
             --image-ids imageTag=${{ github.head_ref }}
+      - name: Cleanup image
+        run: |
+          aws ecr batch-delete-image \
+            --repository-name rtma-vis/data \
+            --image-ids imageTag=${{ github.head_ref }}
diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml
@@ -1,4 +1,4 @@
-name: "API Service Build"
+name: "Application Container Builds"
 on:
   push:
     tags:
@@ -7,17 +7,19 @@ on:
     branches: [main]
     # Path filters aren't evaluated for tags - https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore
     paths:
-      - "services/api/**"
+      - "src/**/*.py"
+      - "src/**/*.html"
       - ".github/workflows/**"
       - ".github/scripts/**"
   pull_request:
     paths:
-      - "services/api/**"
+      - "src/**/*.py"
+      - "src/**/*.html"
       - ".github/workflows/**"
       - ".github/scripts/**"
   workflow_dispatch: # Manually
 env:
-  REGISTRY: ghcr.io/noaa-gsl/unified-graphics/api
+  REGISTRY: ghcr.io/noaa-gsl/unified-graphics
 
 jobs:
   lint:
@@ -28,20 +30,16 @@ jobs:
         run: pipx install poetry
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.9"
       - name: Install dependencies
-        working-directory: services/api
         run: |
-          poetry env use 3.11
+          poetry env use 3.9
           poetry install
       - name: Lint with Black
-        working-directory: services/api
         run: poetry run black --check .
       - name: Lint with Flake8
-        working-directory: services/api
         run: poetry run flake8 --count --show-source --statistics .
       - name: Lint with isort
-        working-directory: services/api
         run: poetry run isort --check .
   type-check:
     runs-on: ubuntu-latest
@@ -51,14 +49,12 @@ jobs:
         run: pipx install poetry
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.9"
       - name: Install dependencies
-        working-directory: services/api
         run: |
-          poetry env use 3.11
+          poetry env use 3.9
           poetry install
       - name: Check Types with mypy
-        working-directory: services/api
         run: poetry run mypy src/
   test:
     runs-on: ubuntu-latest
@@ -70,14 +66,12 @@ jobs:
         run: pipx install poetry
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.9"
       - name: Install dependencies
-        working-directory: services/api
         run: |
-          poetry env use 3.11
+          poetry env use 3.9
           poetry install
       - name: Test
-        working-directory: services/api
         run: |
           poetry run coverage run -m pytest
           poetry run coverage report
@@ -86,7 +80,7 @@ jobs:
         uses: irongut/[email protected]
         with:
           header: API Coverage
-          filename: services/api/coverage.xml
+          filename: coverage.xml
           badge: true
           fail_below_min: true
           format: markdown
@@ -101,7 +95,7 @@ jobs:
         with:
           recreate: true
           path: code-coverage-results.md
-  build:
+  build_app:
     runs-on: ubuntu-latest
     needs: [lint, type-check, test]
     permissions:
@@ -112,7 +106,7 @@ jobs:
         run: python3 ./.github/scripts/extract_git_ref.py # Provides env.BRANCH
       - name: Build & tag image
         run: |
-          docker build -t ${{ env.REGISTRY }}:${{ env.BRANCH }} services/api
+          docker build -t ${{ env.REGISTRY }}/api:${{ env.BRANCH }} -f docker/app/Dockerfile .
       - name: Login to GHCR
         uses: docker/login-action@v2
         with:
@@ -121,18 +115,63 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Push image
         run: |
-          docker push ${{ env.REGISTRY }}:${{ env.BRANCH }}
-  scan:
+          docker push ${{ env.REGISTRY }}/api:${{ env.BRANCH }}
+  build_diag_etl:
     runs-on: ubuntu-latest
-    needs: build
+    needs: [lint, type-check, test]
+    permissions:
+      packages: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Extract branch/tag name
+        run: python3 ./.github/scripts/extract_git_ref.py # Provides env.BRANCH
+      - name: Build & tag image
+        run: |
+          docker build -t ${{ env.REGISTRY }}/data:${{ env.BRANCH }} -f docker/diag-etl/Dockerfile .
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Push image
+        run: |
+          docker push ${{ env.REGISTRY }}/data:${{ env.BRANCH }}
+  scan_app:
+    runs-on: ubuntu-latest
+    needs: build_app
+    steps:
+      - uses: actions/checkout@v3
+      - name: Extract branch/tag name
+        run: python3 ./.github/scripts/extract_git_ref.py # Provides env.BRANCH
+      - name: Scan image with Trivy
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "${{ env.REGISTRY }}/api:${{ env.BRANCH }}"
+          format: "sarif"
+          output: "trivy-results.sarif"
+          ignore-unfixed: true
+          severity: "CRITICAL,HIGH"
+          exit-code: "1"
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload Trivy scan results to GitHub Security tab
+        if: always()
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-results.sarif"
+  scan_diag_etl:
+    runs-on: ubuntu-latest
+    needs: build_diag_etl
     steps:
       - uses: actions/checkout@v3
       - name: Extract branch/tag name
         run: python3 ./.github/scripts/extract_git_ref.py # Provides env.BRANCH
       - name: Scan image with Trivy
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "${{ env.REGISTRY }}:${{ env.BRANCH }}"
+          image-ref: "${{ env.REGISTRY }}/data:${{ env.BRANCH }}"
           format: "sarif"
           output: "trivy-results.sarif"
           ignore-unfixed: true
@@ -151,7 +190,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: vlab
     concurrency: vlab
-    needs: scan
+    needs: [scan_app, scan_diag_etl]
     steps:
       - uses: actions/checkout@v3
       - name: Extract branch/tag name
@@ -172,6 +211,9 @@ jobs:
         uses: aws-actions/amazon-ecr-login@v1
       - name: retag image and push
         run: |
-          docker pull ${{ env.REGISTRY }}:${{ env.BRANCH }}
-          docker tag ${{ env.REGISTRY }}:${{ env.BRANCH }} ${{ secrets.AWS_REGISTRY }}/api:${{ env.BRANCH }}
+          docker pull ${{ env.REGISTRY }}/api:${{ env.BRANCH }}
+          docker tag ${{ env.REGISTRY }}/api:${{ env.BRANCH }} ${{ secrets.AWS_REGISTRY }}/api:${{ env.BRANCH }}
           docker push ${{ secrets.AWS_REGISTRY }}/api:${{ env.BRANCH }}
+          docker pull ${{ env.REGISTRY }}/data:${{ env.BRANCH }}
+          docker tag ${{ env.REGISTRY }}/data:${{ env.BRANCH }} ${{ secrets.AWS_REGISTRY }}/data:${{ env.BRANCH }}
+          docker push ${{ secrets.AWS_REGISTRY }}/data:${{ env.BRANCH }}
diff --git a/.github/workflows/data-cleanup-pr-images.yaml b/.github/workflows/data-cleanup-pr-images.yaml