From 2d8152fad49cb8296b69d3f2414a32d80d286ff0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?=
 <17177420+wozniakpl@users.noreply.github.com>
Date: Tue, 14 May 2024 13:48:27 +0200
Subject: [PATCH 1/4] add ci workflow

---
 .github/workflows/ci.yml | 201 +++++++++++++++++++++++++++++++++++++++
 .gitignore               |   2 +-
 .pdm-python              |   1 +
 3 files changed, 203 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/ci.yml
 create mode 100644 .pdm-python

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 00000000..1034a91d
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,201 @@
+name: CI
+
+on:
+  push:
+    branches:
+    - develop
+    - master
+  pull_request:
+    branches:
+    - develop
+    - master
+
+jobs:
+    build_and_push_dev:
+        runs-on: ubuntu-latest
+        steps:
+          - name: Set up Docker Buildx
+            uses: docker/setup-buildx-action@v1
+          - name: Checkout code
+            uses: actions/checkout@v2
+          - name: DockerHub login
+            uses: docker/login-action@v1
+            with:
+              username: ${{ secrets.DOCKERHUB_USERNAME }}
+              password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+          - name: Push dev
+            run: |
+              docker buildx create --use
+              docker buildx build \
+                --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-${{ github.sha }}-dev \
+                --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-latest-dev \
+                --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-${{ github.sha }}-dev \
+                --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-latest-dev \
+                -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}-dev \
+                -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-latest-dev \
+                -f ./docker/Dockerfile \
+                --target dev \
+                --push \
+                ./docker
+                
+    black:
+      runs-on: ubuntu-latest
+      needs: [build_and_push_dev]
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v2
+        - name: DockerHub login
+          uses: docker/login-action@v1
+          with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+        - name: Check
+          run: |
+            docker run --rm -i \
+              ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}-dev \
+              black . --check
+
+    flake8:
+      runs-on: ubuntu-latest
+      needs: [build_and_push_dev]
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v2
+        - name: DockerHub login
+          uses: docker/login-action@v1
+          with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+        - name: Check
+          run: |
+            docker run --rm -i \
+              ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}-dev \
+              flake8 .
+
+    unit_tests:
+      runs-on: ubuntu-latest
+      needs: [build_and_push_dev]
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v2
+        - name: DockerHub login
+          uses: docker/login-action@v1
+          with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+        - name: Unit tests
+          run: |
+            backend_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}-dev docker compose \
+              -f ./ops/compose.ci-test.yml \
+              up --exit-code-from backend
+
+    build_and_push_prd:
+      needs: [build_and_push_dev]
+      runs-on: ubuntu-latest
+      steps:
+        - name: Set up Docker Buildx
+          uses: docker/setup-buildx-action@v1
+        - name: Checkout code
+          uses: actions/checkout@v2
+        - name: DockerHub login
+          uses: docker/login-action@v1
+          with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+        - name: Push prd
+          run: |
+            docker buildx create --use
+
+            # Base part of the command
+            build_command="docker buildx build \
+              --progress=plain \
+              --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-${{ github.sha }}-dev \
+              --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-latest-dev \
+              --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-${{ github.sha }}-prd \
+              --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-latest-prd \
+              --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-${{ github.sha }}-prd \
+              --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-latest-prd \
+              -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}-prd \
+              -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }} \
+              -f ./docker/Dockerfile \
+              --target prd \
+              --push ./"
+            
+            if [ "${{ github.ref }}" = "refs/heads/master" ]; then
+              version=$(python3 -c "import sys; version=None; [version:=line.split('=')[1].strip().strip('\"') for line in open('pyproject.toml', 'r') if line.strip().startswith('version =')]; print(version if version else sys.exit(1))")
+              tagged_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope:country-report-$version
+              build_command="$build_command -t $tagged_image"
+            fi
+            
+            eval $build_command
+
+    trivy:
+      runs-on: ubuntu-latest
+      needs: [build_and_push_prd]
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v2
+        - name: DockerHub login
+          uses: docker/login-action@v1
+          with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_TOKEN }}
+        - name: Run Trivy vulnerability scanner
+          uses: aquasecurity/trivy-action@master
+          with:
+            image-ref: '${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}'
+            format: 'table'
+            exit-code: '0'
+            ignore-unfixed: true
+            vuln-type: 'os,library'
+            severity: 'CRITICAL,HIGH'
+
+    deploy:
+      runs-on: ubuntu-latest
+      needs: [unit_tests, black, flake8]
+      if: |
+        github.event_name == 'push' &&
+        (
+          github.ref == 'refs/heads/develop' ||
+          github.ref == 'refs/heads/master'
+        )
+      steps:
+        - name: Trigger deploy
+          run: |
+            if [ ${{ github.ref }} == 'refs/heads/develop' ]; then
+              pipelineId=1149
+            elif [ ${{ github.ref }} == 'refs/heads/staging' ]; then
+              pipelineId=1286
+            elif [ ${{ github.ref }} == 'refs/heads/master' ]; then
+              pipelineId=1233
+            else
+              echo "No pipeline to trigger for ref ${{ github.ref }}"
+              exit 0
+            fi
+
+            IFS=',' read -ra pipelines <<< "$pipelineId"
+            for pipeline in "${pipelines[@]}"; do
+              jsonBody='{"variables": {"sha": {"isSecret": false, "value": "${{ github.sha }}"}, "tag": {"isSecret": false, "value": "country-report-${{ github.sha }}"}}}'
+              contentLength=$(echo -n $jsonBody | wc -c)
+              project=ICTD-HCT-MIS
+              organization=unicef
+  
+              echo Triggering deploy for pipeline $pipeline
+              echo JSON body: $jsonBody
+  
+              curl -f -v -L \
+                -u ":${{ secrets.AZURE_PAT }}" \
+                -H "Content-Type: application/json" \
+                -H "Content-Length: $contentLength" \
+                -d "$jsonBody" \
+                https://dev.azure.com/$organization/$project/_apis/pipelines/$pipeline/runs?api-version=7.1-preview.1
+              if [ $? -ne 0 ]; then
+                echo "Failed to trigger deploy for pipeline $pipeline"
+                exit 1
+              fi
+            done
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index e0969fe4..f35e78c9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
-.*
+.vscode
 ~*
 node_modules
 *.pyc
diff --git a/.pdm-python b/.pdm-python
new file mode 100644
index 00000000..715374e6
--- /dev/null
+++ b/.pdm-python
@@ -0,0 +1 @@
+/code/.venv/bin/python
\ No newline at end of file

From c51664b31396518c7f46132ba8e36b8e24cc5d8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?=
 <17177420+wozniakpl@users.noreply.github.com>
Date: Tue, 14 May 2024 13:48:38 +0200
Subject: [PATCH 2/4] rm azure pipeline

---
 .gitignore  |   1 +
 .pdm-python |   1 -
 ops/ci.yaml | 223 ----------------------------------------------------
 3 files changed, 1 insertion(+), 224 deletions(-)
 delete mode 100644 .pdm-python
 delete mode 100644 ops/ci.yaml

diff --git a/.gitignore b/.gitignore
index f35e78c9..54740acc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 .vscode
+.pdm_python
 ~*
 node_modules
 *.pyc
diff --git a/.pdm-python b/.pdm-python
deleted file mode 100644
index 715374e6..00000000
--- a/.pdm-python
+++ /dev/null
@@ -1 +0,0 @@
-/code/.venv/bin/python
\ No newline at end of file
diff --git a/ops/ci.yaml b/ops/ci.yaml
deleted file mode 100644
index fc75c374..00000000
--- a/ops/ci.yaml
+++ /dev/null
@@ -1,223 +0,0 @@
-trigger:
-  batch: true
-  branches:
-    include:
-    - develop
-    - staging
-    - master
-pr:
-- develop
-- staging
-- master
-resources:
-- repo: self
-
-variables:
-  Docker.backend.repository: "hope-country-report"
-  tag: $(Build.SourceVersion)
-  additionalTag: $(Build.SourceBranchName)
-
-  ${{ if eq(variables['Build.SourceBranchName'], 'develop') }}:
-    Docker.registryConnection: "ICTD-HOPE-DEV-ACR"
-    Docker.url: "uniappsakshopedev.azurecr.io"
-    pipelineId: 1149
-  ${{ elseif eq(variables['Build.SourceBranchName'], 'staging') }}:
-    Docker.registryConnection: "ICTD-HOPE-DEV-ACR"
-    Docker.url: "uniappsakshopedev.azurecr.io"
-    pipelineId: 1286
-  ${{ elseif eq(variables['Build.SourceBranchName'], 'master') }}:
-    Docker.registryConnection: "ICTD-HOPE-PRD-ACR"
-    Docker.url: "uniappsakshope.azurecr.io"
-    pipelineId: 1233
-  ${{ else }}:
-    Docker.registryConnection: "ICTD-HOPE-DEV-ACR"
-    Docker.url: "uniappsakshopedev.azurecr.io"
-    pipelineId: null
-
-stages:
-- stage: build_and_push_backend_dev
-  dependsOn: []
-  displayName: BUILD and PUSH DEV
-  jobs:
-  - job: build_push_backend
-    pool:
-      vmImage: ubuntu-latest
-    displayName: "[BACKEND_BUILD]"
-    steps:
-    - task: Docker@2
-      inputs:
-        containerRegistry: '$(Docker.registryConnection)'
-        command: 'login'
-    - script: |
-        docker buildx create --use
-        docker buildx build \
-        --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:dev-$(additionalTag) \
-        --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:dev-latest \
-        --cache-to=$(Docker.url)/$(Docker.backend.repository)-cache:dev-$(additionalTag) \
-        --cache-to=$(Docker.url)/$(Docker.backend.repository)-cache:dev-latest \
-        -t $(Docker.url)/$(Docker.backend.repository):dev-$(tag) \
-        --target dev \
-        --push \
-        -f docker/Dockerfile \
-        ./
-      displayName: Docker build dev
-
-- stage: check_format
-  displayName: FORMAT
-  dependsOn: build_and_push_backend_dev
-  jobs:
-  - job: tests
-    pool:
-      vmImage: ubuntu-latest
-    displayName: "[DJANGO FORMAT]"
-    steps:
-    - task: Docker@2
-      displayName: "[ACR] Login"
-      inputs:
-        command: login
-        containerRegistry: $(Docker.registryConnection)
-    - task: DockerCompose@0
-      displayName: "Check format"
-      inputs:
-        containerregistrytype: 'Azure Container Registry'
-        azureContainerRegistry: $(Docker.registryConnection)
-        dockerComposeFile: 'ops/compose.ci-test.yml'
-        action: 'Run a Docker Compose command'
-        dockerComposeCommand: 'run --no-deps backend black --check .'
-        dockerComposeFileArgs: |
-          backend_image=$(Docker.url)/$(Docker.backend.repository):dev-$(tag)
-
-- stage: lint_backend
-  displayName: LINT
-  dependsOn: build_and_push_backend_dev
-  jobs:
-  - job: lint
-    pool:
-      vmImage: ubuntu-latest
-    displayName: "[DJANGO LINT]"
-    steps:
-    - task: Docker@2
-      displayName: "[ACR] Login"
-      inputs:
-        command: login
-        containerRegistry: $(Docker.registryConnection)
-    - task: DockerCompose@0
-      displayName: "Run lint"
-      inputs:
-        containerregistrytype: 'Azure Container Registry'
-        azureContainerRegistry: $(Docker.registryConnection)
-        dockerComposeFile: 'ops/compose.ci-test.yml'
-        action: 'Run a Docker Compose command'
-        dockerComposeCommand: 'run --no-deps backend flake8 .'
-        dockerComposeFileArgs: |
-          backend_image=$(Docker.url)/$(Docker.backend.repository):dev-$(tag)
-
-- stage: test_dev
-  displayName: TEST
-  dependsOn: [check_format, lint_backend]
-  jobs:
-  - job: tests
-    pool:
-      vmImage: ubuntu-latest
-    displayName: "[DJANGO TEST]"
-    steps:
-    - task: Docker@2
-      displayName: "[ACR] Login"
-      inputs:
-        command: login
-        containerRegistry: $(Docker.registryConnection)
-    - task: DockerCompose@0
-      displayName: "Run tests"
-      inputs:
-        containerregistrytype: 'Azure Container Registry'
-        azureContainerRegistry: $(Docker.registryConnection)
-        dockerComposeFile: 'ops/compose.ci-test.yml'
-        action: 'Run a Docker Compose command'
-        dockerComposeCommand: 'up --exit-code-from backend'
-        dockerComposeFileArgs: |
-          backend_image=$(Docker.url)/$(Docker.backend.repository):dev-$(tag)
-
-- stage: build_and_push_backend_prd
-  displayName: BUILD AND PUSH PRD
-  dependsOn: [build_and_push_backend_dev]
-  jobs:
-  - job: build_push_backend
-    pool:
-      vmImage: ubuntu-latest
-    displayName: "[BACKEND_BUILD]"
-    steps:
-    - task: Docker@2
-      inputs:
-        containerRegistry: '$(Docker.registryConnection)'
-        command: 'login'
-    - script: |
-        docker buildx create --use
-        docker buildx build \
-        --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:dev-$(additionalTag) \
-        --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:dev-latest \
-        --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:prd-$(additionalTag) \
-        --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:prd-latest \
-        --cache-to=$(Docker.url)/$(Docker.backend.repository)-cache:prd-$(additionalTag) \
-        --cache-to=$(Docker.url)/$(Docker.backend.repository)-cache:prd-latest \
-        -t $(Docker.url)/$(Docker.backend.repository):prd-$(tag) \
-        --target prd \
-        --push \
-        -f docker/Dockerfile \
-        ./
-      displayName: Docker build prd
-
-- stage: trivy
-  dependsOn: [build_and_push_backend_prd]
-  displayName: Trivy
-  jobs:
-  - job: run_trivy
-    pool:
-      vmImage: ubuntu-latest
-    displayName: "[TRIVY]"
-    steps:
-    - task: Docker@2
-      displayName: "[ACR] Login"
-      inputs:
-        command: login
-        containerRegistry: $(Docker.registryConnection)
-    - script: |
-        docker pull $(Docker.url)/$(Docker.backend.repository):prd-$(tag)
-        docker run -v /var/run/docker.sock:/var/run/docker.sock -e TRIVY_EXIT_CODE=2 -e TRIVY_SEVERITY=HIGH,CRITICAL aquasec/trivy:0.47.0 image $(Docker.url)/$(Docker.backend.repository):prd-$(tag)
-      continueOnError: true
-
-
-- stage: trigger_deploy
-  displayName: Trigger deploy
-  dependsOn: [build_and_push_backend_prd, test_dev]
-  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
-  jobs:
-  - job: trigger_deploy
-    pool:
-      vmImage: ubuntu-latest
-    displayName: "[TRIGGER_DEPLOY]"
-    steps:
-    - script: |
-        pipelineId=$(pipelineId)
-        IFS=',' read -ra pipelines <<< "$pipelineId"
-        for pipeline in "${pipelines[@]}"; do
-          jsonBody='{"variables": {"tag": {"isSecret": false, "value": "$(tag)"}}}'
-          contentLength=$(echo -n $jsonBody | wc -c)
-          project=ICTD-HCT-MIS
-          organization=unicef
-
-          echo Triggering deploy for pipeline $pipeline
-          echo JSON body: $jsonBody
-
-          curl -v -L \
-            -u ":$(AZURE_PAT)" \
-            -H "Content-Type: application/json" \
-            -H "Content-Length: $contentLength" \
-            -d "$jsonBody" \
-            https://dev.azure.com/$organization/$project/_apis/pipelines/$pipeline/runs?api-version=7.1-preview.1
-          if [ $? -ne 0 ]; then
-            echo "Failed to trigger deploy for pipeline $pipeline"
-            exit 1
-          fi
-        done
-      displayName: "Trigger deploy"
-      condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))

From 9b404c2fd2f502d1b29b455e7de7b235ed5e3d05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?=
 <17177420+wozniakpl@users.noreply.github.com>
Date: Tue, 14 May 2024 13:53:09 +0200
Subject: [PATCH 3/4] add missing dependency

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1034a91d..a3eb59d1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -157,7 +157,7 @@ jobs:
 
     deploy:
       runs-on: ubuntu-latest
-      needs: [unit_tests, black, flake8]
+      needs: [unit_tests, black, flake8, build_and_push_prd]
       if: |
         github.event_name == 'push' &&
         (

From 781f9e08fb3cb8f635763b794934c30c93f605ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartosz=20Wo=C5=BAniak?=
 <17177420+wozniakpl@users.noreply.github.com>
Date: Tue, 14 May 2024 14:00:00 +0200
Subject: [PATCH 4/4] fix docker build dir

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a3eb59d1..757d0a79 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -37,7 +37,7 @@ jobs:
                 -f ./docker/Dockerfile \
                 --target dev \
                 --push \
-                ./docker
+                ./
                 
     black:
       runs-on: ubuntu-latest