feat(cve-scan-patch): patch image in dedicated GH Action job

sighupio · Oct 4, 2024 · 98fed78 · 98fed78
1 parent b41918d
commit 98fed78
Show file tree

Hide file tree

Showing 3 changed files with 123 additions and 55 deletions.
diff --git a/.github/workflows/cve-scan-and-patching.yml b/.github/workflows/cve-scan-and-patching.yml
@@ -22,12 +22,11 @@ jobs:
           echo "KFD_VERSIONS_JSON=[\"${KFD_VERSIONS//[$'\r\n']/\",\"}\"]" >> "$GITHUB_OUTPUT"
           echo "KFD_VERSIONS_LIST=${KFD_VERSIONS//[$'\r\n']/ }" >> "$GITHUB_OUTPUT"
           echo "TODAY_DATE=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT"
-
     outputs:
       kfd_versions_json: ${{ steps.set_output.outputs.KFD_VERSIONS_JSON }}
       kfd_versions_list: ${{ steps.set_output.outputs.KFD_VERSIONS_JSON }}
       today_date: ${{ steps.set_output.outputs.TODAY_DATE }}
-  scan-pre-patch:
+  scan_pre_patch:
     runs-on: ubuntu-latest
     needs: fetch_kfd_versions
     strategy:
@@ -41,7 +40,7 @@ jobs:
           registry: registry.sighup.io
           username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }}
           password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }}
-      - name: Install furyctl, trivy and copa
+      - name: Install furyctl, trivy
         run: |
           sudo apt-get install wget apt-transport-https gnupg
           wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null
@@ -53,25 +52,43 @@ jobs:
           wget https://github.com/sighupio/furyctl/releases/download/v0.29.8/furyctl-linux-amd64.tar.gz
           tar -xzvf furyctl-linux-amd64.tar.gz -C /usr/local/bin/
           furyctl version
-
-      - name: Execute CVEs scan for KFD ${{ matrix.kfd_version}}
+      - name: Execute CVEs scan for KFD ${{ matrix.kfd_version }}
         id: scan_pre_patch
         run: |
           cd CVEs
           make trivy-download-db
           make scan-pre-patch KFD_VERSIONS="${{ matrix.kfd_version }}"
-
       - name: publish CVE scan pre patch output files for KFD ${{ matrix.kfd_version }}
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.kfd_version}}
+          name: ${{ matrix.kfd_version }}
           path: |
             CVEs/${{ matrix.kfd_version }}/images.txt
             CVEs/${{ matrix.kfd_version }}/FURY-CVEs.md
-
+  fetch_kfd_images_to_patch:
+    runs-on: ubuntu-latest
+    needs: scan_pre_patch
+    steps:
+      - uses: actions/checkout@master
+      - name: download CVE scan pre patch output files
+        uses: actions/download-artifact@v4
+        with:
+          path: CVEs
+      - name: Set output
+        id: set_output
+        run: |
+          make concat-multiple-kfd-images-list
+          IMAGES_TO_PATCH=$(cat all_kfd_images.txt)
+          echo "IMAGES_TO_PATCH_JSON=[\"${IMAGES_TO_PATCH//[$'\r\n']/\",\"}\"]" >> "$GITHUB_OUTPUT"
+    outputs:
+      images_to_patch_json: ${{ steps.set_output.outputs.IMAGES_TO_PATCH_JSON }}
   patch:
     runs-on: ubuntu-latest
-    needs: scan-pre-patch
+    needs:
+      - fetch-kfd-images-to-patch
+    strategy:
+      matrix:
+        image_to_patch: ${{ fromJson(needs.fetch_kfd_images_to_patch.outputs.images_to_patch_json) }}
     steps:
       - uses: actions/checkout@master
       - name: Login to Docker Hub
@@ -85,7 +102,7 @@ jobs:
           registry: registry.sighup.io
           username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }}
           password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }}
-      - name: Install furyctl, trivy and copa
+      - name: Install buildkit, trivy and copa
         run: |
           sudo apt-get install wget apt-transport-https gnupg
           wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null
@@ -103,30 +120,25 @@ jobs:
           chmod +x copa
           sudo mv copa /usr/local/bin/
           copa --version
-
-      - name: download CVE scan pre patch output files
-        uses: actions/download-artifact@v4
-        with:
-          path: CVEs
-
       - name: Execute CVEs patching
         id: patching
         run: |
+          IMAGE_TO_PATCH=${{ matrix.image_to_patch }}
           cd CVEs
+          mkdir -p reports
           make trivy-download-db
-          DOCKER_CONFIG="${DOCKER_CONFIG}" make patch
-
-      - name: publish CVE patching report
+          DOCKER_CONFIG="${DOCKER_CONFIG}" make patch -i "${IMAGE_TO_PATCH}" -o reports/${IMAGE_TO_PATCH//[:\/]/_}.patched.md
+      - name: publish CVE patching report for ${{ matrix.image_to_patch }}
         uses: actions/upload-artifact@v4
         with:
-          name: kfd-${{needs.fetch_kfd_versions.outputs.today_date}}-patch-report
+          name: patch-report-${{ matrix.image_to_patch }}.md
           path: |
-            CVEs/PATCHED.md
-            
-
-  scan-post-patch:
+            CVEs/reports/*.patched.md
+  scan_post_patch:
     runs-on: ubuntu-latest
-    needs: patch
+    needs:
+      - fetch_kfd_versions
+      - patch
     strategy:
       matrix:
         kfd_version: ${{ fromJson(needs.fetch_kfd_versions.outputs.kfd_versions_json) }}
@@ -138,37 +150,53 @@ jobs:
           registry: registry.sighup.io
           username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }}
           password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }}
-      - name: Install furyctl, trivy and copa
+      - name: Install trivy
         run: |
           sudo apt-get install wget apt-transport-https gnupg
           wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null
           echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
           sudo apt-get update
           sudo apt-get install trivy
           trivy --version
-
-          wget https://github.com/sighupio/furyctl/releases/download/v0.29.8/furyctl-linux-amd64.tar.gz
-          tar -xzvf furyctl-linux-amd64.tar.gz -C /usr/local/bin/
-          furyctl version
-
-      - name: Execute CVEs scan post patch for KFD ${{ matrix.kfd_version}}
-        id: scan_post_patch
-        run: |
-          cd CVEs
-          make trivy-download-db
-          make scan-post-patch KFD_VERSIONS="${{ matrix.kfd_version }}"
-
       - name: download CVE scan pre patch output files
         uses: actions/download-artifact@v4
         with:
           name: ${{ matrix.kfd_version }}
           path: CVEs
-
-      - name: publish CVE scan post patch output files for KFD ${{ matrix.kfd_version}}
+      - name: Execute CVEs scan post patch for KFD ${{ matrix.kfd_version }}
+        id: scan_post_patch
+        run: |
+          cd CVEs
+          make trivy-download-db
+          make scan-post-patch KFD_VERSIONS="${{ matrix.kfd_version }}"
+      - name: publish CVE scan post patch output files for KFD ${{ matrix.kfd_version }}
         uses: actions/upload-artifact@v4
         with:
           overwrite: true
           name: ${{ matrix.kfd_version }}
           path: |
+            CVEs/${{ matrix.kfd_version }}/images.txt
+            CVEs/${{ matrix.kfd_version }}/images-patched.txt
             CVEs/${{ matrix.kfd_version }}/FURY-CVEs.md
             CVEs/${{ matrix.kfd_version }}/FURY-SECURED-CVEs.md
+  report:
+    runs-on: ubuntu-latest
+    needs:
+      - scan_post_patch
+    steps:
+      - name: download CVE patching output files
+        uses: actions/download-artifact@v4
+        with:
+          path: reports
+          pattern: patch-report-*.md
+          merge-multiple: true
+      - name: publish CVE patching reports
+        uses: actions/upload-artifact@v4
+        with:
+          name: cve-patch-reports-by-image
+          path: |
+            reports/*.patched.md
+      - uses: geekyeggo/delete-artifact@v5
+        with:
+          name: patch-report-*.md
+          failOnError: false
diff --git a/CVEs/Makefile b/CVEs/Makefile
@@ -1,31 +1,38 @@
-.PHONY: scan-pre-patch patch scan-post-patch download-deps kustomize-build-all gen-image-list trivy-download-db scan-vulns
+.PHONY: scan-pre-patch \
+	patch scan-post-patch \
+	download-deps \
+	kustomize-build-all \
+	generate-image-list-from-manifests \
+	trivy-download-db \
+	scan-vulns \
+	concat-multiple-kfd-images-list
 
 KFD_VERSIONS := $(shell find . -type d -name "v*" -maxdepth 1 -mindepth 1 | cut -d/ -f2 | sort )
+DRY_RUN := 0
+PATCH_FILE_IMAGE_LIST_TO_PATCHING := all_kfd_images.txt
 
-ifndef DRY_RUN
-	DRY_RUN := 0
-endif
-
-all: trivy-download-db scan-pre-patch patch scan-post-patch
+all: trivy-download-db scan-pre-patch concat-multiple-kfd-images-list patch scan-post-patch
 
 scan-pre-patch:
 	@for version in $(KFD_VERSIONS); do \
 		( \
 			$(MAKE) download-deps KFD_VERSION=$$version; \
 			$(MAKE) kustomize-build-all KFD_VERSION=$$version; \
-			$(MAKE) gen-image-list KFD_VERSION=$$version; \
+			$(MAKE) generate-image-list-from-manifests KFD_VERSION=$$version; \
 			$(MAKE) scan-vulns KFD_VERSION=$$version LIST_FILE=$$version/images.txt OUTPUT_FILE=$$version/FURY-CVEs.md;  \
 		) & \
 	done; \
 	wait
 
-patch:
+concat-multiple-kfd-images-list:
 	@for version in $(KFD_VERSIONS); do \
-      cat $$version/images.txt >> global_images.tmp.txt; \
+	  cat $$version/images.txt >> $(PATCH_FILE_IMAGE_LIST_TO_PATCHING).tmp; \
 	done
-	cat global_images.tmp.txt | sort -u > global_images.txt
-	rm global_images.tmp.txt
-	DOCKER_CONFIG="${DOCKER_CONFIG}" DRY_RUN=$(DRY_RUN) ../scripts/patch_images_with_copacetic.sh
+	cat "$(PATCH_FILE_IMAGE_LIST_TO_PATCHING).tmp" | sort -u > $(PATCH_FILE_IMAGE_LIST_TO_PATCHING)
+	rm $(PATCH_FILE_IMAGE_LIST_TO_PATCHING).tmp
+
+patch:
+	DOCKER_CONFIG="${DOCKER_CONFIG}" DRY_RUN=$(DRY_RUN) ../scripts/patch_images_with_copacetic.sh -i "$(IMAGE_TO_PATCH)" -l "$(PATCH_FILE_IMAGE_LIST_TO_PATCHING)" -o "$(PATCH_REPORT_OUTPUT_FILE)"
 
 scan-post-patch:
 	@for version in $(KFD_VERSIONS); do \
@@ -51,7 +58,7 @@ kustomize-build-all:
 			`find $(KFD_VERSION)/.furyctl/bin -name kustomize -type f` build `dirname $$kustomize` >> $(KFD_VERSION)/built.yaml; \
 	done
 
-gen-image-list:
+generate-image-list-from-manifests:
 	awk '$$0 ~ /registry.sighup.io/ {if($$1 == "image:"){print $$2}}' $(KFD_VERSION)/built.yaml | sort -u > $(KFD_VERSION)/images.tmp.txt  #match image:
 	awk '$$0 ~ /registry.sighup.io/ {if($$2 == "image:"){print $$3}}' $(KFD_VERSION)/built.yaml| sort -u >> $(KFD_VERSION)/images.tmp.txt  #match - image:
 	awk -F= '$$2 ~ /registry.sighup.io/ {print $$2}' $(KFD_VERSION)/built.yaml | sort -u >> $(KFD_VERSION)/images.tmp.txt 				 #match --ARG=IMAGE

diff --git a/scripts/patch_images_with_copacetic.sh b/scripts/patch_images_with_copacetic.sh
@@ -1,11 +1,39 @@
 #!/bin/bash
 
+IMAGE_TO_PATCH=
+FILE_WITH_IMAGES_LIST_TO_PATCH=
+PATCH_REPORT_OUTPUT_FILE=
+
+function usage() {
+    echo "Usage: $0 -i IMAGE_TO_PATCH | -l FILE_WITH_IMAGES_LIST_TO_PATCH [-o PATCH_REPORT_OUTPUT_FILE ]"
+}
+
+while getopts ":i:l:o:" o; do
+    case "${o}" in
+        i)
+            IMAGE_TO_PATCH=${OPTARG}
+            ;;
+        l)
+            FILE_WITH_IMAGES_LIST_TO_PATCH=${OPTARG}
+            ;;
+        o)
+            PATCH_REPORT_OUTPUT_FILE=${OPTARG}
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+[[ -z $FILE_WITH_IMAGES_LIST_TO_PATCH ]] && FILE_WITH_IMAGES_LIST_TO_PATCH=images.txt
+[[ -z $PATCH_REPORT_OUTPUT_FILE ]] && PATCH_REPORT_OUTPUT_FILE=PATCHED.md
+
 TRIVY_SCAN_OUTPUT_DIR=.patching/scan
 COPA_PATCH_OUTPUT_DIR=.patching/patch
 DOCKERFILE_OUTPUT_DIR=.patching/dockerfile
 LOG_OUTPUT_DIR=.patching/log
-FILE_WITH_IMAGES_LIST_TO_PATCH=global_images.txt
-PATCH_REPORT_OUTPUT_FILE=PATCHED.md
 PATCH_ERROR_OUTPUT_FILE="$LOG_OUTPUT_DIR/patch-error.log"
 
 if [ -z "$(docker ps -f name=buildkitd -q)" ]
@@ -22,6 +50,9 @@ mkdir -p "$TRIVY_SCAN_OUTPUT_DIR" "$COPA_PATCH_OUTPUT_DIR" "$DOCKERFILE_OUTPUT_D
 echo -n "" > "${PATCH_ERROR_OUTPUT_FILE}"
 
 {
+[[ -n $IMAGE_TO_PATCH ]] && printf "# %s\n\n" $IMAGE_TO_PATCH
+printf "Last updated %s\n\n" "$(date +'%Y-%m-%d')";
+printf "## CVEs patched\n\n" ;
 echo "| Source Image | Source Image Hash |CVE | Severity | Description | Patched Image| Patched Image Hash |"
 echo "| --- | --- | --- | --- |--- | --- | --- |"
 } > "${PATCH_REPORT_OUTPUT_FILE}"
@@ -115,6 +146,8 @@ function patch_image() {
   echo ""
 }
 
+[[ -n $IMAGE_TO_PATCH ]] && patch_image "$IMAGE_TO_PATCH" && exit 0
+
 while IFS= read -r image; do
   patch_image "$image"
 done < "$FILE_WITH_IMAGES_LIST_TO_PATCH"