Upload Build Provenance Metadata

.github/workflows/deploy-2-start.yml

@@ -34,11 +34,15 @@ on:
         description: The root SBD that is being used as the modulefile name
 env:
   SPACK_YAML_MODULEFILE_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ inputs.root-sbd }}
+  METADATA_PATH: /opt/metadata
 jobs:
   deploy-to-environment:
     name: Deploy to ${{ inputs.deployment-environment }}
     runs-on: ubuntu-latest
     environment: ${{ inputs.deployment-environment }}
+    outputs:
+      packages-version: ${{ steps.versions.outputs.packages }}
+      config-version: ${{ steps.versions.outputs.config }}
     steps:
       # Deployment
       - uses: actions/checkout@v4
@@ -95,32 +99,99 @@ jobs:
 
           # Obtain metadata
           spack find --paths > ${{ vars.SPACK_LOCATION }}/var/spack/environments/${{ inputs.env-name }}/spack.location
-          spack find --format '{name}@{version} {prefix}' | jq --raw-input --null-input '[inputs | split(" ") | {(.[0]):(.[1])}] | add' > ${{ vars.SPACK_LOCATION }}/var/spack/environments/${{ inputs.env-name }}/spack.location.json
+          spack find --format '{hash} {prefix}' | jq --raw-input --null-input '[inputs | split(" ") | {(.[0]): (.[1])}] | add' > ${{ vars.SPACK_LOCATION }}/var/spack/environments/${{ inputs.env-name }}/spack.location.json
 
           spack env deactivate
           echo "$(date): Deployed ${{ inputs.model }} ${{ inputs.version }} with spack-packages ${{ steps.versions.outputs.packages }}, spack-config ${{ steps.versions.outputs.config }}" >> ${{ vars.SPACK_RELEASE_LOCATION }}/release.log
           EOT
 
       # Release
       - name: Get Release Metadata
-        if: inputs.type == 'release'
         run: |
           rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
             '${{ secrets.USER}}@${{ secrets.HOST_DATA }}:${{ vars.SPACK_LOCATION }}/var/spack/environments/${{ inputs.env-name }}/spack.*' \
             ./${{ inputs.env-name }}
 
+      - name: Upload Metadata Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ inputs.env-name }}
+          path: ./${{ inputs.env-name }}/*
+          overwrite: true
+
+  release:
+    name: Create Release
+    if: inputs.type == 'release'
+    needs:
+      - deploy-to-environment
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.release.outputs.url }}
+      created-at: ${{ steps.metadata.outputs.created-at }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download Metadata Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.env-name }}
+          path: ${{ env.METADATA_PATH }}
+
       - name: Create Release
-        if: inputs.type == 'release'
-        uses: softprops/action-gh-release@v0.1.15
+        id: release
+        uses: softprops/action-gh-release@69320dbe05506a9a39fc8ae11030b214ec2d1f87  # v2.0.5
         with:
           tag_name: ${{ inputs.version }}
           name: ${{ inputs.model}} ${{ inputs.version }}
           body: |
-            This release of ${{ inputs.model }} ${{ inputs.version }} uses [spack-packages ${{ steps.versions.outputs.packages }}](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ steps.versions.outputs.packages }}) and [spack-config ${{ steps.versions.outputs.config }}](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ steps.versions.outputs.config }}).
+            This release of ${{ inputs.model }} ${{ inputs.version }} uses [spack-packages ${{ needs.deploy-to-environment.outputs.packages-version }}](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ needs.deploy-to-environment.outputs.packages-version }}) and [spack-config ${{ needs.deploy-to-environment.outputs.config-version }}](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ needs.deploy-to-environment.outputs.config-version }}).
           generate_release_notes: true
           fail_on_unmatched_files: true
           files: |
-            ./${{ inputs.env-name }}/spack.yaml
-            ./${{ inputs.env-name }}/spack.lock
-            ./${{ inputs.env-name }}/spack.location
-            ./${{ inputs.env-name }}/spack.location.json
+            ./${{ env.METADATA_PATH }}/spack.yaml
+            ./${{ env.METADATA_PATH }}/spack.lock
+            ./${{ env.METADATA_PATH }}/spack.location
+            ./${{ env.METADATA_PATH }}/spack.location.json
+
+      - name: Release Metadata
+        id: metadata
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: echo "created-at=$(gh release view --json createdAt --jq '.createdAt')" >> $GITHUB_OUTPUT
+
+  build-db:
+    name: Build DB Metadata Upload
+    if: inputs.type == 'release'
+    needs:
+      - release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Metadata Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.env-name }}
+          path: ${{ env.METADATA_PATH }}
+
+      - name: Checkout Upload Script
+        uses: actions/checkout@v4
+        with:
+          repository: access-nri/build-cd
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: pip
+
+      - name: Install Build Metadata Script Requirements
+        run: pip install -r tools/release_provenance/requirements.txt
+
+      - name: Upload Build Metadata
+        env:
+          BUILD_DB_CONNECTION_STR: ${{ secrets.BUILD_DB_CONNECTION_STR }}
+          OUTPUT_PATH: ./metadata_output
+        run: |
+          ./scripts/generate-build-metadata.bash ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} ${{ env.METADATA_PATH }} ${{ env.OUTPUT_PATH }} ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }}
+
+          echo "Attempting upload of build_metadata.json"
+          python ./tools/release_provenance/save_release.py "${{ env.OUTPUT_PATH }}/build_metadata.json"

.gitignore

@@ -0,0 +1,2 @@
+# Files generated during generate-build-metadata tests
+tests/scripts/generate-build-metadata/outputs

scripts/generate-build-metadata.bash

@@ -0,0 +1,90 @@
+#!/bin/bash
+set -x
+set -e
+
+### INPUTS ###
+# URL for the associated GitHub Release of $model_name.
+release_url=$1
+# Timestamp for the creation of the $release_url.
+release_time=$2
+# Path to the dir containing the spack.{lock,location.json}.
+json_dir=$3
+# directory that contains the <package>.json files
+output_dir=$4
+# Name of the model (or root-sbd) (eg. access-om2)
+model_name=$5
+
+# the rest of the model components for the given $model_name
+# (eg. for access-om2 there would be mom5, cice5, etc...)
+shift 5
+packages=( "$@" )
+
+### SCRIPT ###
+mkdir -p "$output_dir"
+
+spack=$(jq \
+  '{
+    version: .spack.version,
+    commit: .spack.commit
+  }' "$json_dir/spack.lock")
+
+model=$(jq \
+  --arg model "$model_name" \
+  --arg release_url "$release_url" \
+  --arg release_time "$release_time" \
+  --argjson spack "$spack" \
+  '.concrete_specs | to_entries[] | select(.value.name == $model)
+  | {
+      spack_hash: .key,
+      spec: (.value.name + "@" + .value.version),
+      created_at: $release_time,
+      release_url: $release_url,
+      spack_version: $spack
+  }' "$json_dir/spack.lock"
+)
+
+# construction of the initial build_metadata.json
+jq --null-input \
+  --argjson model "$model" \
+  '{
+    model_build: $model,
+    component_build: [],
+  }' > "$output_dir/build_metadata.json"
+
+for pkg in "${packages[@]}"; do
+  pkg_hash=$(jq --raw-output \
+    --arg pkg "$pkg" \
+    '.concrete_specs | to_entries[] | select(.value.name == $pkg) | .key' \
+    "$json_dir/spack.lock"
+  )
+
+  echo "Hash of $pkg is $pkg_hash"
+
+  install_path=$(jq --raw-output \
+    --arg pkg_hash "$pkg_hash" \
+    'to_entries[] | select(.key == $pkg_hash) | .value.path' \
+    "$json_dir/spack.location.json"
+  )
+
+  component=$(jq \
+    --arg pkg "$pkg" \
+    --arg install_path "$install_path" \
+    '.concrete_specs | to_entries[] | select(.value.name == $pkg)
+    | {
+      spack_hash: .key,
+      spec: (.value.name + "@" + .value.version),
+      install_path: $install_path
+    }' "$json_dir/spack.lock"
+  )
+
+  # piecewise construction of the entire build_metadata.json for each
+  # build_component
+  jq \
+    --argjson component "$component" \
+    '.component_build += [$component]' \
+    "$output_dir/build_metadata.json" > "$output_dir/build_metadata.json.tmp"
+
+  mv "$output_dir/build_metadata.json.tmp" "$output_dir/build_metadata.json"
+
+  cat "$output_dir/build_metadata.json"
+done

tests/scripts/generate-build-metadata/inputs/spack.location.json

@@ -0,0 +1,14 @@
+{
+    "ucmkt2gy72vvikhstgvoft65wnskr7rd": {
+        "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/access-om2-git.2024.03.0=2024.03.0-ucmkt2gy72vvikhstgvoft65wnskr7rd"
+    },
+    "v3zncpqjj2gyseudbwiudolcjq3k3leo": {
+        "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/cice5-git.2023.10.19=2023.10.19-v3zncpqjj2gyseudbwiudolcjq3k3leo"
+    },
+    "qji4nlmr6utrribaiyhewe4je6mifguz": {
+        "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/mom5-git.2023.11.09=2023.11.09-qji4nlmr6utrribaiyhewe4je6mifguz"
+    },
+    "i3inxzaihefr3rqljoovhyevwai6bsff": {
+        "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/netcdf-c-4.7.4-i3inxzaihefr3rqljoovhyevwai6bsff"
+    }
+}