From 5415ae054ea7496f939667d07a4a11c7e742a95a Mon Sep 17 00:00:00 2001 From: Utkarsh Gupta <53311121+utkarshgupta95@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:43:12 +1000 Subject: [PATCH] added release_url to component build (#113) * added release_url to component build * Send git attributes of vars.BUILD_DB_PACKAGES to generate-build-metadata.bash * Added /releases section to git url * deploy-2-start.yml: Added tag to the end of the pkg_release_url * deploy-2-start.yml: Enable and activate spack * deploy-2-start.yml: Get version from spack.lock * deploy-2-start.yml: Added further comments --------- Co-authored-by: Tommy Gatti --- .github/workflows/deploy-2-start.yml | 51 +++++++++++++++++-- scripts/generate-build-metadata.bash | 10 +++- tools/release_provenance/models.py | 1 + tools/release_provenance/save_release.py | 1 + .../release_provenance/test_release_data.json | 6 ++- 5 files changed, 61 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index 68f1f91..3e89f1d 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -114,20 +114,61 @@ jobs: spack env activate ${{ inputs.env-name }} spack --debug install --fresh ${{ vars.SPACK_INSTALL_PARALLEL_JOBS }} || exit $? spack module tcl refresh -y + EOT + + - name: Get metadata from ${{ inputs.deployment-environment }} + env: + SPACK_ENV_PATH: ${{ steps.path.outputs.spack }}/var/spack/environments/${{ inputs.env-name }} + run: | + ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + . ${{ steps.path.outputs.spack-config }}/spack-enable.bash + spack env activate ${{ inputs.env-name }} + + spack find --paths > ${{ env.SPACK_ENV_PATH }}/spack.location + spack find --format '{hash} {prefix}' | jq --raw-input --null-input '[inputs | split(" ") | {(.[0]): (.[1])}] | add' > ${{ env.SPACK_ENV_PATH }}/spack.location.json + + # Get the repos associated with the packages for the build database + jq -n '{}' > ${{ env.SPACK_ENV_PATH }}/build-db-pkgs.json + for pkg in ${{ vars.BUILD_DB_PACKAGES }}; do + # TODO: Is there a way to get the git attribute without concretizing? + pkg_repo_url=$(spack python -c "import spack.spec; print(spack.spec.Spec('$pkg').concretized().package.git)") - # Obtain metadata - spack find --paths > ${{ steps.path.outputs.spack }}/var/spack/environments/${{ inputs.env-name }}/spack.location - spack find --format '{hash} {prefix}' | jq --raw-input --null-input '[inputs | split(" ") | {(.[0]): (.[1])}] | add' > ${{ steps.path.outputs.spack }}/var/spack/environments/${{ inputs.env-name }}/spack.location.json + # We get the version of $pkg from spack.lock, and then strip + # potential 'git.' and '=VERSION' parts from 'git.TAG=VERSION' + version=$(jq --compact-output --raw-output \ + --arg p "$pkg" \ + '.concrete_specs | to_entries[].value | select(.name == $p) | .version + | match("^(?:git.)?([^=]*)") + | .captures[0].string' \ + ${{ env.SPACK_ENV_PATH}}/spack.lock + ) + + # Example: + # pkg_repo_url = https://github.com/ACCESS-NRI/MOM5.git, which is then stripped of the '.git'. + # version = 2024.08.11, giving + # pkg_release_url = https://github.com/ACCESS-NRI/MOM5/releases/tag/2024.08.11 + pkg_release_url="${pkg_repo_url%.*}/releases/tag/${version}" + + echo "$pkg pkg_repo_url is $pkg_repo_url, pkg_release_url is $pkg_release_url, version is $version" + jq \ + --arg p "$pkg" \ + --arg r "$pkg_release_url" \ + '. += {($p): ($r)}' \ + ${{ env.SPACK_ENV_PATH }}/build-db-pkgs.json > ${{ env.SPACK_ENV_PATH }}/build-db-pkgs.json.tmp + mv -f ${{ env.SPACK_ENV_PATH }}/build-db-pkgs.json.tmp ${{ env.SPACK_ENV_PATH }}/build-db-pkgs.json + done spack env deactivate - echo "$(date): Deployed ${{ inputs.model }} ${{ inputs.version }} with spack-packages ${{ steps.versions.outputs.packages }}, spack-config ${{ steps.versions.outputs.config }}" >> ${{ steps.path.outputs.root }}/release.log EOT # Release - name: Get Release Metadata + env: + # TODO: Can we put both envs above in a $GITHUB_ENV file instead? + SPACK_ENV_PATH: ${{ steps.path.outputs.spack }}/var/spack/environments/${{ inputs.env-name }} run: | rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ - '${{ secrets.USER}}@${{ secrets.HOST_DATA }}:${{ steps.path.outputs.spack }}/var/spack/environments/${{ inputs.env-name }}/spack.*' \ + '${{ secrets.USER}}@${{ secrets.HOST_DATA }}:${{ env.SPACK_ENV_PATH }}/spack.*' \ ./${{ inputs.env-name }} - name: Upload Metadata Artifact diff --git a/scripts/generate-build-metadata.bash b/scripts/generate-build-metadata.bash index e1db7ad..abfd68d 100755 --- a/scripts/generate-build-metadata.bash +++ b/scripts/generate-build-metadata.bash @@ -75,14 +75,22 @@ for pkg in "${packages[@]}"; do "$json_dir/spack.location.json" ) + release_url=$(jq --raw-output \ + --arg pkg "$pkg" \ + '.[$pkg]' \ + "$json_dir/build-db-pkgs.json" + ) + component=$(jq \ --arg pkg "$pkg" \ --arg install_path "$install_path" \ + --arg release_url "$release_url" \ '.concrete_specs | to_entries[] | select(.value.name == $pkg) | { spack_hash: .key, spec: (.value.name + "@" + .value.version), - install_path: $install_path + install_path: $install_path, + release_url: $release_url }' "$json_dir/spack.lock" ) diff --git a/tools/release_provenance/models.py b/tools/release_provenance/models.py index 548d2be..af10a12 100644 --- a/tools/release_provenance/models.py +++ b/tools/release_provenance/models.py @@ -26,6 +26,7 @@ class ComponentBuild(Base): spack_hash = Column(String, primary_key=True, index=True) spec = Column(String, nullable=False) install_path = Column(String, nullable=False, unique=True) + release_url = Column(Text, nullable=False, unique=True) model_build = relationship('ModelBuild', secondary="model_component", back_populates='component_build') class ModelStatusEnum(enum.Enum): diff --git a/tools/release_provenance/save_release.py b/tools/release_provenance/save_release.py index b7d4823..855a4b9 100644 --- a/tools/release_provenance/save_release.py +++ b/tools/release_provenance/save_release.py @@ -19,6 +19,7 @@ def get_component_build(component_build_data_list, model_build): component_build.install_path = component_build_data["install_path"] component_build.spack_hash = component_build_data["spack_hash"] component_build.spec = component_build_data["spec"] + component_build.release_url = component_build_data["release_url"] component_build.model_build.append(model_build) component_build_list.append(component_build) else: diff --git a/tools/release_provenance/test_release_data.json b/tools/release_provenance/test_release_data.json index c0512d6..272bb06 100644 --- a/tools/release_provenance/test_release_data.json +++ b/tools/release_provenance/test_release_data.json @@ -3,12 +3,14 @@ { "spack_hash": "ewcdbrfukblyjxpkhd3mfkj4yxfolal8", "spec": "mom5@git.2023.11.09=2023.11.09", - "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.280/" + "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.280/", + "release_url": "https://github.com/ACCESS-NRI/mom5/releases" }, { "spack_hash": "ewcdbrfukblyjxpkhd3mfkj4yxfolal9", "spec": "mom5@git.2023.11.09=2023.11.09", - "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.1.5.283/" + "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.1.5.283/", + "release_url": "https://github.com/ACCESS-NRI/mom5-example/releases" } ], "model_build": {