Skip to content

CD - Syft

CD - Syft #317

Workflow file for this run

name: CD - Syft
on:
schedule:
- cron: "0 12 * * 0" # At 12:00 UTC on every Sunday
workflow_dispatch:
inputs:
skip_tests:
description: "If true, skip pre-release tests"
required: false
default: "false"
type: choice
options:
- false
- true
release_platform:
description: "Release Platform"
required: true
default: "REAL_PYPI"
type: choice
options:
- REAL_PYPI
- TEST_PYPI
- REAL_AND_TEST_PYPI
# Prevents concurrent runs of the same workflow
# while the previous run is still in progress
concurrency:
group: "CD - Syft"
cancel-in-progress: false
jobs:
release-checks:
if: github.repository == 'OpenMined/PySyft' # don't run on forks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
# The steps ensure that the cron job is able to run only for
# for beta releases and not for stable releases
# For stable release, we always have to manually run the workflow
- name: Check if the release is a beta release
run: |
if [[ $(python packages/grid/VERSION) != *"beta"* && ${{github.event_name}} == 'schedule' ]]; then
echo "Cron Job could only be run for beta releases"
exit 1
fi
# Print the github user triggering the workflow
- name: Github User triggering the workflow
run: |
echo "Github User: ${{ github.actor }}"
call-pr-tests-linting:
needs: [release-checks]
if: github.repository == 'OpenMined/PySyft' && needs.release-checks.result == 'success' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
uses: OpenMined/PySyft/.github/workflows/pr-tests-linting.yml@dev
call-pr-tests-syft:
needs: [release-checks]
if: github.repository == 'OpenMined/PySyft' && needs.release-checks.result == 'success' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
uses: OpenMined/PySyft/.github/workflows/pr-tests-syft.yml@dev
call-pr-tests-stack:
needs: [release-checks]
if: github.repository == 'OpenMined/PySyft' && needs.release-checks.result == 'success' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
uses: OpenMined/PySyft/.github/workflows/pr-tests-stack.yml@dev
secrets: inherit
build-and-push-docker-images:
needs:
[
call-pr-tests-linting,
call-pr-tests-syft,
call-pr-tests-stack,
release-checks,
]
if: always() && needs.release-checks.result == 'success' && (needs.call-pr-tests-linting.result == 'success' && needs.call-pr-tests-syft.result == 'success' && needs.call-pr-tests-stack.result == 'success' || github.event.inputs.skip_tests == 'true')
strategy:
matrix:
runner: [sh-arc-linux-x64, sh-arc-linux-arm64]
runs-on: ${{ matrix.runner }}
outputs:
release_tag: ${{ steps.get_release_tag.outputs.release_tag }}
grid_version: ${{ steps.release_metadata.outputs.grid_version }}
steps:
- uses: actions/checkout@v4
# actions/setup-python doesn't yet support ARM
- name: Setup Python on x64
if: ${{ !endsWith(matrix.runner, '-arm64') }}
uses: actions/setup-python@v5
with:
python-version: "3.12"
# Currently psutil package requires gcc to be installed on arm
# for building psutil from source
# as linux/aarch64 wheels not avaialble for psutil
# We could remove once we have aarch64 wheels
# https://github.com/giampaolo/psutil/issues/1972
- name: Install Metadata packages for arm64
if: ${{ endsWith(matrix.runner, '-arm64') }}
run: |
sudo apt update -y
sudo apt install software-properties-common -y
sudo apt install gcc -y
- name: Setup Python on arm64
if: ${{ endsWith(matrix.runner, '-arm64') }}
uses: deadsnakes/[email protected]
with:
python-version: "3.12"
- name: Install Git
run: |
sudo apt-get update
sudo apt-get install git -y
- name: Check python version
run: |
python --version
python3 --version
which python
- name: Install dependencies
run: |
pip install --upgrade pip uv==0.1.35 bump2version tox tox-uv==1.5.1
uv --version
- name: Get Release tag
id: get_release_tag
run: |
if [[ $(python packages/grid/VERSION) == *"beta"* ]]; then
echo "release_tag=beta" >> $GITHUB_OUTPUT
else
echo "release_tag=latest" >> $GITHUB_OUTPUT
fi
- name: Bump the Version
if: steps.get_release_tag.outputs.release_tag == 'beta'
run: |
ls **/VERSION | xargs -I {} python {}
cat packages/grid/devspace.yaml | grep '0\.'
bump2version prenum --allow-dirty --no-commit
ls **/VERSION | xargs -I {} python {}
cat packages/grid/devspace.yaml | grep '0\.'
- name: Generate Release Metadata
id: release_metadata
run: |
if [[ ${{matrix.runner}} == *"x64"* ]]; then
echo "release_platform=linux/amd64" >> $GITHUB_OUTPUT
echo "short_release_platform=amd64" >> $GITHUB_OUTPUT
else
echo "release_platform=linux/arm64" >> $GITHUB_OUTPUT
echo "short_release_platform=arm64" >> $GITHUB_OUTPUT
fi
echo "grid_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
# TODO: Optimize redundant bump protocol version checks
- name: Check and Bump Protocol Version
run: |
if [[ "${{ steps.get_release_tag.outputs.release_tag }}" == "latest" ]]; then
export BUMP=True
fi
tox -e syft.protocol.check
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_LOGIN }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push `grid-backend` image to DockerHub
id: grid-backend-build
uses: docker/build-push-action@v5
with:
context: ./packages
file: ./packages/grid/backend/backend.dockerfile
platforms: ${{ steps.release_metadata.outputs.release_platform }}
target: backend
outputs: type=image,name=openmined/grid-backend,push-by-digest=true,name-canonical=true,push=true
cache-from: type=registry,ref=openmined/grid-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }}
cache-to: type=registry,ref=openmined/grid-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max
- name: Export digest for grid-backend
run: |
mkdir -p /tmp/digests/grid-backend
digest="${{ steps.grid-backend-build.outputs.digest }}"
touch "/tmp/digests/grid-backend/${digest#sha256:}"
- name: Build and push `grid-frontend` image to DockerHub
id: grid-frontend-build
uses: docker/build-push-action@v5
with:
context: ./packages/grid/frontend
file: ./packages/grid/frontend/frontend.dockerfile
platforms: ${{ steps.release_metadata.outputs.release_platform }}
outputs: type=image,name=openmined/grid-frontend,push-by-digest=true,name-canonical=true,push=true
target: grid-ui-development
cache-from: type=registry,ref=openmined/grid-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform }}
cache-to: type=registry,ref=openmined/grid-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max
- name: Export digest for grid-frontend
run: |
mkdir -p /tmp/digests/grid-frontend
digest="${{ steps.grid-frontend-build.outputs.digest }}"
touch "/tmp/digests/grid-frontend/${digest#sha256:}"
- name: Build and push `grid-seaweedfs` image to DockerHub
id: grid-seaweedfs-build
uses: docker/build-push-action@v5
with:
context: ./packages/grid/seaweedfs
file: ./packages/grid/seaweedfs/seaweedfs.dockerfile
platforms: ${{ steps.release_metadata.outputs.release_platform }}
outputs: type=image,name=openmined/grid-seaweedfs,push-by-digest=true,name-canonical=true,push=true
cache-from: type=registry,ref=openmined/grid-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform }}
cache-to: type=registry,ref=openmined/grid-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max
- name: Export digest for grid-seaweedfs
run: |
mkdir -p /tmp/digests/grid-seaweedfs
digest="${{ steps.grid-seaweedfs-build.outputs.digest }}"
touch "/tmp/digests/grid-seaweedfs/${digest#sha256:}"
- name: Build and push `syft` image to registry
id: syft-build
uses: docker/build-push-action@v5
with:
context: ./packages/
file: ./packages/grid/syft-client/syft.Dockerfile
outputs: type=image,name=openmined/syft-client,push-by-digest=true,name-canonical=true,push=true
platforms: ${{ steps.release_metadata.outputs.release_platform }}
cache-from: type=registry,ref=openmined/syft-client:cache-${{ steps.release_metadata.outputs.short_release_platform }}
cache-to: type=registry,ref=openmined/syft-client:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max
- name: Export digest for `syft` image
run: |
mkdir -p /tmp/digests/syft
digest="${{ steps.syft-build.outputs.digest }}"
touch "/tmp/digests/syft/${digest#sha256:}"
- name: Upload digests
uses: actions/upload-artifact@v4
with:
name: digests-${{ steps.release_metadata.outputs.grid_version }}-${{ steps.release_metadata.outputs.short_release_platform }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
#Used to merge x64 and arm64 into one docker image
merge-docker-images:
needs: [build-and-push-docker-images]
if: always() && (needs.build-and-push-docker-images.result == 'success')
runs-on: sh-arc-linux-x64
outputs:
release_tag: ${{ needs.build-and-push-docker-images.outputs.release_tag }}
steps:
- name: Download digests
uses: actions/download-artifact@v4
with:
path: /tmp/digests
pattern: digests-${{ needs.build-and-push-docker-images.outputs.grid_version }}-*
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_LOGIN }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Create manifest list and push for grid-backend
working-directory: /tmp/digests/grid-backend
run: |
docker buildx imagetools create \
-t openmined/grid-backend:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
-t openmined/grid-backend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
$(printf 'openmined/grid-backend@sha256:%s ' *)
- name: Create manifest list and push for grid-frontend
working-directory: /tmp/digests/grid-frontend
run: |
docker buildx imagetools create \
-t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
-t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
$(printf 'openmined/grid-frontend@sha256:%s ' *)
- name: Create manifest list and push for grid-seaweedfs
working-directory: /tmp/digests/grid-seaweedfs
run: |
docker buildx imagetools create \
-t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
-t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
$(printf 'openmined/grid-seaweedfs@sha256:%s ' *)
- name: Create manifest list and push for syft
working-directory: /tmp/digests/syft
run: |
docker buildx imagetools create \
-t openmined/syft-client:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
-t openmined/syft-client:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
$(printf 'openmined/syft-client@sha256:%s ' *)
deploy-syft:
needs: [merge-docker-images]
if: always() && needs.merge-docker-images.result == 'success'
runs-on: ubuntu-latest
permissions:
contents: write # For tag and release notes.
outputs:
syft_version: ${{ steps.release_checks.outputs.syft_version }}
release_platform: ${{ github.event.inputs.release_platform }}
steps:
- name: Permission to home directory
run: |
sudo chown -R $USER:$USER $HOME
- uses: actions/checkout@v4
with:
token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
# free 10GB of space
- name: Remove unnecessary files
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
docker image prune --all --force
docker builder prune --all --force
docker system prune --all --force
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install dependencies
run: |
pip install --upgrade pip uv==0.1.35 tox tox-uv==1.5.1 setuptools wheel twine bump2version PyYAML
uv --version
- name: Bump the Version
if: needs.merge-docker-images.outputs.release_tag == 'beta'
run: |
ls **/VERSION | xargs -I {} python {}
cat packages/grid/devspace.yaml | grep '0\.'
bump2version prenum --allow-dirty --no-commit
ls **/VERSION | xargs -I {} python {}
cat packages/grid/devspace.yaml | grep '0\.'
python packages/hagrid/scripts/update_manifest.py $(python packages/grid/VERSION)
- name: Generate Release Metadata
id: release_checks
run: |
echo "github_release_version=$(python packages/grid/VERSION | sed 's/-beta./b/')" >> $GITHUB_OUTPUT
echo "future_stable_version=$(python packages/grid/VERSION | sed 's/-beta.*//')" >> $GITHUB_OUTPUT
echo "syft_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
if [[ ${{ needs.merge-docker-images.outputs.release_tag}} == "beta" ]]; then
echo "is_beta_release=true" >> $GITHUB_OUTPUT
else
echo "is_beta_release=false" >> $GITHUB_OUTPUT
fi
- name: Check Protocol Version
run: |
tox -e syft.protocol.check
- name: Build Helm Chart
shell: bash
run: |
# install k3d
K3D_VERSION=v5.6.3
wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
mv k3d-linux-amd64 k3d
chmod +x k3d
export PATH=`pwd`:$PATH
k3d version
#Install Devspace
DEVSPACE_VERSION=v6.3.12
curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
chmod +x devspace
devspace version
# Install helm
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
helm version
tox -e syft.build.helm
tox -e syft.package.helm
- name: Check and Bump Protocol Version
run: |
if [[ "${{ needs.merge-docker-images.outputs.release_tag }}" == "latest" ]]; then
export BUMP=True
fi
tox -e syft.protocol.check
- name: Update PyPI Readme
run: |
python scripts/convert_to_pypi_readme.py --input-file packages/syft/README.md --output-file packages/syft/PYPI.md --version ${{ steps.release_checks.outputs.future_stable_version }}
- name: Linting
run: |
tox -e lint || true
- name: Changes to commit to Syft Repo during beta release
if: needs.merge-docker-images.outputs.release_tag == 'beta'
uses: EndBug/add-and-commit@v9
with:
author_name: ${{ secrets.OM_BOT_NAME }}
author_email: ${{ secrets.OM_BOT_EMAIL }}
message: "[syft]bump version"
add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml', 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/syft/src/syft/protocol/releases/', 'packages/grid/backend/worker_cpu.dockerfile','packages/grid/helm/syft/values.yaml','packages/grid/helm/syft']"
- name: Changes to commit to Syft Repo during stable release
if: needs.merge-docker-images.outputs.release_tag == 'latest'
uses: EndBug/add-and-commit@v9
with:
author_name: ${{ secrets.OM_BOT_NAME }}
author_email: ${{ secrets.OM_BOT_EMAIL }}
message: "[syft] bump protocol version"
add: "['packages/syft/src/syft/protocol/protocol_version.json', 'packages/syft/src/syft/protocol/releases/','packages/syft/PYPI.md','packages/grid/helm/repo']"
- name: Scheduled Build and Publish
if: github.event_name == 'schedule'
run: |
tox -e syft.publish
twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
- name: Manual Build and Publish
if: github.event_name != 'schedule'
run: |
tox -e syft.publish
if [[ "${{ github.event.inputs.release_platform }}" == "REAL_PYPI" ]]; then
twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
fi
if [[ "${{ github.event.inputs.release_platform }}" == "TEST_PYPI" ]]; then
twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
fi
if [[ "${{ github.event.inputs.release_platform }}" == "REAL_AND_TEST_PYPI" ]]; then
twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
fi
# Checkout Infra repo (nested)
- name: Checkout Infra Repo
uses: actions/checkout@v4
with:
repository: ${{ secrets.INFRA_REPO }}
ref: "main"
token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
path: infrastructure
# This step will copy the generated K8s manifest files to the correct directory in Infra repo
- name: Copy files to Infra Repo
run: |
if [[ "${{ needs.merge-docker-images.outputs.release_tag }}" == "beta" ]]; then
rm -rf infrastructure/gitops/environments/pre_release/
mkdir -p infrastructure/gitops/environments/pre_release/
cp -R packages/grid/helm/syft/. infrastructure/gitops/environments/pre_release/
else
rm -rf infrastructure/gitops/environments/stable/
mkdir -p infrastructure/gitops/environments/stable/
cp -R packages/grid/helm/syft/. infrastructure/gitops/environments/stable/
fi
- name: Commit changes to Infra Repo
uses: EndBug/add-and-commit@v9
with:
author_name: ${{ secrets.OM_BOT_NAME }}
author_email: ${{ secrets.OM_BOT_EMAIL }}
message: "Update K8s Manifests from Syft Repo"
add: "."
push: "origin main"
cwd: "./infrastructure/"
- name: Create SyftCLI Config assets
run: |
pip install pyyaml
python scripts/create_syftcli_config.py
- name: GitHub Release
if: github.event.inputs.release_platform != 'TEST_PYPI'
uses: softprops/action-gh-release@v2
with:
name: v${{ steps.release_checks.outputs.github_release_version }}
generate_release_notes: true
prerelease: ${{ steps.release_checks.outputs.is_beta_release }}
files: |
./packages/syftcli/manifest.yml
./build/syftcli-config/*
./packages/hagrid/hagrid/manifest_template.yml
tag_name: v${{ steps.release_checks.outputs.github_release_version }}
# Checkout to gh-pages and update helm repo
- name: Checkout to gh-pages
uses: actions/checkout@v4
with:
ref: gh-pages
token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
path: ghpages
- name: Copy helm repo files from Syft Repo
run: |
rm -rf ghpages/helm/*
cp -R packages/grid/helm/repo/. ghpages/helm/
- name: Commit changes to gh-pages
uses: EndBug/add-and-commit@v9
with:
author_name: ${{ secrets.OM_BOT_NAME }}
author_email: ${{ secrets.OM_BOT_EMAIL }}
message: "Update Helm package from Syft Repo"
add: "helm/"
push: "origin gh-pages"
cwd: "./ghpages/"
# Since the post-release tests are dependent on the release being available on PyPI
# we need to wait for PyPI to update before running the post-release tests
- name: Wait for PyPI to update
run: |
sleep 60
# Can we remove the always flag from the below job?
call-cd-post-release-tests:
needs: [deploy-syft]
if: always() && github.repository == 'OpenMined/PySyft' && needs.deploy-syft.result == 'success' # don't run on forks
uses: OpenMined/PySyft/.github/workflows/cd-post-release-tests.yml@dev
with:
syft_version: ${{ needs.deploy-syft.outputs.syft_version }}
release_platform: ${{ github.event.inputs.release_platform}}