Skip to content

CD - Syft

CD - Syft #145

Workflow file for this run

name: CD - Syft
on:
schedule:
- cron: "00 12 * * */7" # At 12:00 UTC on every seven days
workflow_dispatch:
inputs:
skip_tests:
description: "If true, skip tests"
required: false
default: "false"
release_platform:
description: "Release Platform"
required: true
default: "REAL_PYPI"
type: choice
options:
- REAL_PYPI
- TEST_PYPI
- REAL_AND_TEST_PYPI
jobs:
call-pr-tests-linting:
if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
uses: OpenMined/PySyft/.github/workflows/pr-tests-linting.yml@dev
call-pr-tests-syft:
if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
uses: OpenMined/PySyft/.github/workflows/pr-tests-syft.yml@dev
call-pr-tests-stack:
if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
uses: OpenMined/PySyft/.github/workflows/pr-tests-stack.yml@dev
secrets: inherit
deploy-syft:
needs: [call-pr-tests-linting, call-pr-tests-syft, call-pr-tests-stack]
if: always() && (needs.call-pr-tests-linting.result == 'success' && needs.call-pr-tests-syft.result == 'success' && needs.call-pr-tests-stack.result == 'success' || github.event.inputs.skip_tests == 'true')
runs-on: om-ci-16vcpu-ubuntu2204
permissions:
contents: write # For tag and release notes.
steps:
- name: Permission to home directory
run: |
sudo chown -R $USER:$USER $HOME
- uses: actions/checkout@v3
with:
token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: "3.11"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install --upgrade tox setuptools wheel twine bump2version PyYAML
- name: Bump the Version
run: |
ls **/VERSION | xargs -I {} python {}
cat packages/grid/devspace.yaml | grep '0\.'
bump2version prenum --allow-dirty --no-commit
tox -e lint || true
ls **/VERSION | xargs -I {} python {}
cat packages/grid/devspace.yaml | grep '0\.'
python packages/hagrid/scripts/update_manifest.py $(python packages/grid/VERSION)
- name: Build Helm Chart
shell: bash
run: |
# install k3d
K3D_VERSION=v5.6.0
wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
mv k3d-linux-amd64 k3d
chmod +x k3d
export PATH=`pwd`:$PATH
k3d version
DEVSPACE_VERSION=v6.3.3
curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
chmod +x devspace
devspace version
tox -e syft.build.helm
tox -e syft.package.helm
- name: Check and Bump Protocol Version
if: github.event_name == 'schedule'
run: |
tox -e syft.protocol.check
- name: Commit changes to Syft
uses: EndBug/add-and-commit@v9
with:
author_name: ${{ secrets.OM_BOT_NAME }}
author_email: ${{ secrets.OM_BOT_EMAIL }}
message: "[syft]bump version"
add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' , 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/grid/helm/index.yaml']"
- name: Scheduled Build and Publish
if: github.event_name == 'schedule'
run: |
tox -e syft.publish
twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
- name: Manual Build and Publish
if: github.event_name != 'schedule'
run: |
tox -e syft.publish
if [ "${{ github.event.inputs.release_platform }}" = "REAL_PYPI" ]; then
twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
fi
if [ "${{ github.event.inputs.release_platform }}" = "TEST_PYPI" ]; then
twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
fi
if [ "${{ github.event.inputs.release_platform }}" = "REAL_AND_TEST_PYPI" ]; then
twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
fi
- name: Set Grid package version
id: grid-version
shell: bash
run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
# Check if the version is a pre-release
- name: Check if the version is a pre-release and modify version string
id: release_checks
run: |
if [[ $(python packages/grid/VERSION) == *"beta"* ]]; then
echo "is_pre_release=true" >> $GITHUB_OUTPUT
echo "github_release_version=$(python packages/grid/VERSION | sed 's/-beta./b/')" >> $GITHUB_OUTPUT
else
echo "is_pre_release=false" >> $GITHUB_OUTPUT
echo "github_release_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
fi
# Checkout Infra repo (nested)
- name: Checkout Infra Repo
uses: actions/checkout@v3
with:
repository: ${{ secrets.INFRA_REPO }}
ref: "main"
token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
path: infrastructure
# This step will copy the generated K8s manifest files to the correct directory in Infra repo
- name: Copy files to Infra Repo
run: |
if $is_pre_release; then
rm -rf infrastructure/gitops/environments/pre-release/.
cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/pre_release/
else
rm -rf infrastructure/gitops/environments/stable/.
cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/stable/
fi
- name: Commit changes to Infra Repo
uses: EndBug/add-and-commit@v9
with:
author_name: ${{ secrets.OM_BOT_NAME }}
author_email: ${{ secrets.OM_BOT_EMAIL }}
message: "Update K8s Manifests from Syft Repo"
add: "."
push: "origin main"
cwd: "./infrastructure/"
- name: Create SyftCLI Config assets
run: |
pip install pyyaml
python scripts/create_syftcli_config.py
- name: GitHub Release
uses: softprops/action-gh-release@v1
with:
name: v${{ steps.release_checks.outputs.github_release_version }}
generate_release_notes: true
prerelease: ${{ steps.release_checks.outputs.is_pre_release }}
files: |
./packages/syftcli/manifest.yml
./build/syftcli-config/*
./packages/hagrid/hagrid/manifest_template.yml
tag_name: v${{ steps.release_checks.outputs.github_release_version }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_LOGIN }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Docker meta for Grid-Backend
id: meta_grid_backend
uses: docker/metadata-action@v3
with:
images: openmined/grid-backend
tags: |
type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
type=raw,value=beta
- name: Build and push `grid-backend` image to DockerHub
uses: docker/build-push-action@v4
with:
context: ./packages
file: ./packages/grid/backend/backend.dockerfile
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta_grid_backend.outputs.tags }}
target: backend
cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }}
cache-to: type=inline
- name: Docker meta for Grid-Frontend
id: meta_grid_frontend
uses: docker/metadata-action@v3
with:
images: openmined/grid-frontend
tags: |
type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
type=raw,value=beta
- name: Build and push `grid-frontend` image to DockerHub
uses: docker/build-push-action@v4
with:
context: ./packages/grid/frontend
file: ./packages/grid/frontend/frontend.dockerfile
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta_grid_frontend.outputs.tags }}
target: grid-ui-development
cache-from: type=registry,ref= ${{ steps.meta_grid_frontend.outputs.tags }}
cache-to: type=inline
- name: Docker meta for Grid-Enclave
id: meta_grid_enclave
uses: docker/metadata-action@v3
with:
images: openmined/grid-enclave
tags: |
type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
type=raw,value=beta
- name: Build and push `grid-enclave` image to DockerHub
uses: docker/build-push-action@v4
with:
context: ./packages
file: ./packages/grid/worker/worker.dockerfile
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta_grid_enclave.outputs.tags }}
target: worker
cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }}
cache-to: type=inline
- name: Docker meta for Grid-Headscale
id: meta_grid_headscale
uses: docker/metadata-action@v3
with:
images: openmined/grid-vpn-headscale
tags: |
type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
type=raw,value=beta
- name: Build and push `grid-headscale` image to DockerHub
uses: docker/build-push-action@v4
with:
context: ./packages/grid/vpn
file: ./packages/grid/vpn/headscale.dockerfile
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta_grid_headscale.outputs.tags }}
cache-from: type=registry,ref= ${{ steps.meta_grid_headscale.outputs.tags }}
cache-to: type=inline
- name: Docker meta for Grid-Tailscale
id: meta_grid_tailscale
uses: docker/metadata-action@v3
with:
images: openmined/grid-vpn-tailscale
tags: |
type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
type=raw,value=beta
- name: Build and push `grid-tailscale` image to DockerHub
uses: docker/build-push-action@v4
with:
context: ./packages/grid/vpn
file: ./packages/grid/vpn/tailscale.dockerfile
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta_grid_tailscale.outputs.tags }}
cache-from: type=registry,ref= ${{ steps.meta_grid_tailscale.outputs.tags }}
cache-to: type=inline
- name: Docker meta for grid-vpn-iptables
id: meta_grid_vpn_iptables
uses: docker/metadata-action@v3
with:
images: openmined/grid-vpn-iptables
tags: |
type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
type=raw,value=beta
- name: Build and push `grid-vpn-iptables` image to DockerHub
uses: docker/build-push-action@v2
with:
context: ./packages/grid/vpn
file: ./packages/grid/vpn/iptables.dockerfile
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta_grid_vpn_iptables.outputs.tags }}
cache-from: type=registry,ref= ${{ steps.meta_grid_vpn_iptables.outputs.tags }}
cache-to: type=inline
- name: Login to Docker-Helm
run: |
helm registry login registry-1.docker.io -u ${{ secrets.DOCKER_LOGIN }} --password ${{secrets.DOCKER_PASSWORD}}
cd packages/grid/helm/repo && helm push syft-${{ steps.grid-version.outputs.GRID_VERSION }}.tgz oci://registry-1.docker.io/openmined