[WIP] Update CI.yaml #83
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Gitian CI | |
| on: | |
| pull_request: | |
| types: | |
| - labeled | |
| jobs: | |
| build-gitian: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v2' | |
| with: | |
| version: '>= 363.0.0' | |
| - name: Build Gitian | |
| id: gitian | |
| shell: bash | |
| run: | | |
| sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null | |
| echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json | |
| gcloud auth activate-service-account --key-file=json.json | |
| export random=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 12; echo;) | |
| for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do | |
| echo "removing SSH key $i" | |
| gcloud compute os-login ssh-keys remove --key $i || echo "failed to remove key" | |
| done | |
| gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=${{ secrets.GCP_PROJECT_ID_PROD }} --zone=us-central1-a --no-address --network=vpc-${{ secrets.GCP_PROJECT_ID_PROD }} --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@${{ secrets.GCP_PROJECT_ID_PROD }}.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB | |
| export counter=1 | |
| while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]] | |
| do | |
| echo "attempt number: $counter" | |
| export counter=$((counter+1)) | |
| if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all; exit 1; fi | |
| sleep 5 | |
| done | |
| IFS='/' read -r -a array <<< "${{ github.event.label.name }}" | |
| git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git | |
| cd zcash/contrib/gitian-descriptors | |
| wget -c -q https://github.com/mikefarah/yq/releases/download/v4.28.2/yq_linux_amd64 | |
| echo "7e0d59c65be5054a14ff2a76eb12c2d4ec3e5bc2f1dfa03c7356bb35b50bbf41 yq_linux_amd64" | shasum -a 256 -c | |
| chmod +x yq_linux_amd64 | |
| export ZCASH_GITIAN_VERSION=$(cat gitian-linux-parallel.yml | ./yq_linux_amd64 .name) | |
| cd ../../.. | |
| cat <<EOF > ./script.sh | |
| #!/bin/bash | |
| apt -qq update; | |
| apt install ca-certificates curl gnupg lsb-release software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y >/dev/null; | |
| mkdir -m 0755 -p /etc/apt/keyrings; | |
| curl -fsSL https://download.docker.com/linux/debian/gpg -o gpg.asc | |
| echo "1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570 gpg.asc" | shasum -a 256 -c; | |
| sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg --yes < gpg.asc; | |
| echo "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null | |
| apt -qq update; | |
| apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y >/dev/null; | |
| apt-add-repository "deb http://download.virtualbox.org/virtualbox/debian \$(lsb_release -sc) contrib"; | |
| wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc; | |
| echo "49e6801d45f6536232c11be6cdb43fa8e0198538d29d1075a7e10165e1fbafe2 oracle_vbox_2016.asc" | shasum -a 256 -c; | |
| apt-key add oracle_vbox_2016.asc; | |
| apt -qq update | |
| apt install virtualbox-6.1 -y >/dev/null; | |
| eval "\$(direnv hook bash)"; | |
| cd source | |
| cp .env.example .env | |
| cp .envrc.example .envrc | |
| /usr/bin/python3 -m venv ./local/python_venv; | |
| echo "load_prefix local/python_venv" >> .envrc; | |
| export VERSION="${array[2]}" | |
| echo "ZCASH_VERSION=\$VERSION" >> .env; | |
| echo "ZCASH_GIT_REPO_URL=https://github.com/${array[0]}/${array[1]}" >> .env; | |
| cat .env | |
| direnv allow; | |
| /sbin/vboxconfig; | |
| vagrant plugin install --local; | |
| vagrant plugin install --local; | |
| # get keys | |
| gsutil -q rm -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 || echo "" | |
| gsutil -q cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/encrypted_gpg.kms \$HOME/encrypted_gpg.kms | |
| gsutil -q cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/public.asc \$HOME/public.asc | |
| current_dir=\$(pwd) | |
| cd \$HOME | |
| gcloud kms decrypt \ | |
| --key gpg \ | |
| --keyring gpg \ | |
| --location global \ | |
| --plaintext-file private.pgp \ | |
| --ciphertext-file encrypted_gpg.kms | |
| cd \$current_dir | |
| gpg --import \$HOME/private.pgp | |
| export key=\$(gpg --list-secret-keys --keyid-format=long [email protected] | head -n 2 | grep -v sec) | |
| echo "GPG_KEY_ID=\$(gpg --list-keys --with-fingerprint --with-colons | grep fpr: | head -n 1 | sed 's/fpr://g' | sed 's/://g')" >> .env; | |
| echo "GPG_KEY_NAME=sysadmin" >> .env; | |
| git config --global user.name "sysadmin" | |
| git config --global user.email "[email protected]" | |
| # build | |
| direnv allow; | |
| direnv exec \$(pwd) vagrant up zcash-build; | |
| vagrant scp \$HOME/private.pgp : | |
| vagrant ssh zcash-build -c "gpg --import private.pgp" | |
| vagrant ssh zcash-build -c ./gitian-parallel-build.sh || echo "" | |
| vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/sysadmin/*.assert" > assert.txt | |
| tr -d \$'\r' < assert.txt > assert2.txt | |
| echo "#### sigs ####" | |
| for i in \$(cat assert2.txt | grep -E "zcash-*" | grep -v git: | sed 's/ //g' | sed 's/ /-->/g'); do | |
| echo \$i | |
| done | |
| export OS=\$(vagrant ssh zcash-build -c "ls zcash-binaries/\$VERSION" | tr -d '\r') | |
| for i in \$OS; do vagrant ssh zcash-build -c "mkdir \$i; tar Cxvzf \$i zcash-binaries/*/\$i/zcash-*-linux64.tar.gz"; done | |
| vagrant scp :gitian.sigs . | |
| for i in \$OS; | |
| do | |
| mkdir -p debs/\$i; | |
| mkdir -p ./\$i-extract | |
| vagrant ssh zcash-build -c "mkdir /home/vagrant/"\$i"-extract"; | |
| vagrant ssh zcash-build -c "tar -xvf /home/vagrant/zcash-binaries/"\$VERSION"/"\$i"/zcash-*-linux64.tar.gz -C /home/vagrant/"\$i"-extract"; | |
| docker run -d --name \$i debian:\$i bash -c "while true; do sleep 2; done"; | |
| docker exec \$i bash -c "mkdir -p /home/vagrant/\$i-deb-build && cd /home/vagrant/\$i-deb-build && apt -qq update && apt install git dpkg-dev lintian -y >/dev/null && git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git ."; | |
| vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-tx ./\$i-extract/ | |
| vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-fetch-params ./\$i-extract/ | |
| vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcashd ./\$i-extract/ | |
| vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-cli ./\$i-extract/ | |
| vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcashd-wallet-tool ./\$i-extract/ | |
| docker cp ./\$i-extract \$i:/home/vagrant/\$i-deb-build/ | |
| docker exec -w /home/vagrant/\$i-deb-build \$i bash -c "rm -rf src && mv \$i-extract src && ./zcutil/build-debian-package.sh" | |
| docker cp \$i:/tmp/zcbuild ./debs/\$i | |
| docker exec -it \$i bash -c "dpkg -i /tmp/zcbuild/*.deb" | |
| echo #### zcashd --version #### | |
| docker exec -it \$i bash -c "zcashd --version" | |
| done | |
| # sign binaries | |
| vagrant scp :/home/vagrant/zcash-binaries ./ | |
| for i in \$OS; | |
| do | |
| cd ./zcash-binaries/\$VERSION/\$i | |
| for j in \$(ls *linux64.tar.gz); do | |
| mv \$j \$(echo \$j | sed 's/.tar.gz/-debian-'\$i'.tar.gz/g') | |
| done | |
| for j in \$(ls *debug.tar.gz); do | |
| mv \$j \$(echo \$j | sed 's/.tar.gz/-debian-'\$i'.tar.gz/g') | |
| done | |
| gpg -u [email protected] --armor --digest-algo SHA256 --detach-sign *debug-debian-\$i.tar.gz | |
| gpg -u [email protected] --armor --digest-algo SHA256 --detach-sign *linux64-debian-\$i.tar.gz | |
| rm -rf zcash-$(echo $VERSION | sed 's/v//g').tar.gz | |
| gsutil -q -m rsync -r ./ gs://download-downloads/ | |
| cd \$current_dir | |
| done | |
| export final_version=\$(cat assert2.txt | awk '{print \$2}' | grep "desc.yml" | head -n 1 | sed 's/-desc.yml//g') | |
| gsutil -q -m rsync -r ./debs gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/debs | |
| gsutil -q -m rsync -r ./zcash-binaries gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/zcash-binaries | |
| apt install aptly -y >/dev/null | |
| # generate apt | |
| mkdir aptserver | |
| cd aptserver | |
| gsutil -q -m cp -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/pool/main/z/zcash/ . | |
| cd zcash | |
| cp -a ../../debs/bullseye/zcbuild/*.deb \$final_version-amd64-bullseye.deb | |
| cp -a ../../debs/bookworm/zcbuild/*.deb \$final_version-amd64-bookworm.deb | |
| ls \$final_version-amd64-bullseye.deb || exit 1 | |
| ls \$final_version-amd64-bookworm.deb || exit 1 | |
| aptly repo create --distribution buster --comment "" --component main zcash_buster_amd64_repo | |
| aptly repo create --distribution bullseye --comment "" --component main zcash_bullseye_amd64_repo | |
| aptly repo create --distribution bookworm --comment "" --component main zcash_bookworm_amd64_repo | |
| aptly repo create --distribution stretch --comment "" --component main zcash_stretch_amd64_repo | |
| for i in \$(ls *.deb | grep buster); do | |
| aptly repo add zcash_buster_amd64_repo \$i | |
| done | |
| for i in \$(ls *.deb | grep bullseye); do | |
| aptly repo add zcash_bullseye_amd64_repo \$i | |
| done | |
| for i in \$(ls *.deb | grep stretch); do | |
| aptly repo add zcash_stretch_amd64_repo \$i | |
| done | |
| for i in \$(ls *.deb | grep bookworm); do | |
| aptly repo add zcash_bookworm_amd64_repo \$i | |
| done | |
| aptly snapshot create bookworm_snapshot from repo zcash_bookworm_amd64_repo | |
| aptly snapshot create buster_snapshot from repo zcash_buster_amd64_repo | |
| aptly snapshot create bullseye_snapshot from repo zcash_bullseye_amd64_repo | |
| aptly snapshot create stretch_snapshot from repo zcash_stretch_amd64_repo | |
| aptly publish snapshot --distribution buster --component main --architectures amd64 --gpg-key="\$key" --passphrase="" buster_snapshot | |
| aptly publish snapshot --distribution bookworm --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bookworm_snapshot | |
| aptly publish snapshot --distribution bullseye --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bullseye_snapshot | |
| aptly publish snapshot --distribution stretch --component main --architectures amd64 --gpg-key="\$key" --passphrase="" stretch_snapshot | |
| apt install nginx-extras -y >/dev/null | |
| cat << EOH > /etc/nginx/sites-enabled/default | |
| server { | |
| listen 80 default_server; | |
| root /var/www/public; | |
| location / { | |
| autoindex on; | |
| } | |
| server_name _; | |
| } | |
| EOH | |
| # get apt server | |
| cp -a /root/.aptly/public /var/www/ | |
| chown -R www-data:www-data /var/www | |
| /etc/init.d/nginx restart | |
| echo "debug 1" | |
| mkdir \$HOME/mirror | |
| cd \$HOME/mirror | |
| wget -q -r 127.0.0.1 | |
| cp \$HOME/public.asc \$HOME/mirror/127.0.0.1/zcash.asc | |
| cd \$HOME/mirror | |
| gsutil -q -m rsync -r ./127.0.0.1 gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 | |
| cd 127.0.0.1 | |
| if [[ ! ${array[2]} =~ -rc ]] && [[ ${array[2]} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| gsutil -q -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/ | |
| fi | |
| echo "script finished" | |
| EOF | |
| export FAIL=0 | |
| chmod +x ./script.sh | |
| gcloud compute scp ./script.sh --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random: || (echo "error 1" && export FAIL=1) | |
| gcloud compute scp --recurse $(pwd) --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:~/source || (echo "error 2" && export FAIL=1) | |
| gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || (echo "error 3" && export FAIL=1) | |
| if [ $FAIL -eq 1 ]; then | |
| echo "error" | |
| #gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all | |
| exit 1; | |
| fi | |
| # curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' # debug | |
| gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . | |
| rm -rf gitian.sigs/.git | |
| if [[ ! ${array[2]} =~ -rc ]] && [[ ${array[2]} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| mkdir $HOME/.ssh || echo "" | |
| ssh-keyscan github.com >> $HOME/.ssh/known_hosts | |
| echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa | |
| chmod 600 $HOME/.ssh/id_rsa | |
| git clone [email protected]:zcash/gitian.sigs.git sigs | |
| cp -a gitian.sigs/* sigs/ | |
| cd sigs | |
| git config --global user.name "ECC-CI" | |
| git config --global user.email "${{ secrets.BOT_EMAIL }}" | |
| git add . | |
| git commit -am "${{ github.event.label.name }}" | |
| git push # debug | |
| fi | |
| echo "last step" | |
| #gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all # debug |