-
Notifications
You must be signed in to change notification settings - Fork 29
271 lines (240 loc) · 14.2 KB
/
CI.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
name: Gitian CI
on:
pull_request:
types:
- labeled
jobs:
build-gitian:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: 'Set up Cloud SDK'
uses: 'google-github-actions/setup-gcloud@v2'
with:
version: '>= 363.0.0'
- name: Build Gitian
id: gitian
shell: bash
run: |
sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null
echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json
gcloud auth activate-service-account --key-file=json.json
export random=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 12; echo;)
for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do
echo "removing SSH key $i"
gcloud compute os-login ssh-keys remove --key $i || echo "failed to remove key"
done
gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=${{ secrets.GCP_PROJECT_ID_PROD }} --zone=us-central1-a --no-address --network=vpc-${{ secrets.GCP_PROJECT_ID_PROD }} --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@${{ secrets.GCP_PROJECT_ID_PROD }}.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB
export counter=1
while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]]
do
echo "attempt number: $counter"
export counter=$((counter+1))
if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all; exit 1; fi
sleep 5
done
IFS='/' read -r -a array <<< "${{ github.event.label.name }}"
git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git
cd zcash/contrib/gitian-descriptors
wget -c -q https://github.com/mikefarah/yq/releases/download/v4.28.2/yq_linux_amd64
echo "7e0d59c65be5054a14ff2a76eb12c2d4ec3e5bc2f1dfa03c7356bb35b50bbf41 yq_linux_amd64" | shasum -a 256 -c
chmod +x yq_linux_amd64
export ZCASH_GITIAN_VERSION=$(cat gitian-linux-parallel.yml | ./yq_linux_amd64 .name)
cd ../../..
cat <<EOF > ./script.sh
#!/bin/bash
apt -qq update;
apt install ca-certificates curl gnupg lsb-release software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y >/dev/null;
mkdir -m 0755 -p /etc/apt/keyrings;
curl -fsSL https://download.docker.com/linux/debian/gpg -o gpg.asc
echo "1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570 gpg.asc" | shasum -a 256 -c;
sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg --yes < gpg.asc;
echo "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
apt -qq update;
apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y >/dev/null;
apt-add-repository "deb http://download.virtualbox.org/virtualbox/debian \$(lsb_release -sc) contrib";
wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc;
echo "49e6801d45f6536232c11be6cdb43fa8e0198538d29d1075a7e10165e1fbafe2 oracle_vbox_2016.asc" | shasum -a 256 -c;
apt-key add oracle_vbox_2016.asc;
apt -qq update
apt install virtualbox-6.1 -y >/dev/null;
eval "\$(direnv hook bash)";
cd source
cp .env.example .env
cp .envrc.example .envrc
/usr/bin/python3 -m venv ./local/python_venv;
echo "load_prefix local/python_venv" >> .envrc;
export VERSION="${array[2]}"
echo "ZCASH_VERSION=\$VERSION" >> .env;
echo "ZCASH_GIT_REPO_URL=https://github.com/${array[0]}/${array[1]}" >> .env;
cat .env
direnv allow;
/sbin/vboxconfig;
vagrant plugin install --local;
vagrant plugin install --local;
# get keys
gsutil -q rm -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 || echo ""
gsutil -q cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/encrypted_gpg.kms \$HOME/encrypted_gpg.kms
gsutil -q cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/public.asc \$HOME/public.asc
current_dir=\$(pwd)
cd \$HOME
gcloud kms decrypt \
--key gpg \
--keyring gpg \
--location global \
--plaintext-file private.pgp \
--ciphertext-file encrypted_gpg.kms
cd \$current_dir
gpg --import \$HOME/private.pgp
export key=\$(gpg --list-secret-keys --keyid-format=long [email protected] | head -n 2 | grep -v sec)
echo "GPG_KEY_ID=\$(gpg --list-keys --with-fingerprint --with-colons | grep fpr: | head -n 1 | sed 's/fpr://g' | sed 's/://g')" >> .env;
echo "GPG_KEY_NAME=sysadmin" >> .env;
git config --global user.name "sysadmin"
git config --global user.email "[email protected]"
# build
direnv allow;
direnv exec \$(pwd) vagrant up zcash-build;
vagrant scp \$HOME/private.pgp :
vagrant ssh zcash-build -c "gpg --import private.pgp"
vagrant ssh zcash-build -c ./gitian-parallel-build.sh || echo ""
vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/sysadmin/*.assert" > assert.txt
tr -d \$'\r' < assert.txt > assert2.txt
echo "#### sigs ####"
for i in \$(cat assert2.txt | grep -E "zcash-*" | grep -v git: | sed 's/ //g' | sed 's/ /-->/g'); do
echo \$i
done
export OS=\$(vagrant ssh zcash-build -c "ls zcash-binaries/\$VERSION" | tr -d '\r')
for i in \$OS; do vagrant ssh zcash-build -c "mkdir \$i; tar Cxvzf \$i zcash-binaries/*/\$i/zcash-*-linux64.tar.gz"; done
vagrant scp :gitian.sigs .
for i in \$OS;
do
mkdir -p debs/\$i;
mkdir -p ./\$i-extract
vagrant ssh zcash-build -c "mkdir /home/vagrant/"\$i"-extract";
vagrant ssh zcash-build -c "tar -xvf /home/vagrant/zcash-binaries/"\$VERSION"/"\$i"/zcash-*-linux64.tar.gz -C /home/vagrant/"\$i"-extract";
docker run -d --name \$i debian:\$i bash -c "while true; do sleep 2; done";
docker exec \$i bash -c "mkdir -p /home/vagrant/\$i-deb-build && cd /home/vagrant/\$i-deb-build && apt -qq update && apt install git dpkg-dev lintian -y >/dev/null && git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git .";
vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-tx ./\$i-extract/
vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-fetch-params ./\$i-extract/
vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcashd ./\$i-extract/
vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-cli ./\$i-extract/
vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcashd-wallet-tool ./\$i-extract/
docker cp ./\$i-extract \$i:/home/vagrant/\$i-deb-build/
docker exec -w /home/vagrant/\$i-deb-build \$i bash -c "rm -rf src && mv \$i-extract src && ./zcutil/build-debian-package.sh"
docker cp \$i:/tmp/zcbuild ./debs/\$i
docker exec -it \$i bash -c "dpkg -i /tmp/zcbuild/*.deb"
echo #### zcashd --version ####
docker exec -it \$i bash -c "zcashd --version"
done
# sign binaries
vagrant scp :/home/vagrant/zcash-binaries ./
for i in \$OS;
do
cd ./zcash-binaries/\$VERSION/\$i
for j in \$(ls *linux64.tar.gz); do
mv \$j \$(echo \$j | sed 's/.tar.gz/-debian-'\$i'.tar.gz/g')
done
for j in \$(ls *debug.tar.gz); do
mv \$j \$(echo \$j | sed 's/.tar.gz/-debian-'\$i'.tar.gz/g')
done
gpg -u [email protected] --armor --digest-algo SHA256 --detach-sign *debug-debian-\$i.tar.gz
gpg -u [email protected] --armor --digest-algo SHA256 --detach-sign *linux64-debian-\$i.tar.gz
rm -rf zcash-$(echo $VERSION | sed 's/v//g').tar.gz
gsutil -q -m rsync -r ./ gs://download-downloads/
cd \$current_dir
done
export final_version=\$(cat assert2.txt | awk '{print \$2}' | grep "desc.yml" | head -n 1 | sed 's/-desc.yml//g')
gsutil -q -m rsync -r ./debs gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/debs
gsutil -q -m rsync -r ./zcash-binaries gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/zcash-binaries
apt install aptly -y >/dev/null
# generate apt
mkdir aptserver
cd aptserver
gsutil -q -m cp -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/pool/main/z/zcash/ .
cd zcash
cp -a ../../debs/bullseye/zcbuild/*.deb \$final_version-amd64-bullseye.deb
cp -a ../../debs/bookworm/zcbuild/*.deb \$final_version-amd64-bookworm.deb
ls \$final_version-amd64-bullseye.deb || exit 1
ls \$final_version-amd64-bookworm.deb || exit 1
aptly repo create --distribution buster --comment "" --component main zcash_buster_amd64_repo
aptly repo create --distribution bullseye --comment "" --component main zcash_bullseye_amd64_repo
aptly repo create --distribution bookworm --comment "" --component main zcash_bookworm_amd64_repo
aptly repo create --distribution stretch --comment "" --component main zcash_stretch_amd64_repo
for i in \$(ls *.deb | grep buster); do
aptly repo add zcash_buster_amd64_repo \$i
done
for i in \$(ls *.deb | grep bullseye); do
aptly repo add zcash_bullseye_amd64_repo \$i
done
for i in \$(ls *.deb | grep stretch); do
aptly repo add zcash_stretch_amd64_repo \$i
done
for i in \$(ls *.deb | grep bookworm); do
aptly repo add zcash_bookworm_amd64_repo \$i
done
aptly snapshot create bookworm_snapshot from repo zcash_bookworm_amd64_repo
aptly snapshot create buster_snapshot from repo zcash_buster_amd64_repo
aptly snapshot create bullseye_snapshot from repo zcash_bullseye_amd64_repo
aptly snapshot create stretch_snapshot from repo zcash_stretch_amd64_repo
aptly publish snapshot --distribution buster --component main --architectures amd64 --gpg-key="\$key" --passphrase="" buster_snapshot
aptly publish snapshot --distribution bookworm --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bookworm_snapshot
aptly publish snapshot --distribution bullseye --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bullseye_snapshot
aptly publish snapshot --distribution stretch --component main --architectures amd64 --gpg-key="\$key" --passphrase="" stretch_snapshot
apt install nginx-extras -y >/dev/null
cat << EOH > /etc/nginx/sites-enabled/default
server {
listen 80 default_server;
root /var/www/public;
location / {
autoindex on;
}
server_name _;
}
EOH
# get apt server
cp -a /root/.aptly/public /var/www/
chown -R www-data:www-data /var/www
/etc/init.d/nginx restart
echo "debug 1"
mkdir \$HOME/mirror
cd \$HOME/mirror
wget -q -r 127.0.0.1
cp \$HOME/public.asc \$HOME/mirror/127.0.0.1/zcash.asc
cd \$HOME/mirror
gsutil -q -m rsync -r ./127.0.0.1 gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1
cd 127.0.0.1
if [[ ! $version =~ -rc ]] && [[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
gsutil -q -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/
fi
echo "script finished"
EOF
export FAIL=0
chmod +x ./script.sh
gcloud compute scp ./script.sh --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random: || (echo "error 1" && export FAIL=1)
gcloud compute scp --recurse $(pwd) --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:~/source || (echo "error 2" && export FAIL=1)
gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || (echo "error 3" && export FAIL=1)
if [ $FAIL -eq 1 ]; then
echo "error"
#gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all
exit 1;
fi
# curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' # debug
gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs .
rm -rf gitian.sigs/.git
if ! [[ ${array[2]} == *"-rc"* ]]; then
mkdir $HOME/.ssh || echo ""
ssh-keyscan github.com >> $HOME/.ssh/known_hosts
echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa
chmod 600 $HOME/.ssh/id_rsa
git clone [email protected]:zcash/gitian.sigs.git sigs
cp -a gitian.sigs/* sigs/
cd sigs
git config --global user.name "ECC-CI"
git config --global user.email "${{ secrets.BOT_EMAIL }}"
git add .
git commit -am "${{ github.event.label.name }}"
#git push # debug
fi
echo "last step"
#gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all # debug