diff --git a/ci/build.sh b/ci/build.sh index 201c01982..b6feafec1 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -7,7 +7,18 @@ export BUILD_TYPE=DEBUG export ESCAPETEST=yes export EXPLICIT_ROLE=hub export TEST_MACHINE=chroot -# TODO maybe seed the cache? cp -R buildscripts/ci/cache ~/.cache + +set +x # hide secrets +eval $(ssh-agent -s) +if [ -z "$SECRET" ]; then + echo "Need sftp cache ssh secret key. Provide with SECRET env variable" + exit 1 +else + echo "$SECRET" | ssh-add - +fi +ssh-add -l +set -x # stop hiding secrets + time ./buildscripts/build-scripts/build-environment-check time ./buildscripts/build-scripts/install-dependencies time ./buildscripts/build-scripts/configure # 3 minutes locally diff --git a/ci/clean-build-package.sh b/ci/clean-build-package.sh index ca54f7738..a36691cda 100755 --- a/ci/clean-build-package.sh +++ b/ci/clean-build-package.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash +set -ex # clean up docker stuff name=cfengine-build-package # TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine docker stop $name -docker rm $name -docker rmi $name +docker rm -f $name +docker rmi -f $name diff --git a/ci/docker-build-package.sh b/ci/docker-build-package.sh index dfb7eed2f..7e5230c23 100755 --- a/ci/docker-build-package.sh +++ b/ci/docker-build-package.sh @@ -10,44 +10,30 @@ NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT} name=cfengine-build-package label=PACKAGES_HUB_x86_64_linux_ubuntu_20 export JOB_BASE_NAME=label=$label -# todo, check the image against the Dockerfile for up-to-date ness? -docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true -# todo, check if already running and up-to-date? -# send in JOB_BASE_NAME to enable use of retrieved or generated deps cache -docker run -d --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name || true -# copy local caches to docker container -mkdir -p "${NTECH_ROOT}/packages" -mkdir -p "${NTECH_ROOT}/cache" -# pre-seed cache from sftp buildcache if possible -# requires either environment var with private key or mystiko+pass -eval "$(ssh-agent -s)" +docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . + +# add secret key to enable push up to sftp cache set +x # hide secrets if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then - echo "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" | ssh-add - + export SECRET="$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" else - if ! pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec | ssh-add -; then - echo "Need the ssh private key for build artifacts cache, neither env var nor mystiko was available." + if ! export SECRET="$(pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec)"; then + echo "The sftp cache ssh secret key must be provided, either with environment variable GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE or access to mystiko path developers/CFEngine/jenkins/sftp-cache.sec" exit 1 fi fi set -x # done hiding secrets -# clean up any lingering revision file previously generated, if you are changing deps locally and iterating this is important -[ -f "${NTECH_ROOT}/buildscripts/deps-packaging/revision" ] && rm "${NTECH_ROOT}/buildscripts/deps-packaging/revision" -cd "${NTECH_ROOT}/buildscripts/deps-packaging" -# see buildscripts/build-scripts/autogen for a similar workaround to ensure it stays 7 on bootstrap-oslo-dc jobs -git config --add core.abbrev 7 # hack to match smaller commit sha on bootstrap-oslo-dc (debian-9) -revision=$(git log --pretty='format:%h' -1 -- .) -cd - # back to previous directory -PKGS_DIR="${NTECH_ROOT}/cache/buildscripts_cache/pkgs/${label}" -mkdir -p "${PKGS_DIR}" +# send in JOB_BASE_NAME to enable use of retrieved or generated deps cache +docker run -d --env SECRET --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name -# setup host key trust -echo "build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y=" >>~/.ssh/known_hosts +# copy local caches to docker container +mkdir -p "${NTECH_ROOT}/packages" +mkdir -p "${NTECH_ROOT}/cache" -echo -e "cd /export/sftp_dirs_cache/${label}\n get -Ra *${revision}* ${PKGS_DIR}" | \ - sftp -oPubkeyAcceptedKeyTypes=+ssh-rsa -b - jenkins_sftp_cache@build-artifacts-cache.cloud.cfengine.com +# setup host key trust +pubkey="build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y=" # ending with /. in srcpath copies contents to destpath docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache @@ -58,6 +44,10 @@ do docker exec -i $name bash -c "git config --global --add safe.directory /data/$i" done +# add build artifacts host public keys to container for use there +docker exec -i $name bash -c "mkdir -p ~/.ssh" +docker exec -i $name bash -c "echo $pubkey >> ~/.ssh/known_hosts" + docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh' docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'