diff --git a/ci.bash b/ci.bash index 2f1ecfc95..6784b3c24 100644 --- a/ci.bash +++ b/ci.bash @@ -9,6 +9,8 @@ function compile::env : "${JUJU_DEPLOY_CHANNEL:?Must have a channel defined}" : "${JUJU_MODEL:?Must have a model defined}" : "${JUJU_UPDATE_STATUS_INTERVAL:=150s}" + : "${JOB_STAGE:-}" + : "${JOB_REPORTING:-yes}" : "${SERIES:?Must have a release series defined}" : "${SNAP_VERSION:?Must have a snap version defined}" : "${JOB_NAME_CUSTOM:?Must have a job name defined}" @@ -48,6 +50,7 @@ function compile::env kv::set "job_id" "$JOB_ID" kv::set "job_name" "$JOB_NAME_CUSTOM" kv::set "job_name_custom" "$JOB_NAME_CUSTOM" + kv::set "job_stage" "$JOB_STAGE" kv::set "series" "$SERIES" kv::set "arch" "$ARCH" kv::set "snap_version" "$SNAP_VERSION" @@ -164,14 +167,22 @@ function test::execute function test::report { result=$1 - kv::set "result" "$result" touch "meta/result-$result" - python bin/s3 cp "meta/result-$result" "meta/result-$result" + if [ "${JOB_REPORTING}" == "no" ]; then + echo "Job reporting disabled" + else + python bin/s3 cp "meta/result-$result" "meta/result-$result" + fi } function test::capture { + if [ "${JOB_REPORTING}" == "no" ]; then + echo "Job reporting disabled" + return 0 + fi + if which juju-crashdump; then # -s small crashdump by skipping /var/lib/juju # -a debug-layer included debug-layer addon @@ -272,9 +283,7 @@ function ci::cleanup ci::cleanup::before || true test::capture || true - if ! timeout 2m juju destroy-controller --no-prompt --destroy-all-models --destroy-storage "$JUJU_CONTROLLER"; then - timeout 10m juju kill-controller -t 2m0s --no-prompt "$JUJU_CONTROLLER" || true - fi + juju::destroy ci::cleanup::after || true } 2>&1 | sed -u -e "s/^/[$log_name_custom] /" | tee -a "ci.log" } diff --git a/cilib.sh b/cilib.sh index 05ef65313..1f57d6d42 100644 --- a/cilib.sh +++ b/cilib.sh @@ -74,7 +74,14 @@ ci_lxc_push() local lxc_container=$1 local source=$2 local dest=$3 - sudo lxc file push ${source} ${lxc_container}/${dest} + local args=(-p --mode "$(stat -c '0%a' $source)") + if [ "$(stat -c '%U' $source)" == "$(whoami)" ]; then + args+=(--uid 1000) + fi + if [ "$(stat -c '%G' $source)" == "$(whoami)" ]; then + args+=(--gid 1000) + fi + sudo lxc file push "${args[@]}" ${source} ${lxc_container}/${dest} } diff --git a/jobs/microk8s/release.sh b/jobs/microk8s/release.sh new file mode 100644 index 000000000..ea4d8d47f --- /dev/null +++ b/jobs/microk8s/release.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC1090 + +set -x + +############################################################################### +# INITIALIZE +############################################################################### +: "${WORKSPACE:=$(pwd)}" + +. "$WORKSPACE/ci.bash" +. "$WORKSPACE/juju.bash" + +############################################################################### +# FUNCTION OVERRIDES +############################################################################### + +############################################################################### +# ENV +############################################################################### +export ARCH=${ARCH:-amd64} +export TRACKS=${TRACKS:-} +export DRY_RUN=${DRY_RUN:-yes} +export ALWAYS_RELEASE=${ALWAYS_RELEASE:-no} +export TESTS_BRANCH=${TESTS_BRANCH:-} +export PROXY=${PROXY:-} +export RELEASE_CHANNEL=${RELEASE_CHANNEL:-all} + + +SERIES=focal +JUJU_DEPLOY_BUNDLE="${WORKSPACE}/ubuntu.yaml" +JUJU_DEPLOY_CHANNEL=stable +JUJU_CLOUD=aws/us-east-1 +JUJU_CONTROLLER=release-$(identifier::short) +JUJU_VERSION=$(juju --version | cut -f-2 -d.) +CUSTOM_CLOUD=$(echo "$JUJU_CLOUD" | cut -f1 -d/) +JOB_NAME_CUSTOM="release-microk8s-$ARCH" +SNAP_VERSION=stable +JOB_ID=$(identifier) +JOB_REPORTING=no + +function snapcraft::login +{ + snapcraft login --with "${SNAPCRAFTCREDS}" +} + +function gather::channels +{ + if [[ ${RELEASE_CHANNEL} == "all" ]]; then + CHANNELS=("beta" "stable" "pre-release") + else + CHANNELS=($RELEASE_CHANNEL) + fi + echo "Running for channels ${ARCH}/${CHANNELS}" +} + +function juju::deploy::overlay +{ + local constraints + constraints="cores=8 mem=16G root-disk=80G arch=${ARCH}" + if [ "${ARCH}" == "amd64" ] && [ "${CHANNEL}" == "stable" ]; then + constraints+=" instance-type=g3s.xlarge" + fi + + cat << EOF > $JUJU_DEPLOY_BUNDLE +series: $SERIES +applications: + ubuntu: + charm: ubuntu + channel: latest/stable + constraints: $constraints + num_units: 1 +EOF +} + +function juju::deploy +{ + juju deploy -m "$JUJU_CONTROLLER:$JUJU_MODEL" "$JUJU_DEPLOY_BUNDLE" + juju::deploy-report $? "model-deploy" +} + +function test::execute +{ + local juju_full_model="$JUJU_CONTROLLER:$JUJU_MODEL" + export JUJU_UNIT=ubuntu/0 + juju ssh -m "${juju_full_model}" --pty=true $JUJU_UNIT -- 'sudo snap install lxd' + juju ssh -m "${juju_full_model}" --pty=true $JUJU_UNIT -- 'sudo lxd.migrate -yes' || true + juju ssh -m "${juju_full_model}" --pty=true $JUJU_UNIT -- 'sudo lxd init --auto' + if [ "${CHANNEL}" == "pre-release" ]; then + juju ssh -m "${juju_full_model}" --pty=true $JUJU_UNIT -- 'sudo snap install snapcraft --classic' + fi + + case $CHANNEL in + beta) + SCRIPT_NAME="release-to-beta.py" + ;; + stable) + SCRIPT_NAME="release-to-stable.py" + ;; + pre-release) + SCRIPT_NAME="release-pre-release.py" + ;; + esac + + declare -n is_pass=$1 + timeout 6h python jobs/microk8s/${SCRIPT_NAME} + ret=$? + is_pass="True" + if (( ret == 124 )); then + is_pass="Timeout" + elif (( ret > 0 )); then + is_pass="False" + fi +} + +############################################################################### +# START +############################################################################### + +snapcraft::login +gather::channels +for CHANNEL in "${CHANNELS[@]}"; do + export "CHANNEL" + timeout 6h python jobs/microk8s/release-needed.py + if (( $? == 0 )); then + JOB_STAGE="${CHANNEL}-${ARCH}" + JUJU_MODEL="release-microk8s-${JOB_STAGE}-model" + ci::run + fi +done diff --git a/jobs/release-microk8s.yaml b/jobs/release-microk8s.yaml index 52950e780..edda46008 100644 --- a/jobs/release-microk8s.yaml +++ b/jobs/release-microk8s.yaml @@ -2,17 +2,53 @@ - job-template: name: 'release-microk8s-arch-{arch}' + axes: + - axis: + type: slave # wokeignore:rule=slave + name: node + values: + - runner-cloud description: | Tests and releases microk8s. Affects all tracks on {arch}. - project-type: pipeline - pipeline-scm: - scm: - - k8s-jenkins-jenkaas - script-path: jobs/release-microk8s/release-microk8s.groovy + project-type: matrix + scm: + - k8s-jenkins-jenkaas wrappers: - default-job-wrapper - ci-creds + properties: + - block-on-infra + - build-discarder: + num-to-keep: 10 + triggers: + - timed: "@daily" parameters: + - juju-lts + - choice: + name: LXC_APT_LIST + description: |- + comma-separated list of apt packages to install into the container + choices: + - pip,python3-venv,python3-dev,libffi-dev + - choice: + name: LXC_SNAP_LIST + description: |- + comma-separated list of snaps to install into the container + if the snap requires multiple arguments + choices: + - snapcraft --channel=6.x/stable,juju-wait,juju-crashdump,juju --channel=$juju_channel + - choice: + name: LXC_PUSH_LIST + description: |- + comma-separated list of paths to push into the container + choices: + - $SNAPCRAFTCREDS + - choice: + name: LXC_MOUNT_LIST + description: |- + comma-separated list of paths to mount in the container from $HOME to $LXC_HOME + choices: + - .local,.ssh,.aws,snap - choice: name: ARCH description: | @@ -20,7 +56,7 @@ choices: - "{arch}" - choice: - name: CHANNEL + name: RELEASE_CHANNEL description: | Choose a specific release channel choices: @@ -32,8 +68,8 @@ name: TRACKS default: "" description: | - The tracks you want to test and release. For example latest 1.10 - 1.11 1.12. Leave empty to iterate over all tracks. + Space separated list of tracks you want to test and release. + For example "latest 1.10 1.11 1.12". Leave empty to iterate over all tracks. - choice: name: DRY_RUN description: | @@ -59,16 +95,10 @@ default: "" description: | Proxy endpoint - triggers: - - timed: "@daily" - properties: - - build-discarder: - num-to-keep: 10 - - build-blocker: - use-build-blocker: true - blocking-jobs: - - "infra.*" - block-level: 'NODE' + builders: + - run-lxc: + COMMAND: | + bash jobs/microk8s/release.sh - project: name: release-microk8s-arch diff --git a/jobs/release-microk8s/release-microk8s.groovy b/jobs/release-microk8s/release-microk8s.groovy deleted file mode 100644 index e9f725b86..000000000 --- a/jobs/release-microk8s/release-microk8s.groovy +++ /dev/null @@ -1,178 +0,0 @@ - -def destroy_controller(controller) { - return """#!/bin/bash - if ! timeout 4m juju destroy-controller --no-prompt --destroy-all-models --destroy-storage "${controller}"; then - timeout 4m juju kill-controller --no-prompt "${controller}" || true - fi - """ -} - -pipeline { - agent { - label "runner-${params.ARCH}" - } - /* XXX: Global $PATH setting doesn't translate properly in pipelines - https://stackoverflow.com/questions/43987005/jenkins-does-not-recognize-command-sh - */ - environment { - PATH = "/var/lib/jenkins/venvs/ci/bin:/snap/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin" - AWS_REGION = "us-east-1" - JUJU_CLOUD = "aws/us-east-1" - K8STEAMCI = credentials('k8s_team_ci_lp') - CDKBOT_GH = credentials('cdkbot_github') - LPCREDS = credentials('launchpad_creds') - CHARM_CREDS = credentials('charm_creds') - JUJU_CREDS = credentials('juju_creds') - JUJU_CLOUDS = credentials('juju_clouds') - SSOCREDS = credentials('sso_token') - SNAPCRAFTCREDS = credentials('snapcraft_creds') - SNAPCRAFTCPCCREDS = credentials('snapcraft_cpc_creds') - AWS_CREDS = credentials('aws_creds') - SURL_CREDS = credentials('surl-creds') - AWSIAMARN = credentials('aws-iam-arn') - CDKBOTSSHCREDS = credentials('cdkbot_ssh_rsa') - K8STEAMCI_GPG_PUB = credentials('deb-gpg-public') - K8STEAMCI_GPG_PRIVATE= credentials('deb-gpg-private') - K8STEAMCI_GPG_KEY = credentials('deb-gpg-key') - NOTIFY_EMAIL = credentials('microk8s_notify_email') - - } - options { - ansiColor('xterm') - timestamps() - } - stages { - stage("Setup Channels") { - steps { - script { - arch = params.ARCH - channels = ["beta", "stable", "pre-release"].findAll { channel -> - params.CHANNEL == "all" || params.CHANNEL == channel - } - echo "Running for channels ${arch}/${channels}" - } - } - } - stage("Snapcraft login") { - steps { - sh "snapcraft login --with ${SNAPCRAFTCREDS}" - } - } - stage("Setup tox environment") { - steps { - sh """ - tox -c jobs/microk8s/tox.ini -e py38 -- python -c 'print("Tox Environment Ready")' - """ - } - } - stage("Run Release Steps") { - steps { - script { - channels.each { channel -> - stage("Channel ${channel}") { - script { - def job="release-microk8s" - def stage="${channel}-${arch}" - def juju_controller="${job}-${stage}" - def juju_model="${job}-${stage}-model" - def juju_full_model="${juju_controller}:${juju_model}" - def instance_type = "" - def eksd_instance_type = "" - def constraints = "" - def job_name = [ - beta: "release-to-beta.py", - stable: "release-to-stable.py", - "pre-release": "release-pre-release.py", - ] - - try { - sh """ - . jobs/microk8s/.tox/py38/bin/activate - ALWAYS_RELEASE=${params.ALWAYS_RELEASE}\ - TRACKS=${params.TRACKS}\ - CHANNEL=${channel}\ - timeout 6h python jobs/microk8s/release-needed.py - """ - } catch (err) { - return 0 - } - - if (arch == "arm64") { - constraints = "mem=16G cores=8 root-disk=80G arch=${arch}" - eksd_instance_type = "m6g.large" - } else if (arch == "amd64") { - if ("${channel}" == "stable") { - constraints = "instance-type=g3s.xlarge root-disk=80G arch=${arch}" - } else { - constraints = "mem=16G cores=8 root-disk=80G arch=${arch}" - } - eksd_instance_type = "m4.large" - } else { - error("Aborting build due to unknown arch=${arch}") - } - - sh destroy_controller(juju_controller) - sh """#!/bin/bash -x - juju bootstrap "${JUJU_CLOUD}" "${juju_controller}" \ - --model-default test-mode=true \ - --model-default resource-tags="owner=k8sci job=${job} stage=${stage}" \ - --bootstrap-constraints "mem=8G cores=2" - - juju add-model "${juju_model}" -c "${juju_controller}" - - # We deploy 20.04 because the upgrade-path test deploys K8s 1.19 onwards - # that requires old cgroups. - juju deploy -m "${juju_full_model}" --constraints "${constraints}" ubuntu --base ubuntu@20.04 - - juju-wait -e "${juju_full_model}" -w - - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo snap install lxd' - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo lxd.migrate -yes' || true - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo lxd init --auto' - """ - if (channel == "pre-release"){ - sh """ - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo snap install snapcraft --classic' - """ - } - try { - sh """ - . jobs/microk8s/.tox/py38/bin/activate - DRY_RUN=${params.DRY_RUN} ALWAYS_RELEASE=${params.ALWAYS_RELEASE}\ - TESTS_BRANCH=${params.TESTS_BRANCH} TRACKS=${params.TRACKS}\ - PROXY=${params.PROXY} JUJU_UNIT=ubuntu/0\ - JUJU_CONTROLLER=${juju_controller} JUJU_MODEL=${juju_model}\ - timeout 6h python jobs/microk8s/${job_name[channel]} - """ - } catch (err) { - unstable("${job_name[channel]} completed with errors.") - emailext( - to: env.NOTIFY_EMAIL, - subject: "Job '${JOB_NAME}' (${BUILD_NUMBER}) had an on stage ${job_name[channel]}", - body: "Please go to ${BUILD_URL} and verify the build" - ) - } finally { - sh destroy_controller(juju_controller) - } - } - } - } - } - } - } - } - post { - always { - script { - if (getBinding().hasVariable("channels")) { - channels.each { channel -> - def job="release-microk8s" - def stage="${channel}-${arch}" - def juju_controller="${job}-${stage}" - sh destroy_controller(juju_controller) - } - } - } - } - } -} diff --git a/juju.bash b/juju.bash index 5e9436e37..b19ab036a 100644 --- a/juju.bash +++ b/juju.bash @@ -60,6 +60,13 @@ function juju::pip::2.9 fi } +function juju::destroy +{ + if ! timeout 2m juju destroy-controller --no-prompt --destroy-all-models --destroy-storage "$JUJU_CONTROLLER"; then + timeout 10m juju kill-controller -t 2m0s --no-prompt "$JUJU_CONTROLLER" || true + fi +} + function juju::bootstrap { extra_args='--model-default image-stream=daily' @@ -81,13 +88,16 @@ function juju::bootstrap add_model=("--add-model ${JUJU_MODEL}") fi + juju::destroy + TAGS="owner=k8sci job=${JOB_NAME_CUSTOM}" + if [ "${JOB_STAGE}" ]; then TAGS+=" stage=${STAGE}"; fi juju bootstrap "$JUJU_CLOUD" "$JUJU_CONTROLLER" \ ${add_model[@]} \ --debug \ --force --bootstrap-series "$SERIES" \ --bootstrap-constraints arch="${ARCH:-amd64}" \ --model-default test-mode=true \ - --model-default resource-tags=owner=k8sci \ + --model-default resource-tags="${TAGS}" \ --model-default automatically-retry-hooks=true \ --model-default logging-config="=DEBUG" \ $extra_args