diff --git a/jobs/build-charms.yaml b/jobs/build-charms.yaml index 03f5522da..2610aa0c6 100644 --- a/jobs/build-charms.yaml +++ b/jobs/build-charms.yaml @@ -110,8 +110,6 @@ #!/bin/bash set -eux - . $WORKSPACE/jobs/build-charms/charmcraft-lib.sh - sudo chown -R jenkins:jenkins /var/lib/jenkins/.config/ || true IS_FORCE="" @@ -126,14 +124,6 @@ rm -rf "$WORKSPACE/charms" || true - # Cleanup old charmcraft containers - ci_lxc_delete "$JOB_NAME" - - # Configure cleanup routine for exit - export charmcraft_lxc="$JOB_NAME-$BUILD_NUMBER" - trap 'ci_lxc_delete $charmcraft_lxc' EXIT - ci_charmcraft_launch $charmcraft_lxc - set +e EXIT_STATUS=0 diff --git a/jobs/build-charms/builder_local.py b/jobs/build-charms/builder_local.py index 820fc45dc..5ca33c127 100644 --- a/jobs/build-charms/builder_local.py +++ b/jobs/build-charms/builder_local.py @@ -474,6 +474,7 @@ def __init__(self, build_type): ) if response and "Item" in response: self.db = response["Item"] + self.db["pull_layer_manifest"] = None def clean(self): for each in self.clean_dirs: @@ -602,7 +603,10 @@ def download(self, layer_name): return layer_manifest def pull_layers(self): - """Clone all downstream layers to be processed locally when doing charm builds.""" + """Clone all downstream layers to be processed locally when doing local charm builds.""" + layers = self.db.get("pull_layer_manifest") + if layers is not None: + return layers layers_to_pull = [ layer_name for layer in self.layers @@ -611,8 +615,8 @@ def pull_layers(self): ] pool = ThreadPool() results = pool.map(self.download, layers_to_pull) - - self.db["pull_layer_manifest"] = list(results) + self.db["pull_layer_manifest"] = layers = list(results) + return layers @unique @@ -831,8 +835,7 @@ def version_identification(self, source): version_id = [{"rev": version, "url": self.entity}] if version else None elif self.reactive and source == "local": version_id = [ - {k: curr[k] for k in comparisons} - for curr in self.build.db["pull_layer_manifest"] + {k: curr[k] for k in comparisons} for curr in self.build.pull_layers() ] # Check the current git cloned charm repo commit and add that to @@ -915,7 +918,6 @@ def within_channel_bounds(self, to_channels): def charm_build(self): """Perform a build against charm/bundle.""" - lxc = os.environ.get("charmcraft_lxc") ret = SimpleNamespace(ok=False) charm_path = Path(self.src_path, f"{self.name}.charm") if "override-build" in self.opts: @@ -927,6 +929,7 @@ def charm_build(self): echo=self.echo, ) elif self.reactive: + self.build.pull_layers() supported_architectures = ( self.opts.get("architectures") or K8S_CHARM_SUPPORT_ARCHES ) @@ -942,21 +945,8 @@ def charm_build(self): ret.ok = True except sh.ErrorReturnCode as e: self.echo(e.stderr) - elif lxc: - msg = f"Consider moving {self.name} to launchpad builder" - border = "-".join("=" * (len(msg) // 2 + 1)) - self.echo("\n".join((border, msg, border))) - self.echo(f"Building in container {lxc}.") - repository = f"https://github.com/{self.downstream}" - charmcraft_script = ( - "#!/bin/bash -eux\n" - f"source {Path(__file__).parent / 'charmcraft-lib.sh'}\n" - f"ci_charmcraft_pack {lxc} {repository} {self.branch} {self.opts.get('subdir', '')}\n" - f"ci_charmcraft_copy {lxc} {charm_path}\n" - ) - ret = script(charmcraft_script, echo=self.echo) else: - self.echo("No 'charmcraft_lxc' container available") + self.echo("charmcraft pack should be performed on launchpad builder") if not ret.ok: self.echo("Failed to build, aborting") @@ -990,7 +980,7 @@ def push(self, artifact: Artifact): self.tag(f"{self.name}-{artifact.rev}") def tag(self, tag: str) -> bool: - """Tag current commit in this repo with a lightweigh tag.""" + """Tag current commit in this repo with a lightweight tag.""" tagged = False current_sha = self.commit() if repo := self.repository: diff --git a/jobs/build-charms/charmcraft-build.sh b/jobs/build-charms/charmcraft-build.sh deleted file mode 100755 index f2eaf72af..000000000 --- a/jobs/build-charms/charmcraft-build.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -eux - -_DIR="${BASH_SOURCE%/*}" -if [[ ! -d "$_DIR" ]]; then _DIR="$PWD"; fi -. "$_DIR/charmcraft-lib.sh" - -## Requirements -## - LXD (initialized) - -## Configuration -## - REPOSITORY: Repository to pull the charm code from -## - BRANCH: Tag to checkout -## - SUBDIR: Subdirectory under repository where charm exists -## - RELEASE_TO_EDGE: Release uploaded revision to CharmHub edge -## - UPLOAD_CHARM: Upload charm to charmhub.io -## - COPY_CHARM: Copy charm to local path -## - JOB_NAME: from jenkins -## - BUILD_NUMBER: from jenkins - - -## Secrets -## - CHARMCRAFT_AUTH: charmcraft credentials. output of `charmcraft login --export auth; cat auth` - -## default optional variables -JOB_NAME="${JOB_NAME:-unnamed-job}" -BUILD_NUMBER="${BUILD_NUMBER:-0}" -SUBDIR="${SUBDIR:-}" -RELEASE_TO_EDGE="${RELEASE_TO_EDGE:-}" -UPLOAD_CHARM="${UPLOAD_CHARM:-}" -COPY_CHARM="${COPY_CHARM:-}" - -# Cleanup old containers -ci_lxc_delete "${JOB_NAME}" - -# Configure cleanup routine -container="${JOB_NAME}-${BUILD_NUMBER}" -trap 'ci_lxc_delete $container' EXIT - -ci_charmcraft_launch $container -ci_charmcraft_pack $container ${REPOSITORY} ${BRANCH} "${SUBDIR}" -if [[ "$UPLOAD_CHARM" == 'true' ]]; then - ci_charmcraft_release $container $RELEASE_TO_EDGE -fi - -if [[ -n "$COPY_CHARM" ]]; then - ci_charmcraft_copy $container $COPY_CHARM -fi diff --git a/jobs/build-charms/charmcraft-lib.sh b/jobs/build-charms/charmcraft-lib.sh deleted file mode 100644 index e2ca230f9..000000000 --- a/jobs/build-charms/charmcraft-lib.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -set -eux - -if [[ $0 == $BASH_SOURCE ]]; then - echo "$0 should be sourced"; - exit -fi -echo "sourced ${BASH_SOURCE:-$0}" -source $(dirname ${BASH_SOURCE:-$0})/../../cilib.sh - -ci_charmcraft_launch() -{ - # Launch local LXD container to publish to charmcraft - local charmcraft_lxc=$1 - ci_lxc_launch ubuntu:20.04 $charmcraft_lxc - until sudo lxc shell $charmcraft_lxc -- bash -c "snap install charmcraft --classic --channel=2.x/stable"; do - echo 'retrying charmcraft install in 3s...' - sleep 3 - done - - local clone_current="git clone $(git remote -v | grep http | head -n 1 | awk '{print $2}') clone-test" - if sudo lxc shell $charmcraft_lxc -- bash -c "$clone_current"; then - echo "$charmcraft_lxc can clone a repo" - else - echo "$charmcraft_lxc can't clone a repo, try setting proxy..." - sudo lxc shell $charmcraft_lxc -- bash -c 'git config --global --add http.proxy http://squid.internal:3128' - sudo lxc shell $charmcraft_lxc -- bash -c 'git config --global --add https.proxy http://squid.internal:3128' - sudo lxc shell $charmcraft_lxc -- bash -c "$clone_current" - fi - sudo lxc shell $charmcraft_lxc -- bash -c 'rm -rf clone-test' - -} - -ci_charmcraft_pack() -{ - # Build charm - local charmcraft_lxc=$1 - local repository=$2 - local branch=$3 - local subdir=${4:-.} - - sudo lxc shell $charmcraft_lxc -- bash -c "rm -rf /root/*" - sudo lxc shell $charmcraft_lxc -- bash -c "git clone ${repository} -b ${branch} charm" - sudo lxc shell $charmcraft_lxc -- bash -c "cd charm/$subdir; cat version || git rev-parse --short HEAD | tee version" - sudo lxc shell $charmcraft_lxc --env CHARMCRAFT_MANAGED_MODE=1 -- bash -c "cd charm; charmcraft pack -v -p $subdir" -} - -ci_charmcraft_release() -{ - # Upload to CharmHub, and optionally release - local charmcraft_lxc=$1 - local do_release_to_edge=${2:-} - local upload_args=$([[ $do_release_to_edge == 'true' ]] && echo ' --release edge') - sudo lxc shell $charmcraft_lxc --env CHARMCRAFT_AUTH="$CHARMCRAFT_AUTH" -- bash -c "cd charm; charmcraft upload *.charm $upload_args" -} - -ci_charmcraft_copy() -{ - # Copy charm out of the container to a local directory - local charmcraft_lxc=$1 - local copy_destination=$2 - for charm in $(sudo lxc exec $charmcraft_lxc -- bash -c "ls /root/charm/*.charm"); do - echo "Pulling ${charmcraft_lxc}${charm} to ${copy_destination}" - sudo lxc file pull ${charmcraft_lxc}${charm} ${copy_destination} - done -} - -ci_charmcraft_promote() -{ - # Promote an existing charm revision to a channel - local charmcraft_lxc=$1 - local charm=$2 - local revision=$3 - local channel=$4 - sudo lxc shell $charmcraft_lxc --env CHARMCRAFT_AUTH="$CHARMCRAFT_AUTH" -- bash -c "charmcraft release $charm --revision $revision --channel $channel" -} diff --git a/jobs/build-charms/main.py b/jobs/build-charms/main.py index a4ed69bd6..2cd5c09cd 100644 --- a/jobs/build-charms/main.py +++ b/jobs/build-charms/main.py @@ -76,7 +76,6 @@ def build( "force": force, } build_env.clean() - build_env.pull_layers() entities = [] for charm_map in build_env.job_list: diff --git a/tests/unit/build_charms/test_charms.py b/tests/unit/build_charms/test_charms.py index 502d52447..47c3a6cd4 100644 --- a/tests/unit/build_charms/test_charms.py +++ b/tests/unit/build_charms/test_charms.py @@ -294,24 +294,12 @@ def test_build_entity_charm_build( # Operator Charms, build with charmcraft container charm_entity.reactive = False - os.environ["charmcraft_lxc"] = "unnamed-job-0" - charm_entity.artifacts = [] - charm_entity.charm_build() - charm_cmd.build.assert_not_called() - mock_script.assert_called_once_with( - "#!/bin/bash -eux\n" - f"source {CHARMCRAFT_LIB_SH}\n" - f"ci_charmcraft_pack unnamed-job-0 https://github.com/{charm_entity.downstream} main \n" - f"ci_charmcraft_copy unnamed-job-0 {K8S_CI_CHARM}/k8s-ci-charm.charm\n", - echo=charm_entity.echo, - ) - mock_script.reset_mock() - - # Operator Charms, fail build without charmcraft container - del os.environ["charmcraft_lxc"] with pytest.raises(builder_local.BuildException): charm_entity.artifacts = [] charm_entity.charm_build() + charm_cmd.build.assert_not_called() + mock_script.assert_not_called() + mock_script.reset_mock() def test_build_entity_upload( @@ -706,7 +694,6 @@ def test_build_command(mock_build_env, mock_build_entity, main): "to_channel": "edge", "force": True, } - mock_build_env.pull_layers.assert_called_once_with() mock_build_env.save.assert_called_once_with() entity.echo.assert_has_calls(