From 5e6ede0b16c77339f2afadd2f37a6d2ac26b79ec Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Tue, 23 Apr 2024 10:34:58 -0700 Subject: [PATCH] CI: workload-identity native Signed-off-by: Jack Francis --- .gitignore | 8 + Makefile | 38 ++- Tiltfile | 7 +- docs/book/src/topics/workload-identity.md | 2 +- e2e.mk | 17 +- hack/create-dev-cluster.sh | 5 + hack/log/redact.sh | 2 + hack/util.sh | 2 - scripts/cleanup-workload-identity.sh | 33 ++ scripts/kind-with-registry.sh | 166 +++++++-- .../azure-cluster-identity-sp.yaml | 13 + .../azure-cluster-identity.yaml | 5 +- templates/azure-cluster-identity/secret.yaml | 9 - templates/cluster-template-aad.yaml | 10 +- .../cluster-template-aks-clusterclass.yaml | 7 +- templates/cluster-template-aks.yaml | 7 +- templates/cluster-template-azure-bastion.yaml | 10 +- templates/cluster-template-azure-cni-v1.yaml | 10 +- templates/cluster-template-clusterclass.yaml | 7 +- templates/cluster-template-dual-stack.yaml | 10 +- templates/cluster-template-edgezone.yaml | 10 +- templates/cluster-template-ephemeral.yaml | 10 +- templates/cluster-template-flatcar.yaml | 10 +- templates/cluster-template-ipv6.yaml | 10 +- .../cluster-template-machinepool-windows.yaml | 10 +- templates/cluster-template-machinepool.yaml | 10 +- templates/cluster-template-nvidia-gpu.yaml | 10 +- templates/cluster-template-private.yaml | 10 +- templates/cluster-template-windows.yaml | 13 +- templates/cluster-template.yaml | 10 +- templates/flavors/base/cluster-template.yaml | 3 + .../windows/machine-deployment-windows.yaml | 3 + ...luster-template-prow-aks-clusterclass.yaml | 7 +- .../test/ci/cluster-template-prow-aks.yaml | 7 +- .../cluster-template-prow-azure-cni-v1.yaml | 10 +- ...r-template-prow-ci-version-dual-stack.yaml | 10 +- ...cluster-template-prow-ci-version-ipv6.yaml | 10 +- .../ci/cluster-template-prow-ci-version.yaml | 13 +- ...template-prow-clusterclass-ci-default.yaml | 7 +- .../ci/cluster-template-prow-custom-vnet.yaml | 7 +- .../ci/cluster-template-prow-dual-stack.yaml | 7 +- .../ci/cluster-template-prow-edgezone.yaml | 7 +- .../ci/cluster-template-prow-flatcar.yaml | 10 +- .../test/ci/cluster-template-prow-ipv6.yaml | 10 +- ...template-prow-machine-pool-ci-version.yaml | 13 +- ...uster-template-prow-machine-pool-flex.yaml | 10 +- .../cluster-template-prow-machine-pool.yaml | 10 +- .../ci/cluster-template-prow-nvidia-gpu.yaml | 10 +- .../test/ci/cluster-template-prow-spot.yaml | 7 +- ...uster-template-prow-workload-identity.yaml | 321 ------------------ templates/test/ci/cluster-template-prow.yaml | 13 +- .../ci/patches/azureclusteridentity-azwi.yaml | 2 +- .../ci/patches/windows-server-version.yaml | 6 +- .../ci/prow-ci-version/kustomization.yaml | 1 + ...e-deployment-ci-version-control-plane.yaml | 26 ++ ...machine-deployment-ci-version-windows.yaml | 3 + .../machine-pool-ci-version-windows.yaml | 3 + .../prow-private/patches/user-assigned.yaml | 2 - .../prow-workload-identity/kustomization.yaml | 20 -- ...r-template-custom-builds-machine-pool.yaml | 10 +- .../dev/cluster-template-custom-builds.yaml | 6 +- .../test/dev/custom-builds/kustomization.yaml | 1 + .../azure-cluster-identity-user-assigned.yaml | 16 + ...machine-deployment-pr-version-windows.yaml | 2 +- .../machine-deployment-pr-version.yaml | 4 +- test/e2e/azure_test.go | 83 +---- test/e2e/common.go | 1 + test/e2e/config/azure-dev.yaml | 4 +- .../cluster-template-kcp-remediation.yaml | 7 +- .../cluster-template-kcp-scale-in.yaml | 7 +- .../cluster-template-machine-pool.yaml | 7 +- .../cluster-template-md-remediation.yaml | 7 +- .../v1beta1/cluster-template-node-drain.yaml | 7 +- .../v1beta1/cluster-template-upgrades.yaml | 7 +- .../cluster-template/kustomization.yaml | 2 +- 75 files changed, 504 insertions(+), 696 deletions(-) create mode 100755 scripts/cleanup-workload-identity.sh create mode 100644 templates/azure-cluster-identity/azure-cluster-identity-sp.yaml delete mode 100644 templates/azure-cluster-identity/secret.yaml delete mode 100644 templates/test/ci/cluster-template-prow-workload-identity.yaml create mode 100644 templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-control-plane.yaml delete mode 100644 templates/test/ci/prow-workload-identity/kustomization.yaml create mode 100644 templates/test/dev/custom-builds/patches/azure-cluster-identity-user-assigned.yaml diff --git a/.gitignore b/.gitignore index dc7e16b4ed6..525561b7ee6 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,11 @@ release-*/manifests/calico-*.yaml # mentioned in the capz book /sp.json /cluster.yaml + +# CI workload-identity +jwks.json +*.pub +*.key +azure_identity_id +azure_wi_back_compat +openid-configuration.json diff --git a/Makefile b/Makefile index 56256220c23..da7865b9819 100644 --- a/Makefile +++ b/Makefile @@ -89,6 +89,10 @@ KUSTOMIZE_VER := v5.4.1 KUSTOMIZE_BIN := kustomize KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER) +AZWI_VER := v1.2.2 +AZWI_BIN := azwi +AZWI := $(TOOLS_BIN_DIR)/$(AZWI_BIN)-$(AZWI_VER) + MOCKGEN_VER := v0.4.0 MOCKGEN_BIN := mockgen MOCKGEN := $(TOOLS_BIN_DIR)/$(MOCKGEN_BIN)-$(MOCKGEN_VER) @@ -177,6 +181,7 @@ ARTIFACTS ?= $(ROOT_DIR)/_artifacts E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml SKIP_CLEANUP ?= false +AZWI_SKIP_CLEANUP ?= false SKIP_LOG_COLLECTION ?= false # @sonasingh46: Skip creating mgmt cluster for ci as workload identity needs kind cluster # to be created with extra mounts for key pairs which is not yet supported @@ -191,6 +196,11 @@ LDFLAGS := $(shell hack/version.sh) CLUSTER_TEMPLATE ?= cluster-template.yaml export KIND_CLUSTER_NAME ?= capz +export RANDOM_SUFFIX := $(shell /bin/bash -c "echo $$RANDOM") +export AZWI_RESOURCE_GROUP ?= capz-wi-$(RANDOM_SUFFIX) +export CI_RG ?= $(AZWI_RESOURCE_GROUP) +export USER_IDENTITY ?= $(addsuffix $(RANDOM_SUFFIX),$(CI_RG)) +export AZURE_IDENTITY_ID_FILEPATH ?= $(ROOT_DIR)/azure_identity_id ## -------------------------------------- ## Binaries @@ -287,7 +297,7 @@ verify-codespell: codespell ## Verify codespell. ##@ Development: .PHONY: install-tools # populate hack/tools/bin -install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND) +install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND) $(AZWI) .PHONY: create-management-cluster create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create a management cluster. @@ -341,7 +351,10 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create .PHONY: create-workload-cluster create-workload-cluster: $(ENVSUBST) $(KUBECTL) ## Create a workload cluster. # Create workload Cluster. - @if [ -f "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" ]; then \ + @if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \ + export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \ + fi; \ + if [ -f "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" ]; then \ timeout --foreground 300 bash -c "until $(ENVSUBST) < $(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE) | $(KUBECTL) apply -f -; do sleep 5; done"; \ elif [ -f "$(CLUSTER_TEMPLATE)" ]; then \ timeout --foreground 300 bash -c "until $(ENVSUBST) < "$(CLUSTER_TEMPLATE)" | $(KUBECTL) apply -f -; do sleep 5; done"; \ @@ -686,7 +699,13 @@ test-cover: test ## Run tests with code coverage and generate reports. .PHONY: kind-create-bootstrap kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster. - export AZWI=$${AZWI:-true} KIND_CLUSTER_NAME=capz-e2e && ./scripts/kind-with-registry.sh + KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh + +.PHONY: cleanup-workload-identity +cleanup-workload-identity: ## Cleanup CI workload-identity infra + @if ! [ "$(AZWI_SKIP_CLEANUP)" == "true" ]; then \ + ./scripts/cleanup-workload-identity.sh \ + fi ## -------------------------------------- ## Security Scanning @@ -708,6 +727,9 @@ kind-create: $(KUBECTL) ## Create capz kind cluster if needed. .PHONY: tilt-up tilt-up: install-tools kind-create ## Start tilt and build kind cluster if needed. + @if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \ + export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \ + fi; \ CLUSTER_TOPOLOGY=true EXP_CLUSTER_RESOURCE_SET=true EXP_MACHINE_POOL=true EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true EXP_EDGEZONE=true tilt up .PHONY: delete-cluster @@ -792,6 +814,16 @@ $(HELM): ## Put helm into tools folder. ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN) rm -f $(TOOLS_BIN_DIR)/get_helm.sh +$(AZWI): ## Put azwi into tools folder. + mkdir -p $(TOOLS_BIN_DIR) + rm -f "$(TOOLS_BIN_DIR)/$(AZWI_BIN)*" + curl --retry $(CURL_RETRIES) -fsSL -o $(TOOLS_BIN_DIR)/azwi.tar.gz https://github.com/Azure/azure-workload-identity/releases/download/$(AZWI_VER)/azwi-$(AZWI_VER)-$(GOOS)-$(GOARCH).tar.gz + tar -xf "$(TOOLS_BIN_DIR)/azwi.tar.gz" -C $(TOOLS_BIN_DIR) $(AZWI_BIN) + mv "$(TOOLS_BIN_DIR)/$(AZWI_BIN)" $(AZWI) + ln -sf $(AZWI) $(TOOLS_BIN_DIR)/$(AZWI_BIN) + chmod +x $(AZWI) $(TOOLS_BIN_DIR)/$(AZWI_BIN) + rm -f $(TOOLS_BIN_DIR)/azwi.tar.gz + $(KIND): ## Build kind into tools folder. GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) sigs.k8s.io/kind $(KIND_BIN) $(KIND_VER) diff --git a/Tiltfile b/Tiltfile index 0ce5d1d0fd3..64ebb03d001 100644 --- a/Tiltfile +++ b/Tiltfile @@ -35,7 +35,7 @@ settings = { } # Auth keys that need to be loaded from the environment -keys = ["AZURE_SUBSCRIPTION_ID", "AZURE_TENANT_ID", "AZURE_CLIENT_SECRET", "AZURE_CLIENT_ID"] +keys = ["AZURE_SUBSCRIPTION_ID", "AZURE_TENANT_ID", "AZURE_CLIENT_ID"] # Get global settings from tilt-settings.yaml or tilt-settings.json tilt_file = "./tilt-settings.yaml" if os.path.exists("./tilt-settings.yaml") else "./tilt-settings.json" @@ -266,13 +266,10 @@ def create_identity_secret(): os.putenv("AZURE_CLUSTER_IDENTITY_SECRET_NAME", "cluster-identity-secret") os.putenv("AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE", "default") - os.putenv("CLUSTER_IDENTITY_NAME", "cluster-identity") + os.putenv("CLUSTER_IDENTITY_NAME", "cluster-identity-ci") os.putenv("ASO_CREDENTIAL_SECRET_NAME", "aso-credentials") - os.putenv("AZURE_CLIENT_SECRET_B64", base64_encode(os.environ.get("AZURE_CLIENT_SECRET"))) - local("cat templates/azure-cluster-identity/secret.yaml | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -", quiet = True, echo_off = True) local("cat templates/flavors/aks-aso/credentials.yaml | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -", quiet = True, echo_off = True) - os.unsetenv("AZURE_CLIENT_SECRET_B64") def create_crs(): # create config maps diff --git a/docs/book/src/topics/workload-identity.md b/docs/book/src/topics/workload-identity.md index 85ddc6456c8..20beb355d1d 100644 --- a/docs/book/src/topics/workload-identity.md +++ b/docs/book/src/topics/workload-identity.md @@ -169,7 +169,7 @@ to give the identity Contributor access to the Azure subscription where the work - At this stage, you can apply this yaml to create a workload cluster. Notes: -- Please follow this [link](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/templates/test/ci/cluster-template-prow-workload-identity.yaml) +- Please follow this [link](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/templates/test/ci/cluster-template-prow.yaml) to see a workload cluster yaml configuration that uses workload identity. - Creating a workload cluster via workload identity will be simplified after [this](https://github.com/kubernetes-sigs/cluster-api-provider-azure/issues/3589) issue is resolved. diff --git a/e2e.mk b/e2e.mk index 8fc3c5a965d..78f321fd1ed 100644 --- a/e2e.mk +++ b/e2e.mk @@ -4,15 +4,18 @@ # long-running E2E jobs every time that file changes ##@ E2E Testing: - .PHONY: test-e2e-run test-e2e-run: generate-e2e-templates install-tools kind-create-bootstrap ## Run e2e tests. - $(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \ - $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \ - -e2e.artifacts-folder="$(ARTIFACTS)" \ - -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ - -e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \ - -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) + @$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \ + if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \ + export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \ + fi; \ + $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \ + -e2e.artifacts-folder="$(ARTIFACTS)" \ + -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ + -e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \ + -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) \ + $(MAKE) cleanup-workload-identity $(MAKE) clean-release-git .PHONY: test-e2e diff --git a/hack/create-dev-cluster.sh b/hack/create-dev-cluster.sh index 09512c026a3..e066dd5e7b3 100755 --- a/hack/create-dev-cluster.sh +++ b/hack/create-dev-cluster.sh @@ -38,6 +38,11 @@ export AZURE_VNET_NAME=${CLUSTER_NAME}-vnet export AZURE_LOCATION="${AZURE_LOCATION:-southcentralus}" export AZURE_RESOURCE_GROUP=${CLUSTER_NAME} +AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:=}" +AZURE_TENANT_ID="${AZURE_TENANT_ID:=}" +AZURE_CLIENT_ID="${AZURE_CLIENT_ID:=}" +AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET:=}" + AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')" AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')" AZURE_CLIENT_ID_B64="$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')" diff --git a/hack/log/redact.sh b/hack/log/redact.sh index 7ef522a7aee..4fb2ec4c389 100755 --- a/hack/log/redact.sh +++ b/hack/log/redact.sh @@ -28,10 +28,12 @@ redact_vars=( "${AZURE_SUBSCRIPTION_ID:-}" "${AZURE_TENANT_ID:-}" "${AZURE_JSON_B64:-}" + "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}" "$(echo -n "${AZURE_SUBSCRIPTION_ID:-}" | base64 | tr -d '\n')" "$(echo -n "${AZURE_TENANT_ID:-}" | base64 | tr -d '\n')" "$(echo -n "${AZURE_CLIENT_ID:-}" | base64 | tr -d '\n')" "$(echo -n "${AZURE_CLIENT_SECRET:-}" | base64 | tr -d '\n')" + "$(echo -n "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}" | base64 | tr -d '\n')" ) for log_file in "${log_files[@]}"; do diff --git a/hack/util.sh b/hack/util.sh index cff56f4839f..6a4fc236ac8 100755 --- a/hack/util.sh +++ b/hack/util.sh @@ -108,6 +108,4 @@ capz::util::generate_ssh_key() { capz::util::ensure_azure_envs() { : "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}" : "${AZURE_TENANT_ID:?Environment variable empty or not defined.}" - : "${AZURE_CLIENT_ID:?Environment variable empty or not defined.}" - : "${AZURE_CLIENT_SECRET:?Environment variable empty or not defined.}" } diff --git a/scripts/cleanup-workload-identity.sh b/scripts/cleanup-workload-identity.sh new file mode 100755 index 00000000000..3e32081dab9 --- /dev/null +++ b/scripts/cleanup-workload-identity.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# Install kubectl and kind +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +# shellcheck source=hack/ensure-azcli.sh +source "${REPO_ROOT}/hack/ensure-azcli.sh" + +AZWI_RESOURCE_GROUP="${AZWI_RESOURCE_GROUP:-}" + +if [[ -z "${AZWI_RESOURCE_GROUP}" ]]; then + echo AZWI_RESOURCE_GROUP environment variable must be set + exit 1 +fi + +echo "Cleaning up CI workload-identity infra..." +az group delete --no-wait -y -n "${AZWI_RESOURCE_GROUP}" diff --git a/scripts/kind-with-registry.sh b/scripts/kind-with-registry.sh index a4e5933207f..a9ecfaf936e 100755 --- a/scripts/kind-with-registry.sh +++ b/scripts/kind-with-registry.sh @@ -19,13 +19,29 @@ set -o pipefail # Install kubectl and kind REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +# shellcheck source=hack/ensure-azcli.sh +source "${REPO_ROOT}/hack/ensure-azcli.sh" +# shellcheck source=hack/ensure-tags.sh +source "${REPO_ROOT}/hack/ensure-tags.sh" + KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl" KIND="${REPO_ROOT}/hack/tools/bin/kind" -AZWI_ENABLED=${AZWI:-} +AZWI="${REPO_ROOT}/hack/tools/bin/azwi" +AZWI_ENABLED="${AZWI_ENABLED:-true}" +RANDOM_SUFFIX="${RANDOM_SUFFIX:-$RANDOM}" +export AZWI_STORAGE_ACCOUNT="oidcissuer${RANDOM_SUFFIX}" +export AZWI_STORAGE_CONTAINER="\$web" +export AZWI_LOCATION="${AZURE_LOCATION:-southcentralus}" +export SERVICE_ACCOUNT_ISSUER="${SERVICE_ACCOUNT_ISSUER:-}" +export SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH="${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH:-}" +export SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH="${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH:-}" +export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY="${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}" +export AZURE_IDENTITY_ID_FILEPATH="${AZURE_IDENTITY_ID_FILEPATH:-$REPO_ROOT/azure_identity_id}" make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}" # Export desired cluster name; default is "capz" KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-capz}" +CONFORMANCE_FLAVOR="${CONFORMANCE_FLAVOR:-}" export KIND_CLUSTER_NAME if [[ "$("${KIND}" get clusters)" =~ .*"${KIND_CLUSTER_NAME}".* ]]; then @@ -42,44 +58,156 @@ if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true registry:2 fi -# To use workload identity, service account signing key pairs base64 encoded should be exposed via the -# env variables. The function creates the key pair files after reading it from the env variables. +# Environment variable inputs +# SERVICE_ACCOUNT_ISSUER - BYO existing service account issuer +# Accepts a URI string, e.g., https://${AZWI_STORAGE_ACCOUNT}.blob.core.windows.net/${AZWI_STORAGE_CONTAINER}/ +# Assumes that the required openid and jwks artifacts exist in the well-known locations at this URI +# SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH - A local filepath to a 2048 bit RSA public key +# Defaults to capz-wi-sa.pub, must exist locally and match the signed artifacts if using BYO +# SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH - A local filepath to a 2048 bit RSA private key +# Defaults to capz-wi-sa.key, must exist locally and match the signed artifacts if using BYO +# If the above keypair filepaths environment variables are not included, a keypair will be created at runtime +# Note: if a new keypair is created at runtime then you must not BYO service account issuer +# AZWI_RESOURCE_GROUP - Azure resource group where Workload Identity infra lives +# AZWI_LOCATION - Azure location for Workload Identity infra +# AZWI_STORAGE_ACCOUNT - Storage account in resource group $AZWI_RESOURCE_GROUP containing required artifacts +# Must be configured for static website hosting +# AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY - BYO existing user-assigned identity +# Should be a UUID that represents the clientID of the identity object +# USER_IDENTITY - Name to use when creating a new user-assigned identity +# Required if not bringing your own identity via $AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY +# AZURE_IDENTITY_ID_FILEPATH - A local filepath to store the newly created user-assigned identity if not bringing your own function checkAZWIENVPreReqsAndCreateFiles() { - if [[ -z "${SERVICE_ACCOUNT_SIGNING_PUB}" ]]; then - echo "'SERVICE_ACCOUNT_SIGNING_PUB' is not set." - exit 1 + if [[ -z "${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH}" || -z "${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH}" ]]; then + export SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH="${REPO_ROOT}/capz-wi-sa.pub" + export SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH="${REPO_ROOT}/capz-wi-sa.key" fi + if [ -z "${SERVICE_ACCOUNT_ISSUER}" ]; then + # check if user is logged into azure cli + if ! az account show > /dev/null 2>&1; then + echo "Please login to Azure CLI using 'az login'" + exit 1 + fi - if [[ -z "${SERVICE_ACCOUNT_SIGNING_KEY}" ]]; then - echo "'SERVICE_ACCOUNT_SIGNING_KEY' is not set." - exit 1 + if [ -z "${AZWI_RESOURCE_GROUP}" ]; then + echo "AZWI_RESOURCE_GROUP environment variable required - Azure resource group to store required Workload Identity artifacts" + exit 1 + fi + if [ "$(az group exists --name "${AZWI_RESOURCE_GROUP}" --output tsv)" == 'false' ]; then + echo "Creating resource group '${AZWI_RESOURCE_GROUP}' in '${AZWI_LOCATION}'" + az group create --name "${AZWI_RESOURCE_GROUP}" --location "${AZWI_LOCATION}" --output none --only-show-errors --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" + fi + # Ensure that our connection to storage is inherited from the existing Azure login context + unset AZURE_STORAGE_KEY + unset AZURE_STORAGE_ACCOUNT + if ! az storage account show --name "${AZWI_STORAGE_ACCOUNT}" --resource-group "${AZWI_RESOURCE_GROUP}" > /dev/null 2>&1; then + echo "Creating storage account '${AZWI_STORAGE_ACCOUNT}' in '${AZWI_RESOURCE_GROUP}'" + az storage account create --resource-group "${AZWI_RESOURCE_GROUP}" --name "${AZWI_STORAGE_ACCOUNT}" --output none --only-show-errors --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" + az storage blob service-properties update --account-name "${AZWI_STORAGE_ACCOUNT}" --static-website + fi + if ! az storage container show --name "${AZWI_STORAGE_CONTAINER}" --account-name "${AZWI_STORAGE_ACCOUNT}" > /dev/null 2>&1; then + echo "Creating storage container '${AZWI_STORAGE_CONTAINER}' in '${AZWI_STORAGE_ACCOUNT}'" + az storage container create --name "${AZWI_STORAGE_CONTAINER}" --account-name "${AZWI_STORAGE_ACCOUNT}" --output none --only-show-errors + fi + SERVICE_ACCOUNT_ISSUER=$(az storage account show --name "${AZWI_STORAGE_ACCOUNT}" -o json | jq -r .primaryEndpoints.web) + export SERVICE_ACCOUNT_ISSUER + AZWI_OPENID_CONFIG_FILEPATH="${REPO_ROOT}/openid-configuration.json" + cat < "${AZWI_OPENID_CONFIG_FILEPATH}" +{ + "issuer": "${SERVICE_ACCOUNT_ISSUER}", + "jwks_uri": "${SERVICE_ACCOUNT_ISSUER}openid/v1/jwks", + "response_types_supported": [ + "id_token" + ], + "subject_types_supported": [ + "public" + ], + "id_token_signing_alg_values_supported": [ + "RS256" + ] +} +EOF + openssl genrsa -out "${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH}" 2048 + openssl rsa -in "${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH}" -pubout -out "${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH}" + AZWI_JWKS_JSON_FILEPATH="${REPO_ROOT}/jwks.json" + "${AZWI}" jwks --public-keys "${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH}" --output-file "${AZWI_JWKS_JSON_FILEPATH}" + echo "Uploading openid-configuration document to '${AZWI_STORAGE_ACCOUNT}' storage account" + upload_to_blob "${AZWI_OPENID_CONFIG_FILEPATH}" ".well-known/openid-configuration" + echo "Uploading jwks document to '${AZWI_STORAGE_ACCOUNT}' storage account" + upload_to_blob "${AZWI_JWKS_JSON_FILEPATH}" "openid/v1/jwks" + echo "Removing key access on storage account as no further data writes are required" + az storage account update -n "${AZWI_STORAGE_ACCOUNT}" -g "${AZWI_RESOURCE_GROUP}" --subscription "${AZURE_SUBSCRIPTION_ID}" --allow-shared-key-access=false --output none --only-show-errors + fi + if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then + if [ -z "${USER_IDENTITY}" ]; then + echo "USER_IDENTITY environment variable required if not bringing your own identity via AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY" + exit 1 + fi + az identity create -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" -l "${AZWI_LOCATION}" --output none --only-show-errors --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" + AZURE_IDENTITY_ID=$(az identity show -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" --query clientId -o tsv) + AZURE_IDENTITY_ID_PRINCIPAL_ID=$(az identity show -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" --query principalId -o tsv) + echo "${AZURE_IDENTITY_ID}" > "${AZURE_IDENTITY_ID_FILEPATH}" + until az role assignment create --assignee-object-id "${AZURE_IDENTITY_ID_PRINCIPAL_ID}" --role "Contributor" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal --output none --only-show-errors; do + sleep 5 + done + az identity federated-credential create -n "capz-federated-identity" \ + --identity-name "${USER_IDENTITY}" \ + -g "${AZWI_RESOURCE_GROUP}" \ + --issuer "${SERVICE_ACCOUNT_ISSUER}" \ + --subject "system:serviceaccount:capz-system:capz-manager" --output none --only-show-errors + az identity federated-credential create -n "aso-federated-identity" \ + --identity-name "${USER_IDENTITY}" \ + -g "${AZWI_RESOURCE_GROUP}" \ + --issuer "${SERVICE_ACCOUNT_ISSUER}" \ + --subject "system:serviceaccount:capz-system:azureserviceoperator-default" --output none --only-show-errors fi - mkdir -p "$HOME"/azwi/creds - echo "${SERVICE_ACCOUNT_SIGNING_PUB}" > "$HOME"/azwi/creds/sa.pub - echo "${SERVICE_ACCOUNT_SIGNING_KEY}" > "$HOME"/azwi/creds/sa.key - SERVICE_ACCOUNT_ISSUER="${SERVICE_ACCOUNT_ISSUER:-https://oidcissuercapzci.blob.core.windows.net/oidc-capzci/}" +} + +function upload_to_blob() { + local file_path=$1 + local blob_name=$2 + + echo "Uploading ${file_path} to '${AZWI_STORAGE_ACCOUNT}' storage account" + az storage blob upload \ + --container-name "${AZWI_STORAGE_CONTAINER}" \ + --file "${file_path}" \ + --name "${blob_name}" \ + --account-name "${AZWI_STORAGE_ACCOUNT}" \ + --output none --only-show-errors } # This function create a kind cluster for Workload identity which requires key pairs path # to be mounted on the kind cluster and hence extra mount flags are required. function createKindForAZWI() { - echo "creating azwi kind" + echo "creating workload-identity-enabled kind configuration" + if [ -n "${CONFORMANCE_FLAVOR}" ] && [ -n "${SERVICE_ACCOUNT_SIGNING_PUB}" ] && [ -n "${SERVICE_ACCOUNT_SIGNING_KEY}" ]; then + echo "using pre-existing service-account-issuer for kind cluster" + KIND_SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH="${REPO_ROOT}/kind-wi-sa.pub" + KIND_SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH="${REPO_ROOT}/kind-wi-sa.key" + echo "${SERVICE_ACCOUNT_SIGNING_PUB}" > "${KIND_SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH}" + echo "${SERVICE_ACCOUNT_SIGNING_KEY}" > "${KIND_SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH}" + KIND_SERVICE_ACCOUNT_ISSUER="https://oidcissuercapzci.blob.core.windows.net/oidc-capzci/" + else + KIND_SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH="${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH}" + KIND_SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH="${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH}" + KIND_SERVICE_ACCOUNT_ISSUER="${SERVICE_ACCOUNT_ISSUER}" + fi cat <