diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index cc8604c2616..77f966e7d53 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -110,9 +110,10 @@ spec: systemctl stop kubelet declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity for BINARY in "$${BINARIES[@]}"; do echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login done systemctl restart kubelet @@ -193,6 +194,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + identity: UserAssigned image: marketplace: offer: capi @@ -203,6 +205,8 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -239,6 +243,7 @@ metadata: spec: template: spec: + identity: UserAssigned image: marketplace: offer: capi @@ -249,6 +254,8 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} vmExtensions: - name: CustomScript protectedSettings: @@ -285,9 +292,10 @@ spec: systemctl stop kubelet declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity for BINARY in "$${BINARIES[@]}"; do echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login done systemctl restart kubelet @@ -343,6 +351,7 @@ spec: runtime: containerd windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} spec: + identity: UserAssigned image: marketplace: offer: capi-windows @@ -355,6 +364,8 @@ spec: storageAccountType: Premium_LRS osType: Windows sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 @@ -415,6 +426,16 @@ spec: sc.exe start sshd path: C:/collect-hns-crashes.ps1 permissions: "0744" + - content: | + Write-Host "Installing Azure CLI" + $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + Write-Host "Version Azure CLI:" + az version + path: C:/install-az-cli.ps1 + permissions: "0744" - content: | $ErrorActionPreference = 'Stop' @@ -423,12 +444,18 @@ spec: $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" if($$KUBE_GIT_VERSION -ne "") { + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + # Install Binaries $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" + echo "running az login via identity" + az login --identity foreach ( $$binary in $$binaries ) { echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login } } @@ -460,6 +487,7 @@ spec: - powershell C:/create-temp-folder.ps1 - powershell C:/replace-containerd.ps1 - powershell C:/collect-hns-crashes.ps1 + - powershell C:/install-az-cli.ps1 - powershell C:/replace-pr-binaries.ps1 users: - groups: Administrators diff --git a/templates/test/dev/custom-builds/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml b/templates/test/dev/custom-builds/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml index 0bd59793f47..4ceacf8311c 100644 --- a/templates/test/dev/custom-builds/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml +++ b/templates/test/dev/custom-builds/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml @@ -1,3 +1,16 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + Write-Host "Installing Azure CLI" + $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + Write-Host "Version Azure CLI:" + az version + path: C:/install-az-cli.ps1 + permissions: "0744" - op: add path: /spec/template/spec/files/- value: @@ -9,12 +22,18 @@ $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" if($$KUBE_GIT_VERSION -ne "") { + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + # Install Binaries $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" + echo "running az login via identity" + az login --identity foreach ( $$binary in $$binaries ) { echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login } } @@ -29,6 +48,10 @@ kube-proxy.exe --version path: C:/replace-pr-binaries.ps1 permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + powershell C:/install-az-cli.ps1 - op: add path: /spec/template/spec/preKubeadmCommands/- value: diff --git a/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml b/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml index 42be3a83049..fa7eb66b773 100644 --- a/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml +++ b/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml @@ -10,9 +10,10 @@ systemctl stop kubelet declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity for BINARY in "$${BINARIES[@]}"; do echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login done systemctl restart kubelet diff --git a/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml b/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml index 85d67a7f03b..37734acfc23 100644 --- a/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml +++ b/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml @@ -10,9 +10,10 @@ systemctl stop kubelet declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity for BINARY in "$${BINARIES[@]}"; do echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login done systemctl restart kubelet diff --git a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml index 19e44aee327..31e1df753bf 100644 --- a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml +++ b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml @@ -6,6 +6,9 @@ metadata: spec: template: spec: + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} image: # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. # latest binaries and images will get replaced to the desired version by the script above. diff --git a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml index 8e9b0bc0233..40cf4c70840 100644 --- a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml +++ b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml @@ -5,6 +5,9 @@ metadata: spec: template: spec: + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} image: # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. # latest binaries and images will get replaced to the desired version by the script above. @@ -21,6 +24,9 @@ metadata: spec: template: spec: + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} image: # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. # latest binaries and images will get replaced to the desired version by the script above.