From dd3b9bd2f9f7fe6b558df3be500447a6ac609a29 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Tue, 29 Oct 2024 14:06:57 -0400 Subject: [PATCH 1/5] ci: Update build.yml workflow to support parallel builds --- .github/workflows/build.yml | 58 ++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6041fe474b..2e38e125d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -166,13 +166,15 @@ jobs: - name: Launch build VM id: azlaunch + timeout-minutes: 10 run: | echo "Authenticating to Azure..." az login --service-principal -u ${{ secrets.AZ_BUILD_APP_ID }} -p ${{ secrets.AZ_BUILD_PWD }} --tenant ${{ secrets.AZ_BUILD_TENANT_ID }} + echo "Creating VM..." vminfo=$(az vm create \ --resource-group ghaDatatracker \ - --name tmpGhaBuildVM \ + --name tmpGhaBuildVM-${{ github.run_number }} \ --image Ubuntu2204 \ --admin-username azureuser \ --generate-ssh-keys \ @@ -183,11 +185,19 @@ jobs: --os-disk-size-gb 100 \ --eviction-policy Delete \ --nic-delete-option Delete \ + --os-disk-delete-option Delete \ --output tsv \ --query "publicIpAddress") echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT" echo "VM Public IP: $vminfo" cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key + + echo "Fetching SSH host public keys..." + until ssh-keyscan -t rsa $vminfo 2> /dev/null + do + echo "Will try again in 5 seconds..." + sleep 5 + done ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - name: Remote SSH into Build VM @@ -353,29 +363,29 @@ jobs: shell: pwsh run: | echo "Destroying VM..." - az vm delete -g ghaDatatracker -n tmpGhaBuildVM --yes --force-deletion true - - $resourceOrderRemovalOrder = [ordered]@{ - "Microsoft.Compute/virtualMachines" = 0 - "Microsoft.Compute/disks" = 1 - "Microsoft.Network/networkInterfaces" = 2 - "Microsoft.Network/publicIpAddresses" = 3 - "Microsoft.Network/networkSecurityGroups" = 4 - "Microsoft.Network/virtualNetworks" = 5 - } - echo "Fetching remaining resources..." - $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json - - $orderedResources = $resources - | Sort-Object @{ - Expression = {$resourceOrderRemovalOrder[$_.type]} - Descending = $False - } - - echo "Deleting remaining resources..." - $orderedResources | ForEach-Object { - az resource delete --resource-group ghaDatatracker --ids $_.id --verbose - } + az vm delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }} --yes --force-deletion true + + # $resourceOrderRemovalOrder = [ordered]@{ + # "Microsoft.Compute/virtualMachines" = 0 + # "Microsoft.Compute/disks" = 1 + # "Microsoft.Network/networkInterfaces" = 2 + # "Microsoft.Network/publicIpAddresses" = 3 + # "Microsoft.Network/networkSecurityGroups" = 4 + # "Microsoft.Network/virtualNetworks" = 5 + # } + # echo "Fetching remaining resources..." + # $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json + + # $orderedResources = $resources + # | Sort-Object @{ + # Expression = {$resourceOrderRemovalOrder[$_.type]} + # Descending = $False + # } + + # echo "Deleting remaining resources..." + # $orderedResources | ForEach-Object { + # az resource delete --resource-group ghaDatatracker --ids $_.id --verbose + # } echo "Logout from Azure..." az logout From e530e0af6aa2a4fe60e77a015e1969cc642abd55 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Tue, 29 Oct 2024 15:24:56 -0400 Subject: [PATCH 2/5] ci: add disable selenium toggle to build workflow --- .github/workflows/build.yml | 1 + .github/workflows/tests.yml | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2e38e125d3..71730ebb52 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -139,6 +139,7 @@ jobs: needs: [prepare] with: ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }} + skipSelenium: true targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }} # ----------------------------------------------------------------- diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4e0b31fb14..682300f2ab 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,6 +8,11 @@ on: default: false required: true type: boolean + skipSelenium: + description: 'Skip Selenium Tests' + default: false + required: true + type: boolean targetBaseVersion: description: 'Target Base Image Version' default: latest @@ -47,6 +52,10 @@ jobs: echo "Model changes without migrations found." exit 1 fi + if [[ "x${{ inputs.skipSelenium }}" == "xtrue" ]]; then + echo "Disable selenium tests..." + sudo rm /usr/bin/geckodriver + fi echo "Running tests..." if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then echo "Lower coverage failures will be ignored." From 0e392af870dea31496074aec0d26b1b838f337b7 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Tue, 29 Oct 2024 15:32:14 -0400 Subject: [PATCH 3/5] ci: remove sudo from tests workflow --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 682300f2ab..7ad3c82364 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,7 +54,7 @@ jobs: fi if [[ "x${{ inputs.skipSelenium }}" == "xtrue" ]]; then echo "Disable selenium tests..." - sudo rm /usr/bin/geckodriver + rm /usr/bin/geckodriver fi echo "Running tests..." if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then From 84f2affad9b211545f5e72f708f12db2ee5d461a Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Tue, 29 Oct 2024 16:07:05 -0400 Subject: [PATCH 4/5] ci: Update build.yml workflow --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 71730ebb52..96ba962a78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -158,6 +158,7 @@ jobs: PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} FROM_TAG: ${{needs.prepare.outputs.from_tag}} TO_TAG: ${{needs.prepare.outputs.to_tag}} + TARGET_BASE: ${{needs.prepare.outputs.base_image_version}} steps: - uses: actions/checkout@v4 @@ -218,7 +219,7 @@ jobs: SKIP_TESTS: ${{ github.event.inputs.skiptests }} DEBIAN_FRONTEND: noninteractive BROWSERSLIST_IGNORE_OLD_DATA: 1 - TARGETBASE: ${{ needs.prepare.outputs.base_image_version }} + TARGET_BASE: ${{ env.TARGET_BASE }} with: host: ${{ steps.azlaunch.outputs.ipaddr }} port: 22 @@ -329,7 +330,7 @@ jobs: echo "==========================================================================" echo "Collecting statics..." echo "==========================================================================" - sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:$TARGETBASE sh dev/build/collectstatics.sh + sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:$TARGET_BASE sh dev/build/collectstatics.sh echo "Pushing statics..." cd static aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors From 01700ab37bef2a40a7abf0a810841263badc0513 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Tue, 29 Oct 2024 16:16:45 -0400 Subject: [PATCH 5/5] ci: fix build workflow --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 96ba962a78..d613328312 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -330,7 +330,8 @@ jobs: echo "==========================================================================" echo "Collecting statics..." echo "==========================================================================" - sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:$TARGET_BASE sh dev/build/collectstatics.sh + echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}" + sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh echo "Pushing statics..." cd static aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors