From 924fc9202685418f157e584b135153ac73e2c13a Mon Sep 17 00:00:00 2001 From: adreed-msft <49764384+adreed-msft@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:45:37 -0700 Subject: [PATCH] Run workload identity tests first to avoid timeout (#2833) * Workload identity hook * Remove hook, Run ahead of everything else * Run New E2E separately * drop parallel statements when running up front * Missed parallel statements, add hook. * Always create the log dir --- azure-pipelines.yml | 184 ++++++++++++++++++- e2etest/newe2e_scenario_manager.go | 9 +- e2etest/newe2e_scenario_variation_manager.go | 3 + e2etest/newe2e_suite_manager.go | 40 +++- e2etest/newe2e_workload_hook.go | 40 ++++ e2etest/zt_aanewe2e_testmain_test.go | 3 + e2etest/zt_newe2e_workload_test.go | 2 +- 7 files changed, 269 insertions(+), 12 deletions(-) create mode 100644 e2etest/newe2e_workload_hook.go diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 589a40923..36eca2819 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -215,7 +215,189 @@ jobs: Write-Output "Running tests" # Run tests and pipe output to test.txt - go test -timeout=2h -v ./e2etest | Tee-Object -FilePath test.txt + go test -timeout=2h -v -tags olde2etest ./e2etest | Tee-Object -FilePath test.txt + + # Save the exit code from the previous command + $exitCode = $LASTEXITCODE + + # Print the contents of test.txt + # Get-Content test.txt + + # Print "Generating junit report" + Write-Output "Generating junit report" + + # Pipe info in test.txt to go-junit-report and save output to report.xml + Get-Content test.txt | & "$(go env GOPATH)/bin/go-junit-report" > "${display_name}_report.xml" + + # Print "Formatting coverage directory to legacy txt format" + Write-Output "Formatting coverage directory to legacy txt format" + + # Format coverage data to text format + go tool covdata textfmt -i=coverage -o "${display_name}_coverage.txt" + + # Print "Formatting coverage to json format" + Write-Output "Formatting coverage to json format" + + # Convert coverage.txt to coverage.json + & "$(go env GOPATH)/bin/gocov$suffix" convert "${display_name}_coverage.txt" > "${display_name}_coverage.json" + + # Print "Formatting coverage to xml format" + Write-Output "Formatting coverage to xml format" + + # Convert coverage.json to coverage.xml + Get-Content "${display_name}_coverage.json" | & "$(go env GOPATH)/bin/gocov-xml$suffix" > "${display_name}_coverage.xml" + + # Return the exit code from step 5 + exit $exitCode + env: + AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY) + AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME) + AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS) + AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS) + AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME) + AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY) + AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs' + AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG) + AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG) + AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG) + CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY) + CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256) + AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage' + NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID) + NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name) + NEW_E2E_ENVIRONMENT: "AzurePipeline" + displayName: 'E2E Test $(display_name) - AMD64 with Workload Identity' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish logs' + condition: succeededOrFailed() + inputs: + pathToPublish: '$(System.DefaultWorkingDirectory)/logs' + artifactName: logs + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testRunner: JUnit + testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml + testRunTitle: 'Go on $(display_name)' + + - task: PublishCodeCoverageResults@1 + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml + additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html + + - job: New_E2E_Framework + timeoutInMinutes: 360 + # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-latest' + build_name: 'azcopy_linux_amd64' + display_name: "Linux" + Windows: + imageName: 'windows-latest' + build_name: 'azcopy_windows_amd64.exe' + display_name: "Windows" + type: 'windows' + MacOS: + imageName: 'macos-latest' + build_name: 'azcopy_darwin_amd64' + display_name: "MacOS" + pool: + vmImage: $(imageName) + + steps: + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: 'Install-Module -Name Az.Accounts -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + pwsh: 'true' + displayName: 'Install Powershell Az Module' + - task: GoTool@0 + inputs: + version: $(AZCOPY_GOLANG_VERSION_COVERAGE) + - script: | + go install github.com/jstemmer/go-junit-report@v0.9.1 + go install github.com/axw/gocov/gocov@v1.1.0 + go install github.com/AlekSi/gocov-xml@v1.0.0 + go install github.com/matm/gocov-html@v0.0.0-20200509184451-71874e2e203b + displayName: 'Installing dependencies' + - bash: | + echo "##vso[task.setvariable variable=CGO_ENABLED]0" + displayName: 'Set CGO_ENABLED for Windows' + condition: eq(variables.type, 'windows') + - bash: | + npm install -g azurite + mkdir azurite + azurite --silent --location azurite --debug azurite\debug.log & + displayName: 'Install and Run Azurite' + # Running E2E Tests on AMD64 + - task: AzureCLI@2 + inputs: + azureSubscription: azcopytestworkloadidentity + addSpnToEnvironment: true + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + # Create coverage directory + if (-Not (Test-Path -Path "./coverage")) { + New-Item -Path "./coverage" -ItemType Directory + } + + # Create log directory + if (-Not (Test-Path -Path "${env:AZCOPY_E2E_LOG_OUTPUT}")) { + New-Item -Path "${env:AZCOPY_E2E_LOG_OUTPUT}" -ItemType Directory + } + + # Print "Building executable" + Write-Output "Building executable" + + # Set platform-specific environment variables and tags + $tags = "" + $suffix = "" + $build_name = "" + $display_name = "" + if ($IsWindows) { + $env:GOOS = "windows" + $env:GOARCH = "amd64" + $suffix = ".exe" + $build_name = "azcopy_windows_amd64.exe" + $display_name = "Windows" + } elseif ($IsLinux) { + $env:GOOS = "linux" + $env:GOARCH = "amd64" + $tags = "netgo" + $build_name = "azcopy_linux_amd64" + $display_name = "Linux" + } elseif ($IsMacOS) { + $env:GOOS = "darwin" + $env:GOARCH = "amd64" + $env:CGO_ENABLED = "1" + $build_name = "azcopy_darwin_amd64" + $display_name = "MacOS" + } else { + Write-Error "Unsupported operating system" + exit 1 + } + + # Build the Go program + if ($tags -ne "") { + go build -cover -tags $tags -o $build_name + } else { + go build -cover -o $build_name + } + + # Print "Running tests" + Write-Output "Running tests" + + # Run tests and pipe output to test.txt + go test -timeout=2h -v -run "TestNewE2E/.*" ./e2etest | Tee-Object -FilePath test.txt # Save the exit code from the previous command $exitCode = $LASTEXITCODE diff --git a/e2etest/newe2e_scenario_manager.go b/e2etest/newe2e_scenario_manager.go index 7fe43d14b..1550e1864 100644 --- a/e2etest/newe2e_scenario_manager.go +++ b/e2etest/newe2e_scenario_manager.go @@ -12,6 +12,9 @@ type ScenarioManager struct { testingT *testing.T Func reflect.Value + // Skip the line, don't run parallel. + runNow bool + suite string scenario string @@ -84,6 +87,7 @@ func (sm *ScenarioManager) RunScenario() { }() if !svm.isInvalid { // If we made a real test + svm.runNow = sm.runNow sm.testingT.Run(svm.VariationName(), func(t *testing.T) { svm.t = t svm.callcounts = make(map[string]uint) @@ -93,7 +97,10 @@ func (sm *ScenarioManager) RunScenario() { t.FailNow() } - t.Parallel() + if !svm.runNow { + t.Parallel() + } + svm.Cleanup(func(a ScenarioAsserter) { svm.DeleteCreatedResources() // clean up after ourselves! }) diff --git a/e2etest/newe2e_scenario_variation_manager.go b/e2etest/newe2e_scenario_variation_manager.go index b9106534a..6d116459a 100644 --- a/e2etest/newe2e_scenario_variation_manager.go +++ b/e2etest/newe2e_scenario_variation_manager.go @@ -12,6 +12,9 @@ type ScenarioVariationManager struct { // t is intentionally nil during dryruns. t *testing.T + // runNow disables parallelism in testing, instead running all tests immediately, intended for run-first suites. + runNow bool + // isInvalid is synonymous with Failed. It serves two purposes: // 1. Invalidating dry-runs that would under no deterministic circumstances succeed. // 2. Failing wet-runs that encountered an error or unexpected results. diff --git a/e2etest/newe2e_suite_manager.go b/e2etest/newe2e_suite_manager.go index 9818a1384..f440f7178 100644 --- a/e2etest/newe2e_suite_manager.go +++ b/e2etest/newe2e_suite_manager.go @@ -7,18 +7,24 @@ import ( ) type SuiteManager struct { - testingT *testing.T - Suites map[string]any - ScenarioManagers map[string]any + testingT *testing.T + Suites map[string]any + EarlyRunSuites map[string]any } -var suiteManager = &SuiteManager{Suites: make(map[string]any), ScenarioManagers: make(map[string]any)} +var suiteManager = &SuiteManager{Suites: make(map[string]any), EarlyRunSuites: make(map[string]any)} func (sm *SuiteManager) RegisterSuite(Suite any) { suiteName := reflect.ValueOf(Suite).Elem().Type().Name() sm.Suites[suiteName] = Suite - sm.ScenarioManagers[suiteName] = nil // todo SuiteManager +} + +// Early runs do not run in parallel, and run before anything else. +func (sm *SuiteManager) RegisterEarlyRunSuite(Suite any) { + suiteName := reflect.ValueOf(Suite).Elem().Type().Name() + + sm.EarlyRunSuites[suiteName] = Suite } func (sm *SuiteManager) RunSuites(t *testing.T) { @@ -28,7 +34,11 @@ func (sm *SuiteManager) RunSuites(t *testing.T) { sm.testingT = t - for sName, v := range sm.Suites { + tgt := sm.EarlyRunSuites + early := true +runAllSuites: + + for sName, v := range tgt { sVal := reflect.ValueOf(v) sTyp := reflect.TypeOf(v) mCount := sVal.NumMethod() @@ -61,7 +71,9 @@ func (sm *SuiteManager) RunSuites(t *testing.T) { } t.Run(sName, func(t *testing.T) { - t.Parallel() // todo: env var + if !early { // Early runners must run now. + t.Parallel() // todo: env var + } if setupIdx != -1 { // todo: call setup with suite manager @@ -82,9 +94,13 @@ func (sm *SuiteManager) RunSuites(t *testing.T) { NewFrameworkAsserter(t).AssertNow("Scenario runner panicked (recovered)", NoError{stackTrace: true}, recover()) }() - t.Parallel() + if !early { + t.Parallel() + } - NewScenarioManager(t, sVal.Method(scenarioIdx)).RunScenario() + sm := NewScenarioManager(t, sVal.Method(scenarioIdx)) + sm.runNow = early + sm.RunScenario() _ = scenarioIdx }) @@ -104,4 +120,10 @@ func (sm *SuiteManager) RunSuites(t *testing.T) { } }) } + + if early { // Jump back and run the remaining suites. + early = false + tgt = sm.Suites + goto runAllSuites + } } diff --git a/e2etest/newe2e_workload_hook.go b/e2etest/newe2e_workload_hook.go new file mode 100644 index 000000000..d247233a1 --- /dev/null +++ b/e2etest/newe2e_workload_hook.go @@ -0,0 +1,40 @@ +package e2etest + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-storage-azcopy/v10/common" + "os" +) + +func WorkloadIdentitySetup(a Asserter) { + // Run only in environments that support and are set up for Workload Identity (ex: Azure Pipeline, Azure Kubernetes Service) + if os.Getenv("NEW_E2E_ENVIRONMENT") != "AzurePipeline" { + return // This is OK to skip, because other tests also skip if it isn't present. + } + + workloadInfo := GlobalConfig.E2EAuthConfig.SubscriptionLoginInfo.DynamicOAuth.Workload + // Get the value of the AZURE_FEDERATED_TOKEN environment variable + token := workloadInfo.FederatedToken + a.AssertNow("idToken must be specified to authenticate with workload identity", Empty{Invert: true}, token) + // Write the token to a temporary file + // Create a temporary file to store the token + file, err := os.CreateTemp("", "azure_federated_token.txt") + a.AssertNow("Error creating temporary file", IsNil{}, err) + defer file.Close() + + // Write the token to the temporary file + _, err = file.WriteString(token) + a.AssertNow("Error writing to temporary file", IsNil{}, err) + + tc, err := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ + TenantID: workloadInfo.TenantId, + ClientID: workloadInfo.ClientId, + TokenFilePath: file.Name(), + }) + a.NoError("Workload identity failed to spawn", err, true) + _, err = tc.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{common.StorageScope}, + }) + a.NoError("Workload identity failed to fetch token", err, true) +} diff --git a/e2etest/zt_aanewe2e_testmain_test.go b/e2etest/zt_aanewe2e_testmain_test.go index a0ea73f69..442554ba2 100644 --- a/e2etest/zt_aanewe2e_testmain_test.go +++ b/e2etest/zt_aanewe2e_testmain_test.go @@ -1,3 +1,5 @@ +//go:build !olde2etest + package e2etest import ( @@ -16,6 +18,7 @@ import ( var FrameworkHooks = []TestFrameworkHook{ {HookName: "Config", SetupHook: LoadConfigHook}, + {HookName: "Workload Identity Setup", SetupHook: WorkloadIdentitySetup}, {HookName: "OAuth Cache", SetupHook: SetupOAuthCache}, {HookName: "ARM Client", SetupHook: SetupArmClient, TeardownHook: TeardownArmClient}, {HookName: "Default accts", SetupHook: AccountRegistryInitHook, TeardownHook: AccountRegistryCleanupHook}, diff --git a/e2etest/zt_newe2e_workload_test.go b/e2etest/zt_newe2e_workload_test.go index faf2ff496..50f391f92 100644 --- a/e2etest/zt_newe2e_workload_test.go +++ b/e2etest/zt_newe2e_workload_test.go @@ -7,7 +7,7 @@ import ( ) func init() { - suiteManager.RegisterSuite(&WorkloadIdentitySuite{}) + suiteManager.RegisterEarlyRunSuite(&WorkloadIdentitySuite{}) } type WorkloadIdentitySuite struct{}