Run workload identity tests first to avoid timeout (#2833)

* Workload identity hook

* Remove hook, Run ahead of everything else

* Run New E2E separately

* drop parallel statements when running up front

* Missed parallel statements, add hook.

* Always create the log dir

azure-pipelines.yml

@@ -215,7 +215,189 @@ jobs:
             Write-Output "Running tests"
             
             # Run tests and pipe output to test.txt
-            go test -timeout=2h -v ./e2etest | Tee-Object -FilePath test.txt
+            go test -timeout=2h -v -tags olde2etest ./e2etest | Tee-Object -FilePath test.txt
+            
+            # Save the exit code from the previous command
+            $exitCode = $LASTEXITCODE
+            
+            # Print the contents of test.txt
+            # Get-Content test.txt
+            
+            # Print "Generating junit report"
+            Write-Output "Generating junit report"
+            
+            # Pipe info in test.txt to go-junit-report and save output to report.xml
+            Get-Content test.txt | & "$(go env GOPATH)/bin/go-junit-report" > "${display_name}_report.xml"
+            
+            # Print "Formatting coverage directory to legacy txt format"
+            Write-Output "Formatting coverage directory to legacy txt format"
+            
+            # Format coverage data to text format
+            go tool covdata textfmt -i=coverage -o "${display_name}_coverage.txt"
+            
+            # Print "Formatting coverage to json format"
+            Write-Output "Formatting coverage to json format"
+            
+            # Convert coverage.txt to coverage.json
+            & "$(go env GOPATH)/bin/gocov$suffix" convert "${display_name}_coverage.txt" > "${display_name}_coverage.json"
+            
+            # Print "Formatting coverage to xml format"
+            Write-Output "Formatting coverage to xml format"
+            
+            # Convert coverage.json to coverage.xml
+            Get-Content "${display_name}_coverage.json" | & "$(go env GOPATH)/bin/gocov-xml$suffix" > "${display_name}_coverage.xml"
+            
+            # Return the exit code from step 5
+            exit $exitCode
+        env:
+          AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
+          AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
+          AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS)
+          AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
+          AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
+          AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
+          AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs'
+          AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG)
+          AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG)
+          AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
+          AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG)
+          CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
+          CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
+          AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name)
+          GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage'
+          NEW_E2E_SUBSCRIPTION_ID: $(AZCOPY_NEW_E2E_SUBSCRIPTION_ID)
+          NEW_E2E_AZCOPY_PATH: $(System.DefaultWorkingDirectory)/$(build_name)
+          NEW_E2E_ENVIRONMENT: "AzurePipeline"
+        displayName: 'E2E Test $(display_name) - AMD64 with Workload Identity'
+
+      - task: PublishBuildArtifacts@1
+        displayName: 'Publish logs'
+        condition: succeededOrFailed()
+        inputs:
+          pathToPublish: '$(System.DefaultWorkingDirectory)/logs'
+          artifactName: logs
+
+      - task: PublishTestResults@2
+        condition: succeededOrFailed()
+        inputs:
+          testRunner: JUnit
+          testResultsFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_report.xml
+          testRunTitle: 'Go on $(display_name)'
+
+      - task: PublishCodeCoverageResults@1
+        condition: succeededOrFailed()
+        inputs:
+          codeCoverageTool: Cobertura
+          summaryFileLocation: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.xml
+          additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/**/$(display_name)_coverage.html
+
+  - job: New_E2E_Framework
+    timeoutInMinutes: 360
+    # Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04
+    strategy:
+      matrix:
+        Ubuntu-20:
+          imageName: 'ubuntu-latest'
+          build_name: 'azcopy_linux_amd64'
+          display_name: "Linux"
+        Windows:
+          imageName: 'windows-latest'
+          build_name: 'azcopy_windows_amd64.exe'
+          display_name: "Windows"
+          type: 'windows'
+        MacOS:
+          imageName: 'macos-latest'
+          build_name: 'azcopy_darwin_amd64'
+          display_name: "MacOS"
+    pool:
+      vmImage: $(imageName)
+
+    steps:
+      - task: PowerShell@2
+        inputs:
+          targetType: 'inline'
+          script: 'Install-Module -Name Az.Accounts -Scope CurrentUser -Repository PSGallery -AllowClobber -Force'
+          pwsh: 'true'
+        displayName: 'Install Powershell Az Module'
+      - task: GoTool@0
+        inputs:
+          version: $(AZCOPY_GOLANG_VERSION_COVERAGE)
+      - script: |
+          go install github.com/jstemmer/[email protected]
+          go install github.com/axw/gocov/[email protected]
+          go install github.com/AlekSi/[email protected]
+          go install github.com/matm/[email protected]
+        displayName: 'Installing dependencies'
+      - bash: |
+          echo "##vso[task.setvariable variable=CGO_ENABLED]0"
+        displayName: 'Set CGO_ENABLED for Windows'
+        condition: eq(variables.type, 'windows')
+      - bash: |
+          npm install -g azurite
+          mkdir azurite
+          azurite --silent --location azurite --debug azurite\debug.log &
+        displayName: 'Install and Run Azurite'
+      # Running E2E Tests on AMD64
+      - task: AzureCLI@2
+        inputs:
+          azureSubscription: azcopytestworkloadidentity
+          addSpnToEnvironment: true
+          scriptType: pscore
+          scriptLocation: inlineScript
+          inlineScript: |
+            # Create coverage directory
+            if (-Not (Test-Path -Path "./coverage")) {
+              New-Item -Path "./coverage" -ItemType Directory
+            }
+            
+            # Create log directory
+            if (-Not (Test-Path -Path "${env:AZCOPY_E2E_LOG_OUTPUT}")) {
+              New-Item -Path "${env:AZCOPY_E2E_LOG_OUTPUT}" -ItemType Directory
+            }
+            
+            # Print "Building executable"
+            Write-Output "Building executable"
+            
+            # Set platform-specific environment variables and tags
+            $tags = ""
+            $suffix = ""
+            $build_name = ""
+            $display_name = ""
+            if ($IsWindows) {
+              $env:GOOS = "windows"
+              $env:GOARCH = "amd64"
+              $suffix = ".exe"
+              $build_name = "azcopy_windows_amd64.exe"
+              $display_name = "Windows"
+            } elseif ($IsLinux) {
+              $env:GOOS = "linux"
+              $env:GOARCH = "amd64"
+              $tags = "netgo"
+              $build_name = "azcopy_linux_amd64"
+              $display_name = "Linux"
+            } elseif ($IsMacOS) {
+              $env:GOOS = "darwin"
+              $env:GOARCH = "amd64"
+              $env:CGO_ENABLED = "1"
+              $build_name = "azcopy_darwin_amd64"
+              $display_name = "MacOS"
+            } else {
+              Write-Error "Unsupported operating system"
+              exit 1
+            }
+            
+            # Build the Go program
+            if ($tags -ne "") {
+              go build -cover -tags $tags -o $build_name
+            } else {
+              go build -cover -o $build_name
+            }
+            
+            # Print "Running tests"
+            Write-Output "Running tests"
+            
+            # Run tests and pipe output to test.txt
+            go test -timeout=2h -v -run "TestNewE2E/.*" ./e2etest | Tee-Object -FilePath test.txt
             
             # Save the exit code from the previous command
             $exitCode = $LASTEXITCODE

e2etest/newe2e_scenario_manager.go

@@ -12,6 +12,9 @@ type ScenarioManager struct {
 	testingT *testing.T
 	Func     reflect.Value
 
+	// Skip the line, don't run parallel.
+	runNow bool
+
 	suite    string
 	scenario string
 
@@ -84,6 +87,7 @@ func (sm *ScenarioManager) RunScenario() {
 		}()
 
 		if !svm.isInvalid { // If we made a real test
+			svm.runNow = sm.runNow
 			sm.testingT.Run(svm.VariationName(), func(t *testing.T) {
 				svm.t = t
 				svm.callcounts = make(map[string]uint)
@@ -93,7 +97,10 @@ func (sm *ScenarioManager) RunScenario() {
 					t.FailNow()
 				}
 
-				t.Parallel()
+				if !svm.runNow {
+					t.Parallel()
+				}
+
 				svm.Cleanup(func(a ScenarioAsserter) {
 					svm.DeleteCreatedResources() // clean up after ourselves!
 				})

e2etest/newe2e_scenario_variation_manager.go

@@ -12,6 +12,9 @@ type ScenarioVariationManager struct {
 	// t is intentionally nil during dryruns.
 	t *testing.T
 
+	// runNow disables parallelism in testing, instead running all tests immediately, intended for run-first suites.
+	runNow bool
+
 	// isInvalid is synonymous with Failed. It serves two purposes:
 	// 1. Invalidating dry-runs that would under no deterministic circumstances succeed.
 	// 2. Failing wet-runs that encountered an error or unexpected results.

e2etest/newe2e_suite_manager.go

@@ -7,18 +7,24 @@ import (
 )
 
 type SuiteManager struct {
-	testingT         *testing.T
-	Suites           map[string]any
-	ScenarioManagers map[string]any
+	testingT       *testing.T
+	Suites         map[string]any
+	EarlyRunSuites map[string]any
 }
 
-var suiteManager = &SuiteManager{Suites: make(map[string]any), ScenarioManagers: make(map[string]any)}
+var suiteManager = &SuiteManager{Suites: make(map[string]any), EarlyRunSuites: make(map[string]any)}
 
 func (sm *SuiteManager) RegisterSuite(Suite any) {
 	suiteName := reflect.ValueOf(Suite).Elem().Type().Name()
 
 	sm.Suites[suiteName] = Suite
-	sm.ScenarioManagers[suiteName] = nil // todo SuiteManager
+}
+
+// Early runs do not run in parallel, and run before anything else.
+func (sm *SuiteManager) RegisterEarlyRunSuite(Suite any) {
+	suiteName := reflect.ValueOf(Suite).Elem().Type().Name()
+
+	sm.EarlyRunSuites[suiteName] = Suite
 }
 
 func (sm *SuiteManager) RunSuites(t *testing.T) {
@@ -28,7 +34,11 @@ func (sm *SuiteManager) RunSuites(t *testing.T) {
 
 	sm.testingT = t
 
-	for sName, v := range sm.Suites {
+	tgt := sm.EarlyRunSuites
+	early := true
+runAllSuites:
+
+	for sName, v := range tgt {
 		sVal := reflect.ValueOf(v)
 		sTyp := reflect.TypeOf(v)
 		mCount := sVal.NumMethod()
@@ -61,7 +71,9 @@ func (sm *SuiteManager) RunSuites(t *testing.T) {
 		}
 
 		t.Run(sName, func(t *testing.T) {
-			t.Parallel() // todo: env var
+			if !early { // Early runners must run now.
+				t.Parallel() // todo: env var
+			}
 
 			if setupIdx != -1 {
 				// todo: call setup with suite manager
@@ -82,9 +94,13 @@ func (sm *SuiteManager) RunSuites(t *testing.T) {
 							NewFrameworkAsserter(t).AssertNow("Scenario runner panicked (recovered)", NoError{stackTrace: true}, recover())
 						}()
 
-						t.Parallel()
+						if !early {
+							t.Parallel()
+						}
 
-						NewScenarioManager(t, sVal.Method(scenarioIdx)).RunScenario()
+						sm := NewScenarioManager(t, sVal.Method(scenarioIdx))
+						sm.runNow = early
+						sm.RunScenario()
 
 						_ = scenarioIdx
 					})
@@ -104,4 +120,10 @@ func (sm *SuiteManager) RunSuites(t *testing.T) {
 			}
 		})
 	}
+
+	if early { // Jump back and run the remaining suites.
+		early = false
+		tgt = sm.Suites
+		goto runAllSuites
+	}
 }

e2etest/newe2e_workload_hook.go

@@ -0,0 +1,40 @@
+package e2etest
+
+import (
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"os"
+)
+
+func WorkloadIdentitySetup(a Asserter) {
+	// Run only in environments that support and are set up for Workload Identity (ex: Azure Pipeline, Azure Kubernetes Service)
+	if os.Getenv("NEW_E2E_ENVIRONMENT") != "AzurePipeline" {
+		return // This is OK to skip, because other tests also skip if it isn't present.
+	}
+
+	workloadInfo := GlobalConfig.E2EAuthConfig.SubscriptionLoginInfo.DynamicOAuth.Workload
+	// Get the value of the AZURE_FEDERATED_TOKEN environment variable
+	token := workloadInfo.FederatedToken
+	a.AssertNow("idToken must be specified to authenticate with workload identity", Empty{Invert: true}, token)
+	// Write the token to a temporary file
+	// Create a temporary file to store the token
+	file, err := os.CreateTemp("", "azure_federated_token.txt")
+	a.AssertNow("Error creating temporary file", IsNil{}, err)
+	defer file.Close()
+
+	// Write the token to the temporary file
+	_, err = file.WriteString(token)
+	a.AssertNow("Error writing to temporary file", IsNil{}, err)
+
+	tc, err := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{
+		TenantID:      workloadInfo.TenantId,
+		ClientID:      workloadInfo.ClientId,
+		TokenFilePath: file.Name(),
+	})
+	a.NoError("Workload identity failed to spawn", err, true)
+	_, err = tc.GetToken(ctx, policy.TokenRequestOptions{
+		Scopes: []string{common.StorageScope},
+	})
+	a.NoError("Workload identity failed to fetch token", err, true)
+}

e2etest/zt_aanewe2e_testmain_test.go

@@ -1,3 +1,5 @@
+//go:build !olde2etest
+
 package e2etest
 
 import (
@@ -16,6 +18,7 @@ import (
 
 var FrameworkHooks = []TestFrameworkHook{
 	{HookName: "Config", SetupHook: LoadConfigHook},
+	{HookName: "Workload Identity Setup", SetupHook: WorkloadIdentitySetup},
 	{HookName: "OAuth Cache", SetupHook: SetupOAuthCache},
 	{HookName: "ARM Client", SetupHook: SetupArmClient, TeardownHook: TeardownArmClient},
 	{HookName: "Default accts", SetupHook: AccountRegistryInitHook, TeardownHook: AccountRegistryCleanupHook},

e2etest/zt_newe2e_workload_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func init() {
-	suiteManager.RegisterSuite(&WorkloadIdentitySuite{})
+	suiteManager.RegisterEarlyRunSuite(&WorkloadIdentitySuite{})
 }
 
 type WorkloadIdentitySuite struct{}