Fix to slow performance of PipeBlob uploads (#2821)

* test

* tets

* update

* upload concurrency perf fix

* fix concurrency upload perf

* fixed linting issues, updated readme

* lint

* listing with OAuth support added + e2e test

* updated var name, error format strings

* trigger CI

* fixed linting errors

README.md

@@ -1,5 +1,4 @@
 # AzCopy v10
-
 AzCopy v10 is a command-line utility that you can use to copy data to and from containers and file shares in Azure Storage accounts.
 AzCopy V10 presents easy-to-use commands that are optimized for high performance and throughput.
 

cmd/copy.go

@@ -31,6 +31,7 @@ import (
 	"net/url"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -1347,6 +1348,27 @@ func (cca *CookedCopyCmdArgs) processRedirectionDownload(blobResource common.Res
 func (cca *CookedCopyCmdArgs) processRedirectionUpload(blobResource common.ResourceString, blockSize int64) error {
 	ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion)
 
+	// Use the concurrency environment value
+	concurrencyEnvVar := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue())
+
+	pipingUploadParallelism := pipingUploadParallelism
+	if concurrencyEnvVar != "" {
+		// handle when the concurrency value is AUTO
+		if concurrencyEnvVar == "AUTO" {
+			return errors.New("concurrency auto-tuning is not possible when using redirection transfers (AZCOPY_CONCURRENCY_VALUE = AUTO)")
+		}
+
+		// convert the concurrency value to int
+		concurrencyValue, err := strconv.ParseInt(concurrencyEnvVar, 10, 32)
+
+		//handle the error if the conversion fails
+		if err != nil {
+			return fmt.Errorf("AZCOPY_CONCURRENCY_VALUE is not set to a valid value, an integer is expected (current value: %s): %w", concurrencyEnvVar, err)
+		}
+
+		pipingUploadParallelism = int(concurrencyValue) // Cast to Integer
+	}
+
 	// if no block size is set, then use default value
 	if blockSize == 0 {
 		blockSize = pipingDefaultBlockSize

cmd/list.go

@@ -236,8 +236,6 @@ func (cooked cookedListCmdArgs) handleListContainerCommand() (err error) {
 	// isSource is rather misnomer for canBePublic. We can list public containers, and hence isSource=true
 	if credentialInfo, _, err = GetCredentialInfoForLocation(ctx, cooked.location, source, true, common.CpkOptions{}); err != nil {
 		return fmt.Errorf("failed to obtain credential info: %s", err.Error())
-	} else if cooked.location == cooked.location.File() && source.SAS == "" {
-		return errors.New("azure files requires a SAS token for authentication")
 	} else if credentialInfo.CredentialType.IsAzureOAuth() {
 		uotm := GetUserOAuthTokenManagerInstance()
 		if tokenInfo, err := uotm.GetTokenInfo(ctx); err != nil {

e2etest/zt_newe2e_list_test.go

@@ -14,7 +14,8 @@ func init() {
 type ListSuite struct{}
 
 func (s *ListSuite) Scenario_ListBasic(svm *ScenarioVariationManager) {
-	srcService := GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Blob()}))
+	srcService := GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Blob(),
+		common.ELocation.File()}))
 
 	svm.InsertVariationSeparator(":")
 	body := NewRandomObjectContentContainer(svm, SizeFromString("1K"))
@@ -38,12 +39,15 @@ func (s *ListSuite) Scenario_ListBasic(svm *ScenarioVariationManager) {
 		AzCopyCommand{
 			Verb: AzCopyVerbList,
 			Targets: []ResourceManager{
-				srcObj.Parent().(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{
-					SASTokenOptions: GenericServiceSignatureValues{
-						ContainerName: srcObj.ContainerName(),
-						Permissions:   (&blobsas.ContainerPermissions{Read: true, List: true}).String(),
-					},
-				}),
+				srcObj.Parent().(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm,
+					[]ExplicitCredentialTypes{EExplicitCredentialType.OAuth(),
+						EExplicitCredentialType.SASToken()}), svm,
+					CreateAzCopyTargetOptions{
+						SASTokenOptions: GenericServiceSignatureValues{
+							ContainerName: srcObj.ContainerName(),
+							Permissions:   (&blobsas.ContainerPermissions{Read: true, List: true}).String(),
+						},
+					}),
 			},
 			Flags: ListFlags{},
 		})