Merge branch 'main' into main

ChangeLog.md

@@ -6,6 +6,14 @@
 ### Bug Fixes
 1. Fixed an issue where AzCopy would not persist tokens when logging in via Device Code. ([#2361](https://github.com/Azure/azure-storage-azcopy/issues/2361))
 
+## Version 10.27.1
+
+### Bug Fixes
+1. Reverted a change that resulted in breaking file service transfers and a larger memory footprint. ([#2858](https://github.com/Azure/azure-storage-azcopy/issues/2858)[#2855](https://github.com/Azure/azure-storage-azcopy/issues/2855))
+
+### Dependency updates
+1. github.com/golang-jwt/jwt/v4 v4.5.0 -> v4.5.1 ([#2861](https://github.com/Azure/azure-storage-azcopy/issues/2861))
+
 ## Version 10.27.0
 
 ### New Features

cmd/jobsClean.go

@@ -30,6 +30,8 @@ import (
 	"github.com/Azure/azure-storage-azcopy/v10/common"
 )
 
+var JobsCleanupSuccessMsg = "Successfully removed all jobs."
+
 func init() {
 	type JobsCleanReq struct {
 		withStatus string
@@ -122,10 +124,12 @@ func blindDeleteAllJobFiles() (int, error) {
 	if err != nil {
 		return numPlanFilesRemoved, err
 	}
-
 	// get rid of the logs
 	numLogFilesRemoved, err := removeFilesWithPredicate(azcopyLogPathFolder, func(s string) bool {
-		if strings.HasSuffix(s, ".log") {
+		// Do not remove the current job's log file this will cause the cleanup job to fail.
+		if strings.Contains(s, azcopyCurrentJobID.String()) {
+			return false
+		} else if strings.HasSuffix(s, ".log") {
 			return true
 		}
 		return false

cmd/root.go

@@ -282,6 +282,8 @@ func init() {
 	_ = rootCmd.PersistentFlags().MarkHidden("debug-skip-files")
 }
 
+const versionMetadataUrl = "https://azcopyvnextrelease.z22.web.core.windows.net/releasemetadata/latest_version.txt"
+
 // always spins up a new goroutine, because sometimes the aka.ms URL can't be reached (e.g. a constrained environment where
 // aka.ms is not resolvable to a reachable IP address). In such cases, this routine will run for ever, and the caller should
 // just give up on it.
@@ -290,8 +292,6 @@ func init() {
 func beginDetectNewVersion() chan struct{} {
 	completionChannel := make(chan struct{})
 	go func() {
-		const versionMetadataUrl = "https://azcopyvnextrelease.blob.core.windows.net/releasemetadata/latest_version.txt"
-
 		// step 0: check the Stderr, check local version
 		_, err := os.Stderr.Stat()
 		if err != nil {

cmd/versionChecker_test.go

@@ -216,7 +216,6 @@ func TestCheckReleaseMetadata(t *testing.T) {
 
 	// sanity test for checking if the release metadata exists and can be downloaded
 	options := createClientOptions(nil, nil)
-	versionMetadataUrl := "https://azcopyvnextrelease.blob.core.windows.net/releasemetadata/latest_version.txt"
 
 	blobClient, err := blob.NewClientWithNoCredential(versionMetadataUrl, &blob.ClientOptions{ClientOptions: options})
 	a.NoError(err)

common/folderCreationTracker_interface.go

@@ -7,6 +7,7 @@ package common
 type FolderCreationTracker interface {
 	CreateFolder(folder string, doCreation func() error) error
 	ShouldSetProperties(folder string, overwrite OverwriteOption, prompter Prompter) bool
+	StopTracking(folder string)
 }
 
 type Prompter interface {

common/oauthTokenManager.go

@@ -88,7 +88,7 @@ func newAzcopyHTTPClient() *http.Client {
 				Timeout:   10 * time.Second,
 				KeepAlive: 10 * time.Second,
 				DualStack: true,
-			}).Dial,                   /*Context*/
+			}).Dial, /*Context*/
 			MaxIdleConns:           0, // No limit
 			MaxIdleConnsPerHost:    1000,
 			IdleConnTimeout:        180 * time.Second,
@@ -150,7 +150,7 @@ func (uotm *UserOAuthTokenManager) validateAndPersistLogin(oAuthTokenInfo *OAuth
 		return err
 	}
 	scopes := []string{StorageScope}
-	_, err = tc.GetToken(context.TODO(), policy.TokenRequestOptions{Scopes: scopes})
+	_, err = tc.GetToken(context.TODO(), policy.TokenRequestOptions{Scopes: scopes, EnableCAE: true})
 	if err != nil {
 		return err
 	}
@@ -467,7 +467,7 @@ func (credInfo *OAuthTokenInfo) Refresh(ctx context.Context) (*Token, error) {
 		return nil, err
 	}
 	scopes := []string{StorageScope}
-	t, err := tc.GetToken(ctx, policy.TokenRequestOptions{Scopes: scopes})
+	t, err := tc.GetToken(ctx, policy.TokenRequestOptions{Scopes: scopes, EnableCAE: true})
 	if err != nil {
 		return nil, err
 	}
@@ -518,15 +518,14 @@ type TokenStoreCredential struct {
 // we do not make repeated GetToken calls.
 // This is a temporary fix for issue where we would request a
 // new token from Stg Exp even while they've not yet populated the
-// tokenstore. 
+// tokenstore.
 //
 // This is okay because we use same credential on both source and
 // destination. If we move to a case where the credentials are
 // different, this should be removed.
 //
 // We should move to a method where the token is always read  from
 // tokenstore, and azcopy is invoked after tokenstore is populated.
-//
 var globalTokenStoreCredential *TokenStoreCredential
 var globalTsc sync.Once
 
@@ -709,7 +708,10 @@ func (credInfo *OAuthTokenInfo) GetDeviceCodeCredential() (azcore.TokenCredentia
 
 	if record == (azidentity.AuthenticationRecord{}) {
 		// No stored record; call Authenticate to acquire one
-		record, err = tc.Authenticate(context.TODO(), &policy.TokenRequestOptions{Scopes: []string{StorageScope}})
+		record, err = tc.Authenticate(context.TODO(), &policy.TokenRequestOptions{
+			EnableCAE: true,
+			Scopes:    []string{StorageScope},
+		})
 		if err != nil {
 			return nil, err
 		}

common/util.go

@@ -249,7 +249,7 @@ type ScopedCredential struct {
 }
 
 func (s *ScopedCredential) GetToken(ctx context.Context, _ policy.TokenRequestOptions) (azcore.AccessToken, error) {
-	return s.cred.GetToken(ctx, policy.TokenRequestOptions{Scopes: s.scopes})
+	return s.cred.GetToken(ctx, policy.TokenRequestOptions{Scopes: s.scopes, EnableCAE: true})
 }
 
 type ServiceClient struct {

docker/dockerinstall.sh

@@ -1,40 +1,35 @@
 # Cleanup old installation
-sudo apt remove docker-desktop
+sudo apt remove -y docker-desktop
+sudo apt-get remove -y docker docker-engine docker.io containerd runc
 rm -r $HOME/.docker/desktop
 sudo rm /usr/local/bin/com.docker.cli
-sudo apt purge docker-desktop
+sudo apt purge -y docker-desktop
 sudo apt-get update
 
 # Install certificates and pre-requisites
-sudo apt-get install ca-certificates curl gnupg lsb-release -y
-sudo mkdir -p /etc/apt/keyrings
+sudo apt-get install -y ca-certificates curl gnupg lsb-release
 
 # Create keyring for docker
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg -y
+sudo mkdir -m 0755 -p /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
 
 # Create file for installation
-echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"| sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 
-# Install docker 
-sudo apt-get update
-sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y
+# Install docker
 sudo apt-get update
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
 
-# Resolve permission issues to connect to docker socket
-sudo groupadd docker
-sudo usermod -aG docker $USER
-sudo chown root:docker /var/run/docker.sock
+# Verify installation
+docker --version
+sudo docker run hello-world
 
-# Create the .docker directory if it doesn't exist
-mkdir -p $HOME/.docker
-sudo chown "$USER":"$USER" /home/"$USER"/.docker -R
-sudo chmod g+rwx "$HOME/.docker" -R
 
 # Delete old azcopy image
 docker rmi `docker images | grep azcopy | cut -d " " -f1`
 
 # Remove existing images
-docker system prune -f
+docker system prune -f -a
 
 # Start docker service
 sudo service docker start

e2etest/newe2e_oauth_cache.go

@@ -85,8 +85,9 @@ func (o *OAuthCache) GetAccessToken(scope string) (*AzCoreAccessToken, error) {
 	if !ok || time.Now().Add(time.Minute*3).After(tok.ExpiresOn) {
 		o.mut.Lock()
 		newTok, err := o.tc.GetToken(ctx, policy.TokenRequestOptions{
-			Scopes:   []string{scope},
-			TenantID: o.tenant,
+			Scopes:    []string{scope},
+			TenantID:  o.tenant,
+			EnableCAE: true,
 		})
 		if err != nil {
 			return nil, fmt.Errorf("failed fetching new AccessToken: %w", err)