Merge branch 'main' into version-pipeline-fix

ChangeLog.md

@@ -6,6 +6,14 @@
 ### Bug Fixes
 1. Fixed an issue where AzCopy would not persist tokens when logging in via Device Code. ([#2361](https://github.com/Azure/azure-storage-azcopy/issues/2361))
 
+## Version 10.27.1
+
+### Bug Fixes
+1. Reverted a change that resulted in breaking file service transfers and a larger memory footprint. ([#2858](https://github.com/Azure/azure-storage-azcopy/issues/2858)[#2855](https://github.com/Azure/azure-storage-azcopy/issues/2855))
+
+### Dependency updates
+1. github.com/golang-jwt/jwt/v4 v4.5.0 -> v4.5.1 ([#2861](https://github.com/Azure/azure-storage-azcopy/issues/2861))
+
 ## Version 10.27.0
 
 ### New Features

common/folderCreationTracker_interface.go

@@ -7,6 +7,7 @@ package common
 type FolderCreationTracker interface {
 	CreateFolder(folder string, doCreation func() error) error
 	ShouldSetProperties(folder string, overwrite OverwriteOption, prompter Prompter) bool
+	StopTracking(folder string)
 }
 
 type Prompter interface {

ste/folderCreationTracker.go

@@ -1,6 +1,7 @@
 package ste
 
 import (
+	"fmt"
 	"net/url"
 	"strings"
 	"sync"
@@ -20,9 +21,10 @@ func NewFolderCreationTracker(fpo common.FolderPropertyOption, plan *JobPartPlan
 	case common.EFolderPropertiesOption.AllFolders(),
 		common.EFolderPropertiesOption.AllFoldersExceptRoot():
 		return &jpptFolderTracker{ // This prevents a dependency cycle. Reviewers: Are we OK with this? Can you think of a better way to do it?
-			plan:     plan,
-			mu:       &sync.Mutex{},
-			contents: common.NewTrie(),
+			plan:                   plan,
+			mu:                     &sync.Mutex{},
+			contents:               make(map[string]uint32),
+			unregisteredButCreated: make(map[string]struct{}),
 		}
 	case common.EFolderPropertiesOption.NoFolders():
 		// can't use simpleFolderTracker here, because when no folders are processed,
@@ -46,10 +48,15 @@ func (f *nullFolderTracker) ShouldSetProperties(folder string, overwrite common.
 	panic("wrong type of folder tracker has been instantiated. This type does not do any tracking")
 }
 
+func (f *nullFolderTracker) StopTracking(folder string) {
+	// noop (because we don't track anything)
+}
+
 type jpptFolderTracker struct {
-	plan     IJobPartPlanHeader
-	mu       *sync.Mutex
-	contents *common.Trie
+	plan                   IJobPartPlanHeader
+	mu                     *sync.Mutex
+	contents               map[string]uint32
+	unregisteredButCreated map[string]struct{}
 }
 
 func (f *jpptFolderTracker) RegisterPropertiesTransfer(folder string, transferIndex uint32) {
@@ -60,14 +67,13 @@ func (f *jpptFolderTracker) RegisterPropertiesTransfer(folder string, transferIn
 		return // Never persist to dev-null
 	}
 
-	fNode, _ := f.contents.InsertDirNode(folder)
-	fNode.TransferIndex = transferIndex
+	f.contents[folder] = transferIndex
 
 	// We created it before it was enumerated-- Let's register that now.
-	if fNode.UnregisteredButCreated {
+	if _, ok := f.unregisteredButCreated[folder]; ok {
 		f.plan.Transfer(transferIndex).SetTransferStatus(common.ETransferStatus.FolderCreated(), false)
-		fNode.UnregisteredButCreated = false
 
+		delete(f.unregisteredButCreated, folder)
 	}
 }
 
@@ -79,15 +85,12 @@ func (f *jpptFolderTracker) CreateFolder(folder string, doCreation func() error)
 		return nil // Never persist to dev-null
 	}
 
-	// If the folder has already been created, then we don't need to create it again
-	fNode, addedToTrie := f.contents.InsertDirNode(folder)
-
-	if !addedToTrie && (f.plan.Transfer(fNode.TransferIndex).TransferStatus() == common.ETransferStatus.FolderCreated() ||
-		f.plan.Transfer(fNode.TransferIndex).TransferStatus() == common.ETransferStatus.Success()) {
+	if idx, ok := f.contents[folder]; ok &&
+		f.plan.Transfer(idx).TransferStatus() == (common.ETransferStatus.FolderCreated()) {
 		return nil
 	}
 
-	if fNode.UnregisteredButCreated {
+	if _, ok := f.unregisteredButCreated[folder]; ok {
 		return nil
 	}
 
@@ -96,14 +99,13 @@ func (f *jpptFolderTracker) CreateFolder(folder string, doCreation func() error)
 		return err
 	}
 
-	if !addedToTrie {
+	if idx, ok := f.contents[folder]; ok {
 		// overwrite it's transfer status
-		f.plan.Transfer(fNode.TransferIndex).SetTransferStatus(common.ETransferStatus.FolderCreated(), false)
+		f.plan.Transfer(idx).SetTransferStatus(common.ETransferStatus.FolderCreated(), false)
 	} else {
 		// A folder hasn't been hit in traversal yet.
 		// Recording it in memory is OK, because we *cannot* resume a job that hasn't finished traversal.
-		// We set the value to 0 as we just want to record it in memory
-		fNode.UnregisteredButCreated = true
+		f.unregisteredButCreated[folder] = struct{}{}
 	}
 
 	return nil
@@ -125,9 +127,8 @@ func (f *jpptFolderTracker) ShouldSetProperties(folder string, overwrite common.
 		defer f.mu.Unlock()
 
 		var created bool
-		if fNode, ok := f.contents.GetDirNode(folder); ok {
-			created = f.plan.Transfer(fNode.TransferIndex).TransferStatus() == common.ETransferStatus.FolderCreated() ||
-				f.plan.Transfer(fNode.TransferIndex).TransferStatus() == common.ETransferStatus.Success()
+		if idx, ok := f.contents[folder]; ok {
+			created = f.plan.Transfer(idx).TransferStatus() == common.ETransferStatus.FolderCreated()
 		} else {
 			// This should not happen, ever.
 			// Folder property jobs register with the tracker before they start getting processed.
@@ -157,3 +158,26 @@ func (f *jpptFolderTracker) ShouldSetProperties(folder string, overwrite common.
 		panic("unknown overwrite option")
 	}
 }
+
+func (f *jpptFolderTracker) StopTracking(folder string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	if folder == common.Dev_Null {
+		return // Not possible to track this
+	}
+
+	// no-op, because tracking is now handled by jppt, anyway.
+	if _, ok := f.contents[folder]; ok {
+		delete(f.contents, folder)
+	} else {
+		currentContents := ""
+
+		for k, v := range f.contents {
+			currentContents += fmt.Sprintf("K: %s V: %d\n", k, v)
+		}
+
+		// double should never be hit, but *just in case*.
+		panic(common.NewAzCopyLogSanitizer().SanitizeLogMessage("Folder " + folder + " shouldn't finish tracking until it's been recorded\nCurrent Contents:\n" + currentContents))
+	}
+}