Merge branch 'dev' of github.com:jfrog/jfrog-cli-core into dev

artifactory/commands/buildinfo/adddependencies.go

@@ -262,7 +262,7 @@ func collectPatternMatchingFiles(addDepsParams *specutils.CommonParams, rootPath
 		return nil, err
 	}
 
-	paths, err := fspatterns.ListFiles(rootPath, addDepsParams.IsRecursive(), addDepsParams.IsIncludeDirs(), true, excludePathPattern)
+	paths, err := fspatterns.ListFiles(rootPath, addDepsParams.IsRecursive(), addDepsParams.IsIncludeDirs(), false, true, excludePathPattern)
 	if err != nil {
 		return nil, err
 	}

artifactory/commands/transferconfig/transferconfig.go

@@ -14,6 +14,7 @@ import (
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/generic"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferconfig/configxmlutils"
 	commandsUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
+	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils/precheckrunner"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
 	"github.com/jfrog/jfrog-cli-core/v2/common/commands"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
@@ -212,7 +213,7 @@ func (tcc *TransferConfigCommand) runPreChecks() error {
 		return err
 	}
 
-	return tcc.NewPreChecksRunner(remoteRepositories).Run(context.Background(), tcc.TargetServerDetails)
+	return tcc.NewPreChecksRunner(selectedRepos, remoteRepositories).Run(context.Background(), tcc.TargetServerDetails)
 }
 
 func (tcc *TransferConfigCommand) printWarnings() (err error) {
@@ -296,16 +297,17 @@ func (tcc *TransferConfigCommand) verifyConfigImportPlugin() error {
 }
 
 // Creates the Pre-checks runner for the config import command
-func (tcc *TransferConfigCommand) NewPreChecksRunner(remoteRepositories []interface{}) (runner *commandsUtils.PreCheckRunner) {
-	runner = commandsUtils.NewPreChecksRunner()
+func (tcc *TransferConfigCommand) NewPreChecksRunner(selectedRepos map[utils.RepoType][]services.RepositoryDetails, remoteRepositories []interface{}) (runner *precheckrunner.PreCheckRunner) {
+	runner = precheckrunner.NewPreChecksRunner()
 
 	// Add pre-checks here
-	runner.AddCheck(commandsUtils.NewRemoteRepositoryCheck(&tcc.TargetArtifactoryManager, remoteRepositories))
+	runner.AddCheck(precheckrunner.NewRepositoryNamingCheck(selectedRepos))
+	runner.AddCheck(precheckrunner.NewRemoteRepositoryCheck(&tcc.TargetArtifactoryManager, remoteRepositories))
 
 	return
 }
 
-func (tcc *TransferConfigCommand) getEncryptedItems(selectedSourceRepos map[utils.RepoType][]string) (configXml string, remoteRepositories []interface{}, err error) {
+func (tcc *TransferConfigCommand) getEncryptedItems(selectedSourceRepos map[utils.RepoType][]services.RepositoryDetails) (configXml string, remoteRepositories []interface{}, err error) {
 	reactivateKeyEncryption, err := tcc.DeactivateKeyEncryption()
 	if err != nil {
 		return "", nil, err
@@ -324,10 +326,10 @@ func (tcc *TransferConfigCommand) getEncryptedItems(selectedSourceRepos map[util
 	}
 
 	// Get all remote repositories from the source Artifactory server.
-	if remoteRepositoryNames, ok := selectedSourceRepos[utils.Remote]; ok && len(remoteRepositoryNames) > 0 {
-		remoteRepositories = make([]interface{}, len(remoteRepositoryNames))
-		for i, repoName := range remoteRepositoryNames {
-			if err = tcc.SourceArtifactoryManager.GetRepository(repoName, &remoteRepositories[i]); err != nil {
+	if remoteRepositoriesDetails, ok := selectedSourceRepos[utils.Remote]; ok && len(remoteRepositoriesDetails) > 0 {
+		remoteRepositories = make([]interface{}, len(remoteRepositoriesDetails))
+		for i, remoteRepositoryDetails := range remoteRepositoriesDetails {
+			if err = tcc.SourceArtifactoryManager.GetRepository(remoteRepositoryDetails.Key, &remoteRepositories[i]); err != nil {
 				return
 			}
 		}

artifactory/commands/transferconfig/transferconfig_test.go

@@ -95,7 +95,7 @@ func TestGetConfigXml(t *testing.T) {
 
 	// Test get config xml
 	transferConfigCmd := createTransferConfigCommand(t, serverDetails, nil)
-	configXml, _, err := transferConfigCmd.getEncryptedItems(make(map[utils.RepoType][]string))
+	configXml, _, err := transferConfigCmd.getEncryptedItems(make(map[utils.RepoType][]services.RepositoryDetails))
 	assert.NoError(t, err)
 	assert.Equal(t, "<config></config>", configXml)
 }

artifactory/commands/transferconfigmerge/transferconfigmerge.go

@@ -53,7 +53,7 @@ func (tcmc *TransferConfigMergeCommand) SetExcludeProjectsPatterns(excludeProjec
 
 type mergeEntities struct {
 	projectsToTransfer []accessServices.Project
-	reposToTransfer    map[utils.RepoType][]string
+	reposToTransfer    map[utils.RepoType][]services.RepositoryDetails
 }
 
 type Conflict struct {
@@ -243,8 +243,8 @@ func compareProjects(sourceProject, targetProject accessServices.Project) (*Conf
 	}, nil
 }
 
-func (tcmc *TransferConfigMergeCommand) mergeRepositories(conflicts *[]Conflict) (reposToTransfer map[utils.RepoType][]string, err error) {
-	reposToTransfer = make(map[utils.RepoType][]string)
+func (tcmc *TransferConfigMergeCommand) mergeRepositories(conflicts *[]Conflict) (reposToTransfer map[utils.RepoType][]services.RepositoryDetails, err error) {
+	reposToTransfer = make(map[utils.RepoType][]services.RepositoryDetails)
 	sourceRepos, err := tcmc.SourceArtifactoryManager.GetAllRepositories()
 	if err != nil {
 		return
@@ -286,7 +286,7 @@ func (tcmc *TransferConfigMergeCommand) mergeRepositories(conflicts *[]Conflict)
 			}
 		} else {
 			repoType := utils.RepoTypeFromString(sourceRepo.Type)
-			reposToTransfer[repoType] = append(reposToTransfer[repoType], sourceRepo.Key)
+			reposToTransfer[repoType] = append(reposToTransfer[repoType], sourceRepo)
 		}
 	}
 	return
@@ -351,7 +351,7 @@ func (tcmc *TransferConfigMergeCommand) transferProjectsToTarget(reposToTransfer
 	return
 }
 
-func (tcmc *TransferConfigMergeCommand) decryptAndGetAllRemoteRepositories(remoteRepositoryNames []string) (remoteRepositories []interface{}, err error) {
+func (tcmc *TransferConfigMergeCommand) decryptAndGetAllRemoteRepositories(remoteRepositoriesDetails []services.RepositoryDetails) (remoteRepositories []interface{}, err error) {
 	// Decrypt source Artifactory to get remote repositories with raw text passwords
 	reactivateKeyEncryption, err := tcmc.DeactivateKeyEncryption()
 	if err != nil {
@@ -362,7 +362,11 @@ func (tcmc *TransferConfigMergeCommand) decryptAndGetAllRemoteRepositories(remot
 			err = reactivationErr
 		}
 	}()
-	return tcmc.GetAllRemoteRepositories(remoteRepositoryNames)
+	var remoteRepositoryKeys []string
+	for _, remoteRepositoryDetails := range remoteRepositoriesDetails {
+		remoteRepositoryKeys = append(remoteRepositoryKeys, remoteRepositoryDetails.Key)
+	}
+	return tcmc.GetAllRemoteRepositories(remoteRepositoryKeys)
 }
 
 type projectsMapper struct {

artifactory/commands/transferfiles/delayedartifactshandler.go

@@ -89,19 +89,25 @@ type DelayedArtifactsFile struct {
 }
 
 // Collect all the delayed artifact files that were created up to this point for the repository and transfer their artifacts using handleDelayedArtifactsFiles
-func consumeAllDelayFiles(base phaseBase) error {
+func consumeAllDelayFiles(base phaseBase) (err error) {
 	filesToConsume, err := getDelayFiles([]string{base.repoKey})
-	if err != nil {
-		return err
+	if err != nil || len(filesToConsume) == 0 {
+		return
 	}
 	delayFunctions := getDelayUploadComparisonFunctions(base.repoSummary.PackageType)
-	if len(filesToConsume) > 0 && len(delayFunctions) > 0 {
-		log.Info("Starting to handle delayed artifacts uploads...")
-		if err = handleDelayedArtifactsFiles(filesToConsume, base, delayFunctions[1:]); err == nil {
-			log.Info("Done handling delayed artifacts uploads.")
-		}
+	if len(delayFunctions) == 0 {
+		return
+	}
+
+	log.Info("Starting to handle delayed artifacts uploads...")
+	// Each delay function causes the transfer to skip a specific group of files.
+	// Within the handleDelayedArtifactsFiles function, we recursively remove the first delay function from the slice to transfer the first set of files every time.
+	if err = handleDelayedArtifactsFiles(filesToConsume, base, delayFunctions[1:]); err != nil {
+		return
 	}
-	return err
+
+	log.Info("Done handling delayed artifacts uploads.")
+	return deleteAllFiles(filesToConsume)
 }
 
 // Call consumeAllDelayFiles only if there are no failed transferred files for the repository up to this point.
@@ -182,13 +188,6 @@ func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToCon
 		if err = base.stateManager.ChangeDelayedFilesCountBy(uint64(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil {
 			log.Warn("Couldn't decrease the delayed files counter", err.Error())
 		}
-
-		// Remove the file, so it won't be consumed again.
-		if err = os.Remove(filePath); err != nil {
-			return errorutils.CheckError(err)
-		}
-
-		log.Debug("Done handling delayed artifacts file: '" + filePath + "'. Deleting it...")
 	}
 	return nil
 }

artifactory/commands/transferfiles/fileserror.go

@@ -3,13 +3,11 @@ package transferfiles
 import (
 	"errors"
 	"fmt"
-	"os"
 	"strings"
 	"time"
 
 	"github.com/jfrog/gofrog/parallel"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
-	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 )
 
@@ -33,7 +31,8 @@ func (e *errorsRetryPhase) run() error {
 // Does so by creating and uploading by chunks, and polling on status.
 // Consumed errors files are deleted, new failures are written to new files.
 func (e *errorsRetryPhase) handlePreviousUploadFailures() error {
-	if len(e.errorsFilesToHandle) == 0 {
+	errorsFilesToHandle := e.errorsFilesToHandle
+	if len(errorsFilesToHandle) == 0 {
 		return nil
 	}
 	log.Info("Starting to handle previous upload failures...")
@@ -46,11 +45,13 @@ func (e *errorsRetryPhase) handlePreviousUploadFailures() error {
 	delayAction := func(phase phaseBase, addedDelayFiles []string) error {
 		return consumeAllDelayFiles(phase)
 	}
-	err := e.transferManager.doTransferWithProducerConsumer(action, delayAction)
-	if err == nil {
-		log.Info("Done handling previous upload failures.")
+
+	if err := e.transferManager.doTransferWithProducerConsumer(action, delayAction); err != nil {
+		return err
 	}
-	return err
+
+	log.Info("Done handling previous upload failures.")
+	return deleteAllFiles(errorsFilesToHandle)
 }
 
 func convertUploadStatusToFileRepresentation(statuses []ExtendedFileUploadStatusResponse) (files []api.FileRepresentation) {
@@ -83,24 +84,13 @@ func (e *errorsRetryPhase) handleErrorsFile(errFilePath string, pcWrapper *produ
 	}
 
 	// Upload
-	shouldStop, err := uploadByChunks(convertUploadStatusToFileRepresentation(failedFiles.Errors), uploadChunkChan, e.phaseBase, delayHelper, errorsChannelMng, pcWrapper)
-	if err != nil || shouldStop {
-		return err
-	}
-
-	// Remove the file, so it won't be consumed again.
-	err = os.Remove(errFilePath)
-	if err != nil {
-		return errorutils.CheckError(err)
-	}
-
-	log.Debug("Done handling errors file: '", errFilePath, "'. Deleting it...")
-	return nil
+	_, err = uploadByChunks(convertUploadStatusToFileRepresentation(failedFiles.Errors), uploadChunkChan, e.phaseBase, delayHelper, errorsChannelMng, pcWrapper)
+	return err
 }
 
 func (e *errorsRetryPhase) createErrorFilesHandleFunc(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) errorFileHandlerFunc {
 	return func() parallel.TaskFunc {
-		return func(threadId int) error {
+		return func(int) error {
 			var errList []string
 			var err error
 			for _, errFile := range e.errorsFilesToHandle {

artifactory/commands/transferfiles/fulltransfer.go

@@ -96,8 +96,15 @@ func (m *fullTransferPhase) run() error {
 		if ShouldStop(&m.phaseBase, &delayHelper, errorsChannelMng) {
 			return nil
 		}
-		folderHandler := m.createFolderFullTransferHandlerFunc(*pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng)
-		_, err := pcWrapper.chunkBuilderProducerConsumer.AddTaskWithError(folderHandler(folderParams{relativePath: "."}), pcWrapper.errorsQueue.AddError)
+
+		// Get the directory's node from the snapshot manager, and use information from previous transfer attempts if such exists.
+		node, done, err := m.getAndHandleDirectoryNode(".")
+		if err != nil || done {
+			return err
+		}
+
+		folderHandler := m.createFolderFullTransferHandlerFunc(node, *pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng)
+		_, err = pcWrapper.chunkBuilderProducerConsumer.AddTaskWithError(folderHandler(folderParams{relativePath: "."}), pcWrapper.errorsQueue.AddError)
 		return err
 	}
 	delayAction := func(phase phaseBase, addedDelayFiles []string) error {
@@ -117,17 +124,17 @@ type folderParams struct {
 	relativePath string
 }
 
-func (m *fullTransferPhase) createFolderFullTransferHandlerFunc(pcWrapper producerConsumerWrapper, uploadChunkChan chan UploadedChunk,
+func (m *fullTransferPhase) createFolderFullTransferHandlerFunc(node *reposnapshot.Node, pcWrapper producerConsumerWrapper, uploadChunkChan chan UploadedChunk,
 	delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) folderFullTransferHandlerFunc {
 	return func(params folderParams) parallel.TaskFunc {
 		return func(threadId int) error {
 			logMsgPrefix := clientUtils.GetLogMsgPrefix(threadId, false)
-			return m.transferFolder(params, logMsgPrefix, pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng)
+			return m.transferFolder(node, params, logMsgPrefix, pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng)
 		}
 	}
 }
 
-func (m *fullTransferPhase) transferFolder(params folderParams, logMsgPrefix string, pcWrapper producerConsumerWrapper,
+func (m *fullTransferPhase) transferFolder(node *reposnapshot.Node, params folderParams, logMsgPrefix string, pcWrapper producerConsumerWrapper,
 	uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) (err error) {
 	log.Debug(logMsgPrefix+"Handling folder:", path.Join(m.repoKey, params.relativePath))
 
@@ -139,12 +146,6 @@ func (m *fullTransferPhase) transferFolder(params folderParams, logMsgPrefix str
 		return
 	}
 
-	// Get the directory's node from the snapshot manager, and use information from previous transfer attempts if such exist.
-	node, done, err := m.getAndHandleDirectoryNode(params, logMsgPrefix)
-	if err != nil || done {
-		return err
-	}
-
 	curUploadChunk, err := m.searchAndHandleFolderContents(params, pcWrapper,
 		uploadChunkChan, delayHelper, errorsChannelMng, node)
 	if err != nil {
@@ -227,7 +228,13 @@ func (m *fullTransferPhase) handleFoundChildFolder(params folderParams, pcWrappe
 	item servicesUtils.ResultItem) (err error) {
 	newRelativePath := getFolderRelativePath(item.Name, params.relativePath)
 
-	folderHandler := m.createFolderFullTransferHandlerFunc(pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng)
+	// Get the directory's node from the snapshot manager, and use information from previous transfer attempts if such exists.
+	node, done, err := m.getAndHandleDirectoryNode(newRelativePath)
+	if err != nil || done {
+		return err
+	}
+
+	folderHandler := m.createFolderFullTransferHandlerFunc(node, pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng)
 	_, err = pcWrapper.chunkBuilderProducerConsumer.AddTaskWithError(folderHandler(folderParams{relativePath: newRelativePath}), pcWrapper.errorsQueue.AddError)
 	return
 }
@@ -242,7 +249,7 @@ func (m *fullTransferPhase) handleFoundFile(pcWrapper producerConsumerWrapper,
 		return
 	}
 	// Increment the files count in the directory's node in the snapshot manager, to track its progress.
-	err = node.IncrementFilesCount()
+	err = node.IncrementFilesCount(uint64(file.Size))
 	if err != nil {
 		return
 	}
@@ -289,15 +296,14 @@ func generateFolderContentAqlQuery(repoKey, relativePath string, paginationOffse
 // node - A node in the repository snapshot tree, which represents the current directory.
 // completed - Whether handling the node directory was completed. If it wasn't fully transferred, we start exploring and transferring it from scratch.
 // previousChildren - If the directory requires exploring, previously known children will be added from this map in order to preserve their states and references.
-func (m *fullTransferPhase) getAndHandleDirectoryNode(params folderParams, logMsgPrefix string) (node *reposnapshot.Node, completed bool, err error) {
-	node, err = m.stateManager.LookUpNode(params.relativePath)
+func (m *fullTransferPhase) getAndHandleDirectoryNode(relativePath string) (node *reposnapshot.Node, completed bool, err error) {
+	node, err = m.stateManager.LookUpNode(relativePath)
 	if err != nil {
 		return
 	}
 
 	// If data was not loaded from snapshot, we know that the node is visited for the first time and was not explored.
-	loadedFromSnapshot, err := m.stateManager.WasSnapshotLoaded()
-	if err != nil || !loadedFromSnapshot {
+	if !m.stateManager.WasSnapshotLoaded() {
 		return
 	}
 
@@ -306,7 +312,7 @@ func (m *fullTransferPhase) getAndHandleDirectoryNode(params folderParams, logMs
 		return
 	}
 	if completed {
-		log.Debug(logMsgPrefix+"Skipping completed folder:", path.Join(m.repoKey, params.relativePath))
+		log.Debug("Skipping completed folder:", path.Join(m.repoKey, relativePath))
 		return
 	}
 	// If the node was not completed, we will start exploring it from the beginning.

artifactory/commands/transferfiles/longpropertycheck.go

@@ -9,6 +9,7 @@ import (
 	"github.com/jfrog/gofrog/parallel"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
 	cmdutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
+	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils/precheckrunner"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/progressbar"
 	"github.com/jfrog/jfrog-client-go/artifactory"
@@ -67,7 +68,7 @@ func (lpc *LongPropertyCheck) Name() string {
 	return longPropertyCheckName
 }
 
-func (lpc *LongPropertyCheck) ExecuteCheck(args cmdutils.RunArguments) (passed bool, err error) {
+func (lpc *LongPropertyCheck) ExecuteCheck(args precheckrunner.RunArguments) (passed bool, err error) {
 	// Init producer consumer
 	lpc.producerConsumer = parallel.NewRunner(threadCount, maxThreadCapacity, false)
 	lpc.filesChan = make(chan FileWithLongProperty, threadCount)
@@ -111,7 +112,7 @@ func (lpc *LongPropertyCheck) ExecuteCheck(args cmdutils.RunArguments) (passed b
 
 // Search for long properties in the server and create a search task to find the files that contains them
 // Returns the number of long properties found
-func (lpc *LongPropertyCheck) longPropertiesTaskProducer(progress *progressbar.TasksProgressBar, args cmdutils.RunArguments) int {
+func (lpc *LongPropertyCheck) longPropertiesTaskProducer(progress *progressbar.TasksProgressBar, args precheckrunner.RunArguments) int {
 	// Init
 	serviceManager, err := createTransferServiceManager(args.Context, args.ServerDetails)
 	if err != nil {
@@ -173,7 +174,7 @@ func getSearchAllPropertiesQuery(pageNumber int) string {
 
 // Create a task that fetch from the server the files with the given property.
 // We keep only the files that are at the requested repos and pass them at the files channel
-func createSearchPropertyTask(property Property, repos []string, args cmdutils.RunArguments, filesChan chan FileWithLongProperty, progress *progressbar.TasksProgressBar) parallel.TaskFunc {
+func createSearchPropertyTask(property Property, repos []string, args precheckrunner.RunArguments, filesChan chan FileWithLongProperty, progress *progressbar.TasksProgressBar) parallel.TaskFunc {
 	return func(threadId int) (err error) {
 		serviceManager, err := createTransferServiceManager(args.Context, args.ServerDetails)
 		if err != nil {