Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/dev'
Browse files Browse the repository at this point in the history
  • Loading branch information
yahavi committed Oct 22, 2023
2 parents 45beee8 + 8a053f9 commit 8eb1c37
Show file tree
Hide file tree
Showing 18 changed files with 160 additions and 150 deletions.
3 changes: 0 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,5 @@ jobs:
restore-keys: |
${{ runner.os }}-go-
- name: Lint
run: go vet ./...

- name: Tests
run: go test -v github.com/jfrog/jfrog-cli-core/v2/tests -timeout 0 -race
2 changes: 1 addition & 1 deletion artifactory/commands/npm/publish.go
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ func (npc *NpmPublishCommand) setPackageInfo() error {
}

if fileInfo.IsDir() {
npc.packageInfo, err = biutils.ReadPackageInfoFromPackageJson(npc.publishPath, npc.npmVersion)
npc.packageInfo, err = biutils.ReadPackageInfoFromPackageJsonIfExists(npc.publishPath, npc.npmVersion)
return err
}
log.Debug("The provided path is not a directory, we assume this is a compressed npm package")
Expand Down
8 changes: 5 additions & 3 deletions artifactory/commands/transferfiles/delayedartifactshandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,11 @@ func (w *SplitContentWriter) closeCurrentFile() error {
if err := w.writer.Close(); err != nil {
return err
}
defer func() {
// Reset writer and counter.
w.recordCount = 0
w.writer = nil
}()
if w.writer.GetFilePath() != "" {
fullPath, err := getUniqueErrorOrDelayFilePath(w.dirPath, func() string {
return w.filePrefix
Expand All @@ -384,9 +389,6 @@ func (w *SplitContentWriter) closeCurrentFile() error {
w.fileIndex++
}
}
// Reset writer and counter.
w.recordCount = 0
w.writer = nil
return nil
}

Expand Down
2 changes: 1 addition & 1 deletion artifactory/commands/transferfiles/localgeneratedfilter.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
const (
locallyGeneratedApi = "api/localgenerated/filter/paths"
minArtifactoryVersionForLocallyGenerated = "7.55.0"
maxConcurrentLocallyGeneratedRequests = 2
maxConcurrentLocallyGeneratedRequests = 5
)

// The request and response payload of POST '/api/localgenerated/filter/paths'
Expand Down
7 changes: 4 additions & 3 deletions artifactory/commands/transferfiles/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ type transferDelayAction func(phase phaseBase, addedDelayFiles []string) error

// Transfer files using the 'producer-consumer' mechanism and apply a delay action.
func (ftm *transferManager) doTransferWithProducerConsumer(transferAction transferActionWithProducerConsumerType, delayAction transferDelayAction) error {
ftm.pcDetails = newProducerConsumerWrapper()
// Set the producer-consumer value into the referenced value. This allow the Graceful Stop mechanism to access ftm.pcDetails when needed to stop the transfer.
*ftm.pcDetails = newProducerConsumerWrapper()
return ftm.doTransfer(ftm.pcDetails, transferAction, delayAction)
}

Expand Down Expand Up @@ -203,14 +204,14 @@ type producerConsumerWrapper struct {
errorsQueue *clientUtils.ErrorsQueue
}

func newProducerConsumerWrapper() *producerConsumerWrapper {
func newProducerConsumerWrapper() producerConsumerWrapper {
chunkUploaderProducerConsumer := parallel.NewRunner(GetThreads(), tasksMaxCapacity, false)
chunkBuilderProducerConsumer := parallel.NewRunner(GetThreads(), tasksMaxCapacity, false)
chunkUploaderProducerConsumer.SetFinishedNotification(true)
chunkBuilderProducerConsumer.SetFinishedNotification(true)
errorsQueue := clientUtils.NewErrorsQueue(1)

return &producerConsumerWrapper{
return producerConsumerWrapper{
chunkUploaderProducerConsumer: chunkUploaderProducerConsumer,
chunkBuilderProducerConsumer: chunkBuilderProducerConsumer,
errorsQueue: errorsQueue,
Expand Down
3 changes: 2 additions & 1 deletion artifactory/commands/transferfiles/phase.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ func (pb *phaseBase) setStopSignal(stopSignal chan os.Signal) {
}

func createTransferPhase(i int) transferPhase {
curPhaseBase := phaseBase{phaseId: i}
// Initialize a pointer to an empty producerConsumerWrapper to allow access the real value in StopGracefully
curPhaseBase := phaseBase{phaseId: i, pcDetails: &producerConsumerWrapper{}}
switch i {
case api.Phase1:
return &fullTransferPhase{phaseBase: curPhaseBase}
Expand Down
3 changes: 2 additions & 1 deletion artifactory/commands/transferfiles/phase_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ import (
const zeroUint32 uint32 = 0

func TestStopGracefully(t *testing.T) {
pBase := &phaseBase{pcDetails: newProducerConsumerWrapper()}
pcWrapper := newProducerConsumerWrapper()
pBase := &phaseBase{pcDetails: &pcWrapper}
chunkUploaderProducerConsumer := pBase.pcDetails.chunkUploaderProducerConsumer
chunkBuilderProducerConsumer := pBase.pcDetails.chunkBuilderProducerConsumer
go func() {
Expand Down
6 changes: 4 additions & 2 deletions artifactory/commands/transferfiles/transferfileprogress.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,6 @@ func initTransferProgressMng(allSourceLocalRepos []string, tdc *TransferFilesCom
transfer.runningTime = transfer.transferMng.NewRunningTimeProgressBar()
transfer.speedBar = transfer.transferMng.NewSpeedProgBar()
transfer.timeEstBar = transfer.transferMng.NewTimeEstBar()
transfer.visitedFoldersBar = transfer.transferMng.NewVisitedFoldersBar()
transfer.delayedBar = transfer.transferMng.NewDelayedBar()
// Init global error count for the process
transfer.errorBar = transfer.transferMng.NewErrorBar()
tdc.progressbar = &transfer
Expand All @@ -90,6 +88,8 @@ func (t *TransferProgressMng) NewRepository(name string) {
}
t.emptyLine = t.barsMng.NewHeadlineBar("")
t.currentRepoHeadline = t.barsMng.NewHeadlineBarWithSpinner("Current repository: " + color.Green.Render(name))
t.visitedFoldersBar = t.transferMng.NewVisitedFoldersBar()
t.delayedBar = t.transferMng.NewDelayedBar()
t.transferMng.StopCurrentRepoProgressBars(false)
}

Expand Down Expand Up @@ -209,6 +209,8 @@ func (t *TransferProgressMng) RemoveRepository() {
// Abort all current repository's bars
t.currentRepoHeadline.Abort(true)
t.currentRepoHeadline = nil
t.visitedFoldersBar.GetBar().Abort(true)
t.delayedBar.GetBar().Abort(true)
t.emptyLine.Abort(true)
t.emptyLine = nil
// Abort all phases bars
Expand Down
5 changes: 4 additions & 1 deletion artifactory/commands/transferfiles/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,10 @@ func interruptIfRequested(stopSignal chan os.Signal) error {
return err
}
if exist {
stopSignal <- os.Interrupt
select {
case stopSignal <- os.Interrupt:
default:
}
}
return nil
}
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/google/uuid v1.3.1
github.com/gookit/color v1.5.4
github.com/jedib0t/go-pretty/v6 v6.4.8
github.com/jfrog/build-info-go v1.9.13
github.com/jfrog/build-info-go v1.9.14
github.com/jfrog/gofrog v1.3.1
github.com/jfrog/jfrog-apps-config v1.0.1
github.com/jfrog/jfrog-client-go v1.34.3
Expand Down Expand Up @@ -101,4 +101,4 @@ require (

// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231003120621-90e9d7ea05e9

// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go dev
// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231019085746-e1b192457664
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/jedib0t/go-pretty/v6 v6.4.8 h1:HiNzyMSEpsBaduKhmK+CwcpulEeBrTmxutz4oX/oWkg=
github.com/jedib0t/go-pretty/v6 v6.4.8/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
github.com/jfrog/build-info-go v1.9.13 h1:OeoGzPVK/O4TOUYk35uL4bXg/hleyqMrjGjjmyLOYrg=
github.com/jfrog/build-info-go v1.9.13/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg=
github.com/jfrog/build-info-go v1.9.14 h1:xVezJ16Vpm/boRBn3lI1THCQmkylm+6R4zYWxOQ0NSM=
github.com/jfrog/build-info-go v1.9.14/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg=
github.com/jfrog/gofrog v1.3.1 h1:QqAwQXCVReT724uga1AYqG/ZyrNQ6f+iTxmzkb+YFQk=
github.com/jfrog/gofrog v1.3.1/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0=
github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=
Expand Down
18 changes: 4 additions & 14 deletions utils/progressbar/transferprogressbarmanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,13 @@ type transferLabels struct {
Repositories string
Files string
Storage string
Note string
TransferSpeed string
EstimatedTime string
VisitedFolders string
DelayedFiles string
TransferFailures string
WorkingThreads string
RunningFor string
DiffStorage string
DiffFiles string
FailedStorage string
FailedFiles string
}

func formatString(emoji, key string, windows bool) string {
Expand All @@ -47,23 +42,18 @@ func formatString(emoji, key string, windows bool) string {
return key
}

func initSProgressBarLabels(windows bool) transferLabels {
func initProgressBarLabels(windows bool) transferLabels {
pbs := transferLabels{}
pbs.Repositories = formatString("📦", " Repositories", windows)
pbs.Files = formatString("📄", " Files", windows)
pbs.Storage = formatString("🗄 ", " Storage", windows)
pbs.Note = formatString(" 🟠", " Note: ", windows)
pbs.TransferSpeed = formatString(" ⚡", " Transfer speed: ", windows)
pbs.EstimatedTime = formatString(" ⌛", " Estimated time remaining: ", windows)
pbs.VisitedFolders = formatString(" 📁", " Visited folders: ", windows)
pbs.DelayedFiles = formatString(" ✋", " Delayed files: ", windows)
pbs.TransferFailures = formatString(" ❌", " Transfer failures: ", windows)
pbs.WorkingThreads = formatString(" 🧵", " Working threads: ", windows)
pbs.RunningFor = formatString(" 🏃🏼", " Running for: ", windows)
pbs.DiffStorage = formatString("🗄 ", " Diff Storage", windows)
pbs.DiffFiles = formatString("📄", " Diff Files", windows)
pbs.FailedFiles = formatString("📄", " Failed Files", windows)
pbs.FailedStorage = formatString("🗄 ", " Failed Storage", windows)
pbs.VisitedFolders = formatString(" 📁", " Visited folders: ", windows)
pbs.DelayedFiles = formatString(" ✋", " Delayed files: ", windows)
return pbs
}

Expand All @@ -85,7 +75,7 @@ func InitTransferProgressBarMng(state *state.TransferStateManager, allRepos []st
mng.stateMng = state
mng.allRepos = allRepos
mng.generalShouldStop = false
mng.transferLabels = initSProgressBarLabels(coreutils.IsWindows())
mng.transferLabels = initProgressBarLabels(coreutils.IsWindows())
return
}

Expand Down
81 changes: 49 additions & 32 deletions utils/reposnapshot/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,20 +52,26 @@ func (node *Node) action(action ActionOnNodeFunc) error {

// Convert node to wrapper in order to save it to file.
func (node *Node) convertToWrapper() (wrapper *NodeExportWrapper, err error) {
var children []*Node
err = node.action(func(node *Node) error {
wrapper = &NodeExportWrapper{
Name: node.name,
Completed: node.NodeStatus == Completed,
}
for i := range node.children {
converted, err := node.children[i].convertToWrapper()
if err != nil {
return err
}
wrapper.Children = append(wrapper.Children, converted)
}
children = node.children
return nil
})
if err != nil {
return
}

for i := range children {
converted, err := children[i].convertToWrapper()
if err != nil {
return nil, err
}
wrapper.Children = append(wrapper.Children, converted)
}
return
}

Expand Down Expand Up @@ -107,17 +113,19 @@ func (node *Node) getActualPath() (actualPath string, err error) {
}

// Sets node as completed, clear its contents, notifies parent to check completion.
func (node *Node) setCompleted() error {
return node.action(func(node *Node) error {
func (node *Node) setCompleted() (err error) {
var parent *Node
err = node.action(func(node *Node) error {
node.NodeStatus = Completed
node.children = nil
parent := node.parent
parent = node.parent
node.parent = nil
if parent != nil {
return parent.CheckCompleted()
}
return nil
})
if err == nil && parent != nil {
return parent.CheckCompleted()
}
return
}

// Check if node completed - if done exploring, done handling files, children are completed.
Expand Down Expand Up @@ -233,30 +241,39 @@ func (node *Node) RestartExploring() error {
// For a structure such as repo->dir1->dir2->dir3
// The initial call will be to the root, and for an input of ({"dir1","dir2"}), and the final output will be a pointer to dir2.
func (node *Node) findMatchingNode(childrenDirs []string) (matchingNode *Node, err error) {
err = node.action(func(node *Node) error {
// The node was found in the cache. Let's return it.
if len(childrenDirs) == 0 {
matchingNode = node
return nil
}
// The node was found in the cache. Let's return it.
if len(childrenDirs) == 0 {
matchingNode = node
return
}

// Check if any of the current node's children are parents of the current node.
for i := range node.children {
if node.children[i].name == childrenDirs[0] {
matchingNode, err = node.children[i].findMatchingNode(childrenDirs[1:])
return err
}
// Check if any of the current node's children are parents of the current node.
var children []*Node
err = node.action(func(node *Node) error {
children = node.children
return nil
})
if err != nil {
return
}
for i := range children {
if children[i].name == childrenDirs[0] {
matchingNode, err = children[i].findMatchingNode(childrenDirs[1:])
return
}
}

// None of the current node's children are parents of the current node.
// This means we need to start creating the searched node parents.
newNode := CreateNewNode(childrenDirs[0], node)
newNode.parent = node
// None of the current node's children are parents of the current node.
// This means we need to start creating the searched node parents.
newNode := CreateNewNode(childrenDirs[0], node)
err = node.action(func(node *Node) error {
node.children = append(node.children, newNode)
matchingNode, err = newNode.findMatchingNode(childrenDirs[1:])
return err
return nil
})
return
if err != nil {
return
}
return newNode.findMatchingNode(childrenDirs[1:])
}

func CreateNewNode(dirName string, parent *Node) *Node {
Expand Down
2 changes: 1 addition & 1 deletion xray/commands/audit/sca/npm/npm.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func BuildDependencyTree(params utils.AuditParams) (dependencyTrees []*xrayUtils
if err != nil {
return
}
packageInfo, err := biutils.ReadPackageInfoFromPackageJson(currentDir, npmVersion)
packageInfo, err := biutils.ReadPackageInfoFromPackageJsonIfExists(currentDir, npmVersion)
if err != nil {
return
}
Expand Down
17 changes: 9 additions & 8 deletions xray/commands/audit/sca/python/python_test.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package python

import (
"github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca"
"path/filepath"
"testing"

"github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca"

"github.com/jfrog/build-info-go/utils/pythonutils"
"github.com/stretchr/testify/assert"
)
Expand Down Expand Up @@ -43,7 +44,7 @@ func TestPipDependencyListRequirementsFallback(t *testing.T) {
assert.Contains(t, uniqueDeps, pythonPackageTypeIdentifier+"pexpect:4.7.0")
assert.Contains(t, uniqueDeps, pythonPackageTypeIdentifier+"ptyprocess:0.7.0")
assert.Len(t, rootNode, 1)
if assert.True(t, len(rootNode[0].Nodes) > 2) {
if assert.GreaterOrEqual(t, len(rootNode[0].Nodes), 2) {
childNode := sca.GetAndAssertNode(t, rootNode[0].Nodes, "pexpect:4.7.0")
if childNode != nil {
// Test child module
Expand Down Expand Up @@ -105,16 +106,16 @@ func TestBuildPoetryDependencyList(t *testing.T) {
_, cleanUp := sca.CreateTestWorkspace(t, "poetry-project")
defer cleanUp()
expectedPoetryUniqueDeps := []string{
pythonPackageTypeIdentifier + "wcwidth:0.2.5",
pythonPackageTypeIdentifier + "wcwidth:0.2.8",
pythonPackageTypeIdentifier + "colorama:0.4.6",
pythonPackageTypeIdentifier + "packaging:22.0",
pythonPackageTypeIdentifier + "packaging:23.2",
pythonPackageTypeIdentifier + "python:",
pythonPackageTypeIdentifier + "pluggy:0.13.1",
pythonPackageTypeIdentifier + "py:1.11.0",
pythonPackageTypeIdentifier + "atomicwrites:1.4.1",
pythonPackageTypeIdentifier + "attrs:22.1.0",
pythonPackageTypeIdentifier + "more-itertools:9.0.0",
pythonPackageTypeIdentifier + "numpy:1.24.0",
pythonPackageTypeIdentifier + "attrs:23.1.0",
pythonPackageTypeIdentifier + "more-itertools:10.1.0",
pythonPackageTypeIdentifier + "numpy:1.26.1",
pythonPackageTypeIdentifier + "pytest:5.4.3",
}
// Run getModulesDependencyTrees
Expand All @@ -130,7 +131,7 @@ func TestBuildPoetryDependencyList(t *testing.T) {
childNode := sca.GetAndAssertNode(t, rootNode[0].Nodes, "pytest:5.4.3")
// Test sub child module
if assert.NotNil(t, childNode) {
sca.GetAndAssertNode(t, childNode.Nodes, "packaging:22.0")
sca.GetAndAssertNode(t, childNode.Nodes, "packaging:23.2")
}
}
}
Expand Down
Loading

0 comments on commit 8eb1c37

Please sign in to comment.