diff --git a/.github/workflows/codescene.yml b/.github/workflows/codescene.yml index 05724947..c02d0e0e 100644 --- a/.github/workflows/codescene.yml +++ b/.github/workflows/codescene.yml @@ -2,8 +2,6 @@ name: CodeScene on: pull_request: - branches: - - main jobs: delta-analysis: diff --git a/.github/workflows/debricked.yml b/.github/workflows/debricked.yml index 07d6f16d..8c675c35 100644 --- a/.github/workflows/debricked.yml +++ b/.github/workflows/debricked.yml @@ -24,6 +24,4 @@ jobs: restore-keys: | ${{ runner.os }}-go- - run: | - printf "$(go mod graph)\n\n$(go list -mod=readonly -e -m all)" > .debricked-go-dependencies.txt - - run: | - go run cmd/debricked/main.go scan -t ${{ secrets.DEBRICKED_TOKEN }} -e "pkg/**" + go run cmd/debricked/main.go scan -t ${{ secrets.DEBRICKED_TOKEN }} -e "pkg/**" -e "test/**" --resolve diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a9af4cc1..22557592 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,7 +12,7 @@ jobs: name: 'Push Docker images' strategy: matrix: - stage: [ 'cli', 'scan' ] + stage: [ 'cli', 'scan', 'cli-resolution'] runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.gitignore b/.gitignore index f0e9c351..2b8ea1be 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ debricked dist/ /.debricked-go-dependencies.txt /.env +test/resolve/testdata/pip/requirements.txt.venv/ +test/resolve/testdata/pip/.requirements.txt.debricked.lock +pkg/scan/testdata/npm/yarn.lock +pkg/resolution/pm/gradle/.gradle-init-script.debricked.groovy diff --git a/Makefile b/Makefile index d4eebae4..59783b04 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,35 @@ +.PHONY: install install: bash scripts/install.sh +.PHONY: lint lint: bash scripts/lint.sh +.PHONY: test test: bash scripts/test_cli.sh + +.PHONY: test-docker test-docker: bash scripts/test_docker.sh cli +.PHONY: test-e2e +test-e2e: + bash scripts/test_e2e.sh + +.PHONY: test-e2e-docker docker-build-dev: docker build -f build/docker/Dockerfile -t debricked/cli-dev:latest --target dev . + +.PHONY: docker-build-cli docker-build-cli: docker build -f build/docker/Dockerfile -t debricked/cli:latest --target cli . + +.PHONY: docker-build-scan docker-build-scan: docker build -f build/docker/Dockerfile -t debricked/cli-scan:latest --target scan . + +.PHONY: docker-build-cli-resolution +docker-build-cli-resolution: + docker build -f build/docker/Dockerfile -t debricked/cli-resolution:latest --target cli-resolution . diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index cbffcb16..8ed9c6a2 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -16,3 +16,26 @@ COPY --from=dev /cli/debricked /usr/bin/debricked FROM cli AS scan ENTRYPOINT [ "debricked", "scan" ] + +FROM cli AS cli-resolution +RUN apk --no-cache --update add \ + openjdk8-jre \ + python3 \ + py3-scipy \ + py3-pip \ + go + +ENV MAVEN_VERSION 3.9.0 +ENV MAVEN_HOME /usr/lib/mvn +ENV PATH $MAVEN_HOME/bin:$PATH +RUN wget http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz && \ + tar -zxvf apache-maven-$MAVEN_VERSION-bin.tar.gz && \ + rm apache-maven-$MAVEN_VERSION-bin.tar.gz && \ + mv apache-maven-$MAVEN_VERSION $MAVEN_HOME + +ENV GRADLE_VERSION 8.0.2 +ENV GRADLE_HOME /usr/lib/gradle +ENV PATH $GRADLE_HOME/gradle-$GRADLE_VERSION/bin:$PATH +RUN wget https://services.gradle.org/distributions/gradle-$GRADLE_VERSION-bin.zip && \ + unzip gradle-$GRADLE_VERSION-bin.zip -d $GRADLE_HOME && \ + rm gradle-$GRADLE_VERSION-bin.zip \ diff --git a/cmd/debricked/main.go b/cmd/debricked/main.go index 91b0f34a..3befa613 100644 --- a/cmd/debricked/main.go +++ b/cmd/debricked/main.go @@ -4,12 +4,13 @@ import ( "os" "github.com/debricked/cli/pkg/cmd/root" + "github.com/debricked/cli/pkg/wire" ) var version string // Set at compile time func main() { - if err := root.NewRootCmd(version).Execute(); err != nil { + if err := root.NewRootCmd(version, wire.GetCliContainer()).Execute(); err != nil { os.Exit(1) } } diff --git a/examples/templates/Argo/Go/argo.yml b/examples/templates/Argo/Go/argo.yml deleted file mode 100644 index 155562d3..00000000 --- a/examples/templates/Argo/Go/argo.yml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: debricked- -spec: - entrypoint: debricked - arguments: - parameters: - - name: git-url # For example: https://github.com/debricked/go-templates.git - - name: debricked-token # Consider using kubernetes secrets instead. For more details, see: https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml - - templates: - - name: debricked - inputs: - parameters: - - name: git-url - - name: debricked-token - steps: - - - name: build - template: build - arguments: - parameters: - - name: git-url - value: "{{inputs.parameters.git-url}}" - - - name: scan - template: scan - arguments: - parameters: - - name: git-url - value: "{{inputs.parameters.git-url}}" - - name: debricked-token - value: "{{inputs.parameters.debricked-token}}" - artifacts: - - name: repository - from: "{{steps.build.outputs.artifacts.repository}}" - - - name: build - inputs: - parameters: - - name: git-url - artifacts: - - name: repository - path: /repository - git: # For more details, see: https://github.com/argoproj/argo-workflows/blob/master/examples/input-artifact-git.yaml - repo: "{{inputs.parameters.git-url}}" - outputs: - artifacts: - - name: repository - path: /repository - container: - name: 'go' - image: golang:1.17-alpine - workingDir: /repository - command: [sh, -c] - args: [" - printf \"$(go mod graph)\n\n$(go list -mod=readonly -e -m all)\" >.debricked-go-dependencies.txt - "] - - - name: scan - inputs: - parameters: - - name: debricked-token - - name: git-url - artifacts: - - name: repository - path: /repository - container: - name: 'debricked-scan' - image: debricked/cli - workingDir: /repository - command: - - debricked scan - env: - - name: DEBRICKED_TOKEN - value: "{{inputs.parameters.debricked-token}}" - - name: DEBRICKED_GIT_URL - value: "{{inputs.parameters.git-url}}" diff --git a/examples/templates/Argo/Gradle/argo.yml b/examples/templates/Argo/Gradle/argo.yml deleted file mode 100644 index 111983e2..00000000 --- a/examples/templates/Argo/Gradle/argo.yml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: debricked- -spec: - entrypoint: debricked - arguments: - parameters: - - name: git-url # For example: https://github.com/debricked/go-templates.git - - name: debricked-token # Consider using kubernetes secrets instead. For more details, see: https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml - - templates: - - name: debricked - inputs: - parameters: - - name: git-url - - name: debricked-token - steps: - - - name: build - template: build - arguments: - parameters: - - name: git-url - value: "{{inputs.parameters.git-url}}" - - - name: scan - template: scan - arguments: - parameters: - - name: git-url - value: "{{inputs.parameters.git-url}}" - - name: debricked-token - value: "{{inputs.parameters.debricked-token}}" - artifacts: - - name: repository - from: "{{steps.build.outputs.artifacts.repository}}" - - - name: build - inputs: - parameters: - - name: git-url - artifacts: - - name: repository - path: /repository - git: # For more details, see: https://github.com/argoproj/argo-workflows/blob/master/examples/input-artifact-git.yaml - repo: "{{inputs.parameters.git-url}}" - outputs: - artifacts: - - name: repository - path: /repository - container: - name: 'gradle' - image: gradle:7-jdk11 - workingDir: /repository - command: - - /bin/sh - - '-c' - args: - - ./gradlew dependencies > .debricked-gradle-dependencies.txt - - - name: scan - inputs: - parameters: - - name: debricked-token - - name: git-url - artifacts: - - name: repository - path: /repository - container: - name: 'debricked-scan' - image: debricked/cli - workingDir: /repository - command: - - debricked scan - env: - - name: DEBRICKED_TOKEN - value: "{{inputs.parameters.debricked-token}}" - - name: DEBRICKED_GIT_URL - value: "{{inputs.parameters.git-url}}" diff --git a/examples/templates/Argo/Maven/argo.yml b/examples/templates/Argo/Maven/argo.yml deleted file mode 100644 index 9e394f3d..00000000 --- a/examples/templates/Argo/Maven/argo.yml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: debricked- -spec: - entrypoint: debricked - arguments: - parameters: - - name: git-url # For example: https://github.com/debricked/go-templates.git - - name: debricked-token # Consider using kubernetes secrets instead. For more details, see: https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml - - templates: - - name: debricked - inputs: - parameters: - - name: git-url - - name: debricked-token - steps: - - - name: build - template: build - arguments: - parameters: - - name: git-url - value: "{{inputs.parameters.git-url}}" - - - name: scan - template: scan - arguments: - parameters: - - name: git-url - value: "{{inputs.parameters.git-url}}" - - name: debricked-token - value: "{{inputs.parameters.debricked-token}}" - artifacts: - - name: repository - from: "{{steps.build.outputs.artifacts.repository}}" - - - name: build - inputs: - parameters: - - name: git-url - artifacts: - - name: repository - path: /repository - git: # For more details, see: https://github.com/argoproj/argo-workflows/blob/master/examples/input-artifact-git.yaml - repo: "{{inputs.parameters.git-url}}" - outputs: - artifacts: - - name: repository - path: /repository - container: - name: 'maven' - image: maven:3-jdk-11 - workingDir: /repository - command: - - mvn - - dependency:tree - - -DoutputFile=.debricked-maven-dependencies.tgf - - -DoutputType=tgf - - - name: scan - inputs: - parameters: - - name: debricked-token - - name: git-url - artifacts: - - name: repository - path: /repository - container: - name: 'debricked-scan' - image: debricked/cli - workingDir: /repository - command: - - debricked scan - env: - - name: DEBRICKED_TOKEN - value: "{{inputs.parameters.debricked-token}}" - - name: DEBRICKED_GIT_URL - value: "{{inputs.parameters.git-url}}" diff --git a/examples/templates/Argo/README.md b/examples/templates/Argo/README.md index 09e34834..cad032dc 100644 --- a/examples/templates/Argo/README.md +++ b/examples/templates/Argo/README.md @@ -1,5 +1,2 @@ # Argo Workflows -- [Default template](Default/argo.yml) -- [Maven template](Maven/argo.yml) -- [Gradle template](Gradle/argo.yml) -- [Go template](Go/argo.yml) +- [Default template](argo.yml) diff --git a/examples/templates/Argo/Default/argo.yml b/examples/templates/Argo/argo.yml similarity index 96% rename from examples/templates/Argo/Default/argo.yml rename to examples/templates/Argo/argo.yml index 55f5f48b..e584c8bf 100644 --- a/examples/templates/Argo/Default/argo.yml +++ b/examples/templates/Argo/argo.yml @@ -25,7 +25,7 @@ spec: image: debricked/cli workingDir: /repository command: - - debricked scan + - debricked scan --resolve env: - name: DEBRICKED_TOKEN value: "{{inputs.parameters.debricked-token}}" diff --git a/examples/templates/Azure/Go/azure-pipelines.yml b/examples/templates/Azure/Go/azure-pipelines.yml deleted file mode 100644 index 3fc2e6be..00000000 --- a/examples/templates/Azure/Go/azure-pipelines.yml +++ /dev/null @@ -1,24 +0,0 @@ -trigger: - branches: - include: - - '*' # Run on all branches - -resources: - - repo: self - -stages: - - stage: debricked - jobs: - - job: debricked - displayName: Debricked scan - pool: - vmImage: 'ubuntu-latest' - steps: - - script: printf "$(go mod graph)\n\n$(go list -mod=readonly -e -m all)" > .debricked-go-dependencies.txt - displayName: 'go mod graph & go list' - - script: | - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - ./debricked scan - displayName: Debricked scan - env: - DEBRICKED_TOKEN: $(DEBRICKED_TOKEN) diff --git a/examples/templates/Azure/Gradle/azure-pipelines.yml b/examples/templates/Azure/Gradle/azure-pipelines.yml deleted file mode 100644 index 4bca5f92..00000000 --- a/examples/templates/Azure/Gradle/azure-pipelines.yml +++ /dev/null @@ -1,24 +0,0 @@ -trigger: - branches: - include: - - '*' # Run on all branches - -resources: - - repo: self - -stages: - - stage: debricked - jobs: - - job: debricked - displayName: Debricked scan - pool: - vmImage: 'ubuntu-latest' - steps: - - script: sh ./gradlew dependencies > .debricked-gradle-dependencies.txt - displayName: './gradlew dependencies' - - script: | - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - ./debricked scan - displayName: Debricked scan - env: - DEBRICKED_TOKEN: $(DEBRICKED_TOKEN) diff --git a/examples/templates/Azure/Maven/azure-pipelines.yml b/examples/templates/Azure/Maven/azure-pipelines.yml deleted file mode 100644 index 204a8462..00000000 --- a/examples/templates/Azure/Maven/azure-pipelines.yml +++ /dev/null @@ -1,25 +0,0 @@ - -trigger: - branches: - include: - - '*' # Run on all branches - -resources: - - repo: self - -stages: - - stage: debricked - jobs: - - job: debricked - displayName: Debricked scan - pool: - vmImage: 'ubuntu-latest' - steps: - - script: mvn dependency:tree -DoutputFile=.debricked-maven-dependencies.tgf -DoutputType=tgf - displayName: 'mvn dependency:tree' - - script: | - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - ./debricked scan - displayName: Debricked scan - env: - DEBRICKED_TOKEN: $(DEBRICKED_TOKEN) diff --git a/examples/templates/Azure/README.md b/examples/templates/Azure/README.md index 49528d56..f325cdb2 100644 --- a/examples/templates/Azure/README.md +++ b/examples/templates/Azure/README.md @@ -1,5 +1,2 @@ # Azure Pipelines -- [Default template](Default/azure-pipelines.yml) -- [Maven template](Maven/azure-pipelines.yml) -- [Gradle template](Gradle/azure-pipelines.yml) -- [Go template](Go/azure-pipelines.yml) +- [Default template](azure-pipelines.yml) diff --git a/examples/templates/Azure/Default/azure-pipelines.yml b/examples/templates/Azure/azure-pipelines.yml similarity index 92% rename from examples/templates/Azure/Default/azure-pipelines.yml rename to examples/templates/Azure/azure-pipelines.yml index 383b3008..35380a4a 100644 --- a/examples/templates/Azure/Default/azure-pipelines.yml +++ b/examples/templates/Azure/azure-pipelines.yml @@ -16,7 +16,7 @@ stages: steps: - script: | curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - ./debricked scan + ./debricked scan --resolve displayName: Debricked scan env: DEBRICKED_TOKEN: $(DEBRICKED_TOKEN) diff --git a/examples/templates/Bitbucket/README.md b/examples/templates/Bitbucket/README.md index d7b9bba6..f04314c6 100644 --- a/examples/templates/Bitbucket/README.md +++ b/examples/templates/Bitbucket/README.md @@ -1,2 +1,2 @@ # Bitbucket Pipelines -- [Default template](Default/bitbucket-pipelines.yml) +- [Default template](bitbucket-pipelines.yml) diff --git a/examples/templates/Bitbucket/Default/bitbucket-pipelines.yml b/examples/templates/Bitbucket/bitbucket-pipelines.yml similarity index 89% rename from examples/templates/Bitbucket/Default/bitbucket-pipelines.yml rename to examples/templates/Bitbucket/bitbucket-pipelines.yml index 4bbee579..78d428ac 100644 --- a/examples/templates/Bitbucket/Default/bitbucket-pipelines.yml +++ b/examples/templates/Bitbucket/bitbucket-pipelines.yml @@ -6,7 +6,7 @@ test: &test name: Debricked Scan script: - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - ./debricked scan + - ./debricked scan --resolve services: - docker diff --git a/examples/templates/BuildKite/Go/pipeline.yml b/examples/templates/BuildKite/Go/pipeline.yml deleted file mode 100644 index d71feb6d..00000000 --- a/examples/templates/BuildKite/Go/pipeline.yml +++ /dev/null @@ -1,14 +0,0 @@ -steps: - - label: ":go: go mod graph" - command: | - printf "$(go mod graph)\n\n$(go list -mod=readonly -e -m all)" > .debricked-go-dependencies.txt - plugins: - - golang#v2.0.0: - version: 1.17 - artifact_paths: "**/.debricked-go-dependencies.txt" - - wait - - label: ":shield: Debricked" - command: - - buildkite-agent artifact download "**.debricked-go-dependencies.txt" . - - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - ./debricked scan diff --git a/examples/templates/BuildKite/Gradle/pipeline.yml b/examples/templates/BuildKite/Gradle/pipeline.yml deleted file mode 100644 index c5f76981..00000000 --- a/examples/templates/BuildKite/Gradle/pipeline.yml +++ /dev/null @@ -1,16 +0,0 @@ -steps: - - label: ":gradle: ./gradlew dependencies" - command: - - sh ./gradlew dependencies > .debricked-gradle-dependencies.txt - - rm -rf .gradle - plugins: - - docker#v5.3.0: - image: "gradle:jdk11" - workdir: /app - artifact_paths: "**/.debricked-gradle-dependencies.txt" - - wait - - label: ":shield: Debricked" - command: - - buildkite-agent artifact download "**.debricked-maven-dependencies.txt" . - - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - ./debricked scan diff --git a/examples/templates/BuildKite/Maven/pipeline.yml b/examples/templates/BuildKite/Maven/pipeline.yml deleted file mode 100644 index 89d08607..00000000 --- a/examples/templates/BuildKite/Maven/pipeline.yml +++ /dev/null @@ -1,14 +0,0 @@ -steps: - - label: ":maven: mvn dependency:tree" - command: mvn dependency:tree -DoutputFile=.debricked-maven-dependencies.tgf -DoutputType=tgf - plugins: - - docker#v5.3.0: - image: maven - workdir: /app - artifact_paths: "**/.debricked-maven-dependencies.tgf" - - wait - - label: ":shield: Debricked" - command: - - buildkite-agent artifact download "**.debricked-gradle-dependencies.txt" . - - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - ./debricked scan diff --git a/examples/templates/BuildKite/README.md b/examples/templates/BuildKite/README.md index db4c2b31..ac4b2a24 100644 --- a/examples/templates/BuildKite/README.md +++ b/examples/templates/BuildKite/README.md @@ -1,5 +1,2 @@ # BuildKite -- [Default template](Default/pipeline.yml) -- [Maven template](Maven/pipeline.yml) -- [Gradle template](Gradle/pipeline.yml) -- [Go template](Go/pipeline.yml) +- [Default template](pipeline.yml) diff --git a/examples/templates/BuildKite/Default/pipeline.yml b/examples/templates/BuildKite/pipeline.yml similarity index 82% rename from examples/templates/BuildKite/Default/pipeline.yml rename to examples/templates/BuildKite/pipeline.yml index 7b2a4cf5..c41de935 100644 --- a/examples/templates/BuildKite/Default/pipeline.yml +++ b/examples/templates/BuildKite/pipeline.yml @@ -2,4 +2,4 @@ steps: - label: ":shield: Debricked" command: - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - ./debricked scan + - ./debricked scan --resolve diff --git a/examples/templates/CircleCI/Go/config.yml b/examples/templates/CircleCI/Go/config.yml deleted file mode 100644 index 070dae65..00000000 --- a/examples/templates/CircleCI/Go/config.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: 2.1 - -jobs: - build: - docker: - # specify the version you desire here - - image: cimg/go:1.17 - - working_directory: ~/repo - - steps: - - checkout - - run: | - printf "$(go mod graph)\n\n$(go list -mod=readonly -e -m all)" > .debricked-go-dependencies.txt - # It is important that the generated dependency tree files are persisted and attached to the following scan step - - persist_to_workspace: - root: ~/repo - paths: - - '**.debricked-go-dependencies.txt' - # Make sure to add all generated .debricked-go-dependencies.txt files - - scan: - steps: - - checkout - - run: curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - run: ./debricked scan - -workflows: - debricked-scan: - jobs: - - build - - scan: - requires: - - build \ No newline at end of file diff --git a/examples/templates/CircleCI/Gradle/config.yml b/examples/templates/CircleCI/Gradle/config.yml deleted file mode 100644 index 6c01c8fd..00000000 --- a/examples/templates/CircleCI/Gradle/config.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 2.1 - -jobs: - build: - docker: - # specify the version you desire here - - image: circleci/openjdk - - working_directory: ~/repo - - steps: - - checkout - - run: sh ./gradlew dependencies > .debricked-gradle-dependencies.txt - # It is important that the generated dependency tree files are persisted and attached to the following scan step - - persist_to_workspace: - root: ~/repo - paths: - - '**.debricked-gradle-dependencies.txt' - # Make sure to add all generated .debricked-gradle-dependencies.txt files - scan: - steps: - - checkout - - run: curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - run: ./debricked scan - -workflows: - debricked-scan: - jobs: - - build - - scan: - requires: - - build diff --git a/examples/templates/CircleCI/Maven/config.yml b/examples/templates/CircleCI/Maven/config.yml deleted file mode 100644 index 29ec9e7a..00000000 --- a/examples/templates/CircleCI/Maven/config.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 2.1 - -jobs: - build: - docker: - # specify the version you desire here - - image: circleci/openjdk - - working_directory: ~/repo - - steps: - - checkout - - run: mvn dependency:tree -DoutputFile=.debricked-maven-dependencies.tgf -DoutputType=tgf - # It is important that the generated dependency tree files are persisted and attached to the following scan step - - persist_to_workspace: - root: ~/repo - paths: - - '**.debricked-maven-dependencies.tgf' - # Make sure to add all generated .debricked-maven-dependencies.tgf files - scan: - steps: - - checkout - - run: curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - run: ./debricked scan - -workflows: - debricked-scan: - jobs: - - build - - scan: - requires: - - build \ No newline at end of file diff --git a/examples/templates/CircleCI/README.md b/examples/templates/CircleCI/README.md index 6c98bcce..da51ae7b 100644 --- a/examples/templates/CircleCI/README.md +++ b/examples/templates/CircleCI/README.md @@ -1,5 +1,2 @@ # CirleCI -- [Default template](Default/config.yml) -- [Maven template](Maven/config.yml) -- [Gradle template](Gradle/config.yml) -- [Go template](Go/config.yml) +- [Default template](config.yml) diff --git a/examples/templates/CircleCI/Default/config.yml b/examples/templates/CircleCI/config.yml similarity index 100% rename from examples/templates/CircleCI/Default/config.yml rename to examples/templates/CircleCI/config.yml diff --git a/examples/templates/GitHub/Default/debricked.yml b/examples/templates/GitHub/Default/debricked.yml deleted file mode 100644 index b99b99dc..00000000 --- a/examples/templates/GitHub/Default/debricked.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: Debricked scan - -on: [push] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Install Debricked CLI - run: curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - name: debricked scan - run: ./debricked scan - env: - DEBRICKED_TOKEN: ${{ secrets.DEBRICKED_TOKEN }} diff --git a/examples/templates/GitHub/Go/debricked.yml b/examples/templates/GitHub/Go/debricked.yml deleted file mode 100644 index b1a6828c..00000000 --- a/examples/templates/GitHub/Go/debricked.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Debricked scan - -on: [push] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v2 - with: - go-version: '1.17' - - uses: actions/cache@v2 - with: - path: | - ~/Library/Caches/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - run: | - printf "$(go mod graph)\n\n$(go list -mod=readonly -e -m all)" > .debricked-go-dependencies.txt - - name: Install Debricked CLI - run: curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - name: debricked scan - run: ./debricked scan - env: - DEBRICKED_TOKEN: ${{ secrets.DEBRICKED_TOKEN }} diff --git a/examples/templates/GitHub/Gradle/debricked.yml b/examples/templates/GitHub/Gradle/debricked.yml deleted file mode 100644 index 42816113..00000000 --- a/examples/templates/GitHub/Gradle/debricked.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Debricked scan - -on: [push] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v2 - with: - java-version: '11' - distribution: 'adopt' - cache: 'gradle' - - run: sh ./gradlew dependencies > .debricked-gradle-dependencies.txt - - name: Install Debricked CLI - run: curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - name: debricked scan - run: ./debricked scan - env: - DEBRICKED_TOKEN: ${{ secrets.DEBRICKED_TOKEN }} diff --git a/examples/templates/GitHub/Maven/debricked.yml b/examples/templates/GitHub/Maven/debricked.yml deleted file mode 100644 index 024f74cf..00000000 --- a/examples/templates/GitHub/Maven/debricked.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Debricked scan - -on: [push] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v1 - with: - java-version: '13' - - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - run: | - mvn dependency:tree \ - -DoutputFile=.debricked-maven-dependencies.tgf \ - -DoutputType=tgf - - name: Install Debricked CLI - run: curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - name: debricked scan - run: ./debricked scan - env: - DEBRICKED_TOKEN: ${{ secrets.DEBRICKED_TOKEN }} diff --git a/examples/templates/GitHub/README.md b/examples/templates/GitHub/README.md index 6b4ad8e9..ea069dca 100644 --- a/examples/templates/GitHub/README.md +++ b/examples/templates/GitHub/README.md @@ -1,5 +1,2 @@ # GitHub Actions -- [Default template](Default/debricked.yml) -- [Maven template](Maven/debricked.yml) -- [Gradle template](Gradle/debricked.yml) -- [Go template](Go/debricked.yml) +- [Default template](debricked.yml) diff --git a/examples/templates/GitHub/debricked.yml b/examples/templates/GitHub/debricked.yml new file mode 100644 index 00000000..27d1a0ef --- /dev/null +++ b/examples/templates/GitHub/debricked.yml @@ -0,0 +1,25 @@ +name: Debricked scan + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/cache@v3 + with: + path: | + ~/Library/Caches/go-build + ~/go/pkg/mod + ~/.m2/repository + ~/.gradle/caches + ~/.gradle/wrapper + ~/.cache/pip + key: ${{ runner.os }}-debricked-resolution--${{ steps.get-date.outputs.date }} + - name: Install Debricked CLI + run: curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked + - name: debricked scan + run: ./debricked scan --resolve + env: + DEBRICKED_TOKEN: ${{ secrets.DEBRICKED_TOKEN }} \ No newline at end of file diff --git a/examples/templates/GitLab/Go/gitlab-ci.yml b/examples/templates/GitLab/Go/gitlab-ci.yml deleted file mode 100644 index 72e3e668..00000000 --- a/examples/templates/GitLab/Go/gitlab-ci.yml +++ /dev/null @@ -1,19 +0,0 @@ -stages: - - build - - scan - -build: - stage: build - image: go - script: - - printf "$(go mod graph)\n\n$(go list -mod=readonly -e -m all)" > .debricked-go-dependencies.txt - artifacts: - paths: - - .debricked-go-dependencies.txt - expire_in: 1 day - -debricked: - stage: scan - script: - - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - ./debricked scan diff --git a/examples/templates/GitLab/Gradle/gitlab-ci.yml b/examples/templates/GitLab/Gradle/gitlab-ci.yml deleted file mode 100644 index 002eb890..00000000 --- a/examples/templates/GitLab/Gradle/gitlab-ci.yml +++ /dev/null @@ -1,19 +0,0 @@ -stages: - - build - - scan - -build: - stage: build - image: gradle:alpine - script: - - sh ./gradlew dependencies > .debricked-gradle-dependencies.txt - artifacts: - paths: - - .debricked-gradle-dependencies.txt - expire_in: 1 day - -debricked: - stage: scan - script: - - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - ./debricked scan diff --git a/examples/templates/GitLab/Maven/gitlab-ci.yml b/examples/templates/GitLab/Maven/gitlab-ci.yml deleted file mode 100644 index 743687b6..00000000 --- a/examples/templates/GitLab/Maven/gitlab-ci.yml +++ /dev/null @@ -1,21 +0,0 @@ -stages: - - build - - scan - -build: - stage: build - image: maven:3.6.3-jdk-11 - script: - - mvn dependency:tree - -DoutputFile=.debricked-maven-dependencies.tgf - -DoutputType=tgf - artifacts: - paths: - - .debricked-maven-dependencies.tgf - expire_in: 1 day - -debricked: - stage: scan - script: - - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - ./debricked scan diff --git a/examples/templates/GitLab/README.md b/examples/templates/GitLab/README.md index 4110c68c..6c6719a1 100644 --- a/examples/templates/GitLab/README.md +++ b/examples/templates/GitLab/README.md @@ -1,5 +1,2 @@ # GitLab CI/CD -- [Default template](Default/gitlab-ci.yml) -- [Maven template](Maven/gitlab-ci.yml) -- [Gradle template](Gradle/gitlab-ci.yml) -- [Go template](Go/gitlab-ci.yml) +- [Default template](gitlab-ci.yml) diff --git a/examples/templates/GitLab/Default/gitlab-ci.yml b/examples/templates/GitLab/gitlab-ci.yml similarity index 83% rename from examples/templates/GitLab/Default/gitlab-ci.yml rename to examples/templates/GitLab/gitlab-ci.yml index 0fcb6965..cf901551 100644 --- a/examples/templates/GitLab/Default/gitlab-ci.yml +++ b/examples/templates/GitLab/gitlab-ci.yml @@ -4,4 +4,4 @@ debricked: stage: scan script: - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - - ./debricked scan + - ./debricked scan --resolve diff --git a/examples/templates/README.md b/examples/templates/README.md index c46ef16a..664c69eb 100644 --- a/examples/templates/README.md +++ b/examples/templates/README.md @@ -15,4 +15,8 @@ In order for us to analyze all dependencies in your project, their versions, and **Example 1:** If npm is used in your project you will have a `package.json` file, but in order for us to scan all your dependencies we need either `package-lock.json` or `yarn.lock` as well. -**Example 2:** If Maven is used in your project you will have a `pom.xml` file, but in order for us to resolve all your dependencies we need a second file, as Maven does not offer a lock file system. Instead, Maven dependency:tree plugin can be used to create a file called `.debricked-maven-dependencies.tgf` +**Example 2:** If Maven is used in your project you will have a `pom.xml` file, but in order for us to resolve all your dependencies we need a second file, as Maven does not offer a lock file system. Instead, Maven dependency:tree plugin can be used to create a file called `.maven.debricked.lock` + +## Debricked CLI dependency resolution +In all templates the `--resolve` flag is set. That means Debricked CLI will attempt to resolve manifest files that belong to package managers that does not offer lock file systems. +For example, if a `pom.xml` is found by Debricked CLI it will attempt to create `.maven.debricked.lock` automatically. \ No newline at end of file diff --git a/examples/templates/Travis/Default/travis.yml b/examples/templates/Travis/Default/travis.yml deleted file mode 100644 index cb9d5d33..00000000 --- a/examples/templates/Travis/Default/travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -jobs: - include: - - stage: Debricked-scan - on: - branch: "*" - env: - - DEBRICKED_TOKEN=${DEBRICKED_TOKEN} - before_install: curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - script: ./debricked scan diff --git a/examples/templates/Travis/Go/travis.yml b/examples/templates/Travis/Go/travis.yml deleted file mode 100644 index 97c5fbfe..00000000 --- a/examples/templates/Travis/Go/travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: go - -jobs: - include: - - stage: Debricked-scan - on: - branch: "*" - env: - - DEBRICKED_TOKEN=${DEBRICKED_TOKEN} - before_install: - - printf "$(go mod graph)\n\n$(go list -mod=readonly -e -m all)" > .debricked-go-dependencies.txt - - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - cache: - directories: - - $HOME/.cache/go-build - - $HOME/gopath/pkg/mod - script: ./debricked scan diff --git a/examples/templates/Travis/Maven/travis.yml b/examples/templates/Travis/Maven/travis.yml deleted file mode 100644 index 611d4480..00000000 --- a/examples/templates/Travis/Maven/travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -language: java - -jobs: - include: - - stage: Debricked-scan - on: - branch: "*" - env: - - DEBRICKED_TOKEN=${DEBRICKED_TOKEN} - before_install: - - mvn dependency:tree -DoutputFile=.debricked-maven-dependencies.tgf -DoutputType=tgf - - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - cache: - directories: - - $HOME/.m2 - script: ./debricked scan diff --git a/examples/templates/Travis/README.md b/examples/templates/Travis/README.md index 6b6b6abe..e4d514a8 100644 --- a/examples/templates/Travis/README.md +++ b/examples/templates/Travis/README.md @@ -1,5 +1,2 @@ # Travis CI -- [Default template](Default/travis.yml) -- [Maven template](Maven/travis.yml) -- [Gradle template](Gradle/travis.yml) -- [Go template](Go/travis.yml) +- [Default template](travis.yml) diff --git a/examples/templates/Travis/Gradle/travis.yml b/examples/templates/Travis/travis.yml similarity index 52% rename from examples/templates/Travis/Gradle/travis.yml rename to examples/templates/Travis/travis.yml index 9101f437..f8e09c70 100644 --- a/examples/templates/Travis/Gradle/travis.yml +++ b/examples/templates/Travis/travis.yml @@ -1,5 +1,3 @@ -language: java - jobs: include: - stage: Debricked-scan @@ -7,15 +5,16 @@ jobs: branch: "*" env: - DEBRICKED_TOKEN=${DEBRICKED_TOKEN} - before_install: - - ./gradlew dependencies > .debricked-gradle-dependencies.txt - - curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked - #https://docs.travis-ci.com/user/languages/java/#projects-using-gradle + before_install: curl -L https://github.com/debricked/cli/releases/latest/download/cli_linux_x86_64.tar.gz | tar -xz debricked before_cache: - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ cache: directories: + - $HOME/.cache/go-build + - $HOME/gopath/pkg/mod - $HOME/.gradle/caches/ - $HOME/.gradle/wrapper/ - script: ./debricked scan + - $HOME/.m2 + - $HOME/.cache/pip + script: ./debricked scan --resolve diff --git a/go.mod b/go.mod index 0b090b11..980301bf 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/bmatcuk/doublestar/v4 v4.6.0 + github.com/chelnak/ysmrr v0.2.1 github.com/fatih/color v1.15.0 github.com/go-git/go-billy/v5 v5.4.1 github.com/go-git/go-git/v5 v5.6.1 @@ -13,6 +14,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.2 + github.com/vifraa/gopom v0.2.1 ) require ( @@ -46,6 +48,7 @@ require ( github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect golang.org/x/crypto v0.7.0 // indirect diff --git a/go.sum b/go.sum index ac202245..860a504e 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvz github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chelnak/ysmrr v0.2.1 h1:9xLbVcrgnvEFovFAPnDiTCtxHiuLmz03xCg5OUgdOfc= +github.com/chelnak/ysmrr v0.2.1/go.mod h1:9TEgLy2xDMGN62zJm9XZrEWY/fHoGoBslSVEkEpRCXk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -246,11 +248,13 @@ github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -260,6 +264,8 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/vifraa/gopom v0.2.1 h1:MYVMAMyiGzXPPy10EwojzKIL670kl5Zbae+o3fFvQEM= +github.com/vifraa/gopom v0.2.1/go.mod h1:oPa1dcrGrtlO37WPDBm5SqHAT+wTgF8An1Q71Z6Vv4o= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/callgraph/config/config.go b/pkg/callgraph/config/config.go new file mode 100644 index 00000000..fb4c1bd5 --- /dev/null +++ b/pkg/callgraph/config/config.go @@ -0,0 +1,33 @@ +package config + +type IConfig interface { + Language() string + Args() []string + Kwargs() map[string]string +} + +type Config struct { + language string + args []string + kwargs map[string]string +} + +func NewConfig(language string, args []string, kwargs map[string]string) Config { + return Config{ + language, + args, + kwargs, + } +} + +func (c Config) Language() string { + return c.language +} + +func (c Config) Args() []string { + return c.args +} + +func (c Config) Kwargs() map[string]string { + return c.kwargs +} diff --git a/pkg/callgraph/generation.go b/pkg/callgraph/generation.go new file mode 100644 index 00000000..9f7c5557 --- /dev/null +++ b/pkg/callgraph/generation.go @@ -0,0 +1,30 @@ +package callgraph + +import "github.com/debricked/cli/pkg/callgraph/job" + +type IGeneration interface { + Jobs() []job.IJob + HasErr() bool +} + +type Generation struct { + jobs []job.IJob +} + +func NewGeneration(jobs []job.IJob) Generation { + return Generation{jobs} +} + +func (g Generation) Jobs() []job.IJob { + return g.jobs +} + +func (g Generation) HasErr() bool { + for _, j := range g.Jobs() { + if j.Errors().HasError() { + return true + } + } + + return false +} diff --git a/pkg/callgraph/generation_test.go b/pkg/callgraph/generation_test.go new file mode 100644 index 00000000..9631d0d7 --- /dev/null +++ b/pkg/callgraph/generation_test.go @@ -0,0 +1,55 @@ +package callgraph + +import ( + "errors" + "testing" + + "github.com/debricked/cli/pkg/callgraph/job" + "github.com/debricked/cli/pkg/callgraph/job/testdata" + "github.com/stretchr/testify/assert" +) + +const testDir = "dir" + +var testFiles = []string{"file"} + +func TestNewGeneration(t *testing.T) { + res := NewGeneration(nil) + assert.NotNil(t, res) + + res = NewGeneration([]job.IJob{}) + assert.NotNil(t, res) + + res = NewGeneration([]job.IJob{testdata.NewJobMock(testDir, testFiles)}) + assert.NotNil(t, res) + + res = NewGeneration([]job.IJob{testdata.NewJobMock(testDir, testFiles), testdata.NewJobMock(testDir, testFiles)}) + assert.NotNil(t, res) +} + +func TestJobs(t *testing.T) { + res := NewGeneration(nil) + assert.Empty(t, res.Jobs()) + + res.jobs = []job.IJob{} + assert.Len(t, res.Jobs(), 0) + + res.jobs = []job.IJob{testdata.NewJobMock(testDir, testFiles)} + assert.Len(t, res.Jobs(), 1) + + res.jobs = []job.IJob{testdata.NewJobMock(testDir, testFiles), testdata.NewJobMock(testDir, testFiles)} + assert.Len(t, res.Jobs(), 2) +} + +func TestHasError(t *testing.T) { + res := NewGeneration(nil) + assert.False(t, res.HasErr()) + + res.jobs = []job.IJob{testdata.NewJobMock(testDir, testFiles)} + assert.False(t, res.HasErr()) + + jobMock := testdata.NewJobMock(testDir, testFiles) + jobMock.SetErr(errors.New("error")) + res.jobs = append(res.jobs, jobMock) + assert.True(t, res.HasErr()) +} diff --git a/pkg/callgraph/generator.go b/pkg/callgraph/generator.go new file mode 100644 index 00000000..0570c4ff --- /dev/null +++ b/pkg/callgraph/generator.go @@ -0,0 +1,80 @@ +package callgraph + +import ( + "errors" + "fmt" + "runtime" + "time" + + "github.com/debricked/cli/pkg/callgraph/config" + "github.com/debricked/cli/pkg/callgraph/job" + "github.com/debricked/cli/pkg/callgraph/strategy" + "github.com/debricked/cli/pkg/io/finder" +) + +type IGenerator interface { + GenerateWithTimer(paths []string, exclusions []string, configs []config.IConfig, timeout int) error + Generate(paths []string, exclusions []string, configs []config.IConfig, status chan bool) (IGeneration, error) +} + +type Generator struct { + strategyFactory strategy.IFactory + scheduler IScheduler +} + +func NewGenerator( + strategyFactory strategy.IFactory, + scheduler IScheduler, +) Generator { + return Generator{ + strategyFactory, + scheduler, + } +} + +func (r Generator) GenerateWithTimer(paths []string, exclusions []string, configs []config.IConfig, timeout int) error { + status := make(chan bool) + timeoutChan := time.After(time.Duration(timeout) * time.Second) + fmt.Println("Start generation") + go r.Generate(paths, exclusions, configs, status) + select { + case <-status: + fmt.Println("Function completed successfully") + case <-timeoutChan: + fmt.Println("Function timed out") + // use the runtime package to kill the goroutine + runtime.Goexit() + return errors.New("Timeout reached, termingating generate callgraph goroutine") + } + + return nil +} + +func (r Generator) Generate(paths []string, exclusions []string, configs []config.IConfig, status chan bool) (IGeneration, error) { + targetPath := ".debrickedTmpFolder" + debrickedExclusions := []string{targetPath} + exclusions = append(exclusions, debrickedExclusions...) + files, err := finder.FindFiles(paths, exclusions) + finder := finder.Finder{} + + var jobs []job.IJob + for _, config := range configs { + s, strategyErr := r.strategyFactory.Make(config, files, finder) + if strategyErr == nil { + newJobs, err := s.Invoke() + if err != nil { + return nil, err + } + jobs = append(jobs, newJobs...) + } + } + + generation, err := r.scheduler.Schedule(jobs) + + select { + case status <- true: + default: + } + + return generation, err +} diff --git a/pkg/callgraph/generator_test.go b/pkg/callgraph/generator_test.go new file mode 100644 index 00000000..8a9c7fa4 --- /dev/null +++ b/pkg/callgraph/generator_test.go @@ -0,0 +1,80 @@ +package callgraph + +import ( + "errors" + "testing" + + "github.com/debricked/cli/pkg/callgraph/config" + strategyTestdata "github.com/debricked/cli/pkg/callgraph/strategy/testdata" + "github.com/stretchr/testify/assert" +) + +const ( + workers = 10 + goModFile = "go.mod" +) + +func TestNewGenerator(t *testing.T) { + r := NewGenerator( + strategyTestdata.NewStrategyFactoryMock(), + NewScheduler(workers), + ) + assert.NotNil(t, r) +} + +func TestGenerate(t *testing.T) { + r := NewGenerator( + strategyTestdata.NewStrategyFactoryMock(), + NewScheduler(workers), + ) + + var status chan bool + configs := []config.IConfig{ + config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}), + } + res, err := r.Generate([]string{"../../go.mod"}, nil, configs, status) + assert.NotEmpty(t, res.Jobs()) + assert.NoError(t, err) +} + +func TestGenerateInvokeError(t *testing.T) { + r := NewGenerator( + strategyTestdata.NewStrategyFactoryErrorMock(), + NewScheduler(workers), + ) + + var status chan bool + configs := []config.IConfig{ + config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}), + } + _, err := r.Generate([]string{"../../go.mod"}, nil, configs, status) + assert.NotNil(t, err) +} + +func TestGenerateScheduleError(t *testing.T) { + errAssertion := errors.New("error") + r := NewGenerator( + strategyTestdata.NewStrategyFactoryMock(), + SchedulerMock{Err: errAssertion}, + ) + + var status chan bool + configs := []config.IConfig{ + config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}), + } + res, err := r.Generate([]string{"../../go.mod"}, nil, configs, status) + assert.NotEmpty(t, res.Jobs()) + assert.ErrorIs(t, err, errAssertion) +} + +func TestGenerateDirWithoutConfig(t *testing.T) { + r := NewGenerator( + strategyTestdata.NewStrategyFactoryMock(), + SchedulerMock{}, + ) + + var status chan bool + res, err := r.Generate([]string{"invalid-dir"}, nil, nil, status) + assert.Empty(t, res.Jobs()) + assert.NoError(t, err) +} diff --git a/pkg/callgraph/job/base_job.go b/pkg/callgraph/job/base_job.go new file mode 100644 index 00000000..e7a1c9ad --- /dev/null +++ b/pkg/callgraph/job/base_job.go @@ -0,0 +1,53 @@ +package job + +import ( + "errors" + "os/exec" + + err "github.com/debricked/cli/pkg/io/err" +) + +type BaseJob struct { + dir string + files []string + errs err.IErrors + status chan string +} + +func NewBaseJob(dir string, files []string) BaseJob { + return BaseJob{ + dir: dir, + files: files, + errs: err.NewErrors(dir), + status: make(chan string), + } +} + +func (j *BaseJob) GetDir() string { + return j.dir +} + +func (j *BaseJob) GetFiles() []string { + return j.files +} + +func (j *BaseJob) Errors() err.IErrors { + return j.errs +} + +func (j *BaseJob) ReceiveStatus() chan string { + return j.status +} + +func (j *BaseJob) SendStatus(status string) { + j.status <- status +} + +func (j *BaseJob) GetExitError(err error) error { + exitErr, ok := err.(*exec.ExitError) + if !ok { + return err + } + + return errors.New(string(exitErr.Stderr)) +} diff --git a/pkg/callgraph/job/base_job_test.go b/pkg/callgraph/job/base_job_test.go new file mode 100644 index 00000000..b42cd960 --- /dev/null +++ b/pkg/callgraph/job/base_job_test.go @@ -0,0 +1,102 @@ +package job + +import ( + "errors" + "os/exec" + "testing" + + err "github.com/debricked/cli/pkg/io/err" + "github.com/stretchr/testify/assert" +) + +const testDir = "dir" + +var testFiles = []string{"file"} + +func TestNewBaseJob(t *testing.T) { + j := NewBaseJob(testDir, testFiles) + assert.Equal(t, testFiles, j.GetFiles()) + assert.Equal(t, testDir, j.GetDir()) + assert.NotNil(t, j.Errors()) + assert.NotNil(t, j.status) +} + +func TestGetFiles(t *testing.T) { + j := BaseJob{} + j.files = testFiles + assert.Equal(t, testFiles, j.GetFiles()) +} + +func TestGetDir(t *testing.T) { + j := BaseJob{} + j.dir = testDir + assert.Equal(t, testDir, j.GetDir()) +} + +func TestReceiveStatus(t *testing.T) { + j := BaseJob{ + files: testFiles, + dir: testDir, + errs: nil, + status: make(chan string), + } + + statusChan := j.ReceiveStatus() + assert.NotNil(t, statusChan) +} + +func TestErrors(t *testing.T) { + jobErr := errors.New("error") + j := BaseJob{} + j.dir = testDir + j.errs = err.NewErrors(j.dir) + j.errs.Critical(jobErr) + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), jobErr) +} + +func TestSendStatus(t *testing.T) { + j := BaseJob{ + files: testFiles, + dir: testDir, + errs: nil, + status: make(chan string), + } + + go func() { + status := <-j.ReceiveStatus() + assert.Equal(t, "status", status) + }() + + j.SendStatus("status") +} + +func TestDifferentNewBaseJob(t *testing.T) { + differentDir := "testDifferentDir" + differentFiles := []string{"testDifferentFile"} + j := NewBaseJob(differentDir, differentFiles) + assert.NotEqual(t, testFiles, j.GetFiles()) + assert.Equal(t, differentFiles, j.GetFiles()) + assert.NotEqual(t, testDir, j.GetDir()) + assert.Equal(t, differentDir, j.GetDir()) + assert.NotNil(t, j.Errors()) + assert.NotNil(t, j.status) +} + +func TestGetExitErrorWithExitError(t *testing.T) { + err := &exec.ExitError{ + ProcessState: nil, + Stderr: []byte("stderr"), + } + j := BaseJob{} + exitErr := j.GetExitError(err) + assert.ErrorContains(t, exitErr, string(err.Stderr)) +} + +func TestGetExitErrorWithNoneExitError(t *testing.T) { + err := &exec.Error{Err: errors.New("none-exit-err")} + j := BaseJob{} + exitErr := j.GetExitError(err) + assert.ErrorContains(t, exitErr, err.Error()) +} diff --git a/pkg/callgraph/job/job.go b/pkg/callgraph/job/job.go new file mode 100644 index 00000000..6fbfd18c --- /dev/null +++ b/pkg/callgraph/job/job.go @@ -0,0 +1,11 @@ +package job + +import error "github.com/debricked/cli/pkg/io/err" + +type IJob interface { + GetFiles() []string + GetDir() string + Errors() error.IErrors + Run() + ReceiveStatus() chan string +} diff --git a/pkg/callgraph/job/testdata/job_mock.go b/pkg/callgraph/job/testdata/job_mock.go new file mode 100644 index 00000000..72ba0940 --- /dev/null +++ b/pkg/callgraph/job/testdata/job_mock.go @@ -0,0 +1,47 @@ +package testdata + +import ( + "fmt" + + "github.com/debricked/cli/pkg/io/err" +) + +type JobMock struct { + dir string + files []string + errs err.IErrors + status chan string +} + +func (j *JobMock) ReceiveStatus() chan string { + return j.status +} + +func (j *JobMock) GetDir() string { + return j.dir +} + +func (j *JobMock) GetFiles() []string { + return j.files +} + +func (j *JobMock) Errors() err.IErrors { + return j.errs +} + +func (j *JobMock) Run() { + fmt.Println("job mock run") +} + +func NewJobMock(dir string, files []string) *JobMock { + return &JobMock{ + dir: dir, + files: files, + status: make(chan string), + errs: err.NewErrors(dir), + } +} + +func (j *JobMock) SetErr(err err.IError) { + j.errs.Critical(err) +} diff --git a/pkg/callgraph/job/testdata/job_test_util.go b/pkg/callgraph/job/testdata/job_test_util.go new file mode 100644 index 00000000..ed7991b7 --- /dev/null +++ b/pkg/callgraph/job/testdata/job_test_util.go @@ -0,0 +1,31 @@ +package testdata + +import ( + "fmt" + "runtime" + "testing" + + "github.com/debricked/cli/pkg/callgraph/job" + "github.com/debricked/cli/pkg/io/err" + "github.com/stretchr/testify/assert" +) + +func AssertPathErr(t *testing.T, jobErrs err.IErrors) { + var path string + if runtime.GOOS == "windows" { + path = "%PATH%" + } else { + path = "$PATH" + } + errs := jobErrs.GetAll() + assert.Len(t, errs, 1) + err := errs[0] + errMsg := fmt.Sprintf("executable file not found in %s", path) + assert.ErrorContains(t, err, errMsg) +} + +func WaitStatus(j job.IJob) { + for { + <-j.ReceiveStatus() + } +} diff --git a/pkg/callgraph/language/java11/callgraph.go b/pkg/callgraph/language/java11/callgraph.go new file mode 100644 index 00000000..2ffa7e9c --- /dev/null +++ b/pkg/callgraph/language/java11/callgraph.go @@ -0,0 +1,55 @@ +package java + +import ( + "embed" + "io/ioutil" + "os" + "path/filepath" +) + +//go:embed embeded/SootWrapper.jar +var jarCallGraph embed.FS + +type Callgraph struct { + cmdFactory ICmdFactory + workingDirectory string + targetClasses string + targetDir string +} + +func (cg *Callgraph) runCallGraphWithSetup() error { + jarFile, err := jarCallGraph.Open("embeded/SootWrapper.jar") + if err != nil { + return err + } + defer jarFile.Close() + + tempDir, err := ioutil.TempDir("", "jar") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + tempJarFile := filepath.Join(tempDir, "SootWrapper.jar") + + jarBytes, err := ioutil.ReadAll(jarFile) + if err != nil { + return err + } + + err = ioutil.WriteFile(tempJarFile, jarBytes, 0644) + if err != nil { + return err + } + + return cg.runCallGraph(tempJarFile) +} + +func (cg *Callgraph) runCallGraph(callgraphJarPath string) error { + cmd, err := cg.cmdFactory.MakeCallGraphGenerationCmd(callgraphJarPath, cg.workingDirectory, cg.targetClasses, cg.targetDir) + if err != nil { + return err + } + _, err = cmd.Output() + + return err +} diff --git a/pkg/callgraph/language/java11/cmd_factory.go b/pkg/callgraph/language/java11/cmd_factory.go new file mode 100644 index 00000000..6ce350d4 --- /dev/null +++ b/pkg/callgraph/language/java11/cmd_factory.go @@ -0,0 +1,82 @@ +package java + +import ( + "os/exec" +) + +type ICmdFactory interface { + MakeGradleCopyDependenciesCmd(workingDirectory string, gradlew string, groovyFilePath string) (*exec.Cmd, error) + MakeMvnCopyDependenciesCmd(workingDirectory string, targetDir string) (*exec.Cmd, error) + MakeCallGraphGenerationCmd(callgraphJarPath string, workingDirectory string, targetClasses string, dependencyClasses string) (*exec.Cmd, error) +} + +type CmdFactory struct{} + +func (_ CmdFactory) MakeGradleCopyDependenciesCmd( + workingDirectory string, + gradlew string, + groovyFilePath string, +) (*exec.Cmd, error) { + path, err := exec.LookPath(gradlew) + + // TargetDir already in groovy script + return &exec.Cmd{ + Path: path, + Args: []string{ + gradlew, + "-b", + groovyFilePath, + "-q", + "debrickedCopyDependencies", + }, + Dir: workingDirectory, + }, err +} + +func (_ CmdFactory) MakeMvnCopyDependenciesCmd( + workingDirectory string, + targetDir string, +) (*exec.Cmd, error) { + path, err := exec.LookPath("mvn") + + args := []string{ + "mvn", + "-q", + "-B", + "dependency:copy-dependencies", + "-DoutputDirectory=" + targetDir, + "-DskipTests", + } + + return &exec.Cmd{ + Path: path, + Args: args, + Dir: workingDirectory, + }, err +} + +func (_ CmdFactory) MakeCallGraphGenerationCmd( + callgraphJarPath string, + workingDirectory string, + targetClasses string, + dependencyClasses string, +) (*exec.Cmd, error) { + path, err := exec.LookPath("java") + args := []string{ + "java", + "-jar", + callgraphJarPath, + "-u", + targetClasses, + "-l", + dependencyClasses, + "-f", + ".debricked-call-graph", + } + + return &exec.Cmd{ + Path: path, + Args: args, + Dir: workingDirectory, + }, err +} diff --git a/pkg/callgraph/language/java11/cmd_factory_test.go b/pkg/callgraph/language/java11/cmd_factory_test.go new file mode 100644 index 00000000..71d9ea24 --- /dev/null +++ b/pkg/callgraph/language/java11/cmd_factory_test.go @@ -0,0 +1,47 @@ +package java + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMakeGradleCopyDependenciesCmd(t *testing.T) { + gradlew := "gradlew" + groovyFilePath := "groovyfilename" + cmd, err := CmdFactory{}.MakeGradleCopyDependenciesCmd(dir, gradlew, groovyFilePath) + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "gradlew") + assert.Contains(t, args, "groovyfilename") + assert.ErrorContains(t, err, "executable file not found in") + assert.ErrorContains(t, err, "PATH") +} + +func TestMakeMvnCopyDependenciesCmd(t *testing.T) { + targetDir := "target" + cmd, _ := CmdFactory{}.MakeMvnCopyDependenciesCmd(dir, targetDir) + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "mvn") + assert.Contains(t, args, "-q") + assert.Contains(t, args, "-B") + assert.Contains(t, args, "dependency:copy-dependencies") + assert.Contains(t, args, "-DoutputDirectory=target") +} + +func TestMakeCallGraphGenerationCmd(t *testing.T) { + jarPath := "jarpath" + targetClasses := "targetclasses" + dependencyClasses := "dependencypath" + cmd, err := CmdFactory{}.MakeCallGraphGenerationCmd(jarPath, dir, targetClasses, dependencyClasses) + + assert.NoError(t, err) + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "java") + assert.Contains(t, args, "-jar") + assert.Contains(t, args, "jarpath") + assert.Contains(t, args, "targetclasses") + assert.Contains(t, args, "dependencypath") +} diff --git a/pkg/callgraph/language/java11/embeded/SootWrapper.jar b/pkg/callgraph/language/java11/embeded/SootWrapper.jar new file mode 100644 index 00000000..dea12e1d Binary files /dev/null and b/pkg/callgraph/language/java11/embeded/SootWrapper.jar differ diff --git a/pkg/callgraph/language/java11/embeded/gradle-script.groovy b/pkg/callgraph/language/java11/embeded/gradle-script.groovy new file mode 100644 index 00000000..040e00fb --- /dev/null +++ b/pkg/callgraph/language/java11/embeded/gradle-script.groovy @@ -0,0 +1,8 @@ + + +allprojects{ + task debrickedCopyDependencies(type: Copy) { + into ".debrickedTmpDir" + from configurations.default + } +} \ No newline at end of file diff --git a/pkg/callgraph/language/java11/job.go b/pkg/callgraph/language/java11/job.go new file mode 100644 index 00000000..dc52d95f --- /dev/null +++ b/pkg/callgraph/language/java11/job.go @@ -0,0 +1,89 @@ +package java + +import ( + "embed" + "os" + "os/exec" + "path" + + conf "github.com/debricked/cli/pkg/callgraph/config" + "github.com/debricked/cli/pkg/callgraph/job" + gfinder "github.com/debricked/cli/pkg/io/finder/gradle" + ioWriter "github.com/debricked/cli/pkg/io/writer" +) + +const ( + maven = "maven" + gradle = "gradle" +) + +type Job struct { + job.BaseJob + cmdFactory ICmdFactory + config conf.IConfig +} + +func NewJob(dir string, files []string, cmdFactory ICmdFactory, writer ioWriter.IFileWriter, config conf.IConfig) *Job { + return &Job{ + BaseJob: job.NewBaseJob(dir, files), + cmdFactory: cmdFactory, + config: config, + } +} + +//go:embed embeded/gradle-script.groovy +var gradleInitScript embed.FS + +func (j *Job) Run() { + workingDirectory := j.GetDir() + targetClasses := j.GetFiles()[0] + dependencyDir := ".debrickedTmpFolder" + targetDir := path.Join(workingDirectory, dependencyDir) + pmConfig := j.config.Kwargs()["pm"] + + // If folder doesn't exist, copy dependencies + if _, err := os.Stat(targetDir); os.IsNotExist(err) { + var cmd *exec.Cmd + if pmConfig == gradle { + targetGradlew := path.Join(workingDirectory, "gradlew") + gradlew := "gradle" + if _, err := os.Stat(targetGradlew); os.IsExist(err) { + gradlew = targetGradlew + } + + groovyFilePath := path.Join(workingDirectory, ".debrickedGroovyScript.groovy") + ish := gfinder.NewScriptHandler(groovyFilePath, "embeded/gradle-script.groovy", ioWriter.FileWriter{}) + ish.WriteInitFile() + + cmd, err = j.cmdFactory.MakeGradleCopyDependenciesCmd(workingDirectory, gradlew, groovyFilePath) + } else { + cmd, err = j.cmdFactory.MakeMvnCopyDependenciesCmd(workingDirectory, targetDir) + } + j.SendStatus("copying external dep jars to target folder" + targetDir) + if err != nil { + j.Errors().Critical(err) + + return + } + _, err = cmd.Output() + + if err != nil { + j.Errors().Critical(err) + + return + } + } + + j.SendStatus("generating call graph") + callgraph := Callgraph{ + cmdFactory: j.cmdFactory, + workingDirectory: workingDirectory, + targetClasses: targetClasses, + targetDir: targetDir, + } + err := callgraph.runCallGraphWithSetup() + + if err != nil { + j.Errors().Critical(err) + } +} diff --git a/pkg/callgraph/language/java11/job_test.go b/pkg/callgraph/language/java11/job_test.go new file mode 100644 index 00000000..5f0d0367 --- /dev/null +++ b/pkg/callgraph/language/java11/job_test.go @@ -0,0 +1,147 @@ +package java + +import ( + "errors" + "fmt" + "testing" + + conf "github.com/debricked/cli/pkg/callgraph/config" + jobTestdata "github.com/debricked/cli/pkg/callgraph/job/testdata" + "github.com/debricked/cli/pkg/callgraph/language/java11/testdata" + ioWriter "github.com/debricked/cli/pkg/io/writer" + writerTestdata "github.com/debricked/cli/pkg/io/writer/testdata" + "github.com/stretchr/testify/assert" +) + +const ( + badName = "bad-name" + dir = "dir" +) + +var files = []string{"file"} + +func TestNewJob(t *testing.T) { + cmdFactoryMock := testdata.NewEchoCmdFactory() + writer := ioWriter.FileWriter{} + config := conf.Config{} + j := NewJob(dir, files, cmdFactoryMock, writer, config) + assert.Equal(t, []string{"file"}, j.GetFiles()) + assert.Equal(t, "dir", j.GetDir()) + assert.False(t, j.Errors().HasError()) +} + +// func TestRunMakeGradleCopyDependenciesCmdErr(t *testing.T) { +// cmdErr := errors.New("cmd-error") +// cmdFactoryMock := testdata.NewEchoCmdFactory() +// cmdFactoryMock.GradleCopyDepErr = cmdErr +// fileWriterMock := &writerTestdata.FileWriterMock{} +// config := conf.NewConfig("java", nil, map[string]string{"pm": gradle}) +// j := NewJob(dir, files, cmdFactoryMock, fileWriterMock, config) + +// go jobTestdata.WaitStatus(j) +// j.Run() + +// assert.Len(t, j.Errors().GetAll(), 1) +// assert.Contains(t, j.Errors().GetAll(), cmdErr) +// } + +// func TestRunMakeGradleCopyDependenciesOutputErr(t *testing.T) { +// cmdMock := testdata.NewEchoCmdFactory() +// cmdMock.GradleCopyDepName = badName +// cmdFactoryMock := testdata.NewEchoCmdFactory() +// fileWriterMock := &writerTestdata.FileWriterMock{} +// config := conf.NewConfig("java", nil, map[string]string{"pm": gradle}) +// j := NewJob(dir, files, cmdFactoryMock, fileWriterMock, config) + +// go jobTestdata.WaitStatus(j) +// j.Run() + +// jobTestdata.AssertPathErr(t, j.Errors()) +// } + +func TestRunMakeMavenCopyDependenciesCmdErr(t *testing.T) { + cmdErr := errors.New("cmd-error") + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.MvnCopyDepErr = cmdErr + fileWriterMock := &writerTestdata.FileWriterMock{} + config := conf.NewConfig("java", nil, map[string]string{"pm": maven}) + j := NewJob(dir, files, cmdFactoryMock, fileWriterMock, config) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), cmdErr) +} + +// TODO need to update this with callgraph mock? +// func TestRunMakeMavenCopyDependenciesOutputErr(t *testing.T) { +// cmdMock := testdata.NewEchoCmdFactory() +// cmdMock.MvnCopyDepName = badName +// cmdFactoryMock := testdata.NewEchoCmdFactory() +// fileWriterMock := &writerTestdata.FileWriterMock{} +// config := conf.NewConfig("java", nil, map[string]string{"pm": maven}) +// j := NewJob(dir, files, cmdFactoryMock, fileWriterMock, config) + +// go jobTestdata.WaitStatus(j) +// j.Run() + +// fmt.Println(j.Errors()) +// jobTestdata.AssertPathErr(t, j.Errors()) +// } + +func TestRun(t *testing.T) { + fileWriterMock := &writerTestdata.FileWriterMock{} + cmdFactoryMock := testdata.NewEchoCmdFactory() + config := conf.NewConfig("java", nil, map[string]string{"pm": maven}) + j := NewJob(dir, files, cmdFactoryMock, fileWriterMock, config) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.False(t, j.Errors().HasError()) + fmt.Println(string(fileWriterMock.Contents)) + assert.False(t, false) +} + +// func TestRunCreateErr(t *testing.T) { +// createErr := errors.New("create-error") +// fileWriterMock := &writerTestdata.FileWriterMock{CreateErr: createErr} +// cmdFactoryMock := testdata.NewEchoCmdFactory() +// config := conf.NewConfig("java", nil, map[string]string{"pm": maven}) +// j := NewJob(dir, files, cmdFactoryMock, fileWriterMock, config) + +// go jobTestdata.WaitStatus(j) +// j.Run() + +// assert.Len(t, j.Errors().GetAll(), 1) +// assert.Contains(t, j.Errors().GetAll(), createErr) +// } + +// func TestRunWriteErr(t *testing.T) { +// writeErr := errors.New("write-error") +// fileWriterMock := &writerTestdata.FileWriterMock{WriteErr: writeErr} +// cmdFactoryMock := testdata.NewEchoCmdFactory() +// config := conf.NewConfig("java", nil, map[string]string{"pm": maven}) +// j := NewJob(dir, files, cmdFactoryMock, fileWriterMock, config) + +// go jobTestdata.WaitStatus(j) +// j.Run() + +// assert.Len(t, j.Errors().GetAll(), 1) +// assert.Contains(t, j.Errors().GetAll(), writeErr) +// } + +// func TestRunCloseErr(t *testing.T) { +// closeErr := errors.New("close-error") +// fileWriterMock := &writerTestdata.FileWriterMock{CloseErr: closeErr} +// cmdFactoryMock := testdata.NewEchoCmdFactory() +// config := conf.NewConfig("java", nil, map[string]string{"pm": maven}) +// j := NewJob(dir, files, cmdFactoryMock, fileWriterMock, config) + +// go jobTestdata.WaitStatus(j) +// j.Run() + +// assert.Len(t, j.Errors().GetAll(), 1) +// assert.Contains(t, j.Errors().GetAll(), closeErr) +// } diff --git a/pkg/callgraph/language/java11/language.go b/pkg/callgraph/language/java11/language.go new file mode 100644 index 00000000..c782f4d8 --- /dev/null +++ b/pkg/callgraph/language/java11/language.go @@ -0,0 +1,24 @@ +package java + +const Name = "java" +const StandardVersion = "11" + +type Language struct { + name string + version string +} + +func NewLanguage() Language { + return Language{ + name: Name, + version: StandardVersion, + } +} + +func (language Language) Name() string { + return language.name +} + +func (language Language) Version() string { + return language.version +} diff --git a/pkg/callgraph/language/java11/language_test.go b/pkg/callgraph/language/java11/language_test.go new file mode 100644 index 00000000..bd4cabe0 --- /dev/null +++ b/pkg/callgraph/language/java11/language_test.go @@ -0,0 +1,23 @@ +package java + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewLanguage(t *testing.T) { + pm := NewLanguage() + assert.Equal(t, Name, pm.name) + assert.Equal(t, StandardVersion, pm.version) +} + +func TestName(t *testing.T) { + pm := NewLanguage() + assert.Equal(t, Name, pm.Name()) +} + +func TestVersion(t *testing.T) { + pm := NewLanguage() + assert.Equal(t, StandardVersion, pm.Version()) +} diff --git a/pkg/callgraph/language/java11/stategy.go b/pkg/callgraph/language/java11/stategy.go new file mode 100644 index 00000000..6008ea7d --- /dev/null +++ b/pkg/callgraph/language/java11/stategy.go @@ -0,0 +1,93 @@ +package java + +import ( + "fmt" + "log" + "path/filepath" + + conf "github.com/debricked/cli/pkg/callgraph/config" + "github.com/debricked/cli/pkg/callgraph/job" + "github.com/debricked/cli/pkg/io/finder" + "github.com/debricked/cli/pkg/io/writer" + "github.com/fatih/color" +) + +type Strategy struct { + config conf.IConfig + files []string + finder finder.IFinder +} + +func (s Strategy) Invoke() ([]job.IJob, error) { + var jobs []job.IJob + // Filter relevant files + + if s.config == nil { + strategyWarning("No config is setup") + return jobs, nil + } + + pmConfig := s.config.Kwargs()["pm"] + + var roots []string + var err error + switch pmConfig { + case gradle: + roots, err = s.finder.FindGradleRoots(s.files) + case maven: + roots, err = s.finder.FindMavenRoots(s.files) + default: + roots, err = s.finder.FindMavenRoots(s.files) + } + + if err != nil { + strategyWarning("Error while finding roots: " + err.Error()) + return jobs, nil + } + + // TODO: If we want to build, build jobs need to execute before trying to find javaClassDirs. + // If not, mapping between roots and classes could get wonky + // Perfect time to build after getting roots, and maybe if no classes are found? + + classDirs, _ := s.finder.FindJavaClassDirs(s.files) + absRoots, _ := finder.ConvertPathsToAbsPaths(roots) + absClassDirs, _ := finder.ConvertPathsToAbsPaths(classDirs) + rootClassMapping := finder.MapFilesToDir(absRoots, absClassDirs) + + for _, root := range absRoots { + if _, ok := rootClassMapping[root]; ok == false { + strategyWarning("Root found without related classes, make sure to build your project before running, root: " + root) + } + } + if len(rootClassMapping) == 0 { + return jobs, nil + } + + for rootFile, classDirs := range rootClassMapping { + // For each class paths dir within the root, find GCDPath as entrypoint + classDir := finder.GCDPath(classDirs) + rootDir := filepath.Dir(rootFile) + jobs = append(jobs, NewJob( + rootDir, + []string{classDir}, + CmdFactory{}, + writer.FileWriter{}, + s.config, + ), + ) + } + + return jobs, nil +} + +func NewStrategy(config conf.IConfig, files []string, finder finder.IFinder) Strategy { + return Strategy{config, files, finder} +} + +func strategyWarning(errMsg string) { + err := fmt.Errorf(errMsg) + warningColor := color.New(color.FgYellow, color.Bold).SprintFunc() + defaultOutputWriter := log.Writer() + log.Println(warningColor("Warning: ") + err.Error()) + log.SetOutput(defaultOutputWriter) +} diff --git a/pkg/callgraph/language/java11/strategy_test.go b/pkg/callgraph/language/java11/strategy_test.go new file mode 100644 index 00000000..dfa61aa0 --- /dev/null +++ b/pkg/callgraph/language/java11/strategy_test.go @@ -0,0 +1,82 @@ +package java + +import ( + "os" + "path/filepath" + "testing" + + "github.com/debricked/cli/pkg/callgraph/config" + "github.com/debricked/cli/pkg/io/finder/testdata" + "github.com/stretchr/testify/assert" +) + +func TestNewStrategy(t *testing.T) { + s := NewStrategy(nil, nil, nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy(nil, []string{}, nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy(nil, []string{"file"}, nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 1) + + s = NewStrategy(nil, []string{"file-1", "file-2"}, nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 2) + + conf := config.NewConfig("java", []string{"arg1"}, map[string]string{"kwarg": "val"}) + finder := testdata.NewEmptyFinderMock() + testFiles := []string{"file-1"} + finder.FindMavenRootsNames = testFiles + s = NewStrategy(conf, testFiles, finder) + assert.NotNil(t, s) + assert.Len(t, s.files, 1) + assert.Equal(t, s.config, conf) +} + +func TestInvokeNoFiles(t *testing.T) { + s := NewStrategy(nil, []string{}, nil) + jobs, _ := s.Invoke() + assert.Empty(t, jobs) +} + +func TestInvokeOneFile(t *testing.T) { + conf := config.NewConfig("java", []string{"arg1"}, map[string]string{"kwarg": "val"}) + finder := testdata.NewEmptyFinderMock() + testFiles := []string{"file-1"} + finder.FindMavenRootsNames = testFiles + s := NewStrategy(conf, testFiles, finder) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 0) +} + +func TestInvokeManyFiles(t *testing.T) { + conf := config.NewConfig("java", []string{"arg1"}, map[string]string{"kwarg": "val"}) + finder := testdata.NewEmptyFinderMock() + testFiles := []string{"file-1", "file-2"} + finder.FindMavenRootsNames = testFiles + s := NewStrategy(conf, testFiles, finder) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 0) +} + +func TestInvokeManyFilesWCorrectFilters(t *testing.T) { + conf := config.NewConfig("java", []string{"arg1"}, map[string]string{"kwarg": "val"}) + finder := testdata.NewEmptyFinderMock() + testFiles := []string{"file-1", "file-2", "file-3"} + finder.FindMavenRootsNames = []string{"file-3/pom.xml"} + finder.FindJavaClassDirsNames = []string{"file-3/test.class"} + s := NewStrategy(conf, testFiles, finder) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 1) + for _, job := range jobs { + file, _ := filepath.Abs("file-3/") + dir, _ := filepath.Abs("file-3/") + assert.Equal(t, job.GetFiles(), []string{file + string(os.PathSeparator)}) // Get files return gcd path + assert.Equal(t, job.GetDir(), dir) + + } +} diff --git a/pkg/callgraph/language/java11/testdata/cmd_factory_mock.go b/pkg/callgraph/language/java11/testdata/cmd_factory_mock.go new file mode 100644 index 00000000..32076ff1 --- /dev/null +++ b/pkg/callgraph/language/java11/testdata/cmd_factory_mock.go @@ -0,0 +1,32 @@ +package testdata + +import "os/exec" + +type CmdFactoryMock struct { + GradleCopyDepName string + GradleCopyDepErr error + MvnCopyDepName string + MvnCopyDepErr error + CallGraphGenName string + CallGraphGenErr error +} + +func NewEchoCmdFactory() CmdFactoryMock { + return CmdFactoryMock{ + GradleCopyDepName: "echo", + MvnCopyDepName: "echo", + CallGraphGenName: "echo", + } +} + +func (f CmdFactoryMock) MakeGradleCopyDependenciesCmd(_ string, _ string, _ string) (*exec.Cmd, error) { + return exec.Command(f.GradleCopyDepName, "GradleCopyDep"), f.GradleCopyDepErr +} + +func (f CmdFactoryMock) MakeMvnCopyDependenciesCmd(_ string, _ string) (*exec.Cmd, error) { + return exec.Command(f.MvnCopyDepName, "MvnCopyDep"), f.MvnCopyDepErr +} + +func (f CmdFactoryMock) MakeCallGraphGenerationCmd(_ string, _ string, _ string, _ string) (*exec.Cmd, error) { + return exec.Command(f.CallGraphGenName, "CallGraphGen"), f.CallGraphGenErr +} diff --git a/pkg/callgraph/language/language.go b/pkg/callgraph/language/language.go new file mode 100644 index 00000000..bf44bc4e --- /dev/null +++ b/pkg/callgraph/language/language.go @@ -0,0 +1,14 @@ +package language + +import java "github.com/debricked/cli/pkg/callgraph/language/java11" + +type ILanguage interface { + Name() string + Version() string +} + +func Languages() []ILanguage { + return []ILanguage{ + java.NewLanguage(), + } +} diff --git a/pkg/callgraph/language/language_test.go b/pkg/callgraph/language/language_test.go new file mode 100644 index 00000000..46c41d49 --- /dev/null +++ b/pkg/callgraph/language/language_test.go @@ -0,0 +1,24 @@ +package language + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLanguages(t *testing.T) { + langs := Languages() + langNames := []string{ + "java", + } + + for _, langName := range langNames { + t.Run(langName, func(t *testing.T) { + contains := false + for _, pm := range langs { + contains = contains || pm.Name() == langName + } + assert.Truef(t, contains, "failed to assert that %s was returned in Languages()", langName) + }) + } +} diff --git a/pkg/callgraph/scheduler.go b/pkg/callgraph/scheduler.go new file mode 100644 index 00000000..a01347b4 --- /dev/null +++ b/pkg/callgraph/scheduler.go @@ -0,0 +1,84 @@ +package callgraph + +import ( + "sync" + + "github.com/chelnak/ysmrr" + "github.com/debricked/cli/pkg/callgraph/job" + "github.com/debricked/cli/pkg/tui" +) + +type IScheduler interface { + Schedule(jobs []job.IJob) (IGeneration, error) +} + +type queueItem struct { + job job.IJob + spinner *ysmrr.Spinner +} + +type Scheduler struct { + workers int + queue chan queueItem + waitGroup sync.WaitGroup + spinnerManager tui.ISpinnerManager +} + +const callgraph = "Callgraph" + +func NewScheduler(workers int) *Scheduler { + return &Scheduler{workers: workers, waitGroup: sync.WaitGroup{}} +} + +func (scheduler *Scheduler) Schedule(jobs []job.IJob) (IGeneration, error) { + if len(jobs) == 0 { + return NewGeneration(jobs), nil + } + + scheduler.queue = make(chan queueItem, len(jobs)) + scheduler.waitGroup.Add(len(jobs)) + scheduler.spinnerManager = tui.NewSpinnerManager() + scheduler.spinnerManager.Start() + + for _, j := range jobs { + spinner := scheduler.spinnerManager.AddSpinner(callgraph, j.GetDir()) + scheduler.queue <- queueItem{ + job: j, + spinner: spinner, + } + } + + jobIteration := 0 + // Run it in sequence + for item := range scheduler.queue { + jobIteration += 1 + go scheduler.updateStatus(item) + item.job.Run() + scheduler.finish(item) + scheduler.waitGroup.Done() + if jobIteration == len(jobs) { + close(scheduler.queue) + } + } + + scheduler.spinnerManager.Stop() + + return NewGeneration(jobs), nil +} + +func (scheduler *Scheduler) updateStatus(item queueItem) { + for { + msg := <-item.job.ReceiveStatus() + tui.SetSpinnerMessage(item.spinner, callgraph, item.job.GetDir(), msg) + } +} + +func (scheduler *Scheduler) finish(item queueItem) { + if item.job.Errors().HasError() { + tui.SetSpinnerMessage(item.spinner, callgraph, item.job.GetDir(), "failed") + item.spinner.Error() + } else { + tui.SetSpinnerMessage(item.spinner, callgraph, item.job.GetDir(), "done") + item.spinner.Complete() + } +} diff --git a/pkg/callgraph/scheduler_test.go b/pkg/callgraph/scheduler_test.go new file mode 100644 index 00000000..75264e17 --- /dev/null +++ b/pkg/callgraph/scheduler_test.go @@ -0,0 +1,74 @@ +package callgraph + +import ( + "errors" + "testing" + + "github.com/debricked/cli/pkg/callgraph/job" + "github.com/debricked/cli/pkg/callgraph/job/testdata" + "github.com/stretchr/testify/assert" +) + +type SchedulerMock struct { + Err error + JobsMock []job.IJob +} + +func (s SchedulerMock) Schedule(jobs []job.IJob) (IGeneration, error) { + if s.JobsMock != nil { + jobs = s.JobsMock + } + for _, j := range jobs { + j.Run() + } + + return NewGeneration(jobs), s.Err +} + +func TestNewScheduler(t *testing.T) { + s := NewScheduler(10) + assert.NotNil(t, s) +} + +func TestScheduler(t *testing.T) { + s := NewScheduler(10) + res, err := s.Schedule([]job.IJob{testdata.NewJobMock(testDir, testFiles)}) + assert.NoError(t, err) + assert.Len(t, res.Jobs(), 1) + + res, err = s.Schedule([]job.IJob{}) + assert.NoError(t, err) + assert.Len(t, res.Jobs(), 0) + + res, err = s.Schedule(nil) + assert.NoError(t, err) + assert.Len(t, res.Jobs(), 0) + + res, err = s.Schedule([]job.IJob{ + testdata.NewJobMock(testDir, []string{"b/b_file.json"}), + testdata.NewJobMock(testDir, []string{"a/b_file.json"}), + testdata.NewJobMock(testDir, []string{"b/a_file.json"}), + testdata.NewJobMock(testDir, []string{"a/a_file.json"}), + testdata.NewJobMock(testDir, []string{"a/a_file.json"}), + }) + assert.NoError(t, err) + jobs := res.Jobs() + + assert.Len(t, jobs, 5) + for _, j := range jobs { + assert.False(t, j.Errors().HasError()) + } +} + +func TestScheduleJobErr(t *testing.T) { + s := NewScheduler(10) + jobMock := testdata.NewJobMock(testDir, testFiles) + jobErr := errors.New("job-error") + jobMock.SetErr(jobErr) + res, err := s.Schedule([]job.IJob{jobMock}) + assert.NoError(t, err) + assert.Len(t, res.Jobs(), 1) + j := res.Jobs()[0] + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), jobErr) +} diff --git a/pkg/callgraph/strategy/strategy.go b/pkg/callgraph/strategy/strategy.go new file mode 100644 index 00000000..083742c1 --- /dev/null +++ b/pkg/callgraph/strategy/strategy.go @@ -0,0 +1,9 @@ +package strategy + +import ( + "github.com/debricked/cli/pkg/callgraph/job" +) + +type IStrategy interface { + Invoke() ([]job.IJob, error) +} diff --git a/pkg/callgraph/strategy/strategy_factory.go b/pkg/callgraph/strategy/strategy_factory.go new file mode 100644 index 00000000..0e56056c --- /dev/null +++ b/pkg/callgraph/strategy/strategy_factory.go @@ -0,0 +1,29 @@ +package strategy + +import ( + "fmt" + + conf "github.com/debricked/cli/pkg/callgraph/config" + java "github.com/debricked/cli/pkg/callgraph/language/java11" + "github.com/debricked/cli/pkg/io/finder" +) + +type IFactory interface { + Make(config conf.IConfig, paths []string, finder finder.IFinder) (IStrategy, error) +} + +type Factory struct{} + +func NewStrategyFactory() Factory { + return Factory{} +} + +func (sf Factory) Make(config conf.IConfig, paths []string, finder finder.IFinder) (IStrategy, error) { + name := config.Language() + switch name { + case java.Name: + return java.NewStrategy(config, paths, finder), nil + default: + return nil, fmt.Errorf("failed to make strategy from %s", name) + } +} diff --git a/pkg/callgraph/strategy/testdata/strategy_mock.go b/pkg/callgraph/strategy/testdata/strategy_mock.go new file mode 100644 index 00000000..b689fe0b --- /dev/null +++ b/pkg/callgraph/strategy/testdata/strategy_mock.go @@ -0,0 +1,42 @@ +package testdata + +import ( + "errors" + + "github.com/debricked/cli/pkg/callgraph/config" + "github.com/debricked/cli/pkg/callgraph/job" + "github.com/debricked/cli/pkg/callgraph/job/testdata" + "github.com/debricked/cli/pkg/io/finder" +) + +type StrategyMock struct { + config config.IConfig + files []string + finder finder.IFinder +} + +func NewStrategyMock(config config.IConfig, files []string, finder finder.IFinder) StrategyMock { + return StrategyMock{config, files, finder} +} + +func (s StrategyMock) Invoke() ([]job.IJob, error) { + var jobs []job.IJob + jobs = append(jobs, testdata.NewJobMock("dir", s.files)) + + return jobs, nil +} + +type StrategyErrorMock struct { + config config.IConfig + files []string + finder finder.IFinder +} + +func NewStrategyErrorMock(config config.IConfig, files []string, finder finder.IFinder) StrategyErrorMock { + return StrategyErrorMock{config, files, finder} +} + +func (s StrategyErrorMock) Invoke() ([]job.IJob, error) { + + return nil, errors.New("mock-error") +} diff --git a/pkg/callgraph/strategy/testdata/strategy_mock_factory.go b/pkg/callgraph/strategy/testdata/strategy_mock_factory.go new file mode 100644 index 00000000..689ea0db --- /dev/null +++ b/pkg/callgraph/strategy/testdata/strategy_mock_factory.go @@ -0,0 +1,28 @@ +package testdata + +import ( + "github.com/debricked/cli/pkg/callgraph/config" + "github.com/debricked/cli/pkg/callgraph/strategy" + "github.com/debricked/cli/pkg/io/finder" +) + +type FactoryMock struct{} + +func NewStrategyFactoryMock() FactoryMock { + return FactoryMock{} +} + +func (sf FactoryMock) Make(config config.IConfig, paths []string, finder finder.IFinder) (strategy.IStrategy, error) { + + return NewStrategyMock(config, paths, finder), nil +} + +type FactoryErrorMock struct{} + +func NewStrategyFactoryErrorMock() FactoryErrorMock { + return FactoryErrorMock{} +} + +func (sf FactoryErrorMock) Make(config config.IConfig, paths []string, finder finder.IFinder) (strategy.IStrategy, error) { + return NewStrategyErrorMock(config, paths, finder), nil +} diff --git a/pkg/callgraph/testdata/generator_mock.go b/pkg/callgraph/testdata/generator_mock.go new file mode 100644 index 00000000..6c9e6269 --- /dev/null +++ b/pkg/callgraph/testdata/generator_mock.go @@ -0,0 +1,54 @@ +package testdata + +import ( + "os" + "path/filepath" + + "github.com/debricked/cli/pkg/callgraph" + "github.com/debricked/cli/pkg/callgraph/config" + "github.com/debricked/cli/pkg/callgraph/job" +) + +type GeneratorMock struct { + Err error + files []string +} + +func (r *GeneratorMock) GenerateWithTimer(_ []string, _ []string, _ []config.IConfig, _ int) error { + return r.Err +} + +func (r *GeneratorMock) Generate(_ []string, _ []string, _ []config.IConfig, _ chan bool) (callgraph.IGeneration, error) { + for _, f := range r.files { + createdFile, err := os.Create(f) + if err != nil { + return nil, err + } + + err = createdFile.Close() + if err != nil { + return nil, err + } + } + + return callgraph.NewGeneration([]job.IJob{}), r.Err +} + +func (r *GeneratorMock) SetFiles(files []string) { + r.files = files +} + +func (r *GeneratorMock) CleanUp() error { + for _, f := range r.files { + abs, err := filepath.Abs(f) + if err != nil { + return err + } + err = os.Remove(abs) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/client/deb_client.go b/pkg/client/deb_client.go index 727b1226..82f73133 100644 --- a/pkg/client/deb_client.go +++ b/pkg/client/deb_client.go @@ -13,6 +13,7 @@ type IDebClient interface { Post(uri string, contentType string, body *bytes.Buffer) (*http.Response, error) // Get makes a GET request to one of Debricked's API endpoints Get(uri string, format string) (*http.Response, error) + SetAccessToken(accessToken *string) } type DebClient struct { @@ -23,12 +24,6 @@ type DebClient struct { } func NewDebClient(accessToken *string, httpClient IClient) *DebClient { - if accessToken == nil { - accessToken = new(string) - } - if len(*accessToken) == 0 { - *accessToken = os.Getenv("DEBRICKED_TOKEN") - } host := os.Getenv("DEBRICKED_URI") if len(host) == 0 { host = DefaultDebrickedUri @@ -37,7 +32,7 @@ func NewDebClient(accessToken *string, httpClient IClient) *DebClient { return &DebClient{ host: &host, httpClient: httpClient, - accessToken: accessToken, + accessToken: initAccessToken(accessToken), jwtToken: "", } } @@ -49,3 +44,18 @@ func (debClient *DebClient) Post(uri string, contentType string, body *bytes.Buf func (debClient *DebClient) Get(uri string, format string) (*http.Response, error) { return get(uri, debClient, true, format) } + +func (debClient *DebClient) SetAccessToken(accessToken *string) { + debClient.accessToken = initAccessToken(accessToken) +} + +func initAccessToken(accessToken *string) *string { + if accessToken == nil { + accessToken = new(string) + } + if len(*accessToken) == 0 { + *accessToken = os.Getenv("DEBRICKED_TOKEN") + } + + return accessToken +} diff --git a/pkg/client/deb_client_test.go b/pkg/client/deb_client_test.go index b39992a6..2c231c85 100644 --- a/pkg/client/deb_client_test.go +++ b/pkg/client/deb_client_test.go @@ -11,6 +11,7 @@ import ( "testing" testdataClient "github.com/debricked/cli/pkg/client/testdata/client" + "github.com/stretchr/testify/assert" ) var client *DebClient @@ -163,3 +164,13 @@ func TestAuthenticate(t *testing.T) { t.Errorf("failed to assert that the jwt token was properly set to %s. Got %s", jwtTkn, client.jwtToken) } } + +func TestSetAccessToken(t *testing.T) { + debClient := NewDebClient(nil, testdataClient.NewMock()) + debClient.accessToken = nil + testTkn := "0501ac404fd1823d0d4c047f957637a912d3b94713ee32a6" + + debClient.SetAccessToken(&testTkn) + + assert.Equal(t, &testTkn, debClient.accessToken) +} diff --git a/pkg/client/testdata/deb_client_mock.go b/pkg/client/testdata/deb_client_mock.go index 0562f432..53c06a5c 100644 --- a/pkg/client/testdata/deb_client_mock.go +++ b/pkg/client/testdata/deb_client_mock.go @@ -43,6 +43,10 @@ func (mock *DebClientMock) Post(uri string, format string, body *bytes.Buffer) ( return mock.realDebClient.Post(uri, format, body) } +func (mock *DebClientMock) SetAccessToken(_ *string) { + +} + type MockResponse struct { StatusCode int ResponseBody io.ReadCloser diff --git a/pkg/cmd/callgraph/callgraph.go b/pkg/cmd/callgraph/callgraph.go new file mode 100644 index 00000000..a732ea5e --- /dev/null +++ b/pkg/cmd/callgraph/callgraph.go @@ -0,0 +1,68 @@ +package callgraph + +import ( + "fmt" + "path/filepath" + + "github.com/debricked/cli/pkg/callgraph" + conf "github.com/debricked/cli/pkg/callgraph/config" + "github.com/debricked/cli/pkg/file" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var exclusions = file.DefaultExclusions() + +const ( + ExclusionFlag = "exclusion" +) + +func NewCallgraphCmd(generator callgraph.IGenerator) *cobra.Command { + cmd := &cobra.Command{ + Use: "callgraph [path]", + Short: "Generate a static callgraph for the given directory and subdirectories", + Long: `If a directory is inputted all manifest files without a lock file are resolved. +Example: +$ debricked callgraph +`, + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlags(cmd.Flags()) + }, + RunE: RunE(generator), + } + fileExclusionExample := filepath.Join("*", "**.lock") + dirExclusionExample := filepath.Join("**", "node_modules", "**") + exampleFlags := fmt.Sprintf("-e \"%s\" -e \"%s\"", fileExclusionExample, dirExclusionExample) + cmd.Flags().StringArrayVarP(&exclusions, ExclusionFlag, "e", exclusions, `The following terms are supported to exclude paths: +Special Terms | Meaning +------------- | ------- +"*" | matches any sequence of non-Separator characters +"/**/" | matches zero or multiple directories +"?" | matches any single non-Separator character +"[class]" | matches any single non-Separator character against a class of characters ([see "character classes"]) +"{alt1,...}" | matches a sequence of characters if one of the comma-separated alternatives matches + +Example: +$ debricked files resolve . `+exampleFlags) + + viper.MustBindEnv(ExclusionFlag) + + return cmd +} + +func RunE(callgraph callgraph.IGenerator) func(_ *cobra.Command, args []string) error { + return func(_ *cobra.Command, args []string) error { + if len(args) == 0 { + args = append(args, ".") + } + configs := []conf.IConfig{ + conf.NewConfig("java", []string{}, map[string]string{"pm": "maven"}), + // conf.NewConfig("java", []string{}, map[string]string{"pm": "gradle"}), + } + + // err := callgraph.GenerateWithTimer(args, viper.GetStringSlice(ExclusionFlag), configs, 10) + _, err := callgraph.Generate(args, viper.GetStringSlice(ExclusionFlag), configs, make(chan bool)) + + return err + } +} diff --git a/pkg/cmd/files/files.go b/pkg/cmd/files/files.go index ba4e889e..227e2612 100644 --- a/pkg/cmd/files/files.go +++ b/pkg/cmd/files/files.go @@ -1,14 +1,13 @@ package files import ( - "github.com/debricked/cli/pkg/client" "github.com/debricked/cli/pkg/cmd/files/find" "github.com/debricked/cli/pkg/file" "github.com/spf13/cobra" "github.com/spf13/viper" ) -func NewFilesCmd(debClient *client.IDebClient) *cobra.Command { +func NewFilesCmd(finder file.IFinder) *cobra.Command { cmd := &cobra.Command{ Use: "files", Short: "Analyze files", @@ -18,8 +17,7 @@ func NewFilesCmd(debClient *client.IDebClient) *cobra.Command { }, } - f, _ := file.NewFinder(*debClient) - cmd.AddCommand(find.NewFindCmd(f)) + cmd.AddCommand(find.NewFindCmd(finder)) return cmd } diff --git a/pkg/cmd/files/files_test.go b/pkg/cmd/files/files_test.go index c177cfb3..29418be7 100644 --- a/pkg/cmd/files/files_test.go +++ b/pkg/cmd/files/files_test.go @@ -3,21 +3,19 @@ package files import ( "testing" - "github.com/debricked/cli/pkg/client" - "github.com/debricked/cli/pkg/client/testdata" + "github.com/debricked/cli/pkg/file" "github.com/stretchr/testify/assert" ) func TestNewFilesCmd(t *testing.T) { - var debClient client.IDebClient = testdata.NewDebClientMock() - cmd := NewFilesCmd(&debClient) + finder, _ := file.NewFinder(nil) + cmd := NewFilesCmd(finder) commands := cmd.Commands() nbrOfCommands := 1 assert.Lenf(t, commands, nbrOfCommands, "failed to assert that there were %d sub commands connected", nbrOfCommands) } func TestPreRun(t *testing.T) { - var c client.IDebClient = testdata.NewDebClientMock() - cmd := NewFilesCmd(&c) + cmd := NewFilesCmd(nil) cmd.PreRun(cmd, nil) } diff --git a/pkg/cmd/report/report.go b/pkg/cmd/report/report.go index 86a6411f..8d6b82b5 100644 --- a/pkg/cmd/report/report.go +++ b/pkg/cmd/report/report.go @@ -1,7 +1,6 @@ package report import ( - "github.com/debricked/cli/pkg/client" "github.com/debricked/cli/pkg/cmd/report/license" "github.com/debricked/cli/pkg/cmd/report/vulnerability" licenseReport "github.com/debricked/cli/pkg/report/license" @@ -10,7 +9,10 @@ import ( "github.com/spf13/viper" ) -func NewReportCmd(debClient *client.IDebClient) *cobra.Command { +func NewReportCmd( + licenseReporter licenseReport.Reporter, + vulnerabilityReporter vulnerabilityReport.Reporter, +) *cobra.Command { cmd := &cobra.Command{ Use: "report", Short: "Generate reports", @@ -21,11 +23,8 @@ This is a premium feature. Please visit https://debricked.com/pricing/ for more }, } - lReporter := licenseReport.Reporter{DebClient: *debClient} - cmd.AddCommand(license.NewLicenseCmd(lReporter)) - - vReporter := vulnerabilityReport.Reporter{DebClient: *debClient} - cmd.AddCommand(vulnerability.NewVulnerabilityCmd(vReporter)) + cmd.AddCommand(license.NewLicenseCmd(licenseReporter)) + cmd.AddCommand(vulnerability.NewVulnerabilityCmd(vulnerabilityReporter)) return cmd } diff --git a/pkg/cmd/report/report_test.go b/pkg/cmd/report/report_test.go index 768861d3..09cd54a3 100644 --- a/pkg/cmd/report/report_test.go +++ b/pkg/cmd/report/report_test.go @@ -3,21 +3,21 @@ package report import ( "testing" - "github.com/debricked/cli/pkg/client" - "github.com/debricked/cli/pkg/client/testdata" + "github.com/debricked/cli/pkg/report/license" + "github.com/debricked/cli/pkg/report/vulnerability" "github.com/stretchr/testify/assert" ) func TestNewReportCmd(t *testing.T) { - var c client.IDebClient = testdata.NewDebClientMock() - cmd := NewReportCmd(&c) + cmd := NewReportCmd(license.Reporter{}, vulnerability.Reporter{}) commands := cmd.Commands() nbrOfCommands := 2 assert.Lenf(t, commands, nbrOfCommands, "failed to assert that there were %d sub commands connected", nbrOfCommands) } func TestPreRun(t *testing.T) { - var c client.IDebClient = testdata.NewDebClientMock() - cmd := NewReportCmd(&c) + var licenseReporter license.Reporter + var vulnReporter vulnerability.Reporter + cmd := NewReportCmd(licenseReporter, vulnReporter) cmd.PreRun(cmd, nil) } diff --git a/pkg/cmd/resolve/resolve.go b/pkg/cmd/resolve/resolve.go new file mode 100644 index 00000000..de9c4383 --- /dev/null +++ b/pkg/cmd/resolve/resolve.go @@ -0,0 +1,61 @@ +package resolve + +import ( + "fmt" + "path/filepath" + + "github.com/debricked/cli/pkg/file" + "github.com/debricked/cli/pkg/resolution" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var exclusions = file.DefaultExclusions() + +const ( + ExclusionFlag = "exclusion" +) + +func NewResolveCmd(resolver resolution.IResolver) *cobra.Command { + cmd := &cobra.Command{ + Use: "resolve [path]", + Short: "Resolve manifest files", + Long: `If a directory is inputted all manifest files without a lock file are resolved. +Example: +$ debricked files resolve go.mod pkg/ +`, + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlags(cmd.Flags()) + }, + RunE: RunE(resolver), + } + fileExclusionExample := filepath.Join("*", "**.lock") + dirExclusionExample := filepath.Join("**", "node_modules", "**") + exampleFlags := fmt.Sprintf("-e \"%s\" -e \"%s\"", fileExclusionExample, dirExclusionExample) + cmd.Flags().StringArrayVarP(&exclusions, ExclusionFlag, "e", exclusions, `The following terms are supported to exclude paths: +Special Terms | Meaning +------------- | ------- +"*" | matches any sequence of non-Separator characters +"/**/" | matches zero or multiple directories +"?" | matches any single non-Separator character +"[class]" | matches any single non-Separator character against a class of characters ([see "character classes"]) +"{alt1,...}" | matches a sequence of characters if one of the comma-separated alternatives matches + +Example: +$ debricked files resolve . `+exampleFlags) + + viper.MustBindEnv(ExclusionFlag) + + return cmd +} + +func RunE(resolver resolution.IResolver) func(_ *cobra.Command, args []string) error { + return func(_ *cobra.Command, args []string) error { + if len(args) == 0 { + args = append(args, ".") + } + _, err := resolver.Resolve(args, viper.GetStringSlice(ExclusionFlag)) + + return err + } +} diff --git a/pkg/cmd/resolve/resolve_test.go b/pkg/cmd/resolve/resolve_test.go new file mode 100644 index 00000000..5e8db813 --- /dev/null +++ b/pkg/cmd/resolve/resolve_test.go @@ -0,0 +1,95 @@ +package resolve + +import ( + "errors" + "testing" + + "github.com/debricked/cli/pkg/file" + "github.com/debricked/cli/pkg/file/testdata" + "github.com/debricked/cli/pkg/resolution" + resolveTestdata "github.com/debricked/cli/pkg/resolution/testdata" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +func TestNewResolveCmd(t *testing.T) { + var resolver resolution.IResolver + cmd := NewResolveCmd(resolver) + + commands := cmd.Commands() + nbrOfCommands := 0 + assert.Len(t, commands, nbrOfCommands) + + flags := cmd.Flags() + flagAssertions := map[string]string{} + for name, shorthand := range flagAssertions { + flag := flags.Lookup(name) + assert.NotNil(t, flag) + assert.Equal(t, shorthand, flag.Shorthand) + } + + var flagKeys = []string{ + ExclusionFlag, + } + viperKeys := viper.AllKeys() + for _, flagKey := range flagKeys { + match := false + for _, key := range viperKeys { + if key == flagKey { + match = true + } + } + assert.Truef(t, match, "failed to assert that flag was present: "+flagKey) + } + +} + +func TestRunE(t *testing.T) { + f := testdata.NewFinderMock() + r := &resolveTestdata.ResolverMock{} + groups := file.Groups{} + groups.Add(file.Group{}) + f.SetGetGroupsReturnMock(groups, nil) + runE := RunE(r) + + err := runE(nil, []string{"."}) + + assert.NoError(t, err) +} + +func TestRunENoPath(t *testing.T) { + f := testdata.NewFinderMock() + r := &resolveTestdata.ResolverMock{} + groups := file.Groups{} + groups.Add(file.Group{}) + f.SetGetGroupsReturnMock(groups, nil) + runE := RunE(r) + + err := runE(nil, []string{}) + + assert.NoError(t, err) +} + +func TestRunENoFiles(t *testing.T) { + f := testdata.NewFinderMock() + r := &resolveTestdata.ResolverMock{} + groups := file.Groups{} + groups.Add(file.Group{}) + f.SetGetGroupsReturnMock(groups, nil) + exclusions = []string{} + runE := RunE(r) + + err := runE(nil, []string{"."}) + + assert.NoError(t, err) +} + +func TestRunEError(t *testing.T) { + r := &resolveTestdata.ResolverMock{} + errorAssertion := errors.New("finder-error") + r.Err = errorAssertion + runE := RunE(r) + err := runE(nil, []string{"."}) + + assert.EqualError(t, err, "finder-error", "error doesn't match expected") +} diff --git a/pkg/cmd/root/root.go b/pkg/cmd/root/root.go index 4e2fc0b4..86e393cc 100644 --- a/pkg/cmd/root/root.go +++ b/pkg/cmd/root/root.go @@ -1,10 +1,12 @@ package root import ( - "github.com/debricked/cli/pkg/client" + "github.com/debricked/cli/pkg/cmd/callgraph" "github.com/debricked/cli/pkg/cmd/files" "github.com/debricked/cli/pkg/cmd/report" + "github.com/debricked/cli/pkg/cmd/resolve" "github.com/debricked/cli/pkg/cmd/scan" + "github.com/debricked/cli/pkg/wire" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -13,7 +15,7 @@ var accessToken string const AccessTokenFlag = "access-token" -func NewRootCmd(version string) *cobra.Command { +func NewRootCmd(version string, container *wire.CliContainer) *cobra.Command { rootCmd := &cobra.Command{ Use: "debricked", Short: "Debricked CLI - Keep track of your dependencies!", @@ -35,10 +37,14 @@ Complete documentation is available at https://debricked.com/docs/integrations/c Read more: https://debricked.com/docs/administration/access-tokens.html`, ) - var debClient client.IDebClient = client.NewDebClient(&accessToken, client.NewRetryClient()) - rootCmd.AddCommand(report.NewReportCmd(&debClient)) - rootCmd.AddCommand(files.NewFilesCmd(&debClient)) - rootCmd.AddCommand(scan.NewScanCmd(&debClient)) + var debClient = container.DebClient() + debClient.SetAccessToken(&accessToken) + + rootCmd.AddCommand(report.NewReportCmd(container.LicenseReporter(), container.VulnerabilityReporter())) + rootCmd.AddCommand(files.NewFilesCmd(container.Finder())) + rootCmd.AddCommand(scan.NewScanCmd(container.Scanner())) + rootCmd.AddCommand(resolve.NewResolveCmd(container.Resolver())) + rootCmd.AddCommand(callgraph.NewCallgraphCmd(container.CallgraphGenerator())) rootCmd.CompletionOptions.DisableDefaultCmd = true diff --git a/pkg/cmd/root/root_test.go b/pkg/cmd/root/root_test.go index 0493a307..a96c51c6 100644 --- a/pkg/cmd/root/root_test.go +++ b/pkg/cmd/root/root_test.go @@ -3,14 +3,15 @@ package root import ( "testing" + "github.com/debricked/cli/pkg/wire" "github.com/spf13/viper" "github.com/stretchr/testify/assert" ) func TestNewRootCmd(t *testing.T) { - cmd := NewRootCmd("v0.0.0") + cmd := NewRootCmd("v0.0.0", wire.GetCliContainer()) commands := cmd.Commands() - nbrOfCommands := 3 + nbrOfCommands := 5 if len(commands) != nbrOfCommands { t.Errorf("failed to assert that there were %d sub commands connected", nbrOfCommands) } @@ -34,6 +35,6 @@ func TestNewRootCmd(t *testing.T) { } func TestPreRun(t *testing.T) { - cmd := NewRootCmd("") + cmd := NewRootCmd("", wire.GetCliContainer()) cmd.PreRun(cmd, nil) } diff --git a/pkg/cmd/scan/scan.go b/pkg/cmd/scan/scan.go index 42c3425b..54dac71f 100644 --- a/pkg/cmd/scan/scan.go +++ b/pkg/cmd/scan/scan.go @@ -5,8 +5,6 @@ import ( "fmt" "path/filepath" - "github.com/debricked/cli/pkg/ci" - "github.com/debricked/cli/pkg/client" "github.com/debricked/cli/pkg/file" "github.com/debricked/cli/pkg/scan" "github.com/fatih/color" @@ -21,6 +19,7 @@ var commitAuthor string var repositoryUrl string var integrationName string var exclusions = file.DefaultExclusions() +var resolve bool const ( RepositoryFlag = "repository" @@ -30,16 +29,12 @@ const ( RepositoryUrlFlag = "repository-url" IntegrationFlag = "integration" ExclusionFlag = "exclusion" + ResolveFlag = "resolve" ) var scanCmdError error -func NewScanCmd(c *client.IDebClient) *cobra.Command { - var ciService ci.IService = ci.NewService(nil) - - var s scan.IScanner - s, scanCmdError = scan.NewDebrickedScanner(c, ciService) - +func NewScanCmd(scanner scan.IScanner) *cobra.Command { cmd := &cobra.Command{ Use: "scan [path]", Short: "Start a Debricked dependency scan", @@ -48,7 +43,7 @@ If the given path contains a git repository all flags but "integration" will be PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlags(cmd.Flags()) }, - RunE: RunE(&s), + RunE: RunE(&scanner), } cmd.Flags().StringVarP(&repositoryName, RepositoryFlag, "r", "", "repository name") cmd.Flags().StringVarP(&commitName, CommitFlag, "c", "", "commit hash") @@ -82,6 +77,10 @@ Special Terms | Meaning Examples: $ debricked scan . `+exampleFlags) + + cmd.Flags().BoolVar(&resolve, ResolveFlag, false, `Resolves manifest files that lack lock files. This enables more accurate dependency scanning since the whole dependency tree will be analysed. +For example, if there is a "go.mod" in the target path, its dependencies are going to get resolved onto a lock file, and latter scanned.`) + viper.MustBindEnv(RepositoryFlag) viper.MustBindEnv(CommitFlag) viper.MustBindEnv(BranchFlag) @@ -100,6 +99,7 @@ func RunE(s *scan.IScanner) func(_ *cobra.Command, args []string) error { } options := scan.DebrickedOptions{ Path: path, + Resolve: viper.GetBool(ResolveFlag), Exclusions: viper.GetStringSlice(ExclusionFlag), RepositoryName: viper.GetString(RepositoryFlag), CommitName: viper.GetString(CommitFlag), diff --git a/pkg/cmd/scan/scan_test.go b/pkg/cmd/scan/scan_test.go index 5c12e535..9ce7cea7 100644 --- a/pkg/cmd/scan/scan_test.go +++ b/pkg/cmd/scan/scan_test.go @@ -3,8 +3,6 @@ package scan import ( "testing" - "github.com/debricked/cli/pkg/client" - "github.com/debricked/cli/pkg/client/testdata" "github.com/debricked/cli/pkg/scan" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -12,8 +10,7 @@ import ( ) func TestNewScanCmd(t *testing.T) { - var c client.IDebClient = testdata.NewDebClientMock() - cmd := NewScanCmd(&c) + cmd := NewScanCmd(&scannerMock{}) viperKeys := viper.AllKeys() flags := cmd.Flags() @@ -81,8 +78,7 @@ func TestRunEError(t *testing.T) { } func TestPreRun(t *testing.T) { - var c client.IDebClient = testdata.NewDebClientMock() - cmd := NewScanCmd(&c) + cmd := NewScanCmd(nil) cmd.PreRun(cmd, nil) } diff --git a/pkg/file/finder_test.go b/pkg/file/finder_test.go index b5d3bb67..f71f7f68 100644 --- a/pkg/file/finder_test.go +++ b/pkg/file/finder_test.go @@ -52,6 +52,8 @@ func (mock *debClientMock) Get(_ string, _ string) (*http.Response, error) { return &res, nil } +func (mock *debClientMock) SetAccessToken(_ *string) {} + var finder *Finder func setUp(auth bool) { diff --git a/pkg/io/err/error.go b/pkg/io/err/error.go new file mode 100644 index 00000000..333c8667 --- /dev/null +++ b/pkg/io/err/error.go @@ -0,0 +1,5 @@ +package err + +type IError interface { + error +} diff --git a/pkg/io/err/errors.go b/pkg/io/err/errors.go new file mode 100644 index 00000000..235d756b --- /dev/null +++ b/pkg/io/err/errors.go @@ -0,0 +1,48 @@ +package err + +type IErrors interface { + Warning(err IError) + Critical(err IError) + GetWarningErrors() []IError + GetCriticalErrors() []IError + GetAll() []IError + HasError() bool +} + +type Errors struct { + title string + warningErrs []IError + criticalErrs []IError +} + +func NewErrors(title string) *Errors { + return &Errors{ + title: title, + warningErrs: []IError{}, + criticalErrs: []IError{}, + } +} + +func (errors *Errors) Warning(err IError) { + errors.warningErrs = append(errors.warningErrs, err) +} + +func (errors *Errors) Critical(err IError) { + errors.criticalErrs = append(errors.criticalErrs, err) +} + +func (errors *Errors) GetWarningErrors() []IError { + return errors.warningErrs +} + +func (errors *Errors) GetCriticalErrors() []IError { + return errors.criticalErrs +} + +func (errors *Errors) GetAll() []IError { + return append(errors.warningErrs, errors.criticalErrs...) +} + +func (errors *Errors) HasError() bool { + return len(errors.criticalErrs) > 0 || len(errors.warningErrs) > 0 +} diff --git a/pkg/io/err/errors_test.go b/pkg/io/err/errors_test.go new file mode 100644 index 00000000..78a7ed04 --- /dev/null +++ b/pkg/io/err/errors_test.go @@ -0,0 +1,77 @@ +package err + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewErrors(t *testing.T) { + title := "title" + errors := NewErrors(title) + assert.Equal(t, title, errors.title) + assert.NotNil(t, errors) + assert.Empty(t, errors.criticalErrs) + assert.Empty(t, errors.warningErrs) +} + +func TestWarning(t *testing.T) { + errors := NewErrors("") + warning := fmt.Errorf("error") + errors.Warning(warning) + assert.Empty(t, errors.criticalErrs) + assert.Len(t, errors.warningErrs, 1) + assert.Contains(t, errors.warningErrs, warning) +} + +func TestCritical(t *testing.T) { + errors := NewErrors("") + critical := fmt.Errorf("error") + errors.Critical(critical) + assert.Empty(t, errors.warningErrs) + assert.Len(t, errors.criticalErrs, 1) + assert.Contains(t, errors.criticalErrs, critical) +} + +func TestGetWarningErrors(t *testing.T) { + errors := NewErrors("") + warning := fmt.Errorf("error") + errors.Warning(warning) + assert.Empty(t, errors.GetCriticalErrors()) + assert.Len(t, errors.GetWarningErrors(), 1) + assert.Contains(t, errors.GetWarningErrors(), warning) +} + +func TestGetCriticalErrors(t *testing.T) { + errors := NewErrors("") + critical := fmt.Errorf("error") + errors.Critical(critical) + assert.Empty(t, errors.GetWarningErrors()) + assert.Len(t, errors.GetCriticalErrors(), 1) + assert.Contains(t, errors.GetCriticalErrors(), critical) +} + +func TestGetAll(t *testing.T) { + errors := NewErrors("") + warning := fmt.Errorf("warning") + critical := fmt.Errorf("critical") + errors.Warning(warning) + errors.Critical(critical) + assert.Len(t, errors.GetAll(), 2) + assert.Contains(t, errors.GetAll(), warning) + assert.Contains(t, errors.GetAll(), critical) +} + +func TestHasError(t *testing.T) { + errors := NewErrors("") + assert.False(t, errors.HasError()) + + warning := fmt.Errorf("warning") + errors.Warning(warning) + assert.True(t, errors.HasError()) + + critical := fmt.Errorf("critical") + errors.Warning(critical) + assert.True(t, errors.HasError()) +} diff --git a/pkg/io/finder/finder.go b/pkg/io/finder/finder.go new file mode 100644 index 00000000..884c2037 --- /dev/null +++ b/pkg/io/finder/finder.go @@ -0,0 +1,74 @@ +package finder + +import ( + "path/filepath" + + "github.com/debricked/cli/pkg/io/finder/gradle" + "github.com/debricked/cli/pkg/io/finder/maven" +) + +type IFinder interface { + FindMavenRoots(files []string) ([]string, error) + FindJavaClassDirs(files []string) ([]string, error) + FindGradleRoots(files []string) ([]string, error) +} + +type Finder struct{} + +func (f Finder) FindMavenRoots(files []string) ([]string, error) { + pomFiles := FilterFiles(files, "pom.xml") + ps := maven.PomService{} + rootFiles := ps.GetRootPomFiles(pomFiles) + return rootFiles, nil +} + +func (f Finder) FindJavaClassDirs(files []string) ([]string, error) { + filteredFiles := FilterFiles(files, "*.class") + dirsWithJarFiles := make(map[string]bool) + for _, file := range filteredFiles { + dirsWithJarFiles[filepath.Dir(file)] = true + } + + jarFiles := []string{} + for key := range dirsWithJarFiles { + jarFiles = append(jarFiles, key) + } + + return jarFiles, nil +} + +func (f Finder) FindGradleRoots(files []string) ([]string, error) { + gradleBuildFiles := FilterFiles(files, "gradle.build(.kts)?") + gradleSetup := gradle.NewGradleSetup() + err := gradleSetup.Configure(files) + if err != nil { + + return []string{}, err + } + + gradleMainDirs := make(map[string]bool) + for _, gradleProject := range gradleSetup.GradleProjects { + dir := gradleProject.Dir + if _, ok := gradleMainDirs[dir]; ok { + continue + } + gradleMainDirs[dir] = true + } + for _, file := range gradleBuildFiles { + dir, _ := filepath.Abs(filepath.Dir(file)) + if _, ok := gradleSetup.SubProjectMap[dir]; ok { + continue + } + if _, ok := gradleMainDirs[dir]; ok { + continue + } + gradleMainDirs[dir] = true + } + + roots := []string{} + for key := range gradleMainDirs { + roots = append(roots, key) + } + + return roots, nil +} diff --git a/pkg/io/finder/gradle/.gradle-init-script.debricked.groovy b/pkg/io/finder/gradle/.gradle-init-script.debricked.groovy new file mode 100644 index 00000000..1c1fc223 --- /dev/null +++ b/pkg/io/finder/gradle/.gradle-init-script.debricked.groovy @@ -0,0 +1,18 @@ +def debrickedOutputFile = new File('.debricked.multiprojects.txt') + +allprojects { + task debrickedFindSubProjectPaths() { + String output = project.projectDir + doLast { + synchronized(debrickedOutputFile) { + debrickedOutputFile << output + System.getProperty("line.separator") + } + } + } +} + +allprojects { + task debrickedAllDeps(type: DependencyReportTask) { + outputFile = file('./.debricked-gradle-dependencies.txt') + } +} diff --git a/pkg/io/finder/gradle/embeded/gradle-init-script.groovy b/pkg/io/finder/gradle/embeded/gradle-init-script.groovy new file mode 100644 index 00000000..1c1fc223 --- /dev/null +++ b/pkg/io/finder/gradle/embeded/gradle-init-script.groovy @@ -0,0 +1,18 @@ +def debrickedOutputFile = new File('.debricked.multiprojects.txt') + +allprojects { + task debrickedFindSubProjectPaths() { + String output = project.projectDir + doLast { + synchronized(debrickedOutputFile) { + debrickedOutputFile << output + System.getProperty("line.separator") + } + } + } +} + +allprojects { + task debrickedAllDeps(type: DependencyReportTask) { + outputFile = file('./.debricked-gradle-dependencies.txt') + } +} diff --git a/pkg/io/finder/gradle/err.go b/pkg/io/finder/gradle/err.go new file mode 100644 index 00000000..503ef16c --- /dev/null +++ b/pkg/io/finder/gradle/err.go @@ -0,0 +1,39 @@ +package gradle + +type SetupScriptError struct { + message string +} + +type SetupWalkError struct { + message string +} + +type SetupSubprojectError struct { + message string +} + +func (e SetupScriptError) Error() string { + + return e.message +} + +func (e SetupWalkError) Error() string { + + return e.message +} + +func (e SetupSubprojectError) Error() string { + + return e.message +} + +type SetupError []error + +func (e SetupError) Error() string { + var s string + for _, err := range e { + s += err.Error() + "\n" + } + + return s +} diff --git a/pkg/io/finder/gradle/gradle.go b/pkg/io/finder/gradle/gradle.go new file mode 100644 index 00000000..18e8a6a2 --- /dev/null +++ b/pkg/io/finder/gradle/gradle.go @@ -0,0 +1,206 @@ +package gradle + +import ( + "bufio" + "bytes" + "embed" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + + "github.com/debricked/cli/pkg/io/writer" +) + +const ( + initGradle = "gradle" + multiProjectFilename = ".debricked.multiprojects.txt" + gradleInitScriptFileName = ".gradle-init-script.debricked.groovy" +) + +//go:embed embeded/gradle-init-script.groovy +var gradleInitScript embed.FS + +type ISetup interface { + Configure(files []string) (Setup, error) +} + +type Project struct { + Dir string + Gradlew string + MainBuildFile string +} + +type Setup struct { + GradlewMap map[string]string + SettingsMap map[string]string + SubProjectMap map[string]string + GroovyScriptPath string + GradlewOsName string + SettingsFilenames []string + GradleProjects []Project + MetaFileFinder IMetaFileFinder + Writer writer.IFileWriter + InitScriptHandler IInitScriptHandler + CmdFactory ICmdFactory +} + +func NewGradleSetup() *Setup { + groovyScriptPath, _ := filepath.Abs(gradleInitScriptFileName) + gradlewOsName := "gradlew" + if runtime.GOOS == "windows" { + gradlewOsName = "gradlew.bat" + } + writer := writer.FileWriter{} + ish := InitScriptHandler{groovyScriptPath, "embeded/gradle-init-script.groovy", writer} + + return &Setup{ + GradlewMap: map[string]string{}, + SettingsMap: map[string]string{}, + SubProjectMap: map[string]string{}, + GroovyScriptPath: groovyScriptPath, + GradlewOsName: gradlewOsName, + SettingsFilenames: []string{"settings.gradle", "settings.gradle.kts"}, + GradleProjects: []Project{}, + MetaFileFinder: MetaFileFinder{filepath: FilePath{}}, + Writer: writer, + InitScriptHandler: ish, + CmdFactory: CmdFactory{}, + } +} + +func (gs *Setup) Configure(files []string) error { + err := gs.InitScriptHandler.WriteInitFile() + if err != nil { + + return err + } + settingsMap, gradlewMap, err := gs.MetaFileFinder.Find(files) + gs.GradlewMap = gradlewMap + gs.SettingsMap = settingsMap + if err != nil { + + return err + } + err = gs.setupGradleProjectMappings() + if err != nil && len(err.Error()) > 0 { + + return err + } + + return nil +} + +func (gs *Setup) setupFilePathMappings(files []string) { + for _, file := range files { + dir, _ := filepath.Abs(filepath.Dir(file)) + possibleGradlew := filepath.Join(dir, gs.GradlewOsName) + _, err := os.Stat(possibleGradlew) + if err == nil { + gs.GradlewMap[dir] = possibleGradlew + } + for _, settingsFilename := range gs.SettingsFilenames { + possibleSettings := filepath.Join(dir, settingsFilename) + _, err := os.Stat(possibleSettings) + if err == nil { + gs.SettingsMap[dir] = possibleSettings + } + } + } +} + +func (gs *Setup) setupGradleProjectMappings() error { + var errors SetupError + var settingsDirs []string + for k := range gs.SettingsMap { + settingsDirs = append(settingsDirs, k) + } + sort.Strings(settingsDirs) + for _, dir := range settingsDirs { + if _, ok := gs.SubProjectMap[dir]; ok { + continue + } + gradlew := gs.GetGradleW(dir) + mainFile := gs.SettingsMap[dir] + gradleProject := Project{Dir: dir, Gradlew: gradlew, MainBuildFile: mainFile} + err := gs.setupSubProjectPaths(gradleProject) + + if err != nil { + errors = append(errors, err) + } + gs.GradleProjects = append(gs.GradleProjects, gradleProject) + } + + return SetupSubprojectError{message: errors.Error()} +} + +type ICmdFactory interface { + MakeFindSubGraphCmd(workingDirectory string, gradlew string, initScript string) (*exec.Cmd, error) +} +type CmdFactory struct{} + +func (cf CmdFactory) MakeFindSubGraphCmd(workingDirectory string, gradlew string, initScript string) (*exec.Cmd, error) { + path, err := exec.LookPath(gradlew) + + return &exec.Cmd{ + Path: path, + Args: []string{gradlew, "--init-script", initScript, "debrickedFindSubProjectPaths"}, + Dir: workingDirectory, + }, err +} + +func (gs *Setup) setupSubProjectPaths(gp Project) error { + dependenciesCmd, _ := gs.CmdFactory.MakeFindSubGraphCmd(gp.Dir, gp.Gradlew, gs.GroovyScriptPath) + var stderr bytes.Buffer + dependenciesCmd.Stderr = &stderr + _, err := dependenciesCmd.Output() + dependenciesCmd.Stderr = os.Stderr + if err != nil { + errorOutput := stderr.String() + + return SetupSubprojectError{message: errorOutput + err.Error()} + } + multiProject := filepath.Join(gp.Dir, multiProjectFilename) + file, err := os.Open(multiProject) + if err != nil { + + return SetupSubprojectError{message: err.Error()} + } + defer file.Close() + defer os.Remove(multiProject) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + subProjectPath := scanner.Text() + gs.SubProjectMap[subProjectPath] = gp.Dir + } + + if err := scanner.Err(); err != nil { + return SetupSubprojectError{message: err.Error()} + } + + return nil +} + +func (gs *Setup) GetGradleW(dir string) string { + gradlew := initGradle + val, ok := gs.GradlewMap[dir] + if ok { + gradlew = val + } else { + for dirPath, gradlePath := range gs.GradlewMap { + // potential improvement, sort gradlewMap in longest path first" + rel, err := filepath.Rel(dirPath, dir) + isRelative := !strings.HasPrefix(rel, "..") && rel != ".." + if isRelative && err == nil { + gradlew = gradlePath + + break + } + } + } + + return gradlew +} diff --git a/pkg/io/finder/gradle/gradle_test.go b/pkg/io/finder/gradle/gradle_test.go new file mode 100644 index 00000000..67506cfd --- /dev/null +++ b/pkg/io/finder/gradle/gradle_test.go @@ -0,0 +1,203 @@ +package gradle + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + writerTestdata "github.com/debricked/cli/pkg/io/writer/testdata" + + "github.com/stretchr/testify/assert" +) + +func TestNewGradleSetup(t *testing.T) { + + gs := NewGradleSetup() + assert.NotNil(t, gs) +} + +func TestErrors(t *testing.T) { + + walkError := SetupWalkError{message: "test"} + assert.Equal(t, "test", walkError.Error()) + + scriptError := SetupScriptError{message: "test"} + assert.Equal(t, "test", scriptError.Error()) + + subprojectError := SetupSubprojectError{message: "test"} + assert.Equal(t, "test", subprojectError.Error()) + +} + +func TestSetupFilePathMappings(t *testing.T) { + gs := NewGradleSetup() + files := []string{filepath.Join("testdata", "project", "build.gradle")} + gs.setupFilePathMappings(files) + + assert.Len(t, gs.GradlewMap, 1) + assert.Len(t, gs.SettingsMap, 1) +} + +func TestSetupFilePathMappingsNoFiles(t *testing.T) { + gs := NewGradleSetup() + gs.setupFilePathMappings([]string{}) + + assert.Len(t, gs.GradlewMap, 0) + assert.Len(t, gs.SettingsMap, 0) +} + +func TestSetupFilePathMappingsNoGradlew(t *testing.T) { + gs := NewGradleSetup() + files := []string{filepath.Join("testdata", "project", "subproject", "build.gradle")} + gs.setupFilePathMappings(files) + + assert.Len(t, gs.GradlewMap, 0) + assert.Len(t, gs.SettingsMap, 0) +} + +func TestSetupGradleProjectMappings(t *testing.T) { + gs := NewGradleSetup() + gs.CmdFactory = &mockCmdFactory{} + + gs.SettingsMap = map[string]string{ + filepath.Join("testdata", "project"): filepath.Join("testdata", "project", "settings.gradle"), + } + gs.SubProjectMap = map[string]string{} + err := gs.setupGradleProjectMappings() + // assert GradleSetupSubprojectError + assert.NotNil(t, err) + + assert.Len(t, gs.GradleProjects, 1) +} + +type mockCmdFactory struct { + createFile bool +} + +func (m *mockCmdFactory) MakeFindSubGraphCmd(workingDirectory string, _ string, _ string) (*exec.Cmd, error) { + if m.createFile { + fileName := filepath.Join(workingDirectory, multiProjectFilename) + content := []byte(workingDirectory) + file, err := os.Create(fileName) + if err != nil { + + return nil, err + } + defer file.Close() + _, err = file.Write(content) + if err != nil { + + return nil, err + } + } + // if windows use dir + if runtime.GOOS == "windows" { + // gradlewOsName = "gradlew.bat" + return exec.Command("dir"), nil + } + + return exec.Command("ls"), nil +} + +func TestSetupSubProjectPathsNoFileCreated(t *testing.T) { + gs := NewGradleSetup() + gs.CmdFactory = &mockCmdFactory{createFile: false} + + absPath, _ := filepath.Abs(filepath.Join("testdata", "project")) + gradleProject := Project{Dir: absPath, Gradlew: filepath.Join("testdata", "project", "gradlew")} + err := gs.setupSubProjectPaths(gradleProject) + fmt.Println(err) + assert.NotNil(t, err) + assert.Len(t, gs.SubProjectMap, 0) +} + +func TestSetupSubProjectPaths(t *testing.T) { + gs := NewGradleSetup() + gs.CmdFactory = &mockCmdFactory{createFile: true} + + absPath, _ := filepath.Abs(filepath.Join("testdata", "project")) + gradleProject := Project{Dir: absPath, Gradlew: filepath.Join("testdata", "project", "gradlew")} + err := gs.setupSubProjectPaths(gradleProject) + assert.Nil(t, err) + assert.Len(t, gs.SubProjectMap, 1) + + absPath, _ = filepath.Abs(filepath.Join("testdata", "project", "subproject")) + gradleProject = Project{Dir: absPath, Gradlew: filepath.Join("testdata", "project", "gradlew")} + err = gs.setupSubProjectPaths(gradleProject) + assert.Nil(t, err) + assert.Len(t, gs.SubProjectMap, 2) +} + +func TestSetupSubProjectPathsError(t *testing.T) { + gs := NewGradleSetup() + + absPath, _ := filepath.Abs(filepath.Join("testdata", "project")) + gradleProject := Project{Dir: absPath, Gradlew: filepath.Join("testdata", "project", "gradlew")} + err := gs.setupSubProjectPaths(gradleProject) + + assert.NotNil(t, err) +} + +func TestGetGradleW(t *testing.T) { + gs := NewGradleSetup() + + gs.GradlewMap = map[string]string{ + filepath.Join("testdata", "project"): filepath.Join("testdata", "project", "gradlew"), + } + + gradlew := gs.GetGradleW(filepath.Join("testdata", "project", "subproject")) + + assert.Equal(t, filepath.Join("testdata", "project", "gradlew"), gradlew) + + gradlew = gs.GetGradleW(filepath.Join("testdata", "project")) + + assert.Equal(t, filepath.Join("testdata", "project", "gradlew"), gradlew) +} + +type mockInitScriptHandler struct { + writeInitFileErr error +} + +func (_ mockInitScriptHandler) ReadInitFile() ([]byte, error) { + return gradleInitScript.ReadFile("gradle-init/gradle-init-script.groovy") +} + +func (i mockInitScriptHandler) WriteInitFile() error { + return i.writeInitFileErr +} + +type mockFileHandler struct { + setupWalkErr error +} + +func (f mockFileHandler) Find(_ []string) (map[string]string, map[string]string, error) { + return nil, nil, f.setupWalkErr +} + +func TestConfigureErrors(t *testing.T) { + gs := NewGradleSetup() + gs.Writer = &writerTestdata.FileWriterMock{} + err := gs.Configure([]string{"testdata/project"}) + assert.NotNil(t, err) + + gs.MetaFileFinder = mockFileHandler{setupWalkErr: SetupScriptError{message: "mock error"}} + err = gs.Configure([]string{"testdata/project"}) + assert.Equal(t, "mock error", err.Error()) + + gs.InitScriptHandler = mockInitScriptHandler{writeInitFileErr: SetupScriptError{message: "write-init-file-err"}} + err = gs.Configure([]string{"testdata/project"}) + assert.Equal(t, "write-init-file-err", err.Error()) +} + +func TestConfigure(t *testing.T) { + gs := NewGradleSetup() + gs.Writer = &writerTestdata.FileWriterMock{} + gs.MetaFileFinder = mockFileHandler{setupWalkErr: nil} + gs.InitScriptHandler = mockInitScriptHandler{writeInitFileErr: nil} + + err := gs.Configure([]string{"testdata/project"}) + assert.NoError(t, err) +} diff --git a/pkg/io/finder/gradle/init_script_handler.go b/pkg/io/finder/gradle/init_script_handler.go new file mode 100644 index 00000000..003195b5 --- /dev/null +++ b/pkg/io/finder/gradle/init_script_handler.go @@ -0,0 +1,49 @@ +package gradle + +import ( + "github.com/debricked/cli/pkg/io/writer" +) + +type IInitScriptHandler interface { + ReadInitFile() ([]byte, error) + WriteInitFile() error +} + +type InitScriptHandler struct { + groovyScriptPath string + initPath string + fileWriter writer.IFileWriter +} + +func NewScriptHandler(groovyScriptPath string, initPath string, fileWriter writer.IFileWriter) InitScriptHandler { + return InitScriptHandler{ + groovyScriptPath, + initPath, + fileWriter, + } +} + +func (i InitScriptHandler) ReadInitFile() ([]byte, error) { + return gradleInitScript.ReadFile(i.initPath) +} + +func (i InitScriptHandler) WriteInitFile() error { + content, err := i.ReadInitFile() + if err != nil { + + return SetupScriptError{message: err.Error()} + } + lockFile, err := i.fileWriter.Create(i.groovyScriptPath) + if err != nil { + + return SetupScriptError{message: err.Error()} + } + defer lockFile.Close() + err = i.fileWriter.Write(lockFile, content) + if err != nil { + + return SetupScriptError{message: err.Error()} + } + + return nil +} diff --git a/pkg/io/finder/gradle/init_script_handler_test.go b/pkg/io/finder/gradle/init_script_handler_test.go new file mode 100644 index 00000000..b28f3b06 --- /dev/null +++ b/pkg/io/finder/gradle/init_script_handler_test.go @@ -0,0 +1,28 @@ +package gradle + +// func TestWriteInitFile(t *testing.T) { +// createErr := errors.New("create-error") +// fileWriterMock := &writerTestdata.FileWriterMock{CreateErr: createErr} + +// sf := InitScriptHandler{fileWriter: fileWriterMock} +// err := sf.WriteInitFile() +// assert.Equal(t, SetupScriptError{createErr.Error()}, err) + +// fileWriterMock = &writerTestdata.FileWriterMock{WriteErr: createErr} +// sf = InitScriptHandler{initPath: "file", fileWriter: fileWriterMock} +// err = sf.WriteInitFile() +// assert.Equal(t, SetupScriptError{createErr.Error()}, err) +// } + +// func TestWriteInitFileNoInitFile(t *testing.T) { +// sf := InitScriptHandler{initPath: "file", fileWriter: nil} +// oldGradleInitScript := gradleInitScript +// defer func() { +// gradleInitScript = oldGradleInitScript +// }() +// gradleInitScript = embed.FS{} +// err := sf.WriteInitFile() +// readErr := errors.New("open gradle-init/gradle-init-script.groovy: file does not exist") +// assert.Equal(t, SetupScriptError{readErr.Error()}, err) + +// } diff --git a/pkg/io/finder/gradle/meta_file_finder.go b/pkg/io/finder/gradle/meta_file_finder.go new file mode 100644 index 00000000..e59bb57a --- /dev/null +++ b/pkg/io/finder/gradle/meta_file_finder.go @@ -0,0 +1,82 @@ +package gradle + +import ( + "os" + "path/filepath" +) + +type IMetaFileFinder interface { + Find(paths []string) (map[string]string, map[string]string, error) +} + +type MetaFileFinder struct { + filepath IFilePath +} + +type IFilePath interface { + Walk(root string, walkFn filepath.WalkFunc) error + Base(path string) string + Abs(path string) (string, error) + Dir(path string) string +} + +type FilePath struct{} + +func (fp FilePath) Walk(root string, walkFn filepath.WalkFunc) error { + return filepath.Walk(root, walkFn) +} + +func (fp FilePath) Base(path string) string { + return filepath.Base(path) +} + +func (fp FilePath) Abs(path string) (string, error) { + return filepath.Abs(path) +} + +func (fp FilePath) Dir(path string) string { + return filepath.Dir(path) +} + +func (finder MetaFileFinder) Find(paths []string) (map[string]string, map[string]string, error) { + settings := []string{"settings.gradle", "settings.gradle.kts"} + gradlew := []string{"gradlew"} + settingsMap := map[string]string{} + gradlewMap := map[string]string{} + for _, rootPath := range paths { + err := finder.filepath.Walk( + rootPath, + func(path string, fileInfo os.FileInfo, err error) error { + if err != nil { + + return err + } + if !fileInfo.IsDir() { + for _, setting := range settings { + if setting == finder.filepath.Base(path) { + dir, _ := finder.filepath.Abs(finder.filepath.Dir(path)) + file, _ := finder.filepath.Abs(path) + settingsMap[dir] = file + } + } + + for _, gradle := range gradlew { + if gradle == finder.filepath.Base(path) { + dir, _ := finder.filepath.Abs(finder.filepath.Dir(path)) + file, _ := finder.filepath.Abs(path) + gradlewMap[dir] = file + } + } + } + + return nil + }, + ) + if err != nil { + + return nil, nil, SetupWalkError{message: err.Error()} + } + } + + return settingsMap, gradlewMap, nil +} diff --git a/pkg/io/finder/gradle/meta_file_finder_test.go b/pkg/io/finder/gradle/meta_file_finder_test.go new file mode 100644 index 00000000..0f764b68 --- /dev/null +++ b/pkg/io/finder/gradle/meta_file_finder_test.go @@ -0,0 +1,61 @@ +package gradle + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFind(t *testing.T) { + finder := MetaFileFinder{filepath: FilePath{}} + paths := []string{filepath.Join("testdata", "project")} + sMap, gMap, _ := finder.Find(paths) + + assert.Len(t, sMap, 1) + assert.Len(t, gMap, 1) +} + +func TestFindNoFiles(t *testing.T) { + finder := MetaFileFinder{filepath: FilePath{}} + paths := []string{filepath.Join("testdata", "project", "subproject")} + sMap, gMap, _ := finder.Find(paths) + + assert.Len(t, sMap, 0) + assert.Len(t, gMap, 0) +} + +type mockGradleFilePath struct{} + +func (m mockGradleFilePath) Walk(root string, walkFn filepath.WalkFunc) error { + return errors.New("test") +} + +func (m mockGradleFilePath) Base(path string) string { + return filepath.Base(path) +} + +func (m mockGradleFilePath) Abs(path string) (string, error) { + return filepath.Abs(path) +} + +func (m mockGradleFilePath) Dir(path string) string { + return filepath.Dir(path) +} + +func TestWalkError(t *testing.T) { + finder := MetaFileFinder{filepath: mockGradleFilePath{}} + paths := []string{filepath.Join("testdata", "project", "subproject")} + _, _, err := finder.Find(paths) + assert.EqualError(t, err, SetupWalkError{message: "test"}.Error()) +} + +func TestWalkFuncError(t *testing.T) { + finder := MetaFileFinder{filepath: FilePath{}} + paths := []string{filepath.Join("testdata", "test")} + _, _, err := finder.Find(paths) + + // assert err not nil + assert.NotNil(t, err) +} diff --git a/pkg/io/finder/gradle/testdata/cmd_factory_mock.go b/pkg/io/finder/gradle/testdata/cmd_factory_mock.go new file mode 100644 index 00000000..f8c60b66 --- /dev/null +++ b/pkg/io/finder/gradle/testdata/cmd_factory_mock.go @@ -0,0 +1,34 @@ +package testdata + +import ( + "os/exec" + "strings" +) + +type CmdFactoryMock struct { + Err error + Name string +} + +func (f CmdFactoryMock) MakeDependenciesGraphCmd(dir string, gradlew string, _ string) (*exec.Cmd, error) { + err := f.Err + if gradlew == "gradle" { + err = nil + } + + if f.Err != nil && strings.HasPrefix(f.Err.Error(), "give-error-on-gradle") { + err = f.Err + } + + return exec.Command(f.Name, `MakeDependenciesCmd`), err +} + +// implement the interface +func (f CmdFactoryMock) MakeFindSubGraphCmd(_ string, _ string, _ string) (*exec.Cmd, error) { + return exec.Command(f.Name, `MakeFindSubGraphCmd`), f.Err +} + +// implement the interface +func (f CmdFactoryMock) MakeDependenciesCmd(_ string) (*exec.Cmd, error) { + return exec.Command(f.Name, `MakeDependenciesCmd`), f.Err +} diff --git a/pkg/io/finder/gradle/testdata/project/build.gradle b/pkg/io/finder/gradle/testdata/project/build.gradle new file mode 100644 index 00000000..e69de29b diff --git a/pkg/io/finder/gradle/testdata/project/gradlew b/pkg/io/finder/gradle/testdata/project/gradlew new file mode 100644 index 00000000..e69de29b diff --git a/pkg/io/finder/gradle/testdata/project/gradlew.bat b/pkg/io/finder/gradle/testdata/project/gradlew.bat new file mode 100644 index 00000000..e69de29b diff --git a/pkg/io/finder/gradle/testdata/project/settings.gradle b/pkg/io/finder/gradle/testdata/project/settings.gradle new file mode 100644 index 00000000..e69de29b diff --git a/pkg/io/finder/gradle/testdata/project/subproject/build.gradle b/pkg/io/finder/gradle/testdata/project/subproject/build.gradle new file mode 100644 index 00000000..e69de29b diff --git a/pkg/io/finder/maven/maven.go b/pkg/io/finder/maven/maven.go new file mode 100644 index 00000000..726a1d97 --- /dev/null +++ b/pkg/io/finder/maven/maven.go @@ -0,0 +1,57 @@ +package maven + +import ( + "path/filepath" + + "github.com/vifraa/gopom" +) + +type IPomService interface { + GetRootPomFiles(files []string) []string + ParsePomModules(path string) ([]string, error) +} + +type PomService struct{} + +func (p PomService) ParsePomModules(path string) ([]string, error) { + pom, err := gopom.Parse(path) + + if err != nil { + return nil, err + } + + return pom.Modules, nil +} + +func (p PomService) GetRootPomFiles(files []string) []string { + childMap := make(map[string]bool) + var validFiles []string + var roots []string + + for _, filePath := range files { + modules, err := p.ParsePomModules(filePath) + + if err != nil { + continue + } + + validFiles = append(validFiles, filePath) + + if len(modules) == 0 { + continue + } + + for _, module := range modules { + modulePath := filepath.Join(filepath.Dir(filePath), filepath.Dir(module), filepath.Base(module), "pom.xml") + childMap[modulePath] = true + } + } + + for _, file := range validFiles { + if _, ok := childMap[file]; !ok { + roots = append(roots, file) + } + } + + return roots +} diff --git a/pkg/io/finder/maven/maven_test.go b/pkg/io/finder/maven/maven_test.go new file mode 100644 index 00000000..64270517 --- /dev/null +++ b/pkg/io/finder/maven/maven_test.go @@ -0,0 +1,39 @@ +package maven + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParsePomModules(t *testing.T) { + p := PomService{} + modules, err := p.ParsePomModules("testdata/pom.xml") + assert.Nil(t, err) + assert.Len(t, modules, 5) + correct := []string{"guava", "guava-bom", "guava-gwt", "guava-testlib", "guava-tests"} + assert.Equal(t, correct, modules) + + modules, err = p.ParsePomModules("testdata/notAPom.xml") + + assert.NotNil(t, err) + assert.Len(t, modules, 0) +} + +func TestGetRootPomFiles(t *testing.T) { + pomParent := filepath.Join("testdata", "pom.xml") + pomFail := filepath.Join("testdata", "notAPom.xml") + pomChild := filepath.Join("testdata", "guava", "pom.xml") + + p := PomService{} + files := p.GetRootPomFiles([]string{pomParent, pomFail}) + assert.Len(t, files, 1) + + files = p.GetRootPomFiles([]string{pomParent, pomChild}) + assert.Len(t, files, 1) + assert.Equal(t, pomParent, files[0]) + + files = p.GetRootPomFiles([]string{pomFail}) + assert.Len(t, files, 0) +} diff --git a/pkg/io/finder/maven/testdata/cmd_factory_mock.go b/pkg/io/finder/maven/testdata/cmd_factory_mock.go new file mode 100644 index 00000000..d2172f73 --- /dev/null +++ b/pkg/io/finder/maven/testdata/cmd_factory_mock.go @@ -0,0 +1,16 @@ +package testdata + +import "os/exec" + +type CmdFactoryMock struct { + Err error + Name string + Arg string +} + +func (f CmdFactoryMock) MakeDependencyTreeCmd(_ string) (*exec.Cmd, error) { + if len(f.Arg) == 0 { + f.Arg = `"MakeDependencyTreeCmd"` + } + return exec.Command(f.Name, f.Arg), f.Err +} diff --git a/pkg/io/finder/maven/testdata/guava/pom.xml b/pkg/io/finder/maven/testdata/guava/pom.xml new file mode 100644 index 00000000..150831cc --- /dev/null +++ b/pkg/io/finder/maven/testdata/guava/pom.xml @@ -0,0 +1,253 @@ + + + 4.0.0 + + com.google.guava + guava-parent + HEAD-jre-SNAPSHOT + + guava + bundle + Guava: Google Core Libraries for Java + https://github.com/google/guava + + Guava is a suite of core and expanded libraries that include + utility classes, Google's collections, I/O classes, and + much more. + + + + com.google.guava + failureaccess + 1.0.1 + + + com.google.guava + listenablefuture + 9999.0-empty-to-avoid-conflict-with-guava + + + com.google.code.findbugs + jsr305 + + + org.checkerframework + checker-qual + + + com.google.errorprone + error_prone_annotations + + + com.google.j2objc + j2objc-annotations + + + + + + + + maven-jar-plugin + + + + com.google.common + + + + + + true + org.apache.felix + maven-bundle-plugin + 5.1.8 + + + bundle-manifest + process-classes + + manifest + + + + + + + !com.google.common.base.internal, + !com.google.common.util.concurrent.internal, + com.google.common.* + + + com.google.common.util.concurrent.internal, + javax.annotation;resolution:=optional, + javax.crypto.*;resolution:=optional, + sun.misc.*;resolution:=optional + + https://github.com/google/guava/ + + + + + maven-compiler-plugin + + + maven-source-plugin + + + + maven-dependency-plugin + + + unpack-jdk-sources + generate-sources + unpack-dependencies + + srczip + ${project.build.directory}/jdk-sources + false + + **/module-info.java,**/java/io/FileDescriptor.java + + + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + + + maven-javadoc-plugin + + + + + ${project.build.sourceDirectory}:${project.build.directory}/jdk-sources + + + + + com.azul.tooling.in,com.google.common.base.internal,com.google.common.base.internal.*,com.google.thirdparty.publicsuffix,com.google.thirdparty.publicsuffix.*,com.oracle.*,com.sun.*,java.*,javax.*,jdk,jdk.*,org.*,sun.* + + + + + apiNote + X + + + implNote + X + + + implSpec + X + + + jls + X + + + revised + X + + + spec + X + + + + + + false + + + + + https://static.javadoc.io/com.google.code.findbugs/jsr305/3.0.1/ + ${project.basedir}/javadoc-link/jsr305 + + + https://static.javadoc.io/com.google.j2objc/j2objc-annotations/1.1/ + ${project.basedir}/javadoc-link/j2objc-annotations + + + + https://docs.oracle.com/javase/9/docs/api/ + https://docs.oracle.com/javase/9/docs/api/ + + + + https://checkerframework.org/api/ + ${project.basedir}/javadoc-link/checker-framework + + + + https://errorprone.info/api/latest/ + + + + + attach-docs + + + generate-javadoc-site-report + site + javadoc + + + + + + + + srczip-parent + + + ${java.home}/../src.zip + + + + + jdk + srczip + 999 + system + ${java.home}/../src.zip + true + + + + + srczip-lib + + + ${java.home}/lib/src.zip + + + + + jdk + srczip + 999 + system + ${java.home}/lib/src.zip + true + + + + + + maven-javadoc-plugin + + + ${project.build.sourceDirectory}:${project.build.directory}/jdk-sources/java.base + + + + + + + diff --git a/pkg/io/finder/maven/testdata/notAPom.xml b/pkg/io/finder/maven/testdata/notAPom.xml new file mode 100644 index 00000000..a87187bc --- /dev/null +++ b/pkg/io/finder/maven/testdata/notAPom.xml @@ -0,0 +1,3 @@ +pandas==1.1.1 +# comment +numpy==1.2.3 \ No newline at end of file diff --git a/pkg/io/finder/maven/testdata/pom.xml b/pkg/io/finder/maven/testdata/pom.xml new file mode 100644 index 00000000..1cb2a0be --- /dev/null +++ b/pkg/io/finder/maven/testdata/pom.xml @@ -0,0 +1,541 @@ + + + + 4.0.0 + com.google.guava + guava-parent + HEAD-jre-SNAPSHOT + pom + Guava Maven Parent + Parent for guava artifacts + https://github.com/google/guava + + + %regex[.*.class] + 1.1.3 + 3.29.0 + 1.22 + 3.4.1 + 9+181-r4173-1 + + + 3.2.1 + 1980-02-01T00:00:00Z + UTF-8 + + + + GitHub Issues + https://github.com/google/guava/issues + + 2010 + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + scm:git:https://github.com/google/guava.git + scm:git:git@github.com:google/guava.git + https://github.com/google/guava + + + + kevinb9n + Kevin Bourrillion + kevinb@google.com + Google + http://www.google.com + + owner + developer + + -8 + + + + GitHub Actions + https://github.com/google/guava/actions + + + guava + guava-bom + guava-gwt + guava-testlib + guava-tests + + + + src + test + + + src + + **/*.java + **/*.sw* + + + + + + test + + **/*.java + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-versions + + enforce + + + + + 3.0.5 + + + 1.8.0 + + + + + + + + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + ${java.specification.version} + + + + + + + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + UTF-8 + true + + + -sourcepath + doesnotexist + + -XDcompilePolicy=simple + + + + + com.google.errorprone + error_prone_core + 2.16 + + + + true + + + + maven-jar-plugin + 3.2.0 + + + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + post-integration-test + jar + + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + ${animal.sniffer.version} + + true + + org.codehaus.mojo.signature + java18 + 1.0 + + + + + check-java-version-compatibility + test + + check + + + + + + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + true + true + UTF-8 + UTF-8 + UTF-8 + + -XDignore.symbol.file + -Xdoclint:-html + + true + 8 + ${maven-javadoc-plugin.additionalJOptions} + + + + attach-docs + post-integration-test + jar + + + + + maven-dependency-plugin + 3.1.1 + + + maven-antrun-plugin + 1.6 + + + maven-surefire-plugin + 2.7.2 + + + ${test.include} + + + + + %regex[.*PackageSanityTests.*.class] + + %regex[.*Tester.class] + + %regex[.*[$]\d+.class] + + true + alphabetical + + + -Xmx1536M -Duser.language=hi -Duser.country=IN ${test.add.opens} + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0-M3 + + + + + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + + sonatype-nexus-staging + Nexus Release Repository + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + guava-site + Guava Documentation Site + scp://dummy.server/dontinstall/usestaging + + + + + + com.google.code.findbugs + jsr305 + 3.0.2 + + + org.checkerframework + checker-qual + ${checker-framework.version} + + + org.checkerframework + checker-qual + ${checker-framework.version} + sources + + + com.google.errorprone + error_prone_annotations + 2.18.0 + + + com.google.j2objc + j2objc-annotations + 2.8 + + + junit + junit + 4.13.2 + test + + + org.mockito + mockito-core + 4.11.0 + test + + + com.google.jimfs + jimfs + 1.2 + test + + + com.google.truth + truth + ${truth.version} + test + + + + com.google.guava + guava + + + + + com.google.truth.extensions + truth-java8-extension + ${truth.version} + test + + + + com.google.guava + guava + + + + + com.google.caliper + caliper + 1.0-beta-3 + test + + + + com.google.guava + guava + + + + + + + + sonatype-oss-release + + + + org.apache.maven.plugins + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.0.1 + + + sign-artifacts + verify + + sign + + + + + + + + + + javadocs-jdk11-12 + + [11,13) + + + --no-module-directories + + + + open-jre-modules + + [9,] + + + + + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/sun.security.jca=ALL-UNNAMED + + + + + javac9-for-jdk8 + + 1.8 + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + -J-Xbootclasspath/p:${settings.localRepository}/com/google/errorprone/javac/${javac.version}/javac-${javac.version}.jar + + + + + + + + run-error-prone + + + [11,12),[16,) + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + + -Xplugin:ErrorProne -Xep:NullArgumentForNonNullParameter:OFF -Xep:Java8ApiChecker:ERROR + + + -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + + + + + + + + diff --git a/pkg/io/finder/refiner.go b/pkg/io/finder/refiner.go new file mode 100644 index 00000000..3e265f8a --- /dev/null +++ b/pkg/io/finder/refiner.go @@ -0,0 +1,133 @@ +package finder + +import ( + "os" + "path/filepath" + "strings" +) + +func FindFiles(roots []string, exclusions []string) ([]string, error) { + files := make(map[string]bool) + var err error = nil + + for _, root := range roots { + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + for _, dir := range exclusions { + if info.IsDir() && info.Name() == dir { + return filepath.SkipDir + } + } + + if !info.IsDir() { + files[path] = true + } + + return nil + }) + + if err != nil { + break + } + } + + fileList := make([]string, len(files)) + i := 0 + for k := range files { + fileList[i] = k + i++ + } + + return fileList, err +} + +func FilterFiles(files []string, pattern string) []string { + filteredFiles := []string{} + for _, file := range files { + matched, _ := filepath.Match(pattern, filepath.Base(file)) + if matched { + filteredFiles = append(filteredFiles, file) + } + } + return filteredFiles +} + +func ConvertPathsToAbsPaths(paths []string) ([]string, error) { + absPaths := []string{} + + for _, path := range paths { + path, err := filepath.Abs(path) + + if err != nil { + return []string{}, err + } + + absPaths = append(absPaths, path) + } + + return absPaths, nil +} + +func MapFilesToDir(dirs []string, files []string) map[string][]string { + dirToFilesMap := make(map[string][]string) + + if len(dirs) == 0 { + return dirToFilesMap + } + + for _, file := range files { + longestMatchLength := 0 + var matchingDir string + for _, dir := range dirs { + matchLength := 0 + for i := 0; i < len(file) && i < len(dir); i++ { + if file[i] != dir[i] { + break + } + matchLength++ + } + if matchLength > longestMatchLength { + longestMatchLength = matchLength + matchingDir = dir + } + } + + if _, ok := dirToFilesMap[matchingDir]; ok == false { + dirToFilesMap[matchingDir] = []string{} + } + dirToFilesMap[matchingDir] = append(dirToFilesMap[matchingDir], file) + } + + return dirToFilesMap +} + +func GCDPath(paths []string) string { + var result string + var shortest string + + for i, path := range paths { + if i == 0 || len(path) < len(shortest) { + shortest = path + } + } + + for i := 0; i < len(shortest); i++ { + c := shortest[i] + + if filepath.Separator == c { + dirpath := shortest[:i+1] + for _, path := range paths { + if !strings.HasPrefix(path, dirpath) { + return result + } + } + + result = dirpath + } + } + + return result +} diff --git a/pkg/io/finder/testdata/finder_mock.go b/pkg/io/finder/testdata/finder_mock.go new file mode 100644 index 00000000..8866586a --- /dev/null +++ b/pkg/io/finder/testdata/finder_mock.go @@ -0,0 +1,30 @@ +package testdata + +type FinderMock struct { + FindJavaClassDirsNames []string + FindJavaClassDirsErr error + FindMavenRootsNames []string + FindMavenRootsErr error + FindGradleRootsNames []string + FindGradleRootsErr error +} + +func NewEmptyFinderMock() FinderMock { + return FinderMock{ + FindJavaClassDirsNames: []string{}, + FindMavenRootsNames: []string{}, + FindGradleRootsNames: []string{}, + } +} + +func (f FinderMock) FindJavaClassDirs(_ []string) ([]string, error) { + return f.FindJavaClassDirsNames, f.FindJavaClassDirsErr +} + +func (f FinderMock) FindMavenRoots(_ []string) ([]string, error) { + return f.FindMavenRootsNames, f.FindMavenRootsErr +} + +func (f FinderMock) FindGradleRoots(_ []string) ([]string, error) { + return f.FindGradleRootsNames, f.FindGradleRootsErr +} diff --git a/pkg/io/writer/file_writer.go b/pkg/io/writer/file_writer.go new file mode 100644 index 00000000..c152d77f --- /dev/null +++ b/pkg/io/writer/file_writer.go @@ -0,0 +1,27 @@ +package writer + +import ( + "os" +) + +type IFileWriter interface { + Write(file *os.File, p []byte) error + Create(name string) (*os.File, error) + Close(file *os.File) error +} + +type FileWriter struct{} + +func (fw FileWriter) Create(name string) (*os.File, error) { + return os.Create(name) +} + +func (fw FileWriter) Write(file *os.File, p []byte) error { + _, err := file.Write(p) + + return err +} + +func (fw FileWriter) Close(file *os.File) error { + return file.Close() +} diff --git a/pkg/io/writer/file_writer_test.go b/pkg/io/writer/file_writer_test.go new file mode 100644 index 00000000..b3531ede --- /dev/null +++ b/pkg/io/writer/file_writer_test.go @@ -0,0 +1,47 @@ +package writer + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +var fw = FileWriter{} + +const fileName = "debricked-test.json" + +func TestCreate(t *testing.T) { + testFile, err := fw.Create(fileName) + assert.NoError(t, err) + assert.NotNil(t, testFile) + defer deleteFile(t, testFile) +} + +func TestWrite(t *testing.T) { + content := []byte("{}") + testFile, _ := fw.Create(fileName) + defer deleteFile(t, testFile) + + err := fw.Write(testFile, content) + + assert.NoError(t, err) + fileContents, err := os.ReadFile(fileName) + assert.NoError(t, err) + assert.Equal(t, fileContents, content) +} + +func TestClose(t *testing.T) { + testFile, _ := fw.Create(fileName) + defer deleteFile(t, testFile) + + err := fw.Close(testFile) + + assert.NoError(t, err) +} + +func deleteFile(t *testing.T, file *os.File) { + _ = file.Close() + err := os.Remove(file.Name()) + assert.NoError(t, err) +} diff --git a/pkg/io/writer/testdata/file_writer_mock.go b/pkg/io/writer/testdata/file_writer_mock.go new file mode 100644 index 00000000..8ca080df --- /dev/null +++ b/pkg/io/writer/testdata/file_writer_mock.go @@ -0,0 +1,27 @@ +package writer + +import ( + "os" +) + +type FileWriterMock struct { + file *os.File + Contents []byte + CreateErr error + WriteErr error + CloseErr error +} + +func (fw *FileWriterMock) Create(_ string) (*os.File, error) { + return fw.file, fw.CreateErr +} + +func (fw *FileWriterMock) Write(_ *os.File, bytes []byte) error { + fw.Contents = append(fw.Contents, bytes...) + + return fw.WriteErr +} + +func (fw *FileWriterMock) Close(_ *os.File) error { + return fw.CloseErr +} diff --git a/pkg/resolution/file/file_batch.go b/pkg/resolution/file/file_batch.go new file mode 100644 index 00000000..3ccc47d6 --- /dev/null +++ b/pkg/resolution/file/file_batch.go @@ -0,0 +1,37 @@ +package file + +import "github.com/debricked/cli/pkg/resolution/pm" + +type IBatch interface { + Files() []string + Add(file string) + Pm() pm.IPm +} + +type Batch struct { + files map[string]bool + pm pm.IPm +} + +func NewBatch(pm pm.IPm) Batch { + return Batch{files: make(map[string]bool), pm: pm} +} + +func (b Batch) Files() []string { + var files []string + for file := range b.files { + files = append(files, file) + } + + return files +} + +func (b Batch) Add(file string) { + if ok := b.files[file]; !ok { + b.files[file] = true + } +} + +func (b Batch) Pm() pm.IPm { + return b.pm +} diff --git a/pkg/resolution/file/file_batch_factory.go b/pkg/resolution/file/file_batch_factory.go new file mode 100644 index 00000000..7ac062de --- /dev/null +++ b/pkg/resolution/file/file_batch_factory.go @@ -0,0 +1,49 @@ +package file + +import ( + "path" + "regexp" + + "github.com/debricked/cli/pkg/resolution/pm" +) + +type IBatchFactory interface { + Make(files []string) []IBatch +} + +type BatchFactory struct { + pms []pm.IPm +} + +func NewBatchFactory() BatchFactory { + return BatchFactory{ + pms: pm.Pms(), + } +} + +func (bf BatchFactory) Make(files []string) []IBatch { + batchMap := make(map[string]IBatch) + for _, file := range files { + for _, p := range bf.pms { + for _, manifest := range p.Manifests() { + compiledRegex, _ := regexp.Compile(manifest) + if compiledRegex.MatchString(path.Base(file)) { + batch, ok := batchMap[p.Name()] + if !ok { + batch = NewBatch(p) + batchMap[p.Name()] = batch + } + batch.Add(file) + } + } + } + } + + batches := make([]IBatch, 0, len(batchMap)) + + for _, batch := range batchMap { + batches = append(batches, batch) + } + + return batches +} diff --git a/pkg/resolution/file/file_batch_factory_test.go b/pkg/resolution/file/file_batch_factory_test.go new file mode 100644 index 00000000..29a06620 --- /dev/null +++ b/pkg/resolution/file/file_batch_factory_test.go @@ -0,0 +1,106 @@ +package file + +import ( + "testing" + + "github.com/debricked/cli/pkg/resolution/pm" + "github.com/debricked/cli/pkg/resolution/pm/testdata" + "github.com/stretchr/testify/assert" +) + +func TestNewBatchFactory(t *testing.T) { + bf := NewBatchFactory() + assert.NotNil(t, bf) + + pms := bf.pms + assert.Equal(t, pm.Pms(), pms) +} + +func TestMakeNoPms(t *testing.T) { + bf := BatchFactory{} + batches := bf.Make([]string{"go.mod"}) + assert.Empty(t, batches) +} + +func TestMakeNoFiles(t *testing.T) { + bf := NewBatchFactory() + batches := bf.Make([]string{}) + assert.Empty(t, batches) +} + +func TestMakeNoManifests(t *testing.T) { + bf := BatchFactory{pms: []pm.IPm{testdata.PmMock{}}} + batches := bf.Make([]string{"go.mod"}) + assert.Empty(t, batches) +} + +func TestMakeOneFile(t *testing.T) { + bf := BatchFactory{pms: []pm.IPm{ + testdata.PmMock{ + N: "go", + Ms: []string{"go.mod"}, + }, + }} + batches := bf.Make([]string{"test/go.mod"}) + assert.Len(t, batches, 1) + batch := batches[0] + assert.Len(t, batch.Files(), 1) + file := batch.Files()[0] + assert.Equal(t, "test/go.mod", file) +} + +func TestMakeMultipleFiles(t *testing.T) { + bf := BatchFactory{pms: []pm.IPm{ + testdata.PmMock{ + N: "go", + Ms: []string{"go.mod"}, + }, + }} + batches := bf.Make([]string{"go.mod", "test/go.mod"}) + assert.Len(t, batches, 1) + batch := batches[0] + assert.Len(t, batch.Files(), 2) +} + +func TestMakeMultipleBatches(t *testing.T) { + bf := BatchFactory{pms: []pm.IPm{ + testdata.PmMock{ + N: "go", + Ms: []string{"go.mod"}, + }, + testdata.PmMock{ + N: "mvn", + Ms: []string{"pom.xml"}, + }, + }} + batches := bf.Make([]string{"go.mod", "test/pom.xml"}) + assert.Len(t, batches, 2) + for _, batch := range batches { + assert.Len(t, batch.Files(), 1) + } +} + +func TestMakeMultipleBatchesMultipleFiles(t *testing.T) { + bf := BatchFactory{pms: []pm.IPm{ + testdata.PmMock{ + N: "go", + Ms: []string{"go.mod"}, + }, + testdata.PmMock{ + N: "mvn", + Ms: []string{"pom.xml"}, + }, + }} + batches := bf.Make([]string{"go.mod", "test/pom.xml", "test/sub/go.mod"}) + assert.Len(t, batches, 2) + for _, batch := range batches { + if len(batch.Files()) == 1 { + assert.Contains(t, batch.Files(), "test/pom.xml") + } else if len(batch.Files()) == 2 { + assert.Contains(t, batch.Files(), "go.mod") + assert.Contains(t, batch.Files(), "test/sub/go.mod") + } else { + t.Error("failed to assert number of files in the batch") + } + } +} diff --git a/pkg/resolution/file/file_batch_test.go b/pkg/resolution/file/file_batch_test.go new file mode 100644 index 00000000..19d60096 --- /dev/null +++ b/pkg/resolution/file/file_batch_test.go @@ -0,0 +1,60 @@ +package file + +import ( + "testing" + + "github.com/debricked/cli/pkg/resolution/pm/testdata" + "github.com/stretchr/testify/assert" +) + +func TestNewBatch(t *testing.T) { + b := NewBatch(nil) + assert.NotNil(t, b) + + b = NewBatch(testdata.PmMock{}) + assert.NotNil(t, b) +} + +func TestFiles(t *testing.T) { + b := NewBatch(testdata.PmMock{}) + + files := b.Files() + assert.Empty(t, files) + + b.Add("file-1") + assert.Len(t, b.Files(), 1) + + b.Add("file-1") + assert.Len(t, b.Files(), 1) + + b.Add("file-2") + assert.Len(t, b.Files(), 2) +} + +func TestAdd(t *testing.T) { + b := NewBatch(testdata.PmMock{}) + + filesMap := b.files + assert.Empty(t, filesMap) + + b.Add("file-1") + filesMap = b.files + assert.Len(t, filesMap, 1) + + b.Add("file-1") + filesMap = b.files + assert.Len(t, filesMap, 1) + + b.Add("file-2") + filesMap = b.files + assert.Len(t, filesMap, 2) +} + +func TestPm(t *testing.T) { + b := NewBatch(nil) + assert.Nil(t, b.Pm()) + + pm := testdata.PmMock{} + b = NewBatch(pm) + assert.Equal(t, pm, b.Pm()) +} diff --git a/pkg/resolution/file/testdata/file_batch_factory_mock.go b/pkg/resolution/file/testdata/file_batch_factory_mock.go new file mode 100644 index 00000000..9c81add4 --- /dev/null +++ b/pkg/resolution/file/testdata/file_batch_factory_mock.go @@ -0,0 +1,21 @@ +package testdata + +import ( + "github.com/debricked/cli/pkg/resolution/file" + "github.com/debricked/cli/pkg/resolution/pm" +) + +type BatchFactoryMock struct { + pms []pm.IPm +} + +func NewBatchFactoryMock() BatchFactoryMock { + return BatchFactoryMock{ + pms: pm.Pms(), + } +} + +func (bf BatchFactoryMock) Make(_ []string) []file.IBatch { + + return []file.IBatch{} +} diff --git a/pkg/resolution/file/testdata/file_batch_mock.go b/pkg/resolution/file/testdata/file_batch_mock.go new file mode 100644 index 00000000..69d29d3c --- /dev/null +++ b/pkg/resolution/file/testdata/file_batch_mock.go @@ -0,0 +1 @@ +package testdata diff --git a/pkg/resolution/job/base_job.go b/pkg/resolution/job/base_job.go new file mode 100644 index 00000000..df07acc4 --- /dev/null +++ b/pkg/resolution/job/base_job.go @@ -0,0 +1,45 @@ +package job + +import ( + "errors" + "os/exec" +) + +type BaseJob struct { + file string + errs IErrors + status chan string +} + +func NewBaseJob(file string) BaseJob { + return BaseJob{ + file: file, + errs: NewErrors(file), + status: make(chan string), + } +} + +func (j *BaseJob) GetFile() string { + return j.file +} + +func (j *BaseJob) Errors() IErrors { + return j.errs +} + +func (j *BaseJob) ReceiveStatus() chan string { + return j.status +} + +func (j *BaseJob) SendStatus(status string) { + j.status <- status +} + +func (j *BaseJob) GetExitError(err error) error { + exitErr, ok := err.(*exec.ExitError) + if !ok { + return err + } + + return errors.New(string(exitErr.Stderr)) +} diff --git a/pkg/resolution/job/base_job_test.go b/pkg/resolution/job/base_job_test.go new file mode 100644 index 00000000..3512a749 --- /dev/null +++ b/pkg/resolution/job/base_job_test.go @@ -0,0 +1,87 @@ +package job + +import ( + "errors" + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" +) + +const testFile = "file" + +func TestNewBaseJob(t *testing.T) { + j := NewBaseJob(testFile) + assert.Equal(t, testFile, j.GetFile()) + assert.NotNil(t, j.Errors()) + assert.NotNil(t, j.status) +} + +func TestGetFile(t *testing.T) { + j := BaseJob{} + j.file = testFile + assert.Equal(t, testFile, j.GetFile()) +} + +func TestReceiveStatus(t *testing.T) { + j := BaseJob{ + file: testFile, + errs: nil, + status: make(chan string), + } + + statusChan := j.ReceiveStatus() + assert.NotNil(t, statusChan) +} + +func TestErrors(t *testing.T) { + jobErr := errors.New("error") + j := BaseJob{} + j.file = testFile + j.errs = NewErrors(j.file) + j.errs.Critical(jobErr) + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), jobErr) +} + +func TestSendStatus(t *testing.T) { + j := BaseJob{ + file: testFile, + errs: nil, + status: make(chan string), + } + + go func() { + status := <-j.ReceiveStatus() + assert.Equal(t, "status", status) + }() + + j.SendStatus("status") +} + +func TestDifferentNewBaseJob(t *testing.T) { + differentFileName := "testDifferentFile" + j := NewBaseJob(differentFileName) + assert.NotEqual(t, testFile, j.GetFile()) + assert.Equal(t, differentFileName, j.GetFile()) + assert.NotNil(t, j.Errors()) + assert.NotNil(t, j.status) +} + +func TestGetExitErrorWithExitError(t *testing.T) { + err := &exec.ExitError{ + ProcessState: nil, + Stderr: []byte("stderr"), + } + j := BaseJob{} + exitErr := j.GetExitError(err) + assert.ErrorContains(t, exitErr, string(err.Stderr)) +} + +func TestGetExitErrorWithNoneExitError(t *testing.T) { + err := &exec.Error{Err: errors.New("none-exit-err")} + j := BaseJob{} + exitErr := j.GetExitError(err) + assert.ErrorContains(t, exitErr, err.Error()) +} diff --git a/pkg/resolution/job/error.go b/pkg/resolution/job/error.go new file mode 100644 index 00000000..029daa29 --- /dev/null +++ b/pkg/resolution/job/error.go @@ -0,0 +1,5 @@ +package job + +type IError interface { + error +} diff --git a/pkg/resolution/job/errors.go b/pkg/resolution/job/errors.go new file mode 100644 index 00000000..401c70ea --- /dev/null +++ b/pkg/resolution/job/errors.go @@ -0,0 +1,48 @@ +package job + +type IErrors interface { + Warning(err IError) + Critical(err IError) + GetWarningErrors() []IError + GetCriticalErrors() []IError + GetAll() []IError + HasError() bool +} + +type Errors struct { + title string + warningErrs []IError + criticalErrs []IError +} + +func NewErrors(title string) *Errors { + return &Errors{ + title: title, + warningErrs: []IError{}, + criticalErrs: []IError{}, + } +} + +func (errors *Errors) Warning(err IError) { + errors.warningErrs = append(errors.warningErrs, err) +} + +func (errors *Errors) Critical(err IError) { + errors.criticalErrs = append(errors.criticalErrs, err) +} + +func (errors *Errors) GetWarningErrors() []IError { + return errors.warningErrs +} + +func (errors *Errors) GetCriticalErrors() []IError { + return errors.criticalErrs +} + +func (errors *Errors) GetAll() []IError { + return append(errors.warningErrs, errors.criticalErrs...) +} + +func (errors *Errors) HasError() bool { + return len(errors.criticalErrs) > 0 || len(errors.warningErrs) > 0 +} diff --git a/pkg/resolution/job/errors_test.go b/pkg/resolution/job/errors_test.go new file mode 100644 index 00000000..be3dc9a1 --- /dev/null +++ b/pkg/resolution/job/errors_test.go @@ -0,0 +1,77 @@ +package job + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewErrors(t *testing.T) { + title := "title" + errors := NewErrors(title) + assert.Equal(t, title, errors.title) + assert.NotNil(t, errors) + assert.Empty(t, errors.criticalErrs) + assert.Empty(t, errors.warningErrs) +} + +func TestWarning(t *testing.T) { + errors := NewErrors("") + warning := fmt.Errorf("error") + errors.Warning(warning) + assert.Empty(t, errors.criticalErrs) + assert.Len(t, errors.warningErrs, 1) + assert.Contains(t, errors.warningErrs, warning) +} + +func TestCritical(t *testing.T) { + errors := NewErrors("") + critical := fmt.Errorf("error") + errors.Critical(critical) + assert.Empty(t, errors.warningErrs) + assert.Len(t, errors.criticalErrs, 1) + assert.Contains(t, errors.criticalErrs, critical) +} + +func TestGetWarningErrors(t *testing.T) { + errors := NewErrors("") + warning := fmt.Errorf("error") + errors.Warning(warning) + assert.Empty(t, errors.GetCriticalErrors()) + assert.Len(t, errors.GetWarningErrors(), 1) + assert.Contains(t, errors.GetWarningErrors(), warning) +} + +func TestGetCriticalErrors(t *testing.T) { + errors := NewErrors("") + critical := fmt.Errorf("error") + errors.Critical(critical) + assert.Empty(t, errors.GetWarningErrors()) + assert.Len(t, errors.GetCriticalErrors(), 1) + assert.Contains(t, errors.GetCriticalErrors(), critical) +} + +func TestGetAll(t *testing.T) { + errors := NewErrors("") + warning := fmt.Errorf("warning") + critical := fmt.Errorf("critical") + errors.Warning(warning) + errors.Critical(critical) + assert.Len(t, errors.GetAll(), 2) + assert.Contains(t, errors.GetAll(), warning) + assert.Contains(t, errors.GetAll(), critical) +} + +func TestHasError(t *testing.T) { + errors := NewErrors("") + assert.False(t, errors.HasError()) + + warning := fmt.Errorf("warning") + errors.Warning(warning) + assert.True(t, errors.HasError()) + + critical := fmt.Errorf("critical") + errors.Warning(critical) + assert.True(t, errors.HasError()) +} diff --git a/pkg/resolution/job/job.go b/pkg/resolution/job/job.go new file mode 100644 index 00000000..8e5515e3 --- /dev/null +++ b/pkg/resolution/job/job.go @@ -0,0 +1,8 @@ +package job + +type IJob interface { + GetFile() string + Errors() IErrors + Run() + ReceiveStatus() chan string +} diff --git a/pkg/resolution/job/testdata/job_mock.go b/pkg/resolution/job/testdata/job_mock.go new file mode 100644 index 00000000..ff3fd291 --- /dev/null +++ b/pkg/resolution/job/testdata/job_mock.go @@ -0,0 +1,41 @@ +package testdata + +import ( + "fmt" + + "github.com/debricked/cli/pkg/resolution/job" +) + +type JobMock struct { + file string + errs job.IErrors + status chan string +} + +func (j *JobMock) ReceiveStatus() chan string { + return j.status +} + +func (j *JobMock) GetFile() string { + return j.file +} + +func (j *JobMock) Errors() job.IErrors { + return j.errs +} + +func (j *JobMock) Run() { + fmt.Println("job mock run") +} + +func NewJobMock(file string) *JobMock { + return &JobMock{ + file: file, + status: make(chan string), + errs: job.NewErrors(file), + } +} + +func (j *JobMock) SetErr(err job.IError) { + j.errs.Critical(err) +} diff --git a/pkg/resolution/job/testdata/job_test_util.go b/pkg/resolution/job/testdata/job_test_util.go new file mode 100644 index 00000000..4d1a9526 --- /dev/null +++ b/pkg/resolution/job/testdata/job_test_util.go @@ -0,0 +1,30 @@ +package testdata + +import ( + "fmt" + "runtime" + "testing" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/stretchr/testify/assert" +) + +func AssertPathErr(t *testing.T, jobErrs job.IErrors) { + var path string + if runtime.GOOS == "windows" { + path = "%PATH%" + } else { + path = "$PATH" + } + errs := jobErrs.GetAll() + assert.Len(t, errs, 1) + err := errs[0] + errMsg := fmt.Sprintf("executable file not found in %s", path) + assert.ErrorContains(t, err, errMsg) +} + +func WaitStatus(j job.IJob) { + for { + <-j.ReceiveStatus() + } +} diff --git a/pkg/resolution/pm/gomod/cmd_factory.go b/pkg/resolution/pm/gomod/cmd_factory.go new file mode 100644 index 00000000..3fef7a0b --- /dev/null +++ b/pkg/resolution/pm/gomod/cmd_factory.go @@ -0,0 +1,30 @@ +package gomod + +import "os/exec" + +type ICmdFactory interface { + MakeGraphCmd(workingDirectory string) (*exec.Cmd, error) + MakeListCmd(workingDirectory string) (*exec.Cmd, error) +} + +type CmdFactory struct{} + +func (_ CmdFactory) MakeGraphCmd(workingDirectory string) (*exec.Cmd, error) { + path, err := exec.LookPath("go") + + return &exec.Cmd{ + Path: path, + Args: []string{"go", "mod", "graph"}, + Dir: workingDirectory, + }, err +} + +func (_ CmdFactory) MakeListCmd(workingDirectory string) (*exec.Cmd, error) { + path, err := exec.LookPath("go") + + return &exec.Cmd{ + Path: path, + Args: []string{"go", "list", "-mod=readonly", "-e", "-m", "all"}, + Dir: workingDirectory, + }, err +} diff --git a/pkg/resolution/pm/gomod/cmd_factory_test.go b/pkg/resolution/pm/gomod/cmd_factory_test.go new file mode 100644 index 00000000..3503afa5 --- /dev/null +++ b/pkg/resolution/pm/gomod/cmd_factory_test.go @@ -0,0 +1,28 @@ +package gomod + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMakeGraphCmd(t *testing.T) { + cmd, _ := CmdFactory{}.MakeGraphCmd(".") + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "go") + assert.Contains(t, args, "mod") + assert.Contains(t, args, "graph") +} + +func TestMakeListCmd(t *testing.T) { + cmd, _ := CmdFactory{}.MakeListCmd(".") + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "go") + assert.Contains(t, args, "list") + assert.Contains(t, args, "-mod=readonly") + assert.Contains(t, args, "-e") + assert.Contains(t, args, "-m") + assert.Contains(t, args, "all") +} diff --git a/pkg/resolution/pm/gomod/job.go b/pkg/resolution/pm/gomod/job.go new file mode 100644 index 00000000..996529e2 --- /dev/null +++ b/pkg/resolution/pm/gomod/job.go @@ -0,0 +1,99 @@ +package gomod + +import ( + "path/filepath" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/pm/util" + "github.com/debricked/cli/pkg/resolution/pm/writer" +) + +const ( + fileName = ".gomod.debricked.lock" +) + +type Job struct { + job.BaseJob + cmdFactory ICmdFactory + fileWriter writer.IFileWriter +} + +func NewJob( + file string, + cmdFactory ICmdFactory, + fileWriter writer.IFileWriter, +) *Job { + return &Job{ + BaseJob: job.NewBaseJob(file), + cmdFactory: cmdFactory, + fileWriter: fileWriter, + } +} + +func (j *Job) Run() { + j.SendStatus("creating dependency graph") + + workingDirectory := filepath.Dir(filepath.Clean(j.GetFile())) + + graphCmdOutput, err := j.runGraphCmd(workingDirectory) + if err != nil { + j.Errors().Critical(err) + + return + } + + j.SendStatus("creating dependency version list") + listCmdOutput, err := j.runListCmd(workingDirectory) + if err != nil { + j.Errors().Critical(err) + + return + } + + j.SendStatus("creating lock file") + lockFile, err := j.fileWriter.Create(util.MakePathFromManifestFile(j.GetFile(), fileName)) + if err != nil { + j.Errors().Critical(err) + + return + } + defer util.CloseFile(j, j.fileWriter, lockFile) + + var fileContents []byte + fileContents = append(fileContents, graphCmdOutput...) + fileContents = append(fileContents, []byte("\n")...) + fileContents = append(fileContents, listCmdOutput...) + + err = j.fileWriter.Write(lockFile, fileContents) + if err != nil { + j.Errors().Critical(err) + } +} + +func (j *Job) runGraphCmd(workingDirectory string) ([]byte, error) { + graphCmd, err := j.cmdFactory.MakeGraphCmd(workingDirectory) + if err != nil { + return nil, err + } + + graphCmdOutput, err := graphCmd.Output() + if err != nil { + return nil, j.GetExitError(err) + } + + return graphCmdOutput, nil +} + +func (j *Job) runListCmd(workingDirectory string) ([]byte, error) { + listCmd, err := j.cmdFactory.MakeListCmd(workingDirectory) + if err != nil { + return nil, err + } + + listCmdOutput, err := listCmd.Output() + if err != nil { + return nil, j.GetExitError(err) + } + + return listCmdOutput, nil +} diff --git a/pkg/resolution/pm/gomod/job_test.go b/pkg/resolution/pm/gomod/job_test.go new file mode 100644 index 00000000..c0363309 --- /dev/null +++ b/pkg/resolution/pm/gomod/job_test.go @@ -0,0 +1,125 @@ +package gomod + +import ( + "errors" + "testing" + + jobTestdata "github.com/debricked/cli/pkg/resolution/job/testdata" + "github.com/debricked/cli/pkg/resolution/pm/gomod/testdata" + "github.com/debricked/cli/pkg/resolution/pm/writer" + writerTestdata "github.com/debricked/cli/pkg/resolution/pm/writer/testdata" + "github.com/stretchr/testify/assert" +) + +func TestNewJob(t *testing.T) { + j := NewJob("file", CmdFactory{}, writer.FileWriter{}) + assert.Equal(t, "file", j.GetFile()) + assert.False(t, j.Errors().HasError()) +} + +func TestRunGraphCmdErr(t *testing.T) { + cmdErr := errors.New("cmd-error") + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.MakeGraphCmdErr = cmdErr + j := NewJob("file", cmdFactoryMock, nil) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Contains(t, j.Errors().GetCriticalErrors(), cmdErr) +} + +func TestRunCmdOutputErr(t *testing.T) { + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.GraphCmdName = "bad-name" + j := NewJob("file", cmdFactoryMock, nil) + + go jobTestdata.WaitStatus(j) + + j.Run() + + jobTestdata.AssertPathErr(t, j.Errors()) +} + +func TestRunListCmdErr(t *testing.T) { + cmdErr := errors.New("cmd-error") + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.MakeListCmdErr = cmdErr + j := NewJob("file", cmdFactoryMock, nil) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), cmdErr) +} + +func TestRunListCmdOutputErr(t *testing.T) { + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.ListCmdName = "bad-name" + j := NewJob("file", cmdFactoryMock, nil) + + go jobTestdata.WaitStatus(j) + + j.Run() + + jobTestdata.AssertPathErr(t, j.Errors()) +} + +func TestRunCreateErr(t *testing.T) { + createErr := errors.New("create-error") + fileWriterMock := &writerTestdata.FileWriterMock{CreateErr: createErr} + cmdFactoryMock := testdata.NewEchoCmdFactory() + j := NewJob("file", cmdFactoryMock, fileWriterMock) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), createErr) +} + +func TestRunWriteErr(t *testing.T) { + writeErr := errors.New("write-error") + fileWriterMock := &writerTestdata.FileWriterMock{WriteErr: writeErr} + cmdFactoryMock := testdata.NewEchoCmdFactory() + j := NewJob("file", cmdFactoryMock, fileWriterMock) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), writeErr) +} + +func TestRunCloseErr(t *testing.T) { + closeErr := errors.New("close-error") + fileWriterMock := &writerTestdata.FileWriterMock{CloseErr: closeErr} + cmdFactoryMock := testdata.NewEchoCmdFactory() + j := NewJob("file", cmdFactoryMock, fileWriterMock) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), closeErr) +} + +func TestRun(t *testing.T) { + fileContents := []byte("MakeGraphCmd\n\nMakeListCmd\n") + fileWriterMock := &writerTestdata.FileWriterMock{} + cmdFactoryMock := testdata.NewEchoCmdFactory() + j := NewJob("file", cmdFactoryMock, fileWriterMock) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Empty(t, j.Errors().GetAll()) + assert.Equal(t, fileContents, fileWriterMock.Contents) +} diff --git a/pkg/resolution/pm/gomod/pm.go b/pkg/resolution/pm/gomod/pm.go new file mode 100644 index 00000000..19623e14 --- /dev/null +++ b/pkg/resolution/pm/gomod/pm.go @@ -0,0 +1,23 @@ +package gomod + +const Name = "go" + +type Pm struct { + name string +} + +func NewPm() Pm { + return Pm{ + name: Name, + } +} + +func (pm Pm) Name() string { + return pm.name +} + +func (_ Pm) Manifests() []string { + return []string{ + "go.mod", + } +} diff --git a/pkg/resolution/pm/gomod/pm_test.go b/pkg/resolution/pm/gomod/pm_test.go new file mode 100644 index 00000000..53643b02 --- /dev/null +++ b/pkg/resolution/pm/gomod/pm_test.go @@ -0,0 +1,25 @@ +package gomod + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewPm(t *testing.T) { + pm := NewPm() + assert.Equal(t, Name, pm.name) +} + +func TestName(t *testing.T) { + pm := NewPm() + assert.Equal(t, Name, pm.Name()) +} + +func TestManifests(t *testing.T) { + pm := Pm{} + manifests := pm.Manifests() + assert.Len(t, manifests, 1) + manifest := manifests[0] + assert.Equal(t, "go.mod", manifest) +} diff --git a/pkg/resolution/pm/gomod/strategy.go b/pkg/resolution/pm/gomod/strategy.go new file mode 100644 index 00000000..962dd91d --- /dev/null +++ b/pkg/resolution/pm/gomod/strategy.go @@ -0,0 +1,23 @@ +package gomod + +import ( + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/pm/writer" +) + +type Strategy struct { + files []string +} + +func (s Strategy) Invoke() ([]job.IJob, error) { + var jobs []job.IJob + for _, file := range s.files { + jobs = append(jobs, NewJob(file, CmdFactory{}, writer.FileWriter{})) + } + + return jobs, nil +} + +func NewStrategy(files []string) Strategy { + return Strategy{files} +} diff --git a/pkg/resolution/pm/gomod/strategy_test.go b/pkg/resolution/pm/gomod/strategy_test.go new file mode 100644 index 00000000..57e2444f --- /dev/null +++ b/pkg/resolution/pm/gomod/strategy_test.go @@ -0,0 +1,43 @@ +package gomod + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewStrategy(t *testing.T) { + s := NewStrategy(nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy([]string{}) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy([]string{"file"}) + assert.NotNil(t, s) + assert.Len(t, s.files, 1) + + s = NewStrategy([]string{"file-1", "file-2"}) + assert.NotNil(t, s) + assert.Len(t, s.files, 2) +} + +func TestInvokeNoFiles(t *testing.T) { + s := NewStrategy([]string{}) + jobs, _ := s.Invoke() + assert.Empty(t, jobs) +} + +func TestInvokeOneFile(t *testing.T) { + s := NewStrategy([]string{"file"}) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 1) +} + +func TestInvokeManyFiles(t *testing.T) { + s := NewStrategy([]string{"file-1", "file-2"}) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 2) +} diff --git a/pkg/resolution/pm/gomod/testdata/cmd_factory_mock.go b/pkg/resolution/pm/gomod/testdata/cmd_factory_mock.go new file mode 100644 index 00000000..c2e1d9d0 --- /dev/null +++ b/pkg/resolution/pm/gomod/testdata/cmd_factory_mock.go @@ -0,0 +1,25 @@ +package testdata + +import "os/exec" + +type CmdFactoryMock struct { + GraphCmdName string + MakeGraphCmdErr error + ListCmdName string + MakeListCmdErr error +} + +func NewEchoCmdFactory() CmdFactoryMock { + return CmdFactoryMock{ + GraphCmdName: "echo", + ListCmdName: "echo", + } +} + +func (f CmdFactoryMock) MakeGraphCmd(_ string) (*exec.Cmd, error) { + return exec.Command(f.GraphCmdName, "MakeGraphCmd"), f.MakeGraphCmdErr +} + +func (f CmdFactoryMock) MakeListCmd(_ string) (*exec.Cmd, error) { + return exec.Command(f.ListCmdName, "MakeListCmd"), f.MakeListCmdErr +} diff --git a/pkg/resolution/pm/gradle/cmd_factory.go b/pkg/resolution/pm/gradle/cmd_factory.go new file mode 100644 index 00000000..2e8ff0c9 --- /dev/null +++ b/pkg/resolution/pm/gradle/cmd_factory.go @@ -0,0 +1,32 @@ +package gradle + +import ( + "os/exec" +) + +type ICmdFactory interface { + MakeFindSubGraphCmd(workingDirectory string, gradlew string, initScript string) (*exec.Cmd, error) + MakeDependenciesGraphCmd(workingDirectory string, gradlew string, initScript string) (*exec.Cmd, error) +} + +type CmdFactory struct{} + +func (cf CmdFactory) MakeFindSubGraphCmd(workingDirectory string, gradlew string, initScript string) (*exec.Cmd, error) { + path, err := exec.LookPath(gradlew) + + return &exec.Cmd{ + Path: path, + Args: []string{gradlew, "--init-script", initScript, "debrickedFindSubProjectPaths"}, + Dir: workingDirectory, + }, err +} + +func (cf CmdFactory) MakeDependenciesGraphCmd(workingDirectory string, gradlew string, initScript string) (*exec.Cmd, error) { + path, err := exec.LookPath(gradlew) + + return &exec.Cmd{ + Path: path, + Args: []string{gradlew, "--init-script", initScript, "debrickedAllDeps"}, + Dir: workingDirectory, + }, err +} diff --git a/pkg/resolution/pm/gradle/cmd_factory_test.go b/pkg/resolution/pm/gradle/cmd_factory_test.go new file mode 100644 index 00000000..ac23aa12 --- /dev/null +++ b/pkg/resolution/pm/gradle/cmd_factory_test.go @@ -0,0 +1,27 @@ +package gradle + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMakeFindSubGraphCmd(t *testing.T) { + cmd, _ := CmdFactory{}.MakeFindSubGraphCmd(".", "gradlew", "init.gradle") + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "gradlew") + assert.Contains(t, args, "--init-script") + assert.Contains(t, args, "init.gradle") + assert.Contains(t, args, "debrickedFindSubProjectPaths") +} + +func TestMakeDependenciesGraphCmd(t *testing.T) { + cmd, _ := CmdFactory{}.MakeDependenciesGraphCmd(".", "gradlew", "init.gradle") + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "gradlew") + assert.Contains(t, args, "--init-script") + assert.Contains(t, args, "init.gradle") + assert.Contains(t, args, "debrickedAllDeps") +} diff --git a/pkg/resolution/pm/gradle/gradle-init/gradle-init-script.groovy b/pkg/resolution/pm/gradle/gradle-init/gradle-init-script.groovy new file mode 100644 index 00000000..f3f1a0db --- /dev/null +++ b/pkg/resolution/pm/gradle/gradle-init/gradle-init-script.groovy @@ -0,0 +1,26 @@ +def debrickedOutputFile = new File('.debricked.multiprojects.txt') + +allprojects { + task debrickedFindSubProjectPaths() { + String output = project.projectDir + doLast { + synchronized(debrickedOutputFile) { + debrickedOutputFile << output + System.getProperty("line.separator") + } + } + } +} + +allprojects { + task debrickedAllDeps(type: DependencyReportTask) { + outputFile = file('./.gradle.debricked.lock') + } +} + + +allprojects{ + task debrickedJarsToFolder(type: Copy) { + into ".debrickedTmpDir" + from configurations.default + } +} \ No newline at end of file diff --git a/pkg/resolution/pm/gradle/init_script_handler.go b/pkg/resolution/pm/gradle/init_script_handler.go new file mode 100644 index 00000000..1ed91f05 --- /dev/null +++ b/pkg/resolution/pm/gradle/init_script_handler.go @@ -0,0 +1,37 @@ +package gradle + +import ( + "github.com/debricked/cli/pkg/resolution/pm/writer" +) + +type IInitScriptHandler interface { + ReadInitFile() ([]byte, error) + WriteInitFile(targetFileName string, fileWriter writer.IFileWriter) error +} + +type InitScriptHandler struct{} + +func (_ InitScriptHandler) ReadInitFile() ([]byte, error) { + return gradleInitScript.ReadFile("gradle-init/gradle-init-script.groovy") +} + +func (i InitScriptHandler) WriteInitFile(targetFileName string, fileWriter writer.IFileWriter) error { + content, err := i.ReadInitFile() + if err != nil { + + return SetupScriptError{message: err.Error()} + } + lockFile, err := fileWriter.Create(targetFileName) + if err != nil { + + return SetupScriptError{message: err.Error()} + } + defer lockFile.Close() + err = fileWriter.Write(lockFile, content) + if err != nil { + + return SetupScriptError{message: err.Error()} + } + + return nil +} diff --git a/pkg/resolution/pm/gradle/init_script_handler_test.go b/pkg/resolution/pm/gradle/init_script_handler_test.go new file mode 100644 index 00000000..06cda882 --- /dev/null +++ b/pkg/resolution/pm/gradle/init_script_handler_test.go @@ -0,0 +1,36 @@ +package gradle + +import ( + "embed" + "errors" + "testing" + + writerTestdata "github.com/debricked/cli/pkg/resolution/pm/writer/testdata" + "github.com/stretchr/testify/assert" +) + +func TestWriteInitFile(t *testing.T) { + createErr := errors.New("create-error") + fileWriterMock := &writerTestdata.FileWriterMock{CreateErr: createErr} + + sf := InitScriptHandler{} + err := sf.WriteInitFile("file", fileWriterMock) + assert.Equal(t, SetupScriptError{createErr.Error()}, err) + + fileWriterMock = &writerTestdata.FileWriterMock{WriteErr: createErr} + err = sf.WriteInitFile("file", fileWriterMock) + assert.Equal(t, SetupScriptError{createErr.Error()}, err) +} + +func TestWriteInitFileNoInitFile(t *testing.T) { + sf := InitScriptHandler{} + oldGradleInitScript := gradleInitScript + defer func() { + gradleInitScript = oldGradleInitScript + }() + gradleInitScript = embed.FS{} + err := sf.WriteInitFile("file", nil) + readErr := errors.New("open gradle-init/gradle-init-script.groovy: file does not exist") + assert.Equal(t, SetupScriptError{readErr.Error()}, err) + +} diff --git a/pkg/resolution/pm/gradle/job.go b/pkg/resolution/pm/gradle/job.go new file mode 100644 index 00000000..3373a45c --- /dev/null +++ b/pkg/resolution/pm/gradle/job.go @@ -0,0 +1,78 @@ +package gradle + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/pm/writer" +) + +type Job struct { + job.BaseJob + dir string + gradlew string + groovyInitScript string + cmdFactory ICmdFactory + fileWriter writer.IFileWriter +} + +func NewJob( + file string, + dir string, + gradlew string, + groovyInitScript string, + cmdFactory ICmdFactory, + fileWriter writer.IFileWriter, +) *Job { + + return &Job{ + BaseJob: job.NewBaseJob(file), + dir: dir, + gradlew: gradlew, + groovyInitScript: groovyInitScript, + cmdFactory: cmdFactory, + fileWriter: fileWriter, + } +} + +func (j *Job) Run() { + workingDirectory := filepath.Clean(j.GetDir()) + dependenciesCmd, err := j.cmdFactory.MakeDependenciesGraphCmd(workingDirectory, j.gradlew, j.groovyInitScript) + var permissionErr error + + if err != nil { + if strings.HasSuffix(err.Error(), "gradlew\": permission denied") { + permissionErr = fmt.Errorf("Permission to execute gradlew is not granted, fallback to PATHs gradle installation will be used.\nFull error: %s", err.Error()) + + dependenciesCmd, err = j.cmdFactory.MakeDependenciesGraphCmd(workingDirectory, "gradle", j.groovyInitScript) + } + } + + if err != nil { + if permissionErr != nil { + j.Errors().Critical(permissionErr) + } + j.Errors().Critical(err) + + return + } + + j.SendStatus("creating dependency graph") + _, err = dependenciesCmd.Output() + + if permissionErr != nil { + j.Errors().Warning(permissionErr) + } + + if err != nil { + j.Errors().Critical(j.GetExitError(err)) + + return + } +} + +func (j *Job) GetDir() string { + return j.dir +} diff --git a/pkg/resolution/pm/gradle/job_test.go b/pkg/resolution/pm/gradle/job_test.go new file mode 100644 index 00000000..54b976c3 --- /dev/null +++ b/pkg/resolution/pm/gradle/job_test.go @@ -0,0 +1,132 @@ +package gradle + +import ( + "errors" + "testing" + + jobTestdata "github.com/debricked/cli/pkg/resolution/job/testdata" + "github.com/debricked/cli/pkg/resolution/pm/gradle/testdata" + "github.com/debricked/cli/pkg/resolution/pm/writer" + writerTestdata "github.com/debricked/cli/pkg/resolution/pm/writer/testdata" + "github.com/stretchr/testify/assert" +) + +func TestNewJob(t *testing.T) { + j := NewJob("file", "dir", "nil", "nil", CmdFactory{}, writer.FileWriter{}) + assert.Equal(t, "file", j.GetFile()) + assert.Equal(t, "dir", j.GetDir()) + assert.False(t, j.Errors().HasError()) +} + +func TestRunCmdErr(t *testing.T) { + cmdErr := errors.New("cmd-error") + j := NewJob("file", "dir", "nil", "nil", testdata.CmdFactoryMock{Err: cmdErr}, writer.FileWriter{}) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), cmdErr) +} + +func TestRunCmdOutputErr(t *testing.T) { + fileWriterMock := &writerTestdata.FileWriterMock{CreateErr: errors.New("create-error")} + + j := NewJob("file", "dir", "gradlew", "path", testdata.CmdFactoryMock{Name: "bad-name"}, fileWriterMock) + + go jobTestdata.WaitStatus(j) + + j.Run() + + jobTestdata.AssertPathErr(t, j.Errors()) +} + +func TestRunCreateErr(t *testing.T) { + createErr := errors.New("create-error") + fileWriterMock := &writerTestdata.FileWriterMock{CreateErr: createErr} + j := NewJob("file", "dir", "gradlew", "path", testdata.CmdFactoryMock{Name: "echo", Err: createErr}, fileWriterMock) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), createErr) +} + +func TestRunWriteErr(t *testing.T) { + writeErr := errors.New("write-error") + fileWriterMock := &writerTestdata.FileWriterMock{WriteErr: writeErr} + j := NewJob("file", "dir", "", "", testdata.CmdFactoryMock{Name: "echo", Err: writeErr}, fileWriterMock) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), writeErr) +} + +func TestRunCloseErr(t *testing.T) { + closeErr := errors.New("close-error") + fileWriterMock := &writerTestdata.FileWriterMock{CloseErr: closeErr} + j := NewJob("file", "dir", "gradlew", "path", testdata.CmdFactoryMock{Name: "echo", Err: closeErr}, fileWriterMock) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), closeErr) +} + +func TestRunPermissionFailBeforeOutputErr(t *testing.T) { + permissionErr := errors.New("give-error-on-gradle gradlew\": permission denied") + fileWriterMock := &writerTestdata.FileWriterMock{} + j := NewJob("file", "dir", "gradlew", "path", testdata.CmdFactoryMock{Name: "echo", Err: permissionErr}, fileWriterMock) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 2) +} + +func TestRunPermissionErr(t *testing.T) { + permissionErr := errors.New("asdhjaskdhqwe gradlew\": permission denied") + fileWriterMock := &writerTestdata.FileWriterMock{} + j := NewJob("file", "dir", "gradlew", "path", testdata.CmdFactoryMock{Name: "echo", Err: permissionErr}, fileWriterMock) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) +} + +func TestRunPermissionOutputErr(t *testing.T) { + permissionErr := errors.New("asdhjaskdhqwe gradlew\": permission denied") + otherErr := errors.New("WriteError") + fileWriterMock := &writerTestdata.FileWriterMock{WriteErr: otherErr} + + j := NewJob("file", "dir", "gradlew", "path", testdata.CmdFactoryMock{Name: "bad-name", Err: permissionErr}, fileWriterMock) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Len(t, j.Errors().GetAll(), 2) +} + +func TestRun(t *testing.T) { + fileContents := []byte("MakeDependenciesCmd\n") + fileWriterMock := &writerTestdata.FileWriterMock{Contents: fileContents} + cmdFactoryMock := testdata.CmdFactoryMock{Name: "echo"} + j := NewJob("file", "dir", "gradlew", "path", cmdFactoryMock, fileWriterMock) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.False(t, j.Errors().HasError()) + assert.Equal(t, fileContents, fileWriterMock.Contents) +} diff --git a/pkg/resolution/pm/gradle/meta_file_finder.go b/pkg/resolution/pm/gradle/meta_file_finder.go new file mode 100644 index 00000000..e59bb57a --- /dev/null +++ b/pkg/resolution/pm/gradle/meta_file_finder.go @@ -0,0 +1,82 @@ +package gradle + +import ( + "os" + "path/filepath" +) + +type IMetaFileFinder interface { + Find(paths []string) (map[string]string, map[string]string, error) +} + +type MetaFileFinder struct { + filepath IFilePath +} + +type IFilePath interface { + Walk(root string, walkFn filepath.WalkFunc) error + Base(path string) string + Abs(path string) (string, error) + Dir(path string) string +} + +type FilePath struct{} + +func (fp FilePath) Walk(root string, walkFn filepath.WalkFunc) error { + return filepath.Walk(root, walkFn) +} + +func (fp FilePath) Base(path string) string { + return filepath.Base(path) +} + +func (fp FilePath) Abs(path string) (string, error) { + return filepath.Abs(path) +} + +func (fp FilePath) Dir(path string) string { + return filepath.Dir(path) +} + +func (finder MetaFileFinder) Find(paths []string) (map[string]string, map[string]string, error) { + settings := []string{"settings.gradle", "settings.gradle.kts"} + gradlew := []string{"gradlew"} + settingsMap := map[string]string{} + gradlewMap := map[string]string{} + for _, rootPath := range paths { + err := finder.filepath.Walk( + rootPath, + func(path string, fileInfo os.FileInfo, err error) error { + if err != nil { + + return err + } + if !fileInfo.IsDir() { + for _, setting := range settings { + if setting == finder.filepath.Base(path) { + dir, _ := finder.filepath.Abs(finder.filepath.Dir(path)) + file, _ := finder.filepath.Abs(path) + settingsMap[dir] = file + } + } + + for _, gradle := range gradlew { + if gradle == finder.filepath.Base(path) { + dir, _ := finder.filepath.Abs(finder.filepath.Dir(path)) + file, _ := finder.filepath.Abs(path) + gradlewMap[dir] = file + } + } + } + + return nil + }, + ) + if err != nil { + + return nil, nil, SetupWalkError{message: err.Error()} + } + } + + return settingsMap, gradlewMap, nil +} diff --git a/pkg/resolution/pm/gradle/meta_file_finder_test.go b/pkg/resolution/pm/gradle/meta_file_finder_test.go new file mode 100644 index 00000000..0f764b68 --- /dev/null +++ b/pkg/resolution/pm/gradle/meta_file_finder_test.go @@ -0,0 +1,61 @@ +package gradle + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFind(t *testing.T) { + finder := MetaFileFinder{filepath: FilePath{}} + paths := []string{filepath.Join("testdata", "project")} + sMap, gMap, _ := finder.Find(paths) + + assert.Len(t, sMap, 1) + assert.Len(t, gMap, 1) +} + +func TestFindNoFiles(t *testing.T) { + finder := MetaFileFinder{filepath: FilePath{}} + paths := []string{filepath.Join("testdata", "project", "subproject")} + sMap, gMap, _ := finder.Find(paths) + + assert.Len(t, sMap, 0) + assert.Len(t, gMap, 0) +} + +type mockGradleFilePath struct{} + +func (m mockGradleFilePath) Walk(root string, walkFn filepath.WalkFunc) error { + return errors.New("test") +} + +func (m mockGradleFilePath) Base(path string) string { + return filepath.Base(path) +} + +func (m mockGradleFilePath) Abs(path string) (string, error) { + return filepath.Abs(path) +} + +func (m mockGradleFilePath) Dir(path string) string { + return filepath.Dir(path) +} + +func TestWalkError(t *testing.T) { + finder := MetaFileFinder{filepath: mockGradleFilePath{}} + paths := []string{filepath.Join("testdata", "project", "subproject")} + _, _, err := finder.Find(paths) + assert.EqualError(t, err, SetupWalkError{message: "test"}.Error()) +} + +func TestWalkFuncError(t *testing.T) { + finder := MetaFileFinder{filepath: FilePath{}} + paths := []string{filepath.Join("testdata", "test")} + _, _, err := finder.Find(paths) + + // assert err not nil + assert.NotNil(t, err) +} diff --git a/pkg/resolution/pm/gradle/pm.go b/pkg/resolution/pm/gradle/pm.go new file mode 100644 index 00000000..8c1e8253 --- /dev/null +++ b/pkg/resolution/pm/gradle/pm.go @@ -0,0 +1,24 @@ +package gradle + +const Name = "gradle" + +type Pm struct { + name string +} + +func NewPm() Pm { + return Pm{ + name: Name, + } +} + +func (pm Pm) Name() string { + return pm.name +} + +func (_ Pm) Manifests() []string { + return []string{ + "build.gradle", + "build.gradle.kts", + } +} diff --git a/pkg/resolution/pm/gradle/pm_test.go b/pkg/resolution/pm/gradle/pm_test.go new file mode 100644 index 00000000..b3848b64 --- /dev/null +++ b/pkg/resolution/pm/gradle/pm_test.go @@ -0,0 +1,25 @@ +package gradle + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewPm(t *testing.T) { + pm := NewPm() + assert.Equal(t, Name, pm.name) +} + +func TestName(t *testing.T) { + pm := NewPm() + assert.Equal(t, Name, pm.Name()) +} + +func TestManifests(t *testing.T) { + pm := Pm{} + manifests := pm.Manifests() + assert.Len(t, manifests, 2) + manifest := manifests[0] + assert.Equal(t, "build.gradle", manifest) +} diff --git a/pkg/resolution/pm/gradle/project.go b/pkg/resolution/pm/gradle/project.go new file mode 100644 index 00000000..8a1b4161 --- /dev/null +++ b/pkg/resolution/pm/gradle/project.go @@ -0,0 +1,7 @@ +package gradle + +type Project struct { + dir string + gradlew string + mainBuildFile string +} diff --git a/pkg/resolution/pm/gradle/setup.go b/pkg/resolution/pm/gradle/setup.go new file mode 100644 index 00000000..3b3bcf95 --- /dev/null +++ b/pkg/resolution/pm/gradle/setup.go @@ -0,0 +1,181 @@ +package gradle + +import ( + "bufio" + "bytes" + "embed" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + + "github.com/debricked/cli/pkg/resolution/pm/writer" +) + +const ( + initGradle = "gradle" + multiProjectFilename = ".debricked.multiprojects.txt" + gradleInitScriptFileName = ".gradle-init-script.debricked.groovy" +) + +//go:embed gradle-init/gradle-init-script.groovy +var gradleInitScript embed.FS + +type ISetup interface { + Configure(files []string, paths []string) (Setup, error) +} + +type Setup struct { + gradlewMap map[string]string + settingsMap map[string]string + subProjectMap map[string]string + groovyScriptPath string + gradlewOsName string + settingsFilenames []string + GradleProjects []Project + CmdFactory ICmdFactory + MetaFileFinder IMetaFileFinder + InitScriptHandler IInitScriptHandler + Writer writer.IFileWriter +} + +func NewGradleSetup() *Setup { + groovyScriptPath, _ := filepath.Abs(gradleInitScriptFileName) + gradlewOsName := "gradlew" + if runtime.GOOS == "windows" { + gradlewOsName = "gradlew.bat" + } + + return &Setup{ + gradlewMap: map[string]string{}, + settingsMap: map[string]string{}, + subProjectMap: map[string]string{}, + groovyScriptPath: groovyScriptPath, + gradlewOsName: gradlewOsName, + settingsFilenames: []string{"settings.gradle", "settings.gradle.kts"}, + GradleProjects: []Project{}, + CmdFactory: CmdFactory{}, + MetaFileFinder: MetaFileFinder{filepath: FilePath{}}, + InitScriptHandler: InitScriptHandler{}, + Writer: writer.FileWriter{}, + } +} + +func (gs *Setup) Configure(_ []string, paths []string) (Setup, error) { + err := gs.InitScriptHandler.WriteInitFile(gs.groovyScriptPath, gs.Writer) + if err != nil { + + return *gs, err + } + settingsMap, gradlewMap, err := gs.MetaFileFinder.Find(paths) + gs.gradlewMap = gradlewMap + gs.settingsMap = settingsMap + if err != nil { + + return *gs, err + } + err = gs.setupGradleProjectMappings() + if err != nil && len(err.Error()) > 0 { + return *gs, err + } + + return *gs, nil +} + +func (gs *Setup) setupFilePathMappings(files []string) { + for _, file := range files { + dir, _ := filepath.Abs(filepath.Dir(file)) + possibleGradlew := filepath.Join(dir, gs.gradlewOsName) + _, err := os.Stat(possibleGradlew) + if err == nil { + gs.gradlewMap[dir] = possibleGradlew + } + for _, settingsFilename := range gs.settingsFilenames { + possibleSettings := filepath.Join(dir, settingsFilename) + _, err := os.Stat(possibleSettings) + if err == nil { + gs.settingsMap[dir] = possibleSettings + } + } + } +} + +func (gs *Setup) setupGradleProjectMappings() error { + var errors SetupError + var settingsDirs []string + for k := range gs.settingsMap { + settingsDirs = append(settingsDirs, k) + } + sort.Strings(settingsDirs) + for _, dir := range settingsDirs { + if _, ok := gs.subProjectMap[dir]; ok { + continue + } + gradlew := gs.GetGradleW(dir) + mainFile := gs.settingsMap[dir] + gradleProject := Project{dir: dir, gradlew: gradlew, mainBuildFile: mainFile} + err := gs.setupSubProjectPaths(gradleProject) + + if err != nil { + errors = append(errors, err) + } + gs.GradleProjects = append(gs.GradleProjects, gradleProject) + } + + return SetupSubprojectError{message: errors.Error()} +} + +func (gs *Setup) setupSubProjectPaths(gp Project) error { + dependenciesCmd, _ := gs.CmdFactory.MakeFindSubGraphCmd(gp.dir, gp.gradlew, gs.groovyScriptPath) + var stderr bytes.Buffer + dependenciesCmd.Stderr = &stderr + _, err := dependenciesCmd.Output() + dependenciesCmd.Stderr = os.Stderr + if err != nil { + errorOutput := stderr.String() + + return SetupSubprojectError{message: errorOutput + err.Error()} + } + multiProject := filepath.Join(gp.dir, multiProjectFilename) + file, err := os.Open(multiProject) + if err != nil { + + return SetupSubprojectError{message: err.Error()} + } + defer file.Close() + defer os.Remove(multiProject) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + subProjectPath := scanner.Text() + gs.subProjectMap[subProjectPath] = gp.dir + } + + if err := scanner.Err(); err != nil { + return SetupSubprojectError{message: err.Error()} + } + + return nil +} + +func (gs *Setup) GetGradleW(dir string) string { + gradlew := initGradle + val, ok := gs.gradlewMap[dir] + if ok { + gradlew = val + } else { + for dirPath, gradlePath := range gs.gradlewMap { + // potential improvement, sort gradlewMap in longest path first" + rel, err := filepath.Rel(dirPath, dir) + isRelative := !strings.HasPrefix(rel, "..") && rel != ".." + if isRelative && err == nil { + gradlew = gradlePath + + break + } + } + } + + return gradlew +} diff --git a/pkg/resolution/pm/gradle/setup_err.go b/pkg/resolution/pm/gradle/setup_err.go new file mode 100644 index 00000000..503ef16c --- /dev/null +++ b/pkg/resolution/pm/gradle/setup_err.go @@ -0,0 +1,39 @@ +package gradle + +type SetupScriptError struct { + message string +} + +type SetupWalkError struct { + message string +} + +type SetupSubprojectError struct { + message string +} + +func (e SetupScriptError) Error() string { + + return e.message +} + +func (e SetupWalkError) Error() string { + + return e.message +} + +func (e SetupSubprojectError) Error() string { + + return e.message +} + +type SetupError []error + +func (e SetupError) Error() string { + var s string + for _, err := range e { + s += err.Error() + "\n" + } + + return s +} diff --git a/pkg/resolution/pm/gradle/setup_test.go b/pkg/resolution/pm/gradle/setup_test.go new file mode 100644 index 00000000..e57ee798 --- /dev/null +++ b/pkg/resolution/pm/gradle/setup_test.go @@ -0,0 +1,212 @@ +package gradle + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + writerTestdata "github.com/debricked/cli/pkg/resolution/pm/writer/testdata" + + "github.com/debricked/cli/pkg/resolution/pm/writer" + "github.com/stretchr/testify/assert" +) + +func TestNewGradleSetup(t *testing.T) { + + gs := NewGradleSetup() + assert.NotNil(t, gs) +} + +func TestErrors(t *testing.T) { + + walkError := SetupWalkError{message: "test"} + assert.Equal(t, "test", walkError.Error()) + + scriptError := SetupScriptError{message: "test"} + assert.Equal(t, "test", scriptError.Error()) + + subprojectError := SetupSubprojectError{message: "test"} + assert.Equal(t, "test", subprojectError.Error()) + +} + +func TestSetupFilePathMappings(t *testing.T) { + gs := NewGradleSetup() + files := []string{filepath.Join("testdata", "project", "build.gradle")} + gs.setupFilePathMappings(files) + + assert.Len(t, gs.gradlewMap, 1) + assert.Len(t, gs.settingsMap, 1) +} + +func TestSetupFilePathMappingsNoFiles(t *testing.T) { + gs := NewGradleSetup() + gs.setupFilePathMappings([]string{}) + + assert.Len(t, gs.gradlewMap, 0) + assert.Len(t, gs.settingsMap, 0) +} + +func TestSetupFilePathMappingsNoGradlew(t *testing.T) { + gs := NewGradleSetup() + files := []string{filepath.Join("testdata", "project", "subproject", "build.gradle")} + gs.setupFilePathMappings(files) + + assert.Len(t, gs.gradlewMap, 0) + assert.Len(t, gs.settingsMap, 0) +} + +func TestSetupGradleProjectMappings(t *testing.T) { + gs := NewGradleSetup() + gs.CmdFactory = &mockCmdFactory{} + + gs.settingsMap = map[string]string{ + filepath.Join("testdata", "project"): filepath.Join("testdata", "project", "settings.gradle"), + } + gs.subProjectMap = map[string]string{} + err := gs.setupGradleProjectMappings() + // assert GradleSetupSubprojectError + assert.NotNil(t, err) + + assert.Len(t, gs.GradleProjects, 1) +} + +type mockCmdFactory struct { + createFile bool +} + +func (m *mockCmdFactory) MakeFindSubGraphCmd(workingDirectory string, _ string, _ string) (*exec.Cmd, error) { + if m.createFile { + fileName := filepath.Join(workingDirectory, multiProjectFilename) + content := []byte(workingDirectory) + file, err := os.Create(fileName) + if err != nil { + + return nil, err + } + defer file.Close() + _, err = file.Write(content) + if err != nil { + + return nil, err + } + } + // if windows use dir + if runtime.GOOS == "windows" { + // gradlewOsName = "gradlew.bat" + return exec.Command("dir"), nil + } + + return exec.Command("ls"), nil +} + +func (m *mockCmdFactory) MakeDependenciesGraphCmd(workingDirectory string, _ string, _ string) (*exec.Cmd, error) { + return &exec.Cmd{ + Path: workingDirectory, + Args: []string{"touch", ".debricked.dependencies.graph.txt"}, + Dir: workingDirectory, + }, nil +} + +func TestSetupSubProjectPathsNoFileCreated(t *testing.T) { + gs := NewGradleSetup() + gs.CmdFactory = &mockCmdFactory{createFile: false} + + absPath, _ := filepath.Abs(filepath.Join("testdata", "project")) + gradleProject := Project{dir: absPath, gradlew: filepath.Join("testdata", "project", "gradlew")} + err := gs.setupSubProjectPaths(gradleProject) + fmt.Println(err) + assert.NotNil(t, err) + assert.Len(t, gs.subProjectMap, 0) +} + +func TestSetupSubProjectPaths(t *testing.T) { + gs := NewGradleSetup() + gs.CmdFactory = &mockCmdFactory{createFile: true} + + absPath, _ := filepath.Abs(filepath.Join("testdata", "project")) + gradleProject := Project{dir: absPath, gradlew: filepath.Join("testdata", "project", "gradlew")} + err := gs.setupSubProjectPaths(gradleProject) + assert.Nil(t, err) + assert.Len(t, gs.subProjectMap, 1) + + absPath, _ = filepath.Abs(filepath.Join("testdata", "project", "subproject")) + gradleProject = Project{dir: absPath, gradlew: filepath.Join("testdata", "project", "gradlew")} + err = gs.setupSubProjectPaths(gradleProject) + assert.Nil(t, err) + assert.Len(t, gs.subProjectMap, 2) +} + +func TestSetupSubProjectPathsError(t *testing.T) { + gs := NewGradleSetup() + + absPath, _ := filepath.Abs(filepath.Join("testdata", "project")) + gradleProject := Project{dir: absPath, gradlew: filepath.Join("testdata", "project", "gradlew")} + err := gs.setupSubProjectPaths(gradleProject) + + assert.NotNil(t, err) +} + +func TestGetGradleW(t *testing.T) { + gs := NewGradleSetup() + + gs.gradlewMap = map[string]string{ + filepath.Join("testdata", "project"): filepath.Join("testdata", "project", "gradlew"), + } + + gradlew := gs.GetGradleW(filepath.Join("testdata", "project", "subproject")) + + assert.Equal(t, filepath.Join("testdata", "project", "gradlew"), gradlew) + + gradlew = gs.GetGradleW(filepath.Join("testdata", "project")) + + assert.Equal(t, filepath.Join("testdata", "project", "gradlew"), gradlew) +} + +type mockInitScriptHandler struct { + writeInitFileErr error +} + +func (_ mockInitScriptHandler) ReadInitFile() ([]byte, error) { + return gradleInitScript.ReadFile("gradle-init/gradle-init-script.groovy") +} + +func (i mockInitScriptHandler) WriteInitFile(_ string, _ writer.IFileWriter) error { + return i.writeInitFileErr +} + +type mockFileHandler struct { + setupWalkErr error +} + +func (f mockFileHandler) Find(_ []string) (map[string]string, map[string]string, error) { + return nil, nil, f.setupWalkErr +} + +func TestConfigureErrors(t *testing.T) { + gs := NewGradleSetup() + gs.Writer = &writerTestdata.FileWriterMock{} + _, err := gs.Configure([]string{"testdata/project"}, []string{"testdata/project"}) + assert.NotNil(t, err) + + gs.MetaFileFinder = mockFileHandler{setupWalkErr: SetupScriptError{message: "mock error"}} + _, err = gs.Configure([]string{"testdata/project"}, []string{"testdata/project"}) + assert.Equal(t, "mock error", err.Error()) + + gs.InitScriptHandler = mockInitScriptHandler{writeInitFileErr: SetupScriptError{message: "write-init-file-err"}} + _, err = gs.Configure([]string{"testdata/project"}, []string{"testdata/project"}) + assert.Equal(t, "write-init-file-err", err.Error()) +} + +func TestConfigure(t *testing.T) { + gs := NewGradleSetup() + gs.Writer = &writerTestdata.FileWriterMock{} + gs.MetaFileFinder = mockFileHandler{setupWalkErr: nil} + gs.InitScriptHandler = mockInitScriptHandler{writeInitFileErr: nil} + + _, err := gs.Configure([]string{"testdata/project"}, []string{"testdata/project"}) + assert.NoError(t, err) +} diff --git a/pkg/resolution/pm/gradle/strategy.go b/pkg/resolution/pm/gradle/strategy.go new file mode 100644 index 00000000..311b744b --- /dev/null +++ b/pkg/resolution/pm/gradle/strategy.go @@ -0,0 +1,66 @@ +package gradle + +import ( + "io" + "log" + "os" + "path/filepath" + + "github.com/fatih/color" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/pm/writer" +) + +type Strategy struct { + files []string + paths []string + ErrorWriter io.Writer + GradleSetup ISetup +} + +func (s Strategy) Invoke() ([]job.IJob, error) { + var jobs []job.IJob + fileWriter := writer.FileWriter{} + factory := CmdFactory{} + gradleSetup, err := s.GradleSetup.Configure(s.files, s.paths) + if err != nil { + if _, ok := err.(SetupSubprojectError); ok { + warningColor := color.New(color.FgYellow, color.Bold).SprintFunc() + defaultOutputWriter := log.Writer() + log.SetOutput(s.ErrorWriter) + log.Println(warningColor("Warning:\n") + err.Error()) + log.SetOutput(defaultOutputWriter) + } else { + return nil, err + } + } + gradleMainDirs := make(map[string]bool) + for _, gradleProject := range gradleSetup.GradleProjects { + dir := gradleProject.dir + if _, ok := gradleMainDirs[dir]; ok { + continue + } + gradleMainDirs[dir] = true + jobs = append(jobs, NewJob(gradleProject.mainBuildFile, dir, gradleProject.gradlew, gradleSetup.groovyScriptPath, factory, fileWriter)) + + } + for _, file := range s.files { + dir, _ := filepath.Abs(filepath.Dir(file)) + if _, ok := gradleSetup.subProjectMap[dir]; ok { + continue + } + if _, ok := gradleMainDirs[dir]; ok { + continue + } + gradleMainDirs[dir] = true + gradlew := gradleSetup.GetGradleW(dir) + jobs = append(jobs, NewJob(file, dir, gradlew, gradleSetup.groovyScriptPath, factory, fileWriter)) + } + + return jobs, nil +} + +func NewStrategy(files []string, paths []string) Strategy { + return Strategy{files, paths, os.Stdout, NewGradleSetup()} +} diff --git a/pkg/resolution/pm/gradle/strategy_test.go b/pkg/resolution/pm/gradle/strategy_test.go new file mode 100644 index 00000000..9a84bcdf --- /dev/null +++ b/pkg/resolution/pm/gradle/strategy_test.go @@ -0,0 +1,93 @@ +package gradle + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestNewStrategy(t *testing.T) { + s := NewStrategy(nil, nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy([]string{}, nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy([]string{"file"}, nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 1) + + s = NewStrategy([]string{"file-1", "file-2"}, nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 2) +} + +func TestInvokeNoFiles(t *testing.T) { + s := NewStrategy([]string{}, nil) + jobs, _ := s.Invoke() + assert.Empty(t, jobs) +} + +func TestInvokeOneFile(t *testing.T) { + s := NewStrategy([]string{"file"}, nil) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 1) +} + +func TestInvokeManyFiles(t *testing.T) { + s := NewStrategy([]string{"test/file-1", "test/file-2", "test2/file-2"}, nil) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 2) +} + +// mock for ISetup +type mockGradleSetup struct { + mock.Mock +} + +// mock for Setup +func (m *mockGradleSetup) Configure(_ []string, _ []string) (Setup, error) { + args := m.Called() + + return args.Get(0).(Setup), args.Error(1) +} + +func TestInvokeWalkError(t *testing.T) { + s := NewStrategy([]string{"file"}, []string{"path"}) + mocked := &mockGradleSetup{} + mocked.On("Configure").Return(Setup{}, SetupWalkError{}) + + s.GradleSetup = mocked + jobs, err := s.Invoke() + assert.Empty(t, jobs) + assert.Equal(t, err, SetupWalkError{}) +} + +func TestInvokeSubprojectError(t *testing.T) { + s := NewStrategy([]string{"file"}, []string{"path"}) + mocked := &mockGradleSetup{} + mocked.On("Configure").Return(Setup{}, SetupSubprojectError{}) + s.GradleSetup = mocked + jobs, err := s.Invoke() + assert.Nil(t, err) + assert.Len(t, jobs, 1) + assert.Equal(t, s.ErrorWriter, os.Stdout) +} + +func TestInvokeFoundProject(t *testing.T) { + s := NewStrategy([]string{"file"}, []string{"file"}) + subprojectMap := make(map[string]string) + dir, _ := os.Getwd() + subprojectMap[dir] = "" + mocked := &mockGradleSetup{} + mocked.On("Configure").Return(Setup{GradleProjects: []Project{{dir: dir, gradlew: "gradlew"}}, groovyScriptPath: "", subProjectMap: subprojectMap}, nil) + + s.GradleSetup = mocked + jobs, _ := s.Invoke() + + assert.Len(t, jobs, 1) +} diff --git a/pkg/resolution/pm/gradle/testdata/cmd_factory_mock.go b/pkg/resolution/pm/gradle/testdata/cmd_factory_mock.go new file mode 100644 index 00000000..f8c60b66 --- /dev/null +++ b/pkg/resolution/pm/gradle/testdata/cmd_factory_mock.go @@ -0,0 +1,34 @@ +package testdata + +import ( + "os/exec" + "strings" +) + +type CmdFactoryMock struct { + Err error + Name string +} + +func (f CmdFactoryMock) MakeDependenciesGraphCmd(dir string, gradlew string, _ string) (*exec.Cmd, error) { + err := f.Err + if gradlew == "gradle" { + err = nil + } + + if f.Err != nil && strings.HasPrefix(f.Err.Error(), "give-error-on-gradle") { + err = f.Err + } + + return exec.Command(f.Name, `MakeDependenciesCmd`), err +} + +// implement the interface +func (f CmdFactoryMock) MakeFindSubGraphCmd(_ string, _ string, _ string) (*exec.Cmd, error) { + return exec.Command(f.Name, `MakeFindSubGraphCmd`), f.Err +} + +// implement the interface +func (f CmdFactoryMock) MakeDependenciesCmd(_ string) (*exec.Cmd, error) { + return exec.Command(f.Name, `MakeDependenciesCmd`), f.Err +} diff --git a/pkg/resolution/pm/gradle/testdata/project/build.gradle b/pkg/resolution/pm/gradle/testdata/project/build.gradle new file mode 100644 index 00000000..e69de29b diff --git a/pkg/resolution/pm/gradle/testdata/project/gradlew b/pkg/resolution/pm/gradle/testdata/project/gradlew new file mode 100644 index 00000000..e69de29b diff --git a/pkg/resolution/pm/gradle/testdata/project/gradlew.bat b/pkg/resolution/pm/gradle/testdata/project/gradlew.bat new file mode 100644 index 00000000..e69de29b diff --git a/pkg/resolution/pm/gradle/testdata/project/settings.gradle b/pkg/resolution/pm/gradle/testdata/project/settings.gradle new file mode 100644 index 00000000..e69de29b diff --git a/pkg/resolution/pm/gradle/testdata/project/subproject/build.gradle b/pkg/resolution/pm/gradle/testdata/project/subproject/build.gradle new file mode 100644 index 00000000..e69de29b diff --git a/pkg/resolution/pm/maven/cmd_factory.go b/pkg/resolution/pm/maven/cmd_factory.go new file mode 100644 index 00000000..7f7112a8 --- /dev/null +++ b/pkg/resolution/pm/maven/cmd_factory.go @@ -0,0 +1,25 @@ +package maven + +import "os/exec" + +type ICmdFactory interface { + MakeDependencyTreeCmd(workingDirectory string) (*exec.Cmd, error) +} + +type CmdFactory struct{} + +func (_ CmdFactory) MakeDependencyTreeCmd(workingDirectory string) (*exec.Cmd, error) { + path, err := exec.LookPath("mvn") + + return &exec.Cmd{ + Path: path, + Args: []string{ + "mvn", + "dependency:tree", + "-DoutputFile=.maven.debricked.lock", + "-DoutputType=tgf", + "--fail-at-end", + }, + Dir: workingDirectory, + }, err +} diff --git a/pkg/resolution/pm/maven/cmd_factory_test.go b/pkg/resolution/pm/maven/cmd_factory_test.go new file mode 100644 index 00000000..ab8f18c5 --- /dev/null +++ b/pkg/resolution/pm/maven/cmd_factory_test.go @@ -0,0 +1,18 @@ +package maven + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMakeDependencyTreeCmd(t *testing.T) { + cmd, _ := CmdFactory{}.MakeDependencyTreeCmd(".") + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "mvn") + assert.Contains(t, args, "dependency:tree") + assert.Contains(t, args, "-DoutputFile=.maven.debricked.lock") + assert.Contains(t, args, "-DoutputType=tgf") + assert.Contains(t, args, "--fail-at-end") +} diff --git a/pkg/resolution/pm/maven/job.go b/pkg/resolution/pm/maven/job.go new file mode 100644 index 00000000..6af48b6e --- /dev/null +++ b/pkg/resolution/pm/maven/job.go @@ -0,0 +1,40 @@ +package maven + +import ( + "errors" + "path/filepath" + + "github.com/debricked/cli/pkg/resolution/job" +) + +type Job struct { + job.BaseJob + cmdFactory ICmdFactory +} + +func NewJob(file string, cmdFactory ICmdFactory) *Job { + return &Job{ + BaseJob: job.NewBaseJob(file), + cmdFactory: cmdFactory, + } +} + +func (j *Job) Run() { + workingDirectory := filepath.Dir(filepath.Clean(j.GetFile())) + cmd, err := j.cmdFactory.MakeDependencyTreeCmd(workingDirectory) + if err != nil { + j.Errors().Critical(err) + + return + } + j.SendStatus("creating dependency graph") + var output []byte + output, err = cmd.Output() + if err != nil { + if output == nil { + j.Errors().Critical(err) + } else { + j.Errors().Critical(errors.New(string(output))) + } + } +} diff --git a/pkg/resolution/pm/maven/job_test.go b/pkg/resolution/pm/maven/job_test.go new file mode 100644 index 00000000..0787694a --- /dev/null +++ b/pkg/resolution/pm/maven/job_test.go @@ -0,0 +1,64 @@ +package maven + +import ( + "errors" + "testing" + + jobTestdata "github.com/debricked/cli/pkg/resolution/job/testdata" + "github.com/debricked/cli/pkg/resolution/pm/maven/testdata" + "github.com/stretchr/testify/assert" +) + +func TestNewJob(t *testing.T) { + j := NewJob("file", CmdFactory{}) + assert.Equal(t, "file", j.GetFile()) + assert.False(t, j.Errors().HasError()) +} + +func TestRunCmdErr(t *testing.T) { + cmdErr := errors.New("cmd-error") + j := NewJob("file", testdata.CmdFactoryMock{Err: cmdErr}) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), cmdErr) +} + +func TestRunCmdOutputErr(t *testing.T) { + j := NewJob("file", testdata.CmdFactoryMock{Name: "bad-name"}) + + go jobTestdata.WaitStatus(j) + + j.Run() + + jobTestdata.AssertPathErr(t, j.Errors()) +} + +func TestRunCmdOutputErrNoOutput(t *testing.T) { + j := NewJob("file", testdata.CmdFactoryMock{Name: "go", Arg: "bad-arg"}) + + go jobTestdata.WaitStatus(j) + + j.Run() + + errs := j.Errors().GetAll() + assert.Len(t, errs, 1) + err := errs[0] + + // assert empty because, when Output is executed it will allocate memory for the byte slice to contain the standard output. + // However since no bytes are sent to standard output err will be empty here. + assert.Empty(t, err) +} + +func TestRun(t *testing.T) { + j := NewJob("file", testdata.CmdFactoryMock{Name: "echo"}) + + go jobTestdata.WaitStatus(j) + + j.Run() + + assert.False(t, j.Errors().HasError()) +} diff --git a/pkg/resolution/pm/maven/pm.go b/pkg/resolution/pm/maven/pm.go new file mode 100644 index 00000000..0b005b35 --- /dev/null +++ b/pkg/resolution/pm/maven/pm.go @@ -0,0 +1,23 @@ +package maven + +const Name = "mvn" + +type Pm struct { + name string +} + +func NewPm() Pm { + return Pm{ + name: Name, + } +} + +func (pm Pm) Name() string { + return pm.name +} + +func (_ Pm) Manifests() []string { + return []string{ + "pom.xml", + } +} diff --git a/pkg/resolution/pm/maven/pm_test.go b/pkg/resolution/pm/maven/pm_test.go new file mode 100644 index 00000000..ad802446 --- /dev/null +++ b/pkg/resolution/pm/maven/pm_test.go @@ -0,0 +1,25 @@ +package maven + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewPm(t *testing.T) { + pm := NewPm() + assert.Equal(t, Name, pm.name) +} + +func TestName(t *testing.T) { + pm := NewPm() + assert.Equal(t, Name, pm.Name()) +} + +func TestManifests(t *testing.T) { + pm := Pm{} + manifests := pm.Manifests() + assert.Len(t, manifests, 1) + manifest := manifests[0] + assert.Equal(t, "pom.xml", manifest) +} diff --git a/pkg/resolution/pm/maven/pom_service.go b/pkg/resolution/pm/maven/pom_service.go new file mode 100644 index 00000000..726a1d97 --- /dev/null +++ b/pkg/resolution/pm/maven/pom_service.go @@ -0,0 +1,57 @@ +package maven + +import ( + "path/filepath" + + "github.com/vifraa/gopom" +) + +type IPomService interface { + GetRootPomFiles(files []string) []string + ParsePomModules(path string) ([]string, error) +} + +type PomService struct{} + +func (p PomService) ParsePomModules(path string) ([]string, error) { + pom, err := gopom.Parse(path) + + if err != nil { + return nil, err + } + + return pom.Modules, nil +} + +func (p PomService) GetRootPomFiles(files []string) []string { + childMap := make(map[string]bool) + var validFiles []string + var roots []string + + for _, filePath := range files { + modules, err := p.ParsePomModules(filePath) + + if err != nil { + continue + } + + validFiles = append(validFiles, filePath) + + if len(modules) == 0 { + continue + } + + for _, module := range modules { + modulePath := filepath.Join(filepath.Dir(filePath), filepath.Dir(module), filepath.Base(module), "pom.xml") + childMap[modulePath] = true + } + } + + for _, file := range validFiles { + if _, ok := childMap[file]; !ok { + roots = append(roots, file) + } + } + + return roots +} diff --git a/pkg/resolution/pm/maven/pom_service_test.go b/pkg/resolution/pm/maven/pom_service_test.go new file mode 100644 index 00000000..64270517 --- /dev/null +++ b/pkg/resolution/pm/maven/pom_service_test.go @@ -0,0 +1,39 @@ +package maven + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParsePomModules(t *testing.T) { + p := PomService{} + modules, err := p.ParsePomModules("testdata/pom.xml") + assert.Nil(t, err) + assert.Len(t, modules, 5) + correct := []string{"guava", "guava-bom", "guava-gwt", "guava-testlib", "guava-tests"} + assert.Equal(t, correct, modules) + + modules, err = p.ParsePomModules("testdata/notAPom.xml") + + assert.NotNil(t, err) + assert.Len(t, modules, 0) +} + +func TestGetRootPomFiles(t *testing.T) { + pomParent := filepath.Join("testdata", "pom.xml") + pomFail := filepath.Join("testdata", "notAPom.xml") + pomChild := filepath.Join("testdata", "guava", "pom.xml") + + p := PomService{} + files := p.GetRootPomFiles([]string{pomParent, pomFail}) + assert.Len(t, files, 1) + + files = p.GetRootPomFiles([]string{pomParent, pomChild}) + assert.Len(t, files, 1) + assert.Equal(t, pomParent, files[0]) + + files = p.GetRootPomFiles([]string{pomFail}) + assert.Len(t, files, 0) +} diff --git a/pkg/resolution/pm/maven/strategy.go b/pkg/resolution/pm/maven/strategy.go new file mode 100644 index 00000000..f3e3850f --- /dev/null +++ b/pkg/resolution/pm/maven/strategy.go @@ -0,0 +1,26 @@ +package maven + +import ( + "github.com/debricked/cli/pkg/resolution/job" +) + +type Strategy struct { + files []string + cmdFactory ICmdFactory + pomService IPomService +} + +func NewStrategy(files []string) Strategy { + return Strategy{files, CmdFactory{}, PomService{}} +} + +func (s Strategy) Invoke() ([]job.IJob, error) { + var jobs []job.IJob + s.files = s.pomService.GetRootPomFiles(s.files) + + for _, file := range s.files { + jobs = append(jobs, NewJob(file, s.cmdFactory)) + } + + return jobs, nil +} diff --git a/pkg/resolution/pm/maven/strategy_test.go b/pkg/resolution/pm/maven/strategy_test.go new file mode 100644 index 00000000..91d9deed --- /dev/null +++ b/pkg/resolution/pm/maven/strategy_test.go @@ -0,0 +1,61 @@ +package maven + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +type PomServiceMock struct{} + +func (p PomServiceMock) GetRootPomFiles(files []string) []string { + return files +} + +func (p PomServiceMock) ParsePomModules(_ string) ([]string, error) { + return []string{}, nil +} + +func TestNewStrategy(t *testing.T) { + s := NewStrategy(nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy([]string{}) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy([]string{"file"}) + assert.NotNil(t, s) + assert.Len(t, s.files, 1) + + s = NewStrategy([]string{"file-1", "file-2"}) + assert.NotNil(t, s) + assert.Len(t, s.files, 2) +} + +func TestInvokeNoFiles(t *testing.T) { + s := NewStrategy([]string{}) + + jobs, _ := s.Invoke() + + assert.Empty(t, jobs) +} + +func TestInvokeOneFile(t *testing.T) { + s := NewStrategy([]string{"file"}) + s.pomService = PomServiceMock{} + + jobs, _ := s.Invoke() + + assert.Len(t, jobs, 1) +} + +func TestInvokeManyFiles(t *testing.T) { + s := NewStrategy([]string{"file-1", "file-2"}) + s.pomService = PomServiceMock{} + + jobs, _ := s.Invoke() + + assert.Len(t, jobs, 2) +} diff --git a/pkg/resolution/pm/maven/testdata/cmd_factory_mock.go b/pkg/resolution/pm/maven/testdata/cmd_factory_mock.go new file mode 100644 index 00000000..d2172f73 --- /dev/null +++ b/pkg/resolution/pm/maven/testdata/cmd_factory_mock.go @@ -0,0 +1,16 @@ +package testdata + +import "os/exec" + +type CmdFactoryMock struct { + Err error + Name string + Arg string +} + +func (f CmdFactoryMock) MakeDependencyTreeCmd(_ string) (*exec.Cmd, error) { + if len(f.Arg) == 0 { + f.Arg = `"MakeDependencyTreeCmd"` + } + return exec.Command(f.Name, f.Arg), f.Err +} diff --git a/pkg/resolution/pm/maven/testdata/guava/pom.xml b/pkg/resolution/pm/maven/testdata/guava/pom.xml new file mode 100644 index 00000000..150831cc --- /dev/null +++ b/pkg/resolution/pm/maven/testdata/guava/pom.xml @@ -0,0 +1,253 @@ + + + 4.0.0 + + com.google.guava + guava-parent + HEAD-jre-SNAPSHOT + + guava + bundle + Guava: Google Core Libraries for Java + https://github.com/google/guava + + Guava is a suite of core and expanded libraries that include + utility classes, Google's collections, I/O classes, and + much more. + + + + com.google.guava + failureaccess + 1.0.1 + + + com.google.guava + listenablefuture + 9999.0-empty-to-avoid-conflict-with-guava + + + com.google.code.findbugs + jsr305 + + + org.checkerframework + checker-qual + + + com.google.errorprone + error_prone_annotations + + + com.google.j2objc + j2objc-annotations + + + + + + + + maven-jar-plugin + + + + com.google.common + + + + + + true + org.apache.felix + maven-bundle-plugin + 5.1.8 + + + bundle-manifest + process-classes + + manifest + + + + + + + !com.google.common.base.internal, + !com.google.common.util.concurrent.internal, + com.google.common.* + + + com.google.common.util.concurrent.internal, + javax.annotation;resolution:=optional, + javax.crypto.*;resolution:=optional, + sun.misc.*;resolution:=optional + + https://github.com/google/guava/ + + + + + maven-compiler-plugin + + + maven-source-plugin + + + + maven-dependency-plugin + + + unpack-jdk-sources + generate-sources + unpack-dependencies + + srczip + ${project.build.directory}/jdk-sources + false + + **/module-info.java,**/java/io/FileDescriptor.java + + + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + + + maven-javadoc-plugin + + + + + ${project.build.sourceDirectory}:${project.build.directory}/jdk-sources + + + + + com.azul.tooling.in,com.google.common.base.internal,com.google.common.base.internal.*,com.google.thirdparty.publicsuffix,com.google.thirdparty.publicsuffix.*,com.oracle.*,com.sun.*,java.*,javax.*,jdk,jdk.*,org.*,sun.* + + + + + apiNote + X + + + implNote + X + + + implSpec + X + + + jls + X + + + revised + X + + + spec + X + + + + + + false + + + + + https://static.javadoc.io/com.google.code.findbugs/jsr305/3.0.1/ + ${project.basedir}/javadoc-link/jsr305 + + + https://static.javadoc.io/com.google.j2objc/j2objc-annotations/1.1/ + ${project.basedir}/javadoc-link/j2objc-annotations + + + + https://docs.oracle.com/javase/9/docs/api/ + https://docs.oracle.com/javase/9/docs/api/ + + + + https://checkerframework.org/api/ + ${project.basedir}/javadoc-link/checker-framework + + + + https://errorprone.info/api/latest/ + + + + + attach-docs + + + generate-javadoc-site-report + site + javadoc + + + + + + + + srczip-parent + + + ${java.home}/../src.zip + + + + + jdk + srczip + 999 + system + ${java.home}/../src.zip + true + + + + + srczip-lib + + + ${java.home}/lib/src.zip + + + + + jdk + srczip + 999 + system + ${java.home}/lib/src.zip + true + + + + + + maven-javadoc-plugin + + + ${project.build.sourceDirectory}:${project.build.directory}/jdk-sources/java.base + + + + + + + diff --git a/pkg/resolution/pm/maven/testdata/notAPom.xml b/pkg/resolution/pm/maven/testdata/notAPom.xml new file mode 100644 index 00000000..a87187bc --- /dev/null +++ b/pkg/resolution/pm/maven/testdata/notAPom.xml @@ -0,0 +1,3 @@ +pandas==1.1.1 +# comment +numpy==1.2.3 \ No newline at end of file diff --git a/pkg/resolution/pm/maven/testdata/pom.xml b/pkg/resolution/pm/maven/testdata/pom.xml new file mode 100644 index 00000000..1cb2a0be --- /dev/null +++ b/pkg/resolution/pm/maven/testdata/pom.xml @@ -0,0 +1,541 @@ + + + + 4.0.0 + com.google.guava + guava-parent + HEAD-jre-SNAPSHOT + pom + Guava Maven Parent + Parent for guava artifacts + https://github.com/google/guava + + + %regex[.*.class] + 1.1.3 + 3.29.0 + 1.22 + 3.4.1 + 9+181-r4173-1 + + + 3.2.1 + 1980-02-01T00:00:00Z + UTF-8 + + + + GitHub Issues + https://github.com/google/guava/issues + + 2010 + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + scm:git:https://github.com/google/guava.git + scm:git:git@github.com:google/guava.git + https://github.com/google/guava + + + + kevinb9n + Kevin Bourrillion + kevinb@google.com + Google + http://www.google.com + + owner + developer + + -8 + + + + GitHub Actions + https://github.com/google/guava/actions + + + guava + guava-bom + guava-gwt + guava-testlib + guava-tests + + + + src + test + + + src + + **/*.java + **/*.sw* + + + + + + test + + **/*.java + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-versions + + enforce + + + + + 3.0.5 + + + 1.8.0 + + + + + + + + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + ${java.specification.version} + + + + + + + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + UTF-8 + true + + + -sourcepath + doesnotexist + + -XDcompilePolicy=simple + + + + + com.google.errorprone + error_prone_core + 2.16 + + + + true + + + + maven-jar-plugin + 3.2.0 + + + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + post-integration-test + jar + + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + ${animal.sniffer.version} + + true + + org.codehaus.mojo.signature + java18 + 1.0 + + + + + check-java-version-compatibility + test + + check + + + + + + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + true + true + UTF-8 + UTF-8 + UTF-8 + + -XDignore.symbol.file + -Xdoclint:-html + + true + 8 + ${maven-javadoc-plugin.additionalJOptions} + + + + attach-docs + post-integration-test + jar + + + + + maven-dependency-plugin + 3.1.1 + + + maven-antrun-plugin + 1.6 + + + maven-surefire-plugin + 2.7.2 + + + ${test.include} + + + + + %regex[.*PackageSanityTests.*.class] + + %regex[.*Tester.class] + + %regex[.*[$]\d+.class] + + true + alphabetical + + + -Xmx1536M -Duser.language=hi -Duser.country=IN ${test.add.opens} + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0-M3 + + + + + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + + sonatype-nexus-staging + Nexus Release Repository + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + guava-site + Guava Documentation Site + scp://dummy.server/dontinstall/usestaging + + + + + + com.google.code.findbugs + jsr305 + 3.0.2 + + + org.checkerframework + checker-qual + ${checker-framework.version} + + + org.checkerframework + checker-qual + ${checker-framework.version} + sources + + + com.google.errorprone + error_prone_annotations + 2.18.0 + + + com.google.j2objc + j2objc-annotations + 2.8 + + + junit + junit + 4.13.2 + test + + + org.mockito + mockito-core + 4.11.0 + test + + + com.google.jimfs + jimfs + 1.2 + test + + + com.google.truth + truth + ${truth.version} + test + + + + com.google.guava + guava + + + + + com.google.truth.extensions + truth-java8-extension + ${truth.version} + test + + + + com.google.guava + guava + + + + + com.google.caliper + caliper + 1.0-beta-3 + test + + + + com.google.guava + guava + + + + + + + + sonatype-oss-release + + + + org.apache.maven.plugins + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.0.1 + + + sign-artifacts + verify + + sign + + + + + + + + + + javadocs-jdk11-12 + + [11,13) + + + --no-module-directories + + + + open-jre-modules + + [9,] + + + + + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/sun.security.jca=ALL-UNNAMED + + + + + javac9-for-jdk8 + + 1.8 + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + -J-Xbootclasspath/p:${settings.localRepository}/com/google/errorprone/javac/${javac.version}/javac-${javac.version}.jar + + + + + + + + run-error-prone + + + [11,12),[16,) + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + + -Xplugin:ErrorProne -Xep:NullArgumentForNonNullParameter:OFF -Xep:Java8ApiChecker:ERROR + + + -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + + + + + + + + diff --git a/pkg/resolution/pm/pip/cmd_factory.go b/pkg/resolution/pm/pip/cmd_factory.go new file mode 100644 index 00000000..322d920f --- /dev/null +++ b/pkg/resolution/pm/pip/cmd_factory.go @@ -0,0 +1,90 @@ +package pip + +import ( + "os/exec" + "strings" +) + +type ICmdFactory interface { + MakeCreateVenvCmd(file string) (*exec.Cmd, error) + MakeInstallCmd(command string, file string) (*exec.Cmd, error) + MakeCatCmd(file string) (*exec.Cmd, error) + MakeListCmd(command string) (*exec.Cmd, error) + MakeShowCmd(command string, list []string) (*exec.Cmd, error) +} + +type IExecPath interface { + LookPath(file string) (string, error) +} + +type ExecPath struct { +} + +func (_ ExecPath) LookPath(file string) (string, error) { + return exec.LookPath(file) +} + +type CmdFactory struct { + execPath IExecPath +} + +func (cmdf CmdFactory) MakeCreateVenvCmd(fpath string) (*exec.Cmd, error) { + python, err := cmdf.execPath.LookPath("python3") + pythonCommand := "python3" + if err != nil { + if strings.Contains(err.Error(), "executable file not found in ") { + // Python 3 not found, try Python + python, err = cmdf.execPath.LookPath("python") + pythonCommand = "python" + } + + // If error still is != nil, return + if err != nil { + return nil, err + } + } + + return &exec.Cmd{ + Path: python, + Args: []string{pythonCommand, "-m", "venv", fpath, "--clear"}, + }, nil +} + +func (cmdf CmdFactory) MakeInstallCmd(command string, file string) (*exec.Cmd, error) { + path, err := cmdf.execPath.LookPath(command) + + return &exec.Cmd{ + Path: path, + Args: []string{command, "install", "-r", file}, + }, err +} + +func (cmdf CmdFactory) MakeCatCmd(file string) (*exec.Cmd, error) { + path, err := cmdf.execPath.LookPath("cat") + + return &exec.Cmd{ + Path: path, + Args: []string{"cat", file}, + }, err +} + +func (cmdf CmdFactory) MakeListCmd(command string) (*exec.Cmd, error) { + path, err := cmdf.execPath.LookPath(command) + + return &exec.Cmd{ + Path: path, + Args: []string{"pip", "list"}, + }, err +} + +func (cmdf CmdFactory) MakeShowCmd(command string, list []string) (*exec.Cmd, error) { + path, err := cmdf.execPath.LookPath(command) + + args := []string{command, "show"} + args = append(args, list...) + + return &exec.Cmd{ + Path: path, + Args: args, + }, err +} diff --git a/pkg/resolution/pm/pip/cmd_factory_test.go b/pkg/resolution/pm/pip/cmd_factory_test.go new file mode 100644 index 00000000..5246420b --- /dev/null +++ b/pkg/resolution/pm/pip/cmd_factory_test.go @@ -0,0 +1,120 @@ +package pip + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +type ExecPathMock struct { + python3Error error + pythonError error +} + +func (epm ExecPathMock) LookPath(file string) (string, error) { + if epm.python3Error != nil && file == "python3" { + return "", epm.python3Error + } + + if epm.pythonError != nil && file == "python" { + return "", epm.pythonError + } + + return file, nil +} + +func TestCreateVenvCmd(t *testing.T) { + venvName := "test-file.venv" + cmd, err := CmdFactory{ + execPath: ExecPath{}, + }.MakeCreateVenvCmd(venvName) + assert.NoError(t, err) + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "python3") + assert.Contains(t, args, "-m") + assert.Contains(t, args, "venv") + assert.Contains(t, args, venvName) + assert.Contains(t, args, "--clear") +} + +func TestCreateVenvCmdPython3Error(t *testing.T) { + err := errors.New("executable file not found in $PATH") + execPathMock := ExecPathMock{python3Error: err} + venvName := "test-file-python3-error.venv" + cmd, err := CmdFactory{ + execPath: execPathMock, + }.MakeCreateVenvCmd(venvName) + + assert.NoError(t, err) + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "python") + assert.Contains(t, args, "-m") + assert.Contains(t, args, "venv") + assert.Contains(t, args, venvName) + assert.Contains(t, args, "--clear") +} + +func TestCreateVenvCmdPythonCompletelyMissing(t *testing.T) { + pathErr := errors.New("executable file not found in $PATH") + execPathMock := ExecPathMock{python3Error: pathErr, pythonError: pathErr} + venvName := "test-file-python-missing.venv" + _, err := CmdFactory{ + execPath: execPathMock, + }.MakeCreateVenvCmd(venvName) + + assert.ErrorContains(t, err, "executable file not found in") + assert.ErrorContains(t, err, "PATH") +} + +func TestMakeInstallCmd(t *testing.T) { + fileName := "test-file" + pipCommand := "pip" + cmd, err := CmdFactory{ + execPath: ExecPath{}, + }.MakeInstallCmd(pipCommand, fileName) + assert.NoError(t, err) + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "pip") + assert.Contains(t, args, "install") + assert.Contains(t, args, "-r") + assert.Contains(t, args, fileName) +} + +func TestMakeCatCmd(t *testing.T) { + fileName := "test-file" + cmd, _ := CmdFactory{ + execPath: ExecPath{}, + }.MakeCatCmd(fileName) + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "cat") + assert.Contains(t, args, fileName) +} +func TestMakeListCmd(t *testing.T) { + mockCommand := "mock-cmd" + cmd, _ := CmdFactory{ + execPath: ExecPath{}, + }.MakeListCmd(mockCommand) + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "pip") + assert.Contains(t, args, "list") +} + +func TestMakeShowCmd(t *testing.T) { + input := []string{"package1", "package2"} + mockCommand := "pip" + cmd, _ := CmdFactory{ + execPath: ExecPath{}, + }.MakeShowCmd(mockCommand, input) + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "pip") + assert.Contains(t, args, "show") + assert.Contains(t, args, "package1") + assert.Contains(t, args, "package2") +} diff --git a/pkg/resolution/pm/pip/job.go b/pkg/resolution/pm/pip/job.go new file mode 100644 index 00000000..4d03fdc8 --- /dev/null +++ b/pkg/resolution/pm/pip/job.go @@ -0,0 +1,232 @@ +package pip + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/pm/util" + "github.com/debricked/cli/pkg/resolution/pm/writer" +) + +const ( + lockFileExtension = ".pip.debricked.lock" + pip = "pip" + lockFileDelimiter = "***" +) + +type Job struct { + job.BaseJob + install bool + venvPath string + pipCommand string + cmdFactory ICmdFactory + fileWriter writer.IFileWriter + pipCleaner IPipCleaner +} + +func NewJob( + file string, + install bool, + cmdFactory ICmdFactory, + fileWriter writer.IFileWriter, + pipCleaner IPipCleaner, +) *Job { + return &Job{ + BaseJob: job.NewBaseJob(file), + install: install, + cmdFactory: cmdFactory, + fileWriter: fileWriter, + pipCleaner: pipCleaner, + } +} + +type IPipCleaner interface { + RemoveAll(path string) error +} + +type pipCleaner struct{} + +func (p pipCleaner) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +func (j *Job) Install() bool { + return j.install +} + +func (j *Job) Run() { + if j.install { + j.SendStatus("creating venv") + _, err := j.runCreateVenvCmd() + if err != nil { + j.Errors().Critical(err) + + return + } + + j.SendStatus("installing requirements") + _, err = j.runInstallCmd() + if err != nil { + j.Errors().Critical(err) + + return + } + } + + err := j.writeLockContent() + if err != nil { + j.Errors().Critical(err) + + return + } + + if j.install { + j.SendStatus("removing venv") + err = j.pipCleaner.RemoveAll(j.venvPath) + if err != nil { + j.Errors().Critical(err) + } + } +} + +func (j *Job) writeLockContent() error { + j.SendStatus("generating lock file") + catCmdOutput, err := j.runCatCmd() + if err != nil { + return err + } + + listCmdOutput, err := j.runListCmd() + if err != nil { + return err + } + + installedPackages := j.parsePipList(string(listCmdOutput)) + ShowCmdOutput, err := j.runShowCmd(installedPackages) + if err != nil { + return err + } + + lockFileName := fmt.Sprintf(".%s%s", filepath.Base(j.GetFile()), lockFileExtension) + lockFile, err := j.fileWriter.Create(util.MakePathFromManifestFile(j.GetFile(), lockFileName)) + if err != nil { + return err + } + defer closeFile(j, lockFile) + + var fileContents []string + fileContents = append(fileContents, string(catCmdOutput)) + fileContents = append(fileContents, lockFileDelimiter) + fileContents = append(fileContents, string(listCmdOutput)) + fileContents = append(fileContents, lockFileDelimiter) + fileContents = append(fileContents, string(ShowCmdOutput)) + res := []byte(strings.Join(fileContents, "\n")) + + j.SendStatus("writing lock file") + + return j.fileWriter.Write(lockFile, res) +} + +func (j *Job) runCreateVenvCmd() ([]byte, error) { + venvName := fmt.Sprintf("%s.venv", filepath.Base(j.GetFile())) + fpath := filepath.Join(filepath.Dir(j.GetFile()), venvName) + j.venvPath = fpath + + createVenvCmd, err := j.cmdFactory.MakeCreateVenvCmd(j.venvPath) + if err != nil { + return nil, err + } + + createVenvCmdOutput, err := createVenvCmd.Output() + if err != nil { + return nil, j.GetExitError(err) + } + + return createVenvCmdOutput, nil +} + +func (j *Job) runInstallCmd() ([]byte, error) { + var command string + if j.venvPath != "" { + command = filepath.Join(j.venvPath, "bin", pip) + } else { + command = pip + } + j.pipCommand = command + installCmd, err := j.cmdFactory.MakeInstallCmd(j.pipCommand, j.GetFile()) + if err != nil { + return nil, err + } + + installCmdOutput, err := installCmd.Output() + if err != nil { + return nil, j.GetExitError(err) + } + + return installCmdOutput, nil +} + +func (j *Job) runCatCmd() ([]byte, error) { + listCmd, err := j.cmdFactory.MakeCatCmd(j.GetFile()) + if err != nil { + return nil, err + } + + listCmdOutput, err := listCmd.Output() + if err != nil { + return nil, j.GetExitError(err) + } + + return listCmdOutput, nil +} + +func (j *Job) runListCmd() ([]byte, error) { + listCmd, err := j.cmdFactory.MakeListCmd(j.pipCommand) + if err != nil { + return nil, err + } + + listCmdOutput, err := listCmd.Output() + if err != nil { + return nil, j.GetExitError(err) + } + + return listCmdOutput, nil +} + +func (j *Job) runShowCmd(packages []string) ([]byte, error) { + listCmd, err := j.cmdFactory.MakeShowCmd(j.pipCommand, packages) + if err != nil { + return nil, err + } + + listCmdOutput, err := listCmd.Output() + if err != nil { + return nil, j.GetExitError(err) + } + + return listCmdOutput, nil +} + +func closeFile(job *Job, file *os.File) { + err := job.fileWriter.Close(file) + if err != nil { + job.Errors().Critical(err) + } +} + +func (j *Job) parsePipList(pipListOutput string) []string { + lines := strings.Split(pipListOutput, "\n") + var packages []string + for _, line := range lines[2:] { + fields := strings.Split(line, " ") + if len(fields) > 0 { + packages = append(packages, fields[0]) + } + } + + return packages +} diff --git a/pkg/resolution/pm/pip/job_test.go b/pkg/resolution/pm/pip/job_test.go new file mode 100644 index 00000000..19f010aa --- /dev/null +++ b/pkg/resolution/pm/pip/job_test.go @@ -0,0 +1,274 @@ +package pip + +import ( + "errors" + "fmt" + "os" + "strings" + "testing" + + jobTestdata "github.com/debricked/cli/pkg/resolution/job/testdata" + "github.com/debricked/cli/pkg/resolution/pm/pip/testdata" + "github.com/debricked/cli/pkg/resolution/pm/writer" + writerTestdata "github.com/debricked/cli/pkg/resolution/pm/writer/testdata" + "github.com/stretchr/testify/assert" +) + +const ( + badName = "bad-name" +) + +func TestNewJob(t *testing.T) { + j := NewJob("file", false, CmdFactory{ + execPath: ExecPath{}, + }, writer.FileWriter{}, pipCleaner{}) + assert.Equal(t, "file", j.GetFile()) + assert.False(t, j.Errors().HasError()) +} + +func TestInstall(t *testing.T) { + j := Job{install: true} + assert.Equal(t, true, j.Install()) + + j = Job{install: false} + assert.Equal(t, false, j.Install()) +} + +func TestRunCreateVenvCmdErr(t *testing.T) { + cmdErr := errors.New("cmd-error") + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.MakeCreateVenvErr = cmdErr + fileWriterMock := &writerTestdata.FileWriterMock{} + j := NewJob("file", true, cmdFactoryMock, fileWriterMock, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), cmdErr) +} + +func TestRunCreateVenvCmdOutputErr(t *testing.T) { + cmdMock := testdata.NewEchoCmdFactory() + cmdMock.CreateVenvCmdName = badName + j := NewJob("file", true, cmdMock, nil, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + jobTestdata.AssertPathErr(t, j.Errors()) +} + +func TestRunInstallCmdErr(t *testing.T) { + cmdErr := errors.New("cmd-error") + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.MakeInstallErr = cmdErr + fileWriterMock := &writerTestdata.FileWriterMock{} + j := NewJob("file", true, cmdFactoryMock, fileWriterMock, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), cmdErr) +} + +func TestRunInstallCmdOutputErr(t *testing.T) { + cmdMock := testdata.NewEchoCmdFactory() + cmdMock.InstallCmdName = badName + j := NewJob("file", true, cmdMock, nil, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + jobTestdata.AssertPathErr(t, j.Errors()) +} + +func TestRunCatCmdErr(t *testing.T) { + cmdErr := errors.New("cmd-error") + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.MakeCatErr = cmdErr + fileWriterMock := &writerTestdata.FileWriterMock{} + j := NewJob("file", true, cmdFactoryMock, fileWriterMock, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), cmdErr) +} + +func TestRunCatCmdOutputErr(t *testing.T) { + cmdMock := testdata.NewEchoCmdFactory() + cmdMock.CatCmdName = badName + j := NewJob("file", false, cmdMock, nil, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + jobTestdata.AssertPathErr(t, j.Errors()) +} + +func TestRunListCmdErr(t *testing.T) { + cmdErr := errors.New("cmd-error") + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.MakeListErr = cmdErr + fileWriterMock := &writerTestdata.FileWriterMock{} + j := NewJob("file", true, cmdFactoryMock, fileWriterMock, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), cmdErr) +} + +func TestRunListCmdOutputErr(t *testing.T) { + cmdMock := testdata.NewEchoCmdFactory() + cmdMock.ListCmdName = badName + j := NewJob("file", false, cmdMock, nil, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + jobTestdata.AssertPathErr(t, j.Errors()) +} + +func TestRunShowCmdErr(t *testing.T) { + cmdErr := errors.New("cmd-error") + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.MakeShowErr = cmdErr + fileWriterMock := &writerTestdata.FileWriterMock{} + j := NewJob("file", true, cmdFactoryMock, fileWriterMock, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), cmdErr) +} + +func TestRunShowCmdOutputErr(t *testing.T) { + cmdMock := testdata.NewEchoCmdFactory() + cmdMock.ShowCmdName = badName + j := NewJob("file", false, cmdMock, nil, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + jobTestdata.AssertPathErr(t, j.Errors()) +} + +func TestRun(t *testing.T) { + // Load gt-data + list, err := os.ReadFile("testdata/list.txt") + assert.Nil(t, err) + req, err := os.ReadFile("testdata/requirements.txt") + assert.Nil(t, err) + show, err := os.ReadFile("testdata/show.txt") + assert.Nil(t, err) + + var fileContents []string + fileContents = append(fileContents, string(req)+"\n") + fileContents = append(fileContents, lockFileDelimiter) + fileContents = append(fileContents, string(list)+"\n") + fileContents = append(fileContents, lockFileDelimiter) + fileContents = append(fileContents, string(show)+"\n") + res := []byte(strings.Join(fileContents, "\n")) + + fileWriterMock := &writerTestdata.FileWriterMock{} + cmdFactoryMock := testdata.NewEchoCmdFactory() + j := NewJob("file", true, cmdFactoryMock, fileWriterMock, pipCleaner{}) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.False(t, j.Errors().HasError()) + fmt.Println(string(fileWriterMock.Contents)) + assert.Equal(t, string(res), string(fileWriterMock.Contents)) +} + +func TestRunInstall(t *testing.T) { + cmdFactoryMock := testdata.NewEchoCmdFactory() + fileWriterMock := &writerTestdata.FileWriterMock{} + j := NewJob("file", false, cmdFactoryMock, fileWriterMock, nil) + + _, err := j.runInstallCmd() + assert.NoError(t, err) + + assert.False(t, j.Errors().HasError()) +} + +func TestParsePipList(t *testing.T) { + j := NewJob("file", false, CmdFactory{ + execPath: ExecPath{}, + }, writer.FileWriter{}, pipCleaner{}) + file, err := os.ReadFile("testdata/list.txt") + assert.Nil(t, err) + pipData := string(file) + packages := j.parsePipList(pipData) + gt := []string{"aiohttp", "cryptography", "numpy", "Flask", "open-source-health", "pandas", "tqdm"} + assert.Equal(t, gt, packages) + assert.False(t, j.Errors().HasError()) +} + +func TestRunCreateErr(t *testing.T) { + createErr := errors.New("create-error") + fileWriterMock := &writerTestdata.FileWriterMock{CreateErr: createErr} + cmdMock := testdata.NewEchoCmdFactory() + j := NewJob("file", true, cmdMock, fileWriterMock, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), createErr) +} + +func TestRunWriteErr(t *testing.T) { + writeErr := errors.New("write-error") + fileWriterMock := &writerTestdata.FileWriterMock{WriteErr: writeErr} + cmdMock := testdata.NewEchoCmdFactory() + j := NewJob("file", true, cmdMock, fileWriterMock, nil) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), writeErr) +} + +func TestRunCloseErr(t *testing.T) { + closeErr := errors.New("close-error") + fileWriterMock := &writerTestdata.FileWriterMock{CloseErr: closeErr} + cmdMock := testdata.NewEchoCmdFactory() + j := NewJob("file", true, cmdMock, fileWriterMock, pipCleaner{}) + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), closeErr) +} + +type pipCleanerMock struct { + CleanErr error +} + +func (p *pipCleanerMock) RemoveAll(_ string) error { + return p.CleanErr +} + +func TestRunCleanErr(t *testing.T) { + CleanErr := errors.New("clean-error") + fileWriterMock := &writerTestdata.FileWriterMock{} + cmdMock := testdata.NewEchoCmdFactory() + j := NewJob("file", true, cmdMock, fileWriterMock, nil) + j.pipCleaner = &pipCleanerMock{CleanErr: CleanErr} + + go jobTestdata.WaitStatus(j) + j.Run() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), CleanErr) +} diff --git a/pkg/resolution/pm/pip/pm.go b/pkg/resolution/pm/pip/pm.go new file mode 100644 index 00000000..d8fb51d3 --- /dev/null +++ b/pkg/resolution/pm/pip/pm.go @@ -0,0 +1,23 @@ +package pip + +const Name = "pip" + +type Pm struct { + name string +} + +func NewPm() Pm { + return Pm{ + name: Name, + } +} + +func (pm Pm) Name() string { + return pm.name +} + +func (_ Pm) Manifests() []string { + return []string{ + `requirements.*\.txt$`, + } +} diff --git a/pkg/resolution/pm/pip/pm_test.go b/pkg/resolution/pm/pip/pm_test.go new file mode 100644 index 00000000..1a583e42 --- /dev/null +++ b/pkg/resolution/pm/pip/pm_test.go @@ -0,0 +1,48 @@ +package pip + +import ( + "regexp" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewPm(t *testing.T) { + pm := NewPm() + assert.Equal(t, Name, pm.name) +} + +func TestName(t *testing.T) { + pm := NewPm() + assert.Equal(t, Name, pm.Name()) +} + +func TestManifests(t *testing.T) { + pm := Pm{} + manifests := pm.Manifests() + assert.Len(t, manifests, 1) + manifest := manifests[0] + assert.Equal(t, `requirements.*\.txt$`, manifest) + _, err := regexp.Compile(manifest) + assert.NoError(t, err) + + cases := map[string]bool{ + "requirements.txt": true, + "requirements.dev.txt": true, + "requirements.dev.test.txt": true, + "requirements-dev.test.txt": true, + "requirements-dev-test.txt": true, + "requirements-test.txt": true, + "/dir/requirements.txt": true, + "requirements-test-txt": false, + "requirements-test.txt.dev": false, + "requirements-test.txt.pip.debricked.lock": false, + "requirements.txt.pip.debricked.lock": false, + } + for file, isMatch := range cases { + t.Run(file, func(t *testing.T) { + matched, _ := regexp.MatchString(manifest, file) + assert.Equal(t, isMatch, matched) + }) + } +} diff --git a/pkg/resolution/pm/pip/strategy.go b/pkg/resolution/pm/pip/strategy.go new file mode 100644 index 00000000..bed6ec83 --- /dev/null +++ b/pkg/resolution/pm/pip/strategy.go @@ -0,0 +1,32 @@ +package pip + +import ( + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/pm/writer" +) + +type Strategy struct { + files []string +} + +func (s Strategy) Invoke() ([]job.IJob, error) { + var jobs []job.IJob + for _, file := range s.files { + jobs = append(jobs, NewJob( + file, + true, + CmdFactory{ + execPath: ExecPath{}, + }, + writer.FileWriter{}, + pipCleaner{}, + ), + ) + } + + return jobs, nil +} + +func NewStrategy(files []string) Strategy { + return Strategy{files} +} diff --git a/pkg/resolution/pm/pip/strategy_test.go b/pkg/resolution/pm/pip/strategy_test.go new file mode 100644 index 00000000..993ac8cb --- /dev/null +++ b/pkg/resolution/pm/pip/strategy_test.go @@ -0,0 +1,43 @@ +package pip + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewStrategy(t *testing.T) { + s := NewStrategy(nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy([]string{}) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy([]string{"file"}) + assert.NotNil(t, s) + assert.Len(t, s.files, 1) + + s = NewStrategy([]string{"file-1", "file-2"}) + assert.NotNil(t, s) + assert.Len(t, s.files, 2) +} + +func TestInvokeNoFiles(t *testing.T) { + s := NewStrategy([]string{}) + jobs, _ := s.Invoke() + assert.Empty(t, jobs) +} + +func TestInvokeOneFile(t *testing.T) { + s := NewStrategy([]string{"file"}) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 1) +} + +func TestInvokeManyFiles(t *testing.T) { + s := NewStrategy([]string{"file-1", "file-2"}) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 2) +} diff --git a/pkg/resolution/pm/pip/testdata/cmd_factory_mock.go b/pkg/resolution/pm/pip/testdata/cmd_factory_mock.go new file mode 100644 index 00000000..08aa9e6b --- /dev/null +++ b/pkg/resolution/pm/pip/testdata/cmd_factory_mock.go @@ -0,0 +1,64 @@ +package testdata + +import ( + "os" + "os/exec" +) + +type CmdFactoryMock struct { + CreateVenvCmdName string + MakeCreateVenvErr error + InstallCmdName string + MakeInstallErr error + CatCmdName string + MakeCatErr error + ListCmdName string + MakeListErr error + ShowCmdName string + MakeShowErr error +} + +func NewEchoCmdFactory() CmdFactoryMock { + return CmdFactoryMock{ + CreateVenvCmdName: "echo", + InstallCmdName: "echo", + CatCmdName: "echo", + ListCmdName: "echo", + ShowCmdName: "echo", + } +} + +func (f CmdFactoryMock) MakeCreateVenvCmd(file string) (*exec.Cmd, error) { + return exec.Command(f.CreateVenvCmdName, file), f.MakeCreateVenvErr +} + +func (f CmdFactoryMock) MakeInstallCmd(command string, file string) (*exec.Cmd, error) { + return exec.Command(f.InstallCmdName, file), f.MakeInstallErr +} + +func (f CmdFactoryMock) MakeListCmd(command string) (*exec.Cmd, error) { + fileContent, err := os.ReadFile("testdata/list.txt") + if err != nil { + return nil, err + } + pipData := string(fileContent) + return exec.Command(f.ListCmdName, pipData), f.MakeListErr +} + +func (f CmdFactoryMock) MakeCatCmd(file string) (*exec.Cmd, error) { + fileContent, err := os.ReadFile("testdata/requirements.txt") + if err != nil { + return nil, err + } + requirements := string(fileContent) + return exec.Command(f.CatCmdName, requirements), f.MakeCatErr +} + +func (f CmdFactoryMock) MakeShowCmd(command string, list []string) (*exec.Cmd, error) { + fileContent, err := os.ReadFile("testdata/show.txt") + if err != nil { + return nil, err + } + show := string(fileContent) + return exec.Command(f.ShowCmdName, show), f.MakeShowErr +} diff --git a/pkg/resolution/pm/pip/testdata/list.txt b/pkg/resolution/pm/pip/testdata/list.txt new file mode 100644 index 00000000..7e79e4d8 --- /dev/null +++ b/pkg/resolution/pm/pip/testdata/list.txt @@ -0,0 +1,9 @@ +Package Version Editable project location +----------------------------- ------------ ------------------------------------------------------ +aiohttp 3.7.4 +cryptography 3.4.7 +numpy 1.23.4 +Flask 2.0.3 +open-source-health 0.1 /path/to/folder +pandas 1.4.3 +tqdm 4.63.0 \ No newline at end of file diff --git a/pkg/resolution/pm/pip/testdata/requirements.txt b/pkg/resolution/pm/pip/testdata/requirements.txt new file mode 100644 index 00000000..439cd57c --- /dev/null +++ b/pkg/resolution/pm/pip/testdata/requirements.txt @@ -0,0 +1,12 @@ +Flask==2.1.5 +sentry-sdk==1.5.4 +sentry-sdk[flask] + +pandas>=1.4.0 +# matplotlib +# seaborn +tqdm + + + cryptography>=3.3.2,<4.0.0 +# test diff --git a/pkg/resolution/pm/pip/testdata/show.txt b/pkg/resolution/pm/pip/testdata/show.txt new file mode 100644 index 00000000..0bf9c0df --- /dev/null +++ b/pkg/resolution/pm/pip/testdata/show.txt @@ -0,0 +1,43 @@ +Name: Flask +Version: 2.1.2 +Summary: A simple framework for building complex web applications. +Home-page: https://palletsprojects.com/p/flask +Author: Armin Ronacher +Author-email: armin.ronacher@active-4.com +License: BSD-3-Clause +Location: /path/to/site-packages +Requires: click, importlib-metadata, itsdangerous, Jinja2, Werkzeug +Required-by: Flask-Script, Flask-Compress, Flask-Bcrypt +--- +Name: tqdm +Version: 4.64.0 +Summary: Fast, Extensible Progress Meter +Home-page: https://tqdm.github.io +Author: +Author-email: +License: MPLv2.0, MIT Licences +Location: /path/to/site-packages +Requires: +Required-by: transformers, nltk +--- +Name: pandas +Version: 1.4.2 +Summary: Powerful data structures for data analysis, time series, and statistics +Home-page: https://pandas.pydata.org +Author: The Pandas Development Team +Author-email: pandas-dev@python.org +License: BSD-3-Clause +Location: /path/to/site-packages +Requires: python-dateutil, pytz, numpy +Required-by: xarray, seaborn, hvplot, holoviews +--- +Name: numpy +Version: 1.21.5 +Summary: NumPy is the fundamental package for array computing with Python. +Home-page: https://www.numpy.org +Author: Travis E. Oliphant et al. +Author-email: +License: BSD +Location: /path/to/site-packages +Requires: +Required-by: xarray, transformers, pandas, astropy diff --git a/pkg/resolution/pm/pm.go b/pkg/resolution/pm/pm.go new file mode 100644 index 00000000..07858b03 --- /dev/null +++ b/pkg/resolution/pm/pm.go @@ -0,0 +1,22 @@ +package pm + +import ( + "github.com/debricked/cli/pkg/resolution/pm/gomod" + "github.com/debricked/cli/pkg/resolution/pm/gradle" + "github.com/debricked/cli/pkg/resolution/pm/maven" + "github.com/debricked/cli/pkg/resolution/pm/pip" +) + +type IPm interface { + Name() string + Manifests() []string +} + +func Pms() []IPm { + return []IPm{ + maven.NewPm(), + gradle.NewPm(), + gomod.NewPm(), + pip.NewPm(), + } +} diff --git a/pkg/resolution/pm/pm_test.go b/pkg/resolution/pm/pm_test.go new file mode 100644 index 00000000..5dd4eee0 --- /dev/null +++ b/pkg/resolution/pm/pm_test.go @@ -0,0 +1,26 @@ +package pm + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPms(t *testing.T) { + pms := Pms() + pmNames := []string{ + "mvn", + "go", + "gradle", + } + + for _, pmName := range pmNames { + t.Run(pmName, func(t *testing.T) { + contains := false + for _, pm := range pms { + contains = contains || pm.Name() == pmName + } + assert.Truef(t, contains, "failed to assert that %s was returned in Pms()", pmName) + }) + } +} diff --git a/pkg/resolution/pm/testdata/pm_mock.go b/pkg/resolution/pm/testdata/pm_mock.go new file mode 100644 index 00000000..e282738d --- /dev/null +++ b/pkg/resolution/pm/testdata/pm_mock.go @@ -0,0 +1,14 @@ +package testdata + +type PmMock struct { + N string + Ms []string +} + +func (pm PmMock) Name() string { + return pm.N +} + +func (pm PmMock) Manifests() []string { + return pm.Ms +} diff --git a/pkg/resolution/pm/util/util.go b/pkg/resolution/pm/util/util.go new file mode 100644 index 00000000..25220026 --- /dev/null +++ b/pkg/resolution/pm/util/util.go @@ -0,0 +1,27 @@ +package util + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/pm/writer" +) + +func MakePathFromManifestFile(siblingFile string, fileName string) string { + dir := filepath.Dir(siblingFile) + if strings.EqualFold(string(os.PathSeparator), dir) { + return fmt.Sprintf("%s%s", string(os.PathSeparator), fileName) + } + + return fmt.Sprintf("%s%s%s", dir, string(os.PathSeparator), fileName) +} + +func CloseFile(job job.IJob, fileWriter writer.IFileWriter, file *os.File) { + err := fileWriter.Close(file) + if err != nil { + job.Errors().Critical(err) + } +} diff --git a/pkg/resolution/pm/util/util_test.go b/pkg/resolution/pm/util/util_test.go new file mode 100644 index 00000000..763db86a --- /dev/null +++ b/pkg/resolution/pm/util/util_test.go @@ -0,0 +1,53 @@ +package util + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/job/testdata" + writerTestdata "github.com/debricked/cli/pkg/resolution/pm/writer/testdata" + "github.com/stretchr/testify/assert" +) + +func TestMakePathFromManifestFile(t *testing.T) { + manifestFile := filepath.Join("pkg", "resolution", "pm", "util", "file.json") + path := MakePathFromManifestFile(manifestFile, "file.lock") + lockFile := filepath.Join("pkg", "resolution", "pm", "util", "file.lock") + + assert.Equal(t, lockFile, path) + + path = MakePathFromManifestFile("file.json", "file.lock") + lockFile = fmt.Sprintf(".%s%s", string(os.PathSeparator), "file.lock") + assert.Equal(t, lockFile, path) + + path = MakePathFromManifestFile(string(os.PathSeparator), "file.lock") + assert.Equal(t, fmt.Sprintf("%s%s", string(os.PathSeparator), "file.lock"), path) +} + +func TestCloseFile(t *testing.T) { + var j job.IJob = testdata.NewJobMock("") + fileWriterMock := writerTestdata.FileWriterMock{} + + CloseFile(j, &fileWriterMock, nil) + + assert.False(t, j.Errors().HasError()) +} + +func TestCloseFileErr(t *testing.T) { + var j job.IJob = testdata.NewJobMock("") + fileWriterMock := writerTestdata.FileWriterMock{} + closeErr := errors.New("error") + fileWriterMock.CloseErr = closeErr + + CloseFile(j, &fileWriterMock, nil) + + assert.True(t, j.Errors().HasError()) + criticalErrs := j.Errors().GetCriticalErrors() + assert.Len(t, criticalErrs, 1) + criticalErr := criticalErrs[0] + assert.ErrorIs(t, closeErr, criticalErr) +} diff --git a/pkg/resolution/pm/writer/file_writer.go b/pkg/resolution/pm/writer/file_writer.go new file mode 100644 index 00000000..c152d77f --- /dev/null +++ b/pkg/resolution/pm/writer/file_writer.go @@ -0,0 +1,27 @@ +package writer + +import ( + "os" +) + +type IFileWriter interface { + Write(file *os.File, p []byte) error + Create(name string) (*os.File, error) + Close(file *os.File) error +} + +type FileWriter struct{} + +func (fw FileWriter) Create(name string) (*os.File, error) { + return os.Create(name) +} + +func (fw FileWriter) Write(file *os.File, p []byte) error { + _, err := file.Write(p) + + return err +} + +func (fw FileWriter) Close(file *os.File) error { + return file.Close() +} diff --git a/pkg/resolution/pm/writer/file_writer_test.go b/pkg/resolution/pm/writer/file_writer_test.go new file mode 100644 index 00000000..b3531ede --- /dev/null +++ b/pkg/resolution/pm/writer/file_writer_test.go @@ -0,0 +1,47 @@ +package writer + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +var fw = FileWriter{} + +const fileName = "debricked-test.json" + +func TestCreate(t *testing.T) { + testFile, err := fw.Create(fileName) + assert.NoError(t, err) + assert.NotNil(t, testFile) + defer deleteFile(t, testFile) +} + +func TestWrite(t *testing.T) { + content := []byte("{}") + testFile, _ := fw.Create(fileName) + defer deleteFile(t, testFile) + + err := fw.Write(testFile, content) + + assert.NoError(t, err) + fileContents, err := os.ReadFile(fileName) + assert.NoError(t, err) + assert.Equal(t, fileContents, content) +} + +func TestClose(t *testing.T) { + testFile, _ := fw.Create(fileName) + defer deleteFile(t, testFile) + + err := fw.Close(testFile) + + assert.NoError(t, err) +} + +func deleteFile(t *testing.T, file *os.File) { + _ = file.Close() + err := os.Remove(file.Name()) + assert.NoError(t, err) +} diff --git a/pkg/resolution/pm/writer/testdata/file_writer_mock.go b/pkg/resolution/pm/writer/testdata/file_writer_mock.go new file mode 100644 index 00000000..8ca080df --- /dev/null +++ b/pkg/resolution/pm/writer/testdata/file_writer_mock.go @@ -0,0 +1,27 @@ +package writer + +import ( + "os" +) + +type FileWriterMock struct { + file *os.File + Contents []byte + CreateErr error + WriteErr error + CloseErr error +} + +func (fw *FileWriterMock) Create(_ string) (*os.File, error) { + return fw.file, fw.CreateErr +} + +func (fw *FileWriterMock) Write(_ *os.File, bytes []byte) error { + fw.Contents = append(fw.Contents, bytes...) + + return fw.WriteErr +} + +func (fw *FileWriterMock) Close(_ *os.File) error { + return fw.CloseErr +} diff --git a/pkg/resolution/resolution.go b/pkg/resolution/resolution.go new file mode 100644 index 00000000..d1818b9c --- /dev/null +++ b/pkg/resolution/resolution.go @@ -0,0 +1,30 @@ +package resolution + +import "github.com/debricked/cli/pkg/resolution/job" + +type IResolution interface { + Jobs() []job.IJob + HasErr() bool +} + +type Resolution struct { + jobs []job.IJob +} + +func NewResolution(jobs []job.IJob) Resolution { + return Resolution{jobs} +} + +func (r Resolution) Jobs() []job.IJob { + return r.jobs +} + +func (r Resolution) HasErr() bool { + for _, j := range r.Jobs() { + if j.Errors().HasError() { + return true + } + } + + return false +} diff --git a/pkg/resolution/resolution_test.go b/pkg/resolution/resolution_test.go new file mode 100644 index 00000000..16d58a2e --- /dev/null +++ b/pkg/resolution/resolution_test.go @@ -0,0 +1,51 @@ +package resolution + +import ( + "errors" + "testing" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/job/testdata" + "github.com/stretchr/testify/assert" +) + +func TestNewResolution(t *testing.T) { + res := NewResolution(nil) + assert.NotNil(t, res) + + res = NewResolution([]job.IJob{}) + assert.NotNil(t, res) + + res = NewResolution([]job.IJob{testdata.NewJobMock("")}) + assert.NotNil(t, res) + + res = NewResolution([]job.IJob{testdata.NewJobMock(""), testdata.NewJobMock("")}) + assert.NotNil(t, res) +} + +func TestJobs(t *testing.T) { + res := NewResolution(nil) + assert.Empty(t, res.Jobs()) + + res.jobs = []job.IJob{} + assert.Len(t, res.Jobs(), 0) + + res.jobs = []job.IJob{testdata.NewJobMock("")} + assert.Len(t, res.Jobs(), 1) + + res.jobs = []job.IJob{testdata.NewJobMock(""), testdata.NewJobMock("")} + assert.Len(t, res.Jobs(), 2) +} + +func TestHasError(t *testing.T) { + res := NewResolution(nil) + assert.False(t, res.HasErr()) + + res.jobs = []job.IJob{testdata.NewJobMock("")} + assert.False(t, res.HasErr()) + + jobMock := testdata.NewJobMock("") + jobMock.SetErr(errors.New("error")) + res.jobs = append(res.jobs, jobMock) + assert.True(t, res.HasErr()) +} diff --git a/pkg/resolution/resolver.go b/pkg/resolution/resolver.go new file mode 100644 index 00000000..2acb8e9b --- /dev/null +++ b/pkg/resolution/resolver.go @@ -0,0 +1,124 @@ +package resolution + +import ( + "os" + "path" + + "github.com/debricked/cli/pkg/file" + resolutionFile "github.com/debricked/cli/pkg/resolution/file" + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/strategy" + "github.com/debricked/cli/pkg/tui" +) + +type IResolver interface { + Resolve(paths []string, exclusions []string) (IResolution, error) +} + +type Resolver struct { + finder file.IFinder + batchFactory resolutionFile.IBatchFactory + strategyFactory strategy.IFactory + scheduler IScheduler +} + +func NewResolver( + finder file.IFinder, + batchFactory resolutionFile.IBatchFactory, + strategyFactory strategy.IFactory, + scheduler IScheduler, +) Resolver { + return Resolver{ + finder, + batchFactory, + strategyFactory, + scheduler, + } +} + +func (r Resolver) Resolve(paths []string, exclusions []string) (IResolution, error) { + files, err := r.refinePaths(paths, exclusions) + if err != nil { + return nil, err + } + + pmBatches := r.batchFactory.Make(files) + + var jobs []job.IJob + for _, pmBatch := range pmBatches { + s, strategyErr := r.strategyFactory.Make(pmBatch, paths) + if strategyErr == nil { + newJobs, err := s.Invoke() + if err != nil { + return nil, err + } + jobs = append(jobs, newJobs...) + } + } + + resolution, err := r.scheduler.Schedule(jobs) + + if resolution.HasErr() { + jobErrList := tui.NewJobsErrorList(os.Stdout, resolution.Jobs()) + err = jobErrList.Render() + } + + return resolution, err +} + +func (r Resolver) refinePaths(paths []string, exclusions []string) ([]string, error) { + var fileSet = map[string]bool{} + var dirs []string + for _, arg := range paths { + cleanArg := path.Clean(arg) + if cleanArg == "." { + dirs = append(dirs, cleanArg) + + continue + } + + fileInfo, err := os.Stat(arg) + if err != nil { + return nil, err + } + + if fileInfo.IsDir() { + dirs = append(dirs, path.Clean(arg)) + } else { + fileSet[path.Clean(arg)] = true + } + } + + err := r.searchDirs(fileSet, dirs, exclusions) + if err != nil { + return nil, err + } + + var files []string + for f := range fileSet { + files = append(files, f) + } + + return files, nil +} + +func (r Resolver) searchDirs(fileSet map[string]bool, dirs []string, exclusions []string) error { + for _, dir := range dirs { + fileGroups, err := r.finder.GetGroups( + dir, + exclusions, + false, + file.StrictAll, + ) + if err != nil { + return err + } + for _, fileGroup := range fileGroups.ToSlice() { + if fileGroup.HasFile() && !fileGroup.HasLockFiles() { + fileSet[fileGroup.FilePath] = true + } + } + } + + return nil +} diff --git a/pkg/resolution/resolver_test.go b/pkg/resolution/resolver_test.go new file mode 100644 index 00000000..5b16573f --- /dev/null +++ b/pkg/resolution/resolver_test.go @@ -0,0 +1,212 @@ +package resolution + +import ( + "errors" + "fmt" + "testing" + + "github.com/debricked/cli/pkg/file" + "github.com/debricked/cli/pkg/file/testdata" + resolutionFile "github.com/debricked/cli/pkg/resolution/file" + fileTestdata "github.com/debricked/cli/pkg/resolution/file/testdata" + "github.com/debricked/cli/pkg/resolution/job" + jobTestdata "github.com/debricked/cli/pkg/resolution/job/testdata" + + "github.com/debricked/cli/pkg/resolution/strategy" + strategyTestdata "github.com/debricked/cli/pkg/resolution/strategy/testdata" + "github.com/stretchr/testify/assert" +) + +const ( + workers = 10 + goModFile = "go.mod" +) + +func TestNewResolver(t *testing.T) { + r := NewResolver( + &testdata.FinderMock{}, + resolutionFile.NewBatchFactory(), + strategyTestdata.NewStrategyFactoryMock(), + NewScheduler(workers), + ) + assert.NotNil(t, r) +} + +func TestResolve(t *testing.T) { + r := NewResolver( + &testdata.FinderMock{}, + resolutionFile.NewBatchFactory(), + strategyTestdata.NewStrategyFactoryMock(), + NewScheduler(workers), + ) + + res, err := r.Resolve([]string{"../../go.mod"}, nil) + assert.NotEmpty(t, res.Jobs()) + assert.NoError(t, err) +} + +func TestResolveInvokeError(t *testing.T) { + r := NewResolver( + &testdata.FinderMock{}, + resolutionFile.NewBatchFactory(), + strategyTestdata.NewStrategyFactoryErrorMock(), + NewScheduler(workers), + ) + + _, err := r.Resolve([]string{"../../go.mod"}, nil) + assert.NotNil(t, err) +} + +func TestResolveStrategyError(t *testing.T) { + r := NewResolver( + &testdata.FinderMock{}, + fileTestdata.NewBatchFactoryMock(), + strategy.NewStrategyFactory(), + NewScheduler(workers), + ) + + res, err := r.Resolve([]string{"../../go.mod"}, nil) + assert.Empty(t, res.Jobs()) + assert.NoError(t, err) +} + +func TestResolveScheduleError(t *testing.T) { + errAssertion := errors.New("error") + r := NewResolver( + &testdata.FinderMock{}, + resolutionFile.NewBatchFactory(), + strategyTestdata.NewStrategyFactoryMock(), + SchedulerMock{Err: errAssertion}, + ) + + res, err := r.Resolve([]string{"../../go.mod"}, nil) + assert.NotEmpty(t, res.Jobs()) + assert.ErrorIs(t, err, errAssertion) +} + +func TestResolveDirWithoutManifestFiles(t *testing.T) { + r := NewResolver( + &testdata.FinderMock{}, + resolutionFile.NewBatchFactory(), + strategyTestdata.NewStrategyFactoryMock(), + SchedulerMock{}, + ) + + res, err := r.Resolve([]string{"."}, nil) + assert.Empty(t, res.Jobs()) + assert.NoError(t, err) +} + +func TestResolveInvalidDir(t *testing.T) { + r := NewResolver( + &testdata.FinderMock{}, + resolutionFile.NewBatchFactory(), + strategyTestdata.NewStrategyFactoryMock(), + SchedulerMock{}, + ) + + _, err := r.Resolve([]string{"invalid-dir"}, nil) + assert.Error(t, err) +} + +func TestResolveGetGroupsErr(t *testing.T) { + f := testdata.NewFinderMock() + testErr := errors.New("test") + f.SetGetGroupsReturnMock(file.Groups{}, testErr) + + r := NewResolver( + f, + resolutionFile.NewBatchFactory(), + strategyTestdata.NewStrategyFactoryMock(), + SchedulerMock{}, + ) + + _, err := r.Resolve([]string{"."}, nil) + assert.ErrorIs(t, testErr, err) +} + +func TestResolveDirWithManifestFiles(t *testing.T) { + cases := []string{ + "", + ".", + "./", + "testdata", + "./testdata/../testdata", + "./strategy/testdata/", + "strategy/testdata", + } + f := testdata.NewFinderMock() + groups := file.Groups{} + groups.Add(file.Group{FilePath: goModFile}) + f.SetGetGroupsReturnMock(groups, nil) + + r := NewResolver( + f, + resolutionFile.NewBatchFactory(), + strategyTestdata.NewStrategyFactoryMock(), + SchedulerMock{}, + ) + + for _, dir := range cases { + t.Run(fmt.Sprintf("Case: %s", dir), func(t *testing.T) { + res, err := r.Resolve([]string{dir}, nil) + assert.Len(t, res.Jobs(), 1) + j := res.Jobs()[0] + assert.False(t, j.Errors().HasError()) + assert.Equal(t, goModFile, j.GetFile()) + assert.NoError(t, err) + }) + } +} + +func TestResolveDirWithExclusions(t *testing.T) { + f := testdata.NewFinderMock() + groups := file.Groups{} + groups.Add(file.Group{FilePath: goModFile}) + f.SetGetGroupsReturnMock(groups, nil) + + r := NewResolver( + f, + resolutionFile.NewBatchFactory(), + strategyTestdata.NewStrategyFactoryMock(), + SchedulerMock{}, + ) + + res, err := r.Resolve([]string{"."}, []string{"dir"}) + + assert.Len(t, res.Jobs(), 1) + j := res.Jobs()[0] + assert.False(t, j.Errors().HasError()) + assert.Equal(t, goModFile, j.GetFile()) + assert.NoError(t, err) +} + +func TestResolveHasResolutionErrs(t *testing.T) { + f := testdata.NewFinderMock() + groups := file.Groups{} + groups.Add(file.Group{FilePath: goModFile}) + f.SetGetGroupsReturnMock(groups, nil) + + jobErr := errors.New("job-error") + jobWithErr := jobTestdata.NewJobMock(goModFile) + jobWithErr.Errors().Warning(jobErr) + schedulerMock := SchedulerMock{JobsMock: []job.IJob{jobWithErr}} + + r := NewResolver( + f, + resolutionFile.NewBatchFactory(), + strategyTestdata.NewStrategyFactoryMock(), + schedulerMock, + ) + + res, err := r.Resolve([]string{""}, []string{""}) + + assert.NoError(t, err) + assert.Len(t, res.Jobs(), 1) + j := res.Jobs()[0] + assert.Equal(t, goModFile, j.GetFile()) + assert.True(t, j.Errors().HasError()) + errs := j.Errors().GetAll() + assert.Len(t, errs, 1) + assert.ErrorIs(t, jobErr, errs[0]) +} diff --git a/pkg/resolution/scheduler.go b/pkg/resolution/scheduler.go new file mode 100644 index 00000000..570d1598 --- /dev/null +++ b/pkg/resolution/scheduler.go @@ -0,0 +1,92 @@ +package resolution + +import ( + "sort" + "sync" + + "github.com/chelnak/ysmrr" + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/tui" +) + +type IScheduler interface { + Schedule(jobs []job.IJob) (IResolution, error) +} + +type queueItem struct { + job job.IJob + spinner *ysmrr.Spinner +} + +type Scheduler struct { + workers int + queue chan queueItem + waitGroup sync.WaitGroup + spinnerManager tui.ISpinnerManager +} + +const resolving = "Resolving" + +func NewScheduler(workers int) *Scheduler { + return &Scheduler{workers: workers, waitGroup: sync.WaitGroup{}} +} + +func (scheduler *Scheduler) Schedule(jobs []job.IJob) (IResolution, error) { + scheduler.queue = make(chan queueItem, len(jobs)) + scheduler.waitGroup.Add(len(jobs)) + + scheduler.spinnerManager = tui.NewSpinnerManager() + + for w := 1; w <= scheduler.workers; w++ { + go scheduler.worker() + } + + sort.Slice(jobs, func(i, j int) bool { + return jobs[i].GetFile() < jobs[j].GetFile() + }) + + for _, j := range jobs { + spinner := scheduler.spinnerManager.AddSpinner(resolving, j.GetFile()) + scheduler.queue <- queueItem{ + job: j, + spinner: spinner, + } + } + scheduler.spinnerManager.Start() + + scheduler.waitGroup.Wait() + + scheduler.spinnerManager.Stop() + + close(scheduler.queue) + + return NewResolution(jobs), nil +} + +func (scheduler *Scheduler) worker() { + for item := range scheduler.queue { + go scheduler.updateStatus(item) + + item.job.Run() + + scheduler.finish(item) + + scheduler.waitGroup.Done() + } +} +func (scheduler *Scheduler) updateStatus(item queueItem) { + for { + msg := <-item.job.ReceiveStatus() + tui.SetSpinnerMessage(item.spinner, resolving, item.job.GetFile(), msg) + } +} + +func (scheduler *Scheduler) finish(item queueItem) { + if item.job.Errors().HasError() { + tui.SetSpinnerMessage(item.spinner, resolving, item.job.GetFile(), "failed") + item.spinner.Error() + } else { + tui.SetSpinnerMessage(item.spinner, resolving, item.job.GetFile(), "done") + item.spinner.Complete() + } +} diff --git a/pkg/resolution/scheduler_test.go b/pkg/resolution/scheduler_test.go new file mode 100644 index 00000000..842de1a9 --- /dev/null +++ b/pkg/resolution/scheduler_test.go @@ -0,0 +1,81 @@ +package resolution + +import ( + "errors" + "sort" + "testing" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/job/testdata" + "github.com/stretchr/testify/assert" +) + +type SchedulerMock struct { + Err error + JobsMock []job.IJob +} + +func (s SchedulerMock) Schedule(jobs []job.IJob) (IResolution, error) { + if s.JobsMock != nil { + jobs = s.JobsMock + } + for _, j := range jobs { + j.Run() + } + + return NewResolution(jobs), s.Err +} + +func TestNewScheduler(t *testing.T) { + s := NewScheduler(10) + assert.NotNil(t, s) +} + +func TestSchedule(t *testing.T) { + s := NewScheduler(10) + res, err := s.Schedule([]job.IJob{testdata.NewJobMock("")}) + assert.NoError(t, err) + assert.Len(t, res.Jobs(), 1) + + res, err = s.Schedule([]job.IJob{}) + assert.NoError(t, err) + assert.Len(t, res.Jobs(), 0) + + res, err = s.Schedule(nil) + assert.NoError(t, err) + assert.Len(t, res.Jobs(), 0) + + res, err = s.Schedule([]job.IJob{ + testdata.NewJobMock("b/b_file.json"), + testdata.NewJobMock("a/b_file.json"), + testdata.NewJobMock("b/a_file.json"), + testdata.NewJobMock("a/a_file.json"), + testdata.NewJobMock("a/a_file.json"), + }) + assert.NoError(t, err) + jobs := res.Jobs() + + assert.Len(t, jobs, 5) + for _, j := range jobs { + assert.False(t, j.Errors().HasError()) + } + + sortedJobs := jobs + sort.Slice(sortedJobs, func(i, j int) bool { + return sortedJobs[i].GetFile() < sortedJobs[j].GetFile() + }) + assert.Equal(t, sortedJobs, jobs) +} + +func TestScheduleJobErr(t *testing.T) { + s := NewScheduler(10) + jobMock := testdata.NewJobMock("") + jobErr := errors.New("job-error") + jobMock.SetErr(jobErr) + res, err := s.Schedule([]job.IJob{jobMock}) + assert.NoError(t, err) + assert.Len(t, res.Jobs(), 1) + j := res.Jobs()[0] + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, j.Errors().GetAll(), jobErr) +} diff --git a/pkg/resolution/strategy/strategy.go b/pkg/resolution/strategy/strategy.go new file mode 100644 index 00000000..ffb7d4e0 --- /dev/null +++ b/pkg/resolution/strategy/strategy.go @@ -0,0 +1,9 @@ +package strategy + +import ( + "github.com/debricked/cli/pkg/resolution/job" +) + +type IStrategy interface { + Invoke() ([]job.IJob, error) +} diff --git a/pkg/resolution/strategy/strategy_factory.go b/pkg/resolution/strategy/strategy_factory.go new file mode 100644 index 00000000..b4814128 --- /dev/null +++ b/pkg/resolution/strategy/strategy_factory.go @@ -0,0 +1,37 @@ +package strategy + +import ( + "fmt" + + "github.com/debricked/cli/pkg/resolution/file" + "github.com/debricked/cli/pkg/resolution/pm/gomod" + "github.com/debricked/cli/pkg/resolution/pm/gradle" + "github.com/debricked/cli/pkg/resolution/pm/maven" + "github.com/debricked/cli/pkg/resolution/pm/pip" +) + +type IFactory interface { + Make(pmBatch file.IBatch, paths []string) (IStrategy, error) +} + +type Factory struct{} + +func NewStrategyFactory() Factory { + return Factory{} +} + +func (sf Factory) Make(pmFileBatch file.IBatch, paths []string) (IStrategy, error) { + name := pmFileBatch.Pm().Name() + switch name { + case maven.Name: + return maven.NewStrategy(pmFileBatch.Files()), nil + case gradle.Name: + return gradle.NewStrategy(pmFileBatch.Files(), paths), nil + case gomod.Name: + return gomod.NewStrategy(pmFileBatch.Files()), nil + case pip.Name: + return pip.NewStrategy(pmFileBatch.Files()), nil + default: + return nil, fmt.Errorf("failed to make strategy from %s", name) + } +} diff --git a/pkg/resolution/strategy/strategy_factory_test.go b/pkg/resolution/strategy/strategy_factory_test.go new file mode 100644 index 00000000..bc72bae0 --- /dev/null +++ b/pkg/resolution/strategy/strategy_factory_test.go @@ -0,0 +1,45 @@ +package strategy + +import ( + "testing" + + "github.com/debricked/cli/pkg/resolution/file" + "github.com/debricked/cli/pkg/resolution/pm/gomod" + "github.com/debricked/cli/pkg/resolution/pm/gradle" + "github.com/debricked/cli/pkg/resolution/pm/maven" + "github.com/debricked/cli/pkg/resolution/pm/pip" + "github.com/debricked/cli/pkg/resolution/pm/testdata" + "github.com/stretchr/testify/assert" +) + +func TestNewStrategyFactory(t *testing.T) { + f := NewStrategyFactory() + assert.NotNil(t, f) +} + +func TestMakeErr(t *testing.T) { + f := NewStrategyFactory() + batch := file.NewBatch(testdata.PmMock{N: "test"}) + s, err := f.Make(batch, nil) + assert.Nil(t, s) + assert.ErrorContains(t, err, "failed to make strategy from test") +} + +func TestMake(t *testing.T) { + cases := map[string]IStrategy{ + maven.Name: maven.NewStrategy(nil), + gradle.Name: gradle.NewStrategy(nil, nil), + gomod.Name: gomod.NewStrategy(nil), + pip.Name: pip.NewStrategy(nil), + } + f := NewStrategyFactory() + var batch file.IBatch + for name, strategy := range cases { + batch = file.NewBatch(testdata.PmMock{N: name}) + t.Run(name, func(t *testing.T) { + s, err := f.Make(batch, nil) + assert.NoError(t, err) + assert.Equal(t, strategy, s) + }) + } +} diff --git a/pkg/resolution/strategy/testdata/strategy_mock.go b/pkg/resolution/strategy/testdata/strategy_mock.go new file mode 100644 index 00000000..80bf1ceb --- /dev/null +++ b/pkg/resolution/strategy/testdata/strategy_mock.go @@ -0,0 +1,38 @@ +package testdata + +import ( + "errors" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/job/testdata" +) + +type StrategyMock struct { + files []string +} + +func NewStrategyMock(files []string) StrategyMock { + return StrategyMock{files} +} + +func (s StrategyMock) Invoke() ([]job.IJob, error) { + var jobs []job.IJob + for _, file := range s.files { + jobs = append(jobs, testdata.NewJobMock(file)) + } + + return jobs, nil +} + +type StrategyErrorMock struct { + files []string +} + +func NewStrategyErrorMock(files []string) StrategyErrorMock { + return StrategyErrorMock{files} +} + +func (s StrategyErrorMock) Invoke() ([]job.IJob, error) { + + return nil, errors.New("mock-error") +} diff --git a/pkg/resolution/strategy/testdata/strategy_mock_factory.go b/pkg/resolution/strategy/testdata/strategy_mock_factory.go new file mode 100644 index 00000000..8ed1c163 --- /dev/null +++ b/pkg/resolution/strategy/testdata/strategy_mock_factory.go @@ -0,0 +1,28 @@ +package testdata + +import ( + "github.com/debricked/cli/pkg/resolution/file" + "github.com/debricked/cli/pkg/resolution/strategy" +) + +type FactoryMock struct{} + +func NewStrategyFactoryMock() FactoryMock { + return FactoryMock{} +} + +func (sf FactoryMock) Make(pmFileBatch file.IBatch, paths []string) (strategy.IStrategy, error) { + + return NewStrategyMock(pmFileBatch.Files()), nil +} + +type FactoryErrorMock struct{} + +func NewStrategyFactoryErrorMock() FactoryErrorMock { + return FactoryErrorMock{} +} + +func (sf FactoryErrorMock) Make(pmFileBatch file.IBatch, paths []string) (strategy.IStrategy, error) { + + return NewStrategyErrorMock(pmFileBatch.Files()), nil +} diff --git a/pkg/resolution/testdata/resolver_mock.go b/pkg/resolution/testdata/resolver_mock.go new file mode 100644 index 00000000..c20b7889 --- /dev/null +++ b/pkg/resolution/testdata/resolver_mock.go @@ -0,0 +1,48 @@ +package testdata + +import ( + "github.com/debricked/cli/pkg/resolution" + "github.com/debricked/cli/pkg/resolution/job" + "os" + "path/filepath" +) + +type ResolverMock struct { + Err error + files []string +} + +func (r *ResolverMock) Resolve(_ []string, _ []string) (resolution.IResolution, error) { + for _, f := range r.files { + createdFile, err := os.Create(f) + if err != nil { + return nil, err + } + + err = createdFile.Close() + if err != nil { + return nil, err + } + } + + return resolution.NewResolution([]job.IJob{}), r.Err +} + +func (r *ResolverMock) SetFiles(files []string) { + r.files = files +} + +func (r *ResolverMock) CleanUp() error { + for _, f := range r.files { + abs, err := filepath.Abs(f) + if err != nil { + return err + } + err = os.Remove(abs) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/scan/scanner.go b/pkg/scan/scanner.go index 3a7d920e..a6de7d59 100644 --- a/pkg/scan/scanner.go +++ b/pkg/scan/scanner.go @@ -6,11 +6,14 @@ import ( "os" "path/filepath" + "github.com/debricked/cli/pkg/callgraph" + "github.com/debricked/cli/pkg/callgraph/config" "github.com/debricked/cli/pkg/ci" "github.com/debricked/cli/pkg/ci/env" "github.com/debricked/cli/pkg/client" "github.com/debricked/cli/pkg/file" "github.com/debricked/cli/pkg/git" + "github.com/debricked/cli/pkg/resolution" "github.com/debricked/cli/pkg/tui" "github.com/debricked/cli/pkg/upload" "github.com/fatih/color" @@ -29,13 +32,17 @@ type IOptions interface{} type DebrickedScanner struct { client *client.IDebClient - finder *file.Finder + finder file.IFinder uploader *upload.IUploader ciService ci.IService + resolver resolution.IResolver + callgraph callgraph.IGenerator } type DebrickedOptions struct { Path string + Resolve bool + CallGraph bool Exclusions []string RepositoryName string CommitName string @@ -45,24 +52,22 @@ type DebrickedOptions struct { IntegrationName string } -func NewDebrickedScanner(c *client.IDebClient, ciService ci.IService) (*DebrickedScanner, error) { - finder, err := file.NewFinder(*c) - if err != nil { - return nil, newInitError(err) - } - var u upload.IUploader - u, err = upload.NewUploader(*c) - - if err != nil { - return nil, newInitError(err) - } - +func NewDebrickedScanner( + c *client.IDebClient, + finder file.IFinder, + uploader upload.IUploader, + ciService ci.IService, + resolver resolution.IResolver, + callgraph callgraph.IGenerator, +) *DebrickedScanner { return &DebrickedScanner{ c, finder, - &u, + &uploader, ciService, - }, nil + resolver, + callgraph, + } } func (dScanner *DebrickedScanner) Scan(o IOptions) error { @@ -118,11 +123,30 @@ func (dScanner *DebrickedScanner) Scan(o IOptions) error { } func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject git.MetaObject) (*upload.UploadResult, error) { + if options.Resolve { + _, resErr := dScanner.resolver.Resolve([]string{options.Path}, options.Exclusions) + if resErr != nil { + return nil, resErr + } + } + fileGroups, err := dScanner.finder.GetGroups(options.Path, options.Exclusions, false, file.StrictAll) if err != nil { return nil, err } + if options.CallGraph { + configs := []config.IConfig{ + config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}), + // conf.NewConfig("java", []string{}, map[string]string{"pm": "gradle"}), + } + timeout := 60 + resErr := dScanner.callgraph.GenerateWithTimer([]string{options.Path}, options.Exclusions, configs, timeout) + if resErr != nil { + return nil, resErr + } + } + uploaderOptions := upload.DebrickedOptions{FileGroups: fileGroups, GitMetaObject: gitMetaObject, IntegrationsName: options.IntegrationName} result, err := (*dScanner.uploader).Upload(uploaderOptions) if err != nil { @@ -170,7 +194,3 @@ func MapEnvToOptions(o *DebrickedOptions, env env.Env) { o.Path = env.Filepath } } - -func newInitError(err error) error { - return errors.New("failed to initialize the uploader due to: " + err.Error()) -} diff --git a/pkg/scan/scanner_test.go b/pkg/scan/scanner_test.go index 7e2549e3..993db77e 100644 --- a/pkg/scan/scanner_test.go +++ b/pkg/scan/scanner_test.go @@ -3,6 +3,7 @@ package scan import ( "bytes" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -12,6 +13,8 @@ import ( "strings" "testing" + "github.com/debricked/cli/pkg/callgraph" + callgraphTestdata "github.com/debricked/cli/pkg/callgraph/testdata" "github.com/debricked/cli/pkg/ci" "github.com/debricked/cli/pkg/ci/argo" "github.com/debricked/cli/pkg/ci/azure" @@ -26,6 +29,8 @@ import ( "github.com/debricked/cli/pkg/client/testdata" "github.com/debricked/cli/pkg/file" "github.com/debricked/cli/pkg/git" + "github.com/debricked/cli/pkg/resolution" + resolveTestdata "github.com/debricked/cli/pkg/resolution/testdata" "github.com/debricked/cli/pkg/upload" "github.com/stretchr/testify/assert" ) @@ -44,40 +49,28 @@ var ciService ci.IService = ci.NewService([]ci.ICi{ }) func TestNewDebrickedScanner(t *testing.T) { - var debClient client.IDebClient = testdata.NewDebClientMock() - var ciService ci.IService - s, err := NewDebrickedScanner(&debClient, ciService) - - assert.NoError(t, err) - assert.NotNil(t, s) -} - -func TestNewDebrickedScannerWithError(t *testing.T) { var debClient client.IDebClient - var ciService ci.IService - s, err := NewDebrickedScanner(&debClient, ciService) + var cis ci.IService + var finder file.IFinder + var uploader upload.IUploader + var resolver resolution.IResolver + var generator callgraph.IGenerator + s := NewDebrickedScanner(&debClient, finder, uploader, cis, resolver, generator) - assert.Error(t, err) - assert.Nil(t, s) - assert.ErrorContains(t, err, "failed to initialize the uploader") + assert.NotNil(t, s) } func TestScan(t *testing.T) { if runtime.GOOS == "windows" { t.Skipf("TestScan is skipped due to Windows env") } - var debClient client.IDebClient clientMock := testdata.NewDebClientMock() - addMockedFormatsResponse(clientMock) + addMockedFormatsResponse(clientMock, "yarn\\.lock") addMockedFileUploadResponse(clientMock) addMockedFinishResponse(clientMock, http.StatusNoContent) addMockedStatusResponse(clientMock, http.StatusOK, 50) addMockedStatusResponse(clientMock, http.StatusOK, 100) - debClient = clientMock - - var ciService ci.IService = ci.NewService(nil) - - scanner, _ := NewDebrickedScanner(&debClient, ciService) + scanner := makeScanner(clientMock, nil, nil) path := testdataYarn repositoryName := path @@ -131,7 +124,7 @@ func TestScan(t *testing.T) { func TestScanFailingMetaObject(t *testing.T) { var debClient client.IDebClient = testdata.NewDebClientMock() - scanner, _ := NewDebrickedScanner(&debClient, ciService) + scanner := NewDebrickedScanner(&debClient, nil, nil, ciService, nil, nil) cwd, _ := os.Getwd() path := testdataYarn opts := DebrickedOptions{ @@ -158,11 +151,9 @@ func TestScanFailingMetaObject(t *testing.T) { } func TestScanFailingNoFiles(t *testing.T) { - var debClient client.IDebClient clientMock := testdata.NewDebClientMock() - addMockedFormatsResponse(clientMock) - debClient = clientMock - scanner, _ := NewDebrickedScanner(&debClient, ciService) + addMockedFormatsResponse(clientMock, "yarn\\.lock") + scanner := makeScanner(clientMock, nil, nil) opts := DebrickedOptions{ Path: "", Exclusions: []string{"testdata/**"}, @@ -180,7 +171,7 @@ func TestScanFailingNoFiles(t *testing.T) { func TestScanBadOpts(t *testing.T) { var c client.IDebClient - scanner, _ := NewDebrickedScanner(&c, nil) + scanner := NewDebrickedScanner(&c, nil, nil, nil, nil, nil) var opts IOptions err := scanner.Scan(opts) @@ -192,19 +183,15 @@ func TestScanEmptyResult(t *testing.T) { if runtime.GOOS == "windows" { t.Skipf("TestScan is skipped due to Windows env") } - var debClient client.IDebClient clientMock := testdata.NewDebClientMock() - addMockedFormatsResponse(clientMock) + addMockedFormatsResponse(clientMock, "yarn\\.lock") addMockedFileUploadResponse(clientMock) addMockedFinishResponse(clientMock, http.StatusNoContent) addMockedStatusResponse(clientMock, http.StatusOK, 50) // Create mocked scan result response, 201 is returned when the queue time are too long addMockedStatusResponse(clientMock, http.StatusCreated, 0) - debClient = clientMock - - var ciService ci.IService = ci.NewService(nil) - scanner, _ := NewDebrickedScanner(&debClient, ciService) + scanner := makeScanner(clientMock, nil, nil) path := testdataYarn repositoryName := path commitName := "testdata/yarn-commit" @@ -243,7 +230,7 @@ func TestScanEmptyResult(t *testing.T) { func TestScanInCiWithPathSet(t *testing.T) { var debClient client.IDebClient = testdata.NewDebClientMock() - scanner, _ := NewDebrickedScanner(&debClient, ciService) + scanner := NewDebrickedScanner(&debClient, nil, nil, ciService, nil, nil) cwd, _ := os.Getwd() defer resetWd(t, cwd) path := testdataYarn @@ -265,6 +252,68 @@ func TestScanInCiWithPathSet(t *testing.T) { assert.Contains(t, cwd, testdataYarn) } +func TestScanWithResolve(t *testing.T) { + clientMock := testdata.NewDebClientMock() + addMockedFormatsResponse(clientMock, "yarn\\.lock") + addMockedFileUploadResponse(clientMock) + addMockedFinishResponse(clientMock, http.StatusNoContent) + addMockedStatusResponse(clientMock, http.StatusOK, 100) + + resolverMock := resolveTestdata.ResolverMock{} + resolverMock.SetFiles([]string{"yarn.lock"}) + + scanner := makeScanner(clientMock, &resolverMock, nil) + + cwd, _ := os.Getwd() + defer resetWd(t, cwd) + // Clean up resolution must be done before wd reset, otherwise files cannot be deleted + defer cleanUpResolution(t, resolverMock) + + path := filepath.Join("testdata", "npm") + repositoryName := path + commitName := "testdata/npm-commit" + opts := DebrickedOptions{ + Path: path, + Resolve: true, + Exclusions: nil, + RepositoryName: repositoryName, + CommitName: commitName, + BranchName: "", + CommitAuthor: "", + RepositoryUrl: "", + IntegrationName: "", + } + err := scanner.Scan(opts) + assert.NoError(t, err) + cwd, _ = os.Getwd() + assert.Contains(t, cwd, path) +} + +func TestScanWithResolveErr(t *testing.T) { + clientMock := testdata.NewDebClientMock() + resolutionErr := errors.New("resolution-error") + scanner := makeScanner(clientMock, &resolveTestdata.ResolverMock{Err: resolutionErr}, nil) + cwd, _ := os.Getwd() + defer resetWd(t, cwd) + + path := filepath.Join("testdata", "npm") + repositoryName := path + commitName := "testdata/npm-commit" + opts := DebrickedOptions{ + Path: path, + Resolve: true, + Exclusions: nil, + RepositoryName: repositoryName, + CommitName: commitName, + } + err := scanner.Scan(opts) + assert.ErrorIs(t, err, resolutionErr) +} + +func TestScanWithCallgraphGenerate(t *testing.T) { + +} + func TestMapEnvToOptions(t *testing.T) { dOptionsTemplate := DebrickedOptions{ Path: "path", @@ -457,10 +506,10 @@ func TestSetWorkingDirectory(t *testing.T) { } } -func addMockedFormatsResponse(clientMock *testdata.DebClientMock) { +func addMockedFormatsResponse(clientMock *testdata.DebClientMock, regex string) { formats := []file.Format{{ Regex: "", - LockFileRegexes: []string{"yarn\\.lock"}, + LockFileRegexes: []string{regex}, }} formatsBytes, _ := json.Marshal(formats) formatsMockRes := testdata.MockResponse{ @@ -500,3 +549,24 @@ func resetWd(t *testing.T, wd string) { t.Fatal("Can not read the directory: ", wd) } } + +func makeScanner(clientMock *testdata.DebClientMock, resolverMock *resolveTestdata.ResolverMock, generatorMock *callgraphTestdata.GeneratorMock) *DebrickedScanner { + var debClient client.IDebClient = clientMock + + var finder file.IFinder + finder, _ = file.NewFinder(debClient) + + var uploader upload.IUploader + uploader, _ = upload.NewUploader(debClient) + + var cis ci.IService = ci.NewService(nil) + + return NewDebrickedScanner(&debClient, finder, uploader, cis, resolverMock, generatorMock) +} + +func cleanUpResolution(t *testing.T, resolverMock resolveTestdata.ResolverMock) { + err := resolverMock.CleanUp() + if err != nil { + t.Error(err) + } +} diff --git a/pkg/scan/testdata/npm/package.json b/pkg/scan/testdata/npm/package.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/pkg/scan/testdata/npm/package.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/pkg/tui/resolution_error_list.go b/pkg/tui/resolution_error_list.go new file mode 100644 index 00000000..d8b6bf9e --- /dev/null +++ b/pkg/tui/resolution_error_list.go @@ -0,0 +1,78 @@ +package tui + +import ( + "bytes" + "fmt" + "io" + "strings" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/fatih/color" +) + +const ( + title = "Errors" +) + +type JobsErrorList struct { + mirror io.Writer + jobs []job.IJob +} + +func NewJobsErrorList(mirror io.Writer, jobs []job.IJob) JobsErrorList { + return JobsErrorList{mirror: mirror, jobs: jobs} +} + +func (jobsErrList JobsErrorList) Render() error { + var listBuffer bytes.Buffer + + formattedTitle := fmt.Sprintf("%s\n", color.BlueString(title)) + underlining := fmt.Sprintf(strings.Repeat("-", len(title)+1) + "\n") + listBuffer.Write([]byte(formattedTitle)) + listBuffer.Write([]byte(underlining)) + + for _, j := range jobsErrList.jobs { + jobsErrList.addJob(&listBuffer, j) + } + + _, err := jobsErrList.mirror.Write(listBuffer.Bytes()) + + return err +} + +func (jobsErrList JobsErrorList) addJob(list *bytes.Buffer, job job.IJob) { + var jobString string + if !job.Errors().HasError() { + return + } + + list.Write([]byte(fmt.Sprintf("%s\n", color.YellowString(job.GetFile())))) + + for _, warning := range job.Errors().GetWarningErrors() { + err := jobsErrList.createErrorString(warning, true) + jobString = fmt.Sprintf("* %s:\n\t%s\n", color.YellowString("Warning"), err) + list.Write([]byte(jobString)) + } + + for _, critical := range job.Errors().GetCriticalErrors() { + err := jobsErrList.createErrorString(critical, false) + jobString = fmt.Sprintf("* %s:\n\t%s\n", color.RedString("Critical"), err) + + list.Write([]byte(jobString)) + } +} + +func (jobsErrList JobsErrorList) createErrorString(err error, warning bool) string { + var pipe string + if warning { + pipe = color.YellowString("|") + } else { + pipe = color.RedString("|") + } + errString := err.Error() + errString = pipe + errString + errString = strings.Replace(errString, "\n", fmt.Sprintf("\n\t%s", pipe), -1) + errString = strings.TrimSuffix(errString, pipe) + + return errString +} diff --git a/pkg/tui/resolution_error_list_test.go b/pkg/tui/resolution_error_list_test.go new file mode 100644 index 00000000..48d4a2df --- /dev/null +++ b/pkg/tui/resolution_error_list_test.go @@ -0,0 +1,139 @@ +package tui + +import ( + "bytes" + "errors" + "os" + "testing" + + "github.com/debricked/cli/pkg/resolution/job" + "github.com/debricked/cli/pkg/resolution/job/testdata" + "github.com/stretchr/testify/assert" +) + +func TestNewJobsErrorList(t *testing.T) { + mirror := os.Stdout + errList := NewJobsErrorList(mirror, []job.IJob{}) + assert.NotNil(t, errList) +} + +func TestRenderNoJobs(t *testing.T) { + var listBuffer bytes.Buffer + errList := NewJobsErrorList(&listBuffer, []job.IJob{}) + + err := errList.Render() + + assert.NoError(t, err) + output := listBuffer.String() + assertOutput(t, output, nil) +} + +func TestRenderWarningJob(t *testing.T) { + var listBuffer bytes.Buffer + + warningErr := errors.New("warning-message") + jobMock := testdata.NewJobMock("file") + jobMock.Errors().Warning(warningErr) + errList := NewJobsErrorList(&listBuffer, []job.IJob{jobMock}) + + err := errList.Render() + + assert.NoError(t, err) + output := listBuffer.String() + contains := []string{ + "file", + "\n* ", + "Warning", + "|", + "warning-message\n", + } + assertOutput(t, output, contains) +} + +func TestRenderCriticalJob(t *testing.T) { + var listBuffer bytes.Buffer + + warningErr := errors.New("critical-message") + jobMock := testdata.NewJobMock("file") + jobMock.Errors().Critical(warningErr) + errList := NewJobsErrorList(&listBuffer, []job.IJob{jobMock}) + + err := errList.Render() + + assert.NoError(t, err) + output := listBuffer.String() + contains := []string{ + "file", + "\n* ", + "Critical", + "|", + "critical-message\n", + } + assertOutput(t, output, contains) +} + +func TestRenderCriticalAndWarningJob(t *testing.T) { + var listBuffer bytes.Buffer + + jobMock := testdata.NewJobMock("manifest-file") + + warningErr := errors.New("warning-message") + jobMock.Errors().Warning(warningErr) + + criticalErr := errors.New("critical-message") + jobMock.Errors().Critical(criticalErr) + + errList := NewJobsErrorList(&listBuffer, []job.IJob{jobMock}) + + err := errList.Render() + + assert.NoError(t, err) + output := listBuffer.String() + contains := []string{ + "manifest-file", + "\n* ", + "Critical", + "critical-message\n", + "Warning", + "|", + "warning-message\n", + } + assertOutput(t, output, contains) +} + +func TestRenderCriticalAndWorkingJob(t *testing.T) { + var listBuffer bytes.Buffer + + jobWithErrMock := testdata.NewJobMock("manifest-file") + + criticalErr := errors.New("critical-message") + jobWithErrMock.Errors().Critical(criticalErr) + + jobWorkingMock := testdata.NewJobMock("working-manifest-file") + + errList := NewJobsErrorList(&listBuffer, []job.IJob{jobWithErrMock, jobWorkingMock}) + + err := errList.Render() + + assert.NoError(t, err) + output := listBuffer.String() + contains := []string{ + "manifest-file", + "\n* ", + "Critical", + "|", + "critical-message\n", + } + assertOutput(t, output, contains) + + assert.NotContains(t, output, jobWorkingMock) +} + +func assertOutput(t *testing.T, output string, contains []string) { + assert.Contains(t, output, "Errors") + assert.Contains(t, output, "\n-------\n") + + for _, c := range contains { + assert.Contains(t, output, c) + } +} diff --git a/pkg/tui/spinner_manager.go b/pkg/tui/spinner_manager.go new file mode 100644 index 00000000..c3a4fb0a --- /dev/null +++ b/pkg/tui/spinner_manager.go @@ -0,0 +1,64 @@ +package tui + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/chelnak/ysmrr" + "github.com/chelnak/ysmrr/pkg/colors" + "github.com/fatih/color" +) + +type ISpinnerManager interface { + AddSpinner(action string, file string) *ysmrr.Spinner + Start() + Stop() +} + +type SpinnerManager struct { + spinnerManager ysmrr.SpinnerManager +} + +func NewSpinnerManager() SpinnerManager { + return SpinnerManager{ysmrr.NewSpinnerManager(ysmrr.WithSpinnerColor(colors.FgHiBlue))} +} + +func (sm SpinnerManager) AddSpinner(action string, file string) *ysmrr.Spinner { + spinner := sm.spinnerManager.AddSpinner("") + SetSpinnerMessage(spinner, action, file, "waiting for worker") + + return spinner +} + +func (sm SpinnerManager) Start() { + sm.spinnerManager.Start() +} + +func (sm SpinnerManager) Stop() { + sm.spinnerManager.Stop() +} + +func SetSpinnerMessage(spinner *ysmrr.Spinner, action string, filename string, message string) { + const maxNumberOfChars = 50 + truncatedFilename := filename + if len(truncatedFilename) > maxNumberOfChars { + separator := string(os.PathSeparator) + pathParts := strings.Split(filename, separator) + if len(pathParts) > 3 { + firstDir := pathParts[0] + lastDir := pathParts[len(pathParts)-2] + name := pathParts[len(pathParts)-1] + truncatedFilename = filepath.Join( + firstDir, + "...", + lastDir, + name, + ) + } + + } + file := color.YellowString(truncatedFilename) + spinner.UpdateMessage(fmt.Sprintf("%s %s: %s", action, file, message)) +} diff --git a/pkg/tui/spinner_manager_test.go b/pkg/tui/spinner_manager_test.go new file mode 100644 index 00000000..5c80727a --- /dev/null +++ b/pkg/tui/spinner_manager_test.go @@ -0,0 +1,92 @@ +package tui + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/fatih/color" + "github.com/stretchr/testify/assert" +) + +const resolving = "Resolving" + +func TestNewSpinnerManager(t *testing.T) { + spinnerManager := NewSpinnerManager() + assert.NotNil(t, spinnerManager) +} + +func TestSetSpinnerMessage(t *testing.T) { + spinnerManager := NewSpinnerManager() + message := "test" + spinner := spinnerManager.AddSpinner(resolving, message) + assert.Contains(t, spinner.GetMessage(), fmt.Sprintf("Resolving %s: waiting for worker", color.YellowString(message))) + + fileName := "file-name" + message = "new test message" + + SetSpinnerMessage(spinner, resolving, fileName, message) + assert.Contains(t, spinner.GetMessage(), fmt.Sprintf("Resolving %s: %s", color.YellowString(fileName), message)) +} + +func TestSetDifferentActionSpinnerMessage(t *testing.T) { + spinnerManager := NewSpinnerManager() + message := "test" + action := "Callgraph" + spinner := spinnerManager.AddSpinner(action, message) + assert.Contains(t, spinner.GetMessage(), fmt.Sprintf("Callgraph %s: waiting for worker", color.YellowString(message))) + + fileName := "file-name" + message = "new test message" + + SetSpinnerMessage(spinner, resolving, fileName, message) + assert.Contains(t, spinner.GetMessage(), fmt.Sprintf("Resolving %s: %s", color.YellowString(fileName), message)) +} + +func TestSetSpinnerMessageLongFilenameParts(t *testing.T) { + spinnerManager := NewSpinnerManager() + longFilenameParts := []string{ + "directory", + "sub-directory################################################################", + "file.json", + } + longFileName := filepath.Join(longFilenameParts...) + + spinner := spinnerManager.AddSpinner(resolving, longFileName) + message := spinner.GetMessage() + + assert.Contains(t, message, longFileName) +} + +func TestSetSpinnerMessageLongFilenameManyDirs(t *testing.T) { + spinnerManager := NewSpinnerManager() + longFilenameParts := []string{ + "directory", + "sub-directory", + "sub-directory", + "sub-directory", + "sub-directory", + "sub-directory", + "target-directory", + "file.json", + } + longFileName := filepath.Join(longFilenameParts...) + + truncatedFilenameParts := []string{ + longFilenameParts[0], + "...", + longFilenameParts[len(longFilenameParts)-2], + longFilenameParts[len(longFilenameParts)-1], + } + truncatedFilename := filepath.Join(truncatedFilenameParts...) + spinner := spinnerManager.AddSpinner(resolving, longFileName) + message := spinner.GetMessage() + + assert.Contains(t, message, truncatedFilename) +} + +func TestStartStop(t *testing.T) { + spinnerManager := NewSpinnerManager() + spinnerManager.Start() + spinnerManager.Stop() +} diff --git a/pkg/upload/batch.go b/pkg/upload/batch.go index a8e7df14..607b130d 100644 --- a/pkg/upload/batch.go +++ b/pkg/upload/batch.go @@ -19,6 +19,7 @@ import ( "github.com/debricked/cli/pkg/file" "github.com/debricked/cli/pkg/git" "github.com/debricked/cli/pkg/tui" + "github.com/fatih/color" ) var ( @@ -267,5 +268,5 @@ func getRelativeFilePath(filePath string) string { } func printSuccessfulUpload(f string) { - fmt.Println("Successfully uploaded: ", f) + fmt.Printf("Successfully uploaded: %s\n", color.YellowString(f)) } diff --git a/pkg/upload/uploader_test.go b/pkg/upload/uploader_test.go index aa8e7710..f4d42d60 100644 --- a/pkg/upload/uploader_test.go +++ b/pkg/upload/uploader_test.go @@ -149,3 +149,5 @@ func (mock *debClientMock) Get(_ string, _ string) (*http.Response, error) { return res, nil } + +func (mock *debClientMock) SetAccessToken(_ *string) {} diff --git a/pkg/wire/cli_container.go b/pkg/wire/cli_container.go new file mode 100644 index 00000000..aba74205 --- /dev/null +++ b/pkg/wire/cli_container.go @@ -0,0 +1,139 @@ +package wire + +import ( + "fmt" + + "github.com/debricked/cli/pkg/callgraph" + callgraphStrategy "github.com/debricked/cli/pkg/callgraph/strategy" + "github.com/debricked/cli/pkg/ci" + "github.com/debricked/cli/pkg/client" + "github.com/debricked/cli/pkg/file" + licenseReport "github.com/debricked/cli/pkg/report/license" + vulnerabilityReport "github.com/debricked/cli/pkg/report/vulnerability" + "github.com/debricked/cli/pkg/resolution" + resolutionFile "github.com/debricked/cli/pkg/resolution/file" + "github.com/debricked/cli/pkg/resolution/strategy" + "github.com/debricked/cli/pkg/scan" + "github.com/debricked/cli/pkg/upload" + "github.com/hashicorp/go-retryablehttp" + + "sync" +) + +func GetCliContainer() *CliContainer { + if cliContainer == nil { + cliLock.Lock() + defer cliLock.Unlock() + if cliContainer == nil { + cliContainer = &CliContainer{} + err := cliContainer.wire() + if err != nil { + panic(err) + } + } + } + + return cliContainer +} + +var cliLock = &sync.Mutex{} + +var cliContainer *CliContainer + +func (cc *CliContainer) wire() error { + cc.retryClient = client.NewRetryClient() + cc.debClient = client.NewDebClient(nil, cc.retryClient) + finder, err := file.NewFinder(cc.debClient) + if err != nil { + return wireErr(err) + } + cc.finder = finder + + uploader, err := upload.NewUploader(cc.debClient) + if err != nil { + return wireErr(err) + } + cc.uploader = uploader + + cc.ciService = ci.NewService(nil) + + cc.batchFactory = resolutionFile.NewBatchFactory() + cc.strategyFactory = strategy.NewStrategyFactory() + cc.scheduler = resolution.NewScheduler(10) + cc.resolver = resolution.NewResolver( + cc.finder, + cc.batchFactory, + cc.strategyFactory, + cc.scheduler, + ) + cc.cgStrategyFactory = callgraphStrategy.NewStrategyFactory() + cc.cgScheduler = callgraph.NewScheduler(10) + cc.callgraph = callgraph.NewGenerator( + cc.cgStrategyFactory, + cc.cgScheduler, + ) + + cc.scanner = scan.NewDebrickedScanner( + &cc.debClient, + cc.finder, + cc.uploader, + cc.ciService, + cc.resolver, + cc.callgraph, + ) + + cc.licenseReporter = licenseReport.Reporter{DebClient: cc.debClient} + cc.vulnerabilityReporter = vulnerabilityReport.Reporter{DebClient: cc.debClient} + + return nil +} + +type CliContainer struct { + retryClient *retryablehttp.Client + debClient client.IDebClient + finder file.IFinder + uploader upload.IUploader + ciService ci.IService + scanner scan.IScanner + resolver resolution.IResolver + scheduler resolution.IScheduler + strategyFactory strategy.IFactory + batchFactory resolutionFile.IBatchFactory + licenseReporter licenseReport.Reporter + vulnerabilityReporter vulnerabilityReport.Reporter + callgraph callgraph.IGenerator + cgScheduler callgraph.IScheduler + cgStrategyFactory callgraphStrategy.IFactory +} + +func (cc *CliContainer) DebClient() client.IDebClient { + return cc.debClient +} + +func (cc *CliContainer) Finder() file.IFinder { + return cc.finder +} + +func (cc *CliContainer) Scanner() scan.IScanner { + return cc.scanner +} + +func (cc *CliContainer) Resolver() resolution.IResolver { + return cc.resolver +} + +func (cc *CliContainer) CallgraphGenerator() callgraph.IGenerator { + return cc.callgraph +} + +func (cc *CliContainer) LicenseReporter() licenseReport.Reporter { + return cc.licenseReporter +} + +func (cc *CliContainer) VulnerabilityReporter() vulnerabilityReport.Reporter { + return cc.vulnerabilityReporter +} + +func wireErr(err error) error { + return fmt.Errorf("failed to wire with cli-container. Error %s", err) +} diff --git a/pkg/wire/cli_container_test.go b/pkg/wire/cli_container_test.go new file mode 100644 index 00000000..536716fa --- /dev/null +++ b/pkg/wire/cli_container_test.go @@ -0,0 +1,41 @@ +package wire + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWire(t *testing.T) { + cliContainer = &CliContainer{} + defer resetContainer() + + err := cliContainer.wire() + assert.NoError(t, err) + assertCliContainer(t, cliContainer) +} + +func TestGetCliContainer(t *testing.T) { + assert.Nil(t, cliContainer) + testGetCliContainer(t) +} + +func testGetCliContainer(t *testing.T) { + container := GetCliContainer() + assert.NotNil(t, container) + assert.NotNil(t, cliContainer) + assertCliContainer(t, cliContainer) +} + +func resetContainer() { + cliContainer = nil +} + +func assertCliContainer(t *testing.T, cc *CliContainer) { + assert.NotNil(t, cc.DebClient()) + assert.NotNil(t, cc.Finder()) + assert.NotNil(t, cc.Scanner()) + assert.NotNil(t, cc.Resolver()) + assert.NotNil(t, cc.LicenseReporter()) + assert.NotNil(t, cc.VulnerabilityReporter()) +} diff --git a/pkg/wire/container.go b/pkg/wire/container.go new file mode 100644 index 00000000..c95587c2 --- /dev/null +++ b/pkg/wire/container.go @@ -0,0 +1,5 @@ +package wire + +type IContainer interface { + wire() error +} diff --git a/scripts/install.sh b/scripts/install.sh index fdb09873..4de5b9ef 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# test if git is installed if ! command -v git &> /dev/null then echo -e "Failed to find git, thus also the version. Version will be set to v0.0.0" diff --git a/scripts/test_cli.sh b/scripts/test_cli.sh index 3fb3e1e2..be3e7e6d 100755 --- a/scripts/test_cli.sh +++ b/scripts/test_cli.sh @@ -4,13 +4,13 @@ RED='\033[0;31m' SET='\033[0m' set -e -go test -cover -coverprofile=coverage.out ./... +go test -cover -coverprofile=coverage.out ./pkg/... echo -e "\nChecking test coverage threshold..." regex='[0-9]+\.*[0-9]*' if ! [[ $TEST_COVERAGE_THRESHOLD =~ $regex ]]; then - echo "Failed to find test coverage threshold. Defaults to 90%" - TEST_COVERAGE_THRESHOLD=90 + echo "Failed to find test coverage threshold. Defaults to 95%" + TEST_COVERAGE_THRESHOLD=95 fi echo "Test coverage threshold : $TEST_COVERAGE_THRESHOLD %" if [ ! -f "./coverage.out" ]; then diff --git a/scripts/test_e2e.sh b/scripts/test_e2e.sh new file mode 100644 index 00000000..87050916 --- /dev/null +++ b/scripts/test_e2e.sh @@ -0,0 +1,12 @@ +#!/bin/bash/env + +type="$1" + +case $type in + "pip") + go test ./test/resolve/pip_test.go + ;; + *) + go test ./test/... + ;; +esac diff --git a/test/resolve/pip_test.go b/test/resolve/pip_test.go new file mode 100644 index 00000000..e9016674 --- /dev/null +++ b/test/resolve/pip_test.go @@ -0,0 +1,43 @@ +package resolve + +import ( + "os" + "path/filepath" + "testing" + + "github.com/debricked/cli/pkg/cmd/resolve" + "github.com/debricked/cli/pkg/wire" + "github.com/stretchr/testify/assert" +) + +func TestResolvePip(t *testing.T) { + cases := []struct { + name string + requirementsFile string + expectedFile string + }{ + { + name: "basic requirements.txt", + requirementsFile: "testdata/pip/requirements.txt", + expectedFile: "testdata/pip/expected.lock", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + resolveCmd := resolve.NewResolveCmd(wire.GetCliContainer().Resolver()) + err := resolveCmd.RunE(resolveCmd, []string{c.requirementsFile}) + assert.NoError(t, err) + + lockFileDir := filepath.Dir(c.requirementsFile) + lockFile := filepath.Join(lockFileDir, ".requirements.txt.debricked.lock") + lockFileContents, fileErr := os.ReadFile(lockFile) + assert.NoError(t, fileErr) + + expectedFileContents, fileErr := os.ReadFile(c.expectedFile) + assert.NoError(t, fileErr) + + assert.Equal(t, string(expectedFileContents), string(lockFileContents)) + }) + } +} diff --git a/test/resolve/testdata/pip/expected.lock b/test/resolve/testdata/pip/expected.lock new file mode 100644 index 00000000..90c2475b --- /dev/null +++ b/test/resolve/testdata/pip/expected.lock @@ -0,0 +1,91 @@ +pandas==1.5.1 +# comment + +*** +Package Version +--------------- -------- +numpy 1.24.2 +pandas 1.5.1 +pip 22.0.4 +python-dateutil 2.8.2 +pytz 2022.7.1 +setuptools 58.1.0 +six 1.16.0 + +*** +Name: numpy +Version: 1.24.2 +Summary: Fundamental package for array computing in Python +Home-page: https://www.numpy.org +Author: Travis E. Oliphant et al. +Author-email: +License: BSD-3-Clause +Location: /home/nilszeilon/Programming/cli/test/testdata/pip/requirements.txt.venv/lib/python3.9/site-packages +Requires: +Required-by: pandas +--- +Name: pandas +Version: 1.5.1 +Summary: Powerful data structures for data analysis, time series, and statistics +Home-page: https://pandas.pydata.org +Author: The Pandas Development Team +Author-email: pandas-dev@python.org +License: BSD-3-Clause +Location: /home/nilszeilon/Programming/cli/test/testdata/pip/requirements.txt.venv/lib/python3.9/site-packages +Requires: numpy, python-dateutil, pytz +Required-by: +--- +Name: pip +Version: 22.0.4 +Summary: The PyPA recommended tool for installing Python packages. +Home-page: https://pip.pypa.io/ +Author: The pip developers +Author-email: distutils-sig@python.org +License: MIT +Location: /home/nilszeilon/Programming/cli/test/testdata/pip/requirements.txt.venv/lib/python3.9/site-packages +Requires: +Required-by: +--- +Name: python-dateutil +Version: 2.8.2 +Summary: Extensions to the standard Python datetime module +Home-page: https://github.com/dateutil/dateutil +Author: Gustavo Niemeyer +Author-email: gustavo@niemeyer.net +License: Dual License +Location: /home/nilszeilon/Programming/cli/test/testdata/pip/requirements.txt.venv/lib/python3.9/site-packages +Requires: six +Required-by: pandas +--- +Name: pytz +Version: 2022.7.1 +Summary: World timezone definitions, modern and historical +Home-page: http://pythonhosted.org/pytz +Author: Stuart Bishop +Author-email: stuart@stuartbishop.net +License: MIT +Location: /home/nilszeilon/Programming/cli/test/testdata/pip/requirements.txt.venv/lib/python3.9/site-packages +Requires: +Required-by: pandas +--- +Name: setuptools +Version: 58.1.0 +Summary: Easily download, build, install, upgrade, and uninstall Python packages +Home-page: https://github.com/pypa/setuptools +Author: Python Packaging Authority +Author-email: distutils-sig@python.org +License: UNKNOWN +Location: /home/nilszeilon/Programming/cli/test/testdata/pip/requirements.txt.venv/lib/python3.9/site-packages +Requires: +Required-by: +--- +Name: six +Version: 1.16.0 +Summary: Python 2 and 3 compatibility utilities +Home-page: https://github.com/benjaminp/six +Author: Benjamin Peterson +Author-email: benjamin@python.org +License: MIT +Location: /home/nilszeilon/Programming/cli/test/testdata/pip/requirements.txt.venv/lib/python3.9/site-packages +Requires: +Required-by: python-dateutil diff --git a/test/resolve/testdata/pip/requirements.txt b/test/resolve/testdata/pip/requirements.txt new file mode 100644 index 00000000..538d6831 --- /dev/null +++ b/test/resolve/testdata/pip/requirements.txt @@ -0,0 +1,2 @@ +pandas==1.5.1 +# comment