From 90e024d07dd660e73c2731ffc9380fc73ce301bc Mon Sep 17 00:00:00 2001 From: Brian Wyka <46069296+brianwyka@users.noreply.github.com> Date: Mon, 24 Jan 2022 17:12:17 -0500 Subject: [PATCH] Remote repository scanning improvements (#60) * Remote repository scanning improvements progress * remote-repository: Remote repository file reading improvements for Bitbucket * remote-repository: Fix maven version * remote-repository: Update groovy to 3.0.9 for JDK 17 support * remote-repository: Fix maven version in test * remote-repository: Fix more tests for JDK 17 * remote-repository: Fix Dockerfile ci images references * remote-repository: Fix java8 graalvm version * remote-repository: Vulnerability fixes, Fixes #38, Fixes #59, update ci with docker login and graalvm github actions * remote-repository: Login to GHCR with correct creds * remote-repository: Update static native imate to be -H:+StaticExecutableWithDynamicLibC * remote-repository: Update busybox * remote-repository: Static native image builds on musl, scratch Docker image, disabling Bintray publish * remote-repository: Update github workflows * remote-repository: Fix native image builder and push to ghcr * remote-repository: Fix docker image builder for native image musl, and use github actions for docker layer caching * remote-repository: Push docker builders * remote-repository: Push docker builders * remote-repository: 0.7.0 * remote-repository: Skip native image build for java 8 * remote-repository: Skip native image build for java 8 * remote-repository: Fix java8 build and workflow dependencies * remote-repository: Fix needs syntax * remote-repository: 1.8 reference to identigy java 8 in mvn profile * remote-repository: Skip failsafe integration tests for java8 * remote-repository: Only build dist when not java 8 * remote-repository: Fix dist pom * remote-repository: Don't cache docker builders * remote-repository: Only build dist for java 11, use 20.3.0 for windows native image * Revert to Test Report * remote-repository: Remove git-commitid-plugin, fixes #68 * remote-repository: Fix git scripts format * remote-repository: Archive completion script without 'bash' * remote-repository: Update workflow names and archive linux native image * Fix Linux native image archive path --- .../build-and-push-docker-builders.yml | 81 +++- .github/workflows/maven-ci.yml | 91 ++--- .github/workflows/release.yml | 177 +++++---- .gitignore | 1 + .mvn/wrapper/maven-wrapper.properties | 2 +- NOTICE.txt | 2 +- attribution.txt | 8 +- bom/pom.xml | 2 +- cli/pom.xml | 20 +- cli/scripts/update-picocli.sh | 22 +- .../cli/AbstractRemoteScanCommand.java | 21 +- .../sourcehawk/cli/BitbucketScanCommand.java | 37 +- .../optum/sourcehawk/cli/CommandOptions.java | 11 +- .../sourcehawk/cli/GithubScanCommand.java | 37 +- .../com/optum/sourcehawk/cli/Sourcehawk.java | 2 +- cli/src/main/java/picocli/CommandLine.java | 167 +++++--- .../sourcehawk-cli/native-image.properties | 3 +- .../cli/BitbucketScanCommandSpec.groovy | 14 +- .../cli/BitbucketScanSubCommandSpec.groovy | 18 +- .../cli/CliBaseSpecification.groovy | 2 +- .../cli/GithubScanCommandSpec.groovy | 10 +- .../cli/NativeImageConfigSpec.groovy | 1 - .../flattened/sourcehawk-flattened-base.yml | 2 +- .../sourcehawk-flattened-override.yml | 2 +- cli/src/test/resources/sourcehawk-simple.yml | 22 -- core/pom.xml | 2 +- .../optum/sourcehawk/core/data/RemoteRef.java | 49 +-- .../BitbucketRepositoryFileReader.java | 52 --- .../GithubRepositoryFileReader.java | 54 --- .../repository/LocalRepositoryFileReader.java | 6 + .../RemoteRepositoryFileReader.java | 81 ++-- .../core/repository/RepositoryFileReader.java | 8 + .../sourcehawk/core/data/RemoteRefSpec.groovy | 28 +- .../BitbucketRepositoryFileReaderSpec.groovy | 184 --------- .../GithubRepositoryFileReaderSpec.groovy | 239 ------------ .../RemoteRepositoryFileReaderSpec.groovy | 165 ++++++-- distributions/debian/pom.xml | 219 ++++++----- .../docker-builders/Dockerfile-nativeimage | 22 +- distributions/docker/Dockerfile | 22 +- distributions/docker/README.md | 6 +- distributions/docker/etc/group | 1 + distributions/docker/etc/passwd | 1 + distributions/docker/pom.xml | 79 +++- ...-to-docker-hub.sh => push-docker-image.sh} | 0 .../linux/native-image-builder/Dockerfile | 5 +- distributions/linux/pom.xml | 70 +++- distributions/pom.xml | 13 +- distributions/rpm/Dockerfile | 2 +- distributions/rpm/pom.xml | 357 ++++++++---------- .../scripts/build-and-push-docker-builders.sh | 46 --- .../extract-file-from-docker-container.sh | 3 +- enforcer/core/pom.xml | 2 +- enforcer/file/aot/pom.xml | 2 +- ...wkFileEnforcerRegistryProcessorSpec.groovy | 4 +- enforcer/file/common/pom.xml | 2 +- enforcer/file/core/pom.xml | 2 +- enforcer/file/docker/pom.xml | 2 +- enforcer/file/maven/pom.xml | 4 +- enforcer/file/pom.xml | 2 +- enforcer/file/registry/pom.xml | 4 +- enforcer/pom.xml | 2 +- exec/pom.xml | 4 +- .../sourcehawk/exec/scan/ScanExecutor.java | 45 ++- .../exec/scan/ScanResultFactory.java | 38 +- .../exec/ConfigurationReaderSpec.groovy | 5 +- .../sourcehawk/exec/ExecOptionsSpec.groovy | 11 +- .../resources/sourcehawk-flattened-base.yml | 2 +- pom.xml | 15 +- 68 files changed, 1176 insertions(+), 1439 deletions(-) delete mode 100644 core/src/main/java/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReader.java delete mode 100644 core/src/main/java/com/optum/sourcehawk/core/repository/GithubRepositoryFileReader.java delete mode 100644 core/src/test/groovy/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReaderSpec.groovy delete mode 100644 core/src/test/groovy/com/optum/sourcehawk/core/repository/GithubRepositoryFileReaderSpec.groovy create mode 100644 distributions/docker/etc/group create mode 100644 distributions/docker/etc/passwd rename distributions/docker/scripts/{push-docker-image-to-docker-hub.sh => push-docker-image.sh} (100%) delete mode 100755 distributions/scripts/build-and-push-docker-builders.sh diff --git a/.github/workflows/build-and-push-docker-builders.yml b/.github/workflows/build-and-push-docker-builders.yml index 2793019..c9b2543 100644 --- a/.github/workflows/build-and-push-docker-builders.yml +++ b/.github/workflows/build-and-push-docker-builders.yml @@ -10,12 +10,79 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Git Checkout + uses: actions/checkout@v2 with: ref: ${{ github.event.inputs.ref }} - - name: Build and Push Docker Images to Github Container Registry - shell: bash - run: ./distributions/scripts/build-and-push-docker-builders.sh - env: - DOCKER_USERNAME: ${{ secrets.GIT_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.GIT_PACKAGE_TOKEN }} + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Login to Github Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GIT_USERNAME }} + password: ${{ secrets.GIT_PACKAGE_TOKEN }} + - name: Build and Push graalvm-ce-21.3.0-java11 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-nativeimage + build-args: FROM_VERSION=java11-21.3.0 + tags: ghcr.io/optum/ci/nativeimage:graalvm-ce-21.3.0-java11 + push: true + - name: Build and Push graalvm-ce-21.3.0-java17 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-nativeimage + build-args: FROM_VERSION=java17-21.3.0 + tags: ghcr.io/optum/ci/nativeimage:graalvm-ce-21.3.0-java17 + push: true + - name: Build and Push rpmbuild centos7 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-rpmbuild + build-args: FROM=centos:7 + tags: ghcr.io/optum/ci/rpmbuild:centos7 + push: true + - name: Build and Push rpmbuild centos8 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-rpmbuild + build-args: FROM=centos:8 + tags: ghcr.io/optum/ci/rpmbuild:centos8 + push: true + - name: Build and Push rpmbuild fedora33 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-rpmbuild + build-args: FROM=fedora:33 + tags: ghcr.io/optum/ci/rpmbuild:fedora33 + push: true + - name: Build and Push rpmbuild fedora34 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-rpmbuild + build-args: FROM=fedora:34 + tags: ghcr.io/optum/ci/rpmbuild:fedora34 + push: true + - name: Build and Push rpmbuild fedora35 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-rpmbuild + build-args: FROM=fedora:35 + tags: ghcr.io/optum/ci/rpmbuild:fedora35 + push: true diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index aae2c26..b7fab70 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -41,13 +41,16 @@ jobs: matrix: java: ['8', '11', '17'] steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 + - name: Checkout Source Code + uses: actions/checkout@v2 + - name: Setup Maven Cache + uses: actions/cache@v1 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 - - uses: actions/cache@v1 + - name: Setup Sonar Cache + uses: actions/cache@v1 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar @@ -78,27 +81,33 @@ jobs: SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} SONATYPE_GPG_PASSPHRASE: ${{ secrets.SONATYPE_GPG_PASSPHRASE }} + - name: Archive Linux Native Image + if: success() && (matrix.java == '11') + uses: actions/upload-artifact@v2 + with: + name: linux-native-image + path: distributions/linux/target/sourcehawk - name: Archive Native Image JAR - if: success() + if: success() && (matrix.java == '11') uses: actions/upload-artifact@v2 with: - name: native-image-${{ matrix.java }} + name: native-image-jar path: cli/target/*-native-image.jar - - name: Archive Bash Completion Script - if: success() + - name: Archive Completion Script + if: success() && (matrix.java == '11') uses: actions/upload-artifact@v2 with: - name: bash-completion-script-${{ matrix.java }} - path: cli/target/sourcehawk-bash-completion.sh + name: completion-script + path: cli/target/sourcehawk-completion.sh - name: Archive Manpages - if: success() + if: success() && (matrix.java == '11') uses: actions/upload-artifact@v2 with: - name: manpages-${{ matrix.java }} + name: manpages path: gh-pages/manpages/sourcehawk*.1 - name: Aggregate Coverage Reports id: aggregate_coverage_reports - if: success() + if: success() && (matrix.java == '11') run: echo ::set-output name=JACOCO_XML_REPORT_PATHS::$(find . -name "jacoco.xml" -printf '%P\n' | tr '\r\n' ',') - name: Analyze with SonarCloud if: success() && (github.event_name == 'push' && matrix.java == '11') @@ -118,31 +127,31 @@ jobs: runs-on: macos-latest needs: build steps: - - uses: actions/download-artifact@v2 + - name: Download Native Image JAR + uses: actions/download-artifact@v2 with: - name: native-image-8 + name: native-image-jar path: build - - uses: actions/download-artifact@v2 + - name: Download Completion Script + uses: actions/download-artifact@v2 with: - name: bash-completion-script-8 + name: completion-script path: build - - uses: actions/download-artifact@v2 + - name: Download Manpages + uses: actions/download-artifact@v2 with: - name: manpages-8 + name: manpages path: build - name: Rename Native Image JAR working-directory: build run: mv *.jar native-image.jar - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: 8 - name: Setup GraalVM - uses: DeLaGuardo/setup-graalvm@master + uses: graalvm/setup-graalvm@v1 with: - graalvm-version: 21.2.0.java8 - - name: Setup GraalVM Native Image Tool - run: gu install native-image + version: '21.3.0' + java-version: '11' + components: 'native-image' + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Mac Native Image if: success() working-directory: build @@ -150,7 +159,7 @@ jobs: - name: Create Homebrew Tap Formula Archive if: success() working-directory: build - run: gzip sourcehawk*.1 && tar -czvf sourcehawk-homebrew-tap-formula.tar.gz sourcehawk sourcehawk-bash-completion.sh sourcehawk*.1.gz + run: gzip sourcehawk*.1 && tar -czvf sourcehawk-homebrew-tap-formula.tar.gz sourcehawk sourcehawk-completion.sh sourcehawk*.1.gz - name: Archive Mac Native Image if: success() continue-on-error: true @@ -172,28 +181,26 @@ jobs: build-windows-native-image: runs-on: windows-latest needs: build + continue-on-error: true steps: - - uses: actions/download-artifact@v2 + - name: Download Native Image JAR + uses: actions/download-artifact@v2 with: - name: native-image-11 + name: native-image-jar path: build - name: Rename Native Image JAR working-directory: build run: ren *.jar native-image.jar - - name: Setup GraalVM Native Image and Visual C Build Tools - run: | - Invoke-RestMethod -Uri https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.3.0/graalvm-ce-java11-windows-amd64-21.3.0.zip -OutFile 'graal.zip' - Expand-Archive -path 'graal.zip' -destinationpath '.' - graalvm-ce-java11-21.3.0\bin\gu.cmd install native-image - choco install visualstudio2017-workload-vctools + - name: Setup GraalVM + uses: graalvm/setup-graalvm@v1 + with: + version: '20.3.0' + java-version: '11' + components: 'native-image' + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Windows Native Image if: success() - shell: cmd - run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat" - graalvm-ce-java11-21.3.0\bin\native-image -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime - env: - JAVA_HOME: ./graalvm-ce-java11-21.3.0 + run: native-image.cmd -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime - name: Archive Windows Native Image if: success() continue-on-error: true @@ -204,4 +211,4 @@ jobs: - name: Smoke Test if: success() shell: cmd - run: sourcehawk.exe help \ No newline at end of file + run: sourcehawk.exe help diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 519f563..2d29d58 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,8 @@ jobs: RELEASE_VERSION: ${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }} RELEASE_ASSET_UPLOAD_URL: ${{ steps.create_release.outputs.upload_url }} steps: - - uses: actions/checkout@v2 + - name: Checkout Source Code + uses: actions/checkout@v2 with: ref: ${{ github.event.inputs.ref }} token: ${{ secrets.GITHUB_TOKEN }} @@ -56,18 +57,41 @@ jobs: SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} SONATYPE_GPG_PASSPHRASE: ${{ secrets.SONATYPE_GPG_PASSPHRASE }} + build-java11: + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout Source Code + uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.ref }} + - name: Setup Java and Maven + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Set Maven Project Version + shell: bash + run: | + RELEASE_VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout | tail -1 | tr -d '\r\n') + [[ "$RELEASE_VERSION" == *"-SNAPSHOT" ]] && RELEASE_VERSION="${RELEASE_VERSION%"-SNAPSHOT"}" && ./mvnw --batch-mode versions:set -D removeSnapshot || true + [[ "$(git tag -l $RELEASE_VERSION)" == "$RELEASE_VERSION" ]] && echo "Tag $RELEASE_VERSION already exists" && exit 1 + echo ::set-output name=RELEASE_VERSION::$RELEASE_VERSION + - name: Build Maven Project + if: success() + run: ./mvnw --batch-mode install -D ci.build -D ci.release - name: Archive Native Image JAR if: success() uses: actions/upload-artifact@v2 with: - name: native-image + name: native-image-jar path: cli/target/*-native-image.jar - - name: Archive Bash Completion Script + - name: Archive Completion Script if: success() uses: actions/upload-artifact@v2 with: - name: bash-completion-script - path: cli/target/sourcehawk-bash-completion.sh + name: completion-script + path: cli/target/sourcehawk-completion.sh - name: Archive Manpages if: success() uses: actions/upload-artifact@v2 @@ -106,10 +130,19 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ format('v{0}', steps.set_maven_project_version.outputs.RELEASE_VERSION) }} - release_name: ${{ format('{0} {1}', github.event.repository.name, steps.set_maven_project_version.outputs.RELEASE_VERSION) }} + release_name: ${{ format('{0} {1}', github.event.repository.name, needs.build.outputs.RELEASE_VERSION) }} body_path: CHANGELOG.md draft: ${{ github.event.inputs.draft }} prerelease: ${{ github.event.inputs.prerelease }} + - name: Publish Github Pages + if: success() + continue-on-error: true + uses: jamesives/github-pages-deploy-action@3.7.1 + with: + COMMIT_MESSAGE: ${{ format('Publishing github pages for release version {0}', needs.build.outputs.RELEASE_VERSION) }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages + FOLDER: gh-pages - name: Upload Sourcehawk Linux Executable if: success() continue-on-error: true @@ -119,7 +152,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/linux/target/sourcehawk - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-linux-x86_64 + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-linux-x86_64 asset_content_type: application/octet-stream - name: Upload Sourcehawk Debian Buster Package if: success() @@ -130,7 +163,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/debian/target/sourcehawk-debian-buster.deb - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-debian-buster-amd64.deb + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-debian-buster-amd64.deb asset_content_type: application/octet-stream - name: Upload Sourcehawk Ubuntu Focal Package if: success() @@ -141,7 +174,18 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/debian/target/sourcehawk-ubuntu-focal.deb - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-ubuntu-focal-amd64.deb + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-ubuntu-focal-amd64.deb + asset_content_type: application/octet-stream + - name: Upload Sourcehawk Centos 7 RPM Package + if: success() + continue-on-error: true + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./distributions/rpm/target/sourcehawk-centos-7.rpm + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-1.el7.x86_64.rpm asset_content_type: application/octet-stream - name: Upload Sourcehawk Centos 8 RPM Package if: success() @@ -152,9 +196,9 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/rpm/target/sourcehawk-centos-8.rpm - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.el8.x86_64.rpm + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-1.el8.x86_64.rpm asset_content_type: application/octet-stream - - name: Upload Sourcehawk Fedora 35 RPM Package + - name: Upload Sourcehawk Fedora 33 RPM Package if: success() continue-on-error: true uses: actions/upload-release-asset@v1 @@ -162,59 +206,47 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./distributions/rpm/target/sourcehawk-fedora-35.rpm - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.fc35.x86_64.rpm + asset_path: ./distributions/rpm/target/sourcehawk-fedora-33.rpm + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-1.fc33.x86_64.rpm asset_content_type: application/octet-stream - - name: Publish Github Pages + - name: Upload Sourcehawk Fedora 34 RPM Package if: success() continue-on-error: true - uses: jamesives/github-pages-deploy-action@3.7.1 - with: - COMMIT_MESSAGE: ${{ format('Publishing github pages for release version {0}', steps.set_maven_project_version.outputs.RELEASE_VERSION) }} + uses: actions/upload-release-asset@v1 + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages - FOLDER: gh-pages - build-java11: - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.event.inputs.ref }} - - name: Setup Java and Maven - uses: actions/setup-java@v1 with: - java-version: 11 - - name: Set Maven Project Version - shell: bash - run: | - RELEASE_VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout | tail -1 | tr -d '\r\n') - [[ "$RELEASE_VERSION" == *"-SNAPSHOT" ]] && RELEASE_VERSION="${RELEASE_VERSION%"-SNAPSHOT"}" && ./mvnw --batch-mode versions:set -D removeSnapshot || true - [[ "$(git tag -l $RELEASE_VERSION)" == "$RELEASE_VERSION" ]] && echo "Tag $RELEASE_VERSION already exists" && exit 1 - echo ::set-output name=RELEASE_VERSION::$RELEASE_VERSION - - name: Build Maven Project - if: success() - run: ./mvnw --batch-mode install -D ci.build -D ci.release - - name: Archive Native Image JAR + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./distributions/rpm/target/sourcehawk-fedora-34.rpm + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-1.fc34.x86_64.rpm + asset_content_type: application/octet-stream + - name: Upload Sourcehawk Fedora 35 RPM Package if: success() - uses: actions/upload-artifact@v2 + continue-on-error: true + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - name: native-image-java11 - path: cli/target/*-native-image.jar + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./distributions/rpm/target/sourcehawk-fedora-35.rpm + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-1.fc35.x86_64.rpm + asset_content_type: application/octet-stream build-mac-native-image: runs-on: macos-latest - needs: build + needs: build-java11 steps: - - uses: actions/download-artifact@v2 + - name: Download Native Image JAR + uses: actions/download-artifact@v2 with: - name: native-image + name: native-image-jar path: build - - uses: actions/download-artifact@v2 + - name: Download Completion Script + uses: actions/download-artifact@v2 with: - name: bash-completion-script + name: completion-script path: build - - uses: actions/download-artifact@v2 + - name: Download Manpages + uses: actions/download-artifact@v2 with: name: manpages path: build @@ -224,13 +256,14 @@ jobs: - name: Setup Java uses: actions/setup-java@v1 with: - java-version: 8 + java-version: 11 - name: Setup GraalVM - uses: DeLaGuardo/setup-graalvm@master + uses: graalvm/setup-graalvm@v1 with: - graalvm-version: 21.2.0.java8 - - name: Setup GraalVM Native Image Tool - run: gu install native-image + version: '21.3.0' + java-version: '11' + components: 'native-image' + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Mac Native Image if: success() working-directory: build @@ -242,7 +275,7 @@ jobs: - name: Create Homebrew Tap Formula Archive if: success() working-directory: build - run: gzip sourcehawk*.1 && tar -czvf sourcehawk-homebrew-tap-formula.tar.gz sourcehawk sourcehawk-bash-completion.sh sourcehawk*.1.gz + run: gzip sourcehawk*.1 && tar -czvf sourcehawk-homebrew-tap-formula.tar.gz sourcehawk sourcehawk-completion.sh sourcehawk*.1.gz - name: Upload Sourcehawk Mac Executable if: success() continue-on-error: true @@ -275,34 +308,34 @@ jobs: homebrew-tap: optum/homebrew-tap base-branch: main download-url: ${{ steps.upload_homebrew_tap_formula_archive.outputs.browser_download_url }} - commit-message: Updating sourcehawk formula to latest release version ${{ needs.build.outputs.RELEASE_VERSION }} + commit-message: | + Sourcehawk ${{ needs.build.outputs.RELEASE_VERSION }} + + Updating sourcehawk formula to latest release version env: COMMITTER_TOKEN: ${{ secrets.GIT_COMMITTER_TOKEN }} build-windows-native-image: runs-on: windows-latest - needs: [build, build-java11] + needs: build-java11 steps: - - uses: actions/download-artifact@v2 + - name: Download Native Image JAR + uses: actions/download-artifact@v2 with: - name: native-image-java11 + name: native-image-jar path: build - name: Rename Native Image JAR working-directory: build run: ren *.jar native-image.jar - - name: Setup GraalVM Native Image and Visual C Build Tools - run: | - Invoke-RestMethod -Uri https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.3.0/graalvm-ce-java11-windows-amd64-21.3.0.zip -OutFile 'graal.zip' - Expand-Archive -path 'graal.zip' -destinationpath '.' - graalvm-ce-java11-21.3.0\bin\gu.cmd install native-image - choco install visualstudio2017-workload-vctools + - name: Setup GraalVM + uses: graalvm/setup-graalvm@v1 + with: + version: '20.3.0' + java-version: '11' + components: 'native-image' + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Windows Native Image if: success() - shell: cmd - run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat" - graalvm-ce-java11-21.3.0\bin\native-image -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime - env: - JAVA_HOME: ./graalvm-ce-java11-21.3.0 + run: native-image.cmd -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime - name: Smoke Test if: success() shell: cmd diff --git a/.gitignore b/.gitignore index 0e332c5..6028a98 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ target/ *.log log logs +*.bak ### STS ### .apt_generated diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index e404372..f8b5614 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -1 +1 @@ -distributionUrl = https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip \ No newline at end of file +distributionUrl = https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip \ No newline at end of file diff --git a/NOTICE.txt b/NOTICE.txt index 15afa52..4fc5b89 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,6 +1,6 @@ sourcehawk -Copyright 2021 Optum +Copyright 2022 Optum Project Description: ==================== diff --git a/attribution.txt b/attribution.txt index e9b35ea..fc3dc30 100644 --- a/attribution.txt +++ b/attribution.txt @@ -390,7 +390,7 @@ https://opensource.org/licenses/BSD-2-Clause ------------------------------------------------------------------------------------------------------------------------------- -Package: org.spockframework:spock-core:2.0-M3-groovy-3.0 +Package: org.spockframework:spock-core:2.0-groovy-3.0 License: Apache-2.0 @@ -1847,7 +1847,7 @@ limitations under the License. ------------------------------------------------------------------------------------------------------------------------------- -Package: org.codehaus.groovy:groovy:3.0.4 +Package: org.codehaus.groovy:groovy:3.0.9 License: Apache-2.0 @@ -1955,7 +1955,7 @@ limitations under the License. ------------------------------------------------------------------------------------------------------------------------------- -Package: org.apache.maven:maven-model:3.6.3 +Package: org.apache.maven:maven-model:3.8.4 License: Apache-2.0 @@ -2230,7 +2230,7 @@ http://www.apache.org/licenses/LICENSE-2.0 ------------------------------------------------------------------------------------------------------------------------------- -Package: info.picocli:4.6.1 +Package: info.picocli:4.6.2 License: Apache-2.0 diff --git a/bom/pom.xml b/bom/pom.xml index 35dbd3b..febfdc6 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT sourcehawk-bom diff --git a/cli/pom.xml b/cli/pom.xml index 441e361..a73a1ea 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -8,7 +8,7 @@ sourcehawk com.optum.sourcehawk - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT sourcehawk-cli @@ -23,7 +23,7 @@ com.optum.sourcehawk.cli.Sourcehawk - 0.94 + 0.91 **/picocli/**/*.* @@ -31,7 +31,7 @@ **/picocli/**/*.* - 4.6.1 + 4.6.2 @@ -190,7 +190,7 @@ - generate-bash-completion-script + generate-completion-script process-classes java @@ -207,7 +207,7 @@ --force --completionScript - ${project.build.directory}/${cli.name}-bash-completion.sh + ${project.build.directory}/${cli.name}-completion.sh ${cli.class} @@ -235,7 +235,7 @@ build-helper-maven-plugin - attach-bash-completion-script-artifact + attach-completion-script attach-artifact @@ -243,8 +243,8 @@ - ${project.build.directory}/${cli.name}-bash-completion.sh - bash-completion + ${project.build.directory}/${cli.name}-completion.sh + completion sh @@ -389,11 +389,13 @@ - ${project.build.directory}/${cli.name}-bash-completion.sh + ${project.build.directory}/${cli.name}-completion.sh ${project.parent.basedir}/gh-pages/index.html ${project.parent.basedir}/gh-pages/stylesheets/sourcehawk.css ${project.parent.basedir}/gh-pages/manpages/sourcehawk.1 ${project.parent.basedir}/gh-pages/manpages/scan.1 + ${project.parent.basedir}/gh-pages/manpages/scan-bitbucket.1 + ${project.parent.basedir}/gh-pages/manpages/scan-github.1 ${project.parent.basedir}/gh-pages/manpages/validate-config.1 ${project.parent.basedir}/gh-pages/manpages/fix.1 ${project.parent.basedir}/gh-pages/manpages/flatten-config.1 diff --git a/cli/scripts/update-picocli.sh b/cli/scripts/update-picocli.sh index f4619ee..84011f5 100755 --- a/cli/scripts/update-picocli.sh +++ b/cli/scripts/update-picocli.sh @@ -11,11 +11,11 @@ set -e ######################################################################### # Retrieve Latest Version -VERSION=$(curl -sI https://github.com/remkop/picocli/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d 'v' | tr -d '\r\n') +VERSION=$(curl -ksI https://github.com/remkop/picocli/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d 'v' | tr -d '\r\n') # Global Variables -DIR="$( cd "$( dirname "$( dirname "${BASH_SOURCE[0]}" )")" && pwd )" -ROOT_DIR="$( cd "$( dirname "$( dirname "$( dirname "${BASH_SOURCE[0]}" )")")" && pwd )" +DIR="$(dirname "$(cd -- "$(dirname "$0")"; pwd -P)")" +ROOT_DIR="$(dirname "$(dirname "$(cd -- "$(dirname "$0")"; pwd -P)")")" BASE_URL="https://raw.githubusercontent.com/remkop/picocli" LICENSE_URL="$BASE_URL/v$VERSION/LICENSE" LICENSE_FILE_PATH="$DIR/src/main/resources/META-INF/licenses/picocli.txt" @@ -28,15 +28,19 @@ curl -ksf "$LICENSE_URL" > "$LICENSE_FILE_PATH" curl -ksf "$SOURCE_URL" > "$SOURCE_FILE_PATH" # Add some warning suppression to the java source file -sed -i 's/public\sclass\sCommandLine/@SuppressWarnings({"rawtypes", "deprecation" })\npublic class CommandLine/g' "$SOURCE_FILE_PATH" +sed -i.bak -e 's/public class CommandLine/@SuppressWarnings({"rawtypes", "deprecation" })\npublic class CommandLine/g' \ + -e 's/TODO/TIDO/g' "$SOURCE_FILE_PATH" \ + && rm -rf "$SOURCE_FILE_PATH.bak" + +# Remove TODOs so not highlighted in editor +sed -i.bak 's/TODO/TIDO/g' "$SOURCE_FILE_PATH" # Replace the version in pom.xml file for plugin references -sed -i "s/[-[:alnum:]./]\{1,\}<\/picocli.version>/$VERSION<\/picocli.version>/" "$DIR/pom.xml" +sed -i.bak "s/[-[:alnum:]./]\{1,\}<\/picocli.version>/$VERSION<\/picocli.version>/" "$DIR/pom.xml" \ + && rm -rf "$DIR/pom.xml.bak" # Replace the version in attribution.txt file -sed -i "s/Package: info.picocli:[-[:alnum:]./]\{1,\}/Package: info.picocli:$VERSION/" "$ROOT_DIR/attribution.txt" - -# Remove TODOs so not highlighted in editor -sed -i 's/TODO/TIDO/g' "$SOURCE_FILE_PATH" +sed -i.bak "s/Package: info.picocli:[-[:alnum:]./]\{1,\}/Package: info.picocli:$VERSION/" "$ROOT_DIR/attribution.txt" \ + && rm -rf "$ROOT_DIR/attribution.txt.bak" echo "Picocli updated to version: $VERSION" \ No newline at end of file diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java b/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java index daa1855..fffff82 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java @@ -43,7 +43,7 @@ abstract class AbstractRemoteScanCommand implements Callable { */ @Override public Integer call() { - val parentExecOptions = parentCommand.buildExecOptions(); + val parentExecOptions = parentCommand.buildExecOptions(); // TODO: NPE ?? val execOptionsBuilder = parentExecOptions.toBuilder(); val configFileProvided = Optional.ofNullable(parentCommand.spec) .map(CommandLine.Model.CommandSpec::commandLine) @@ -52,21 +52,14 @@ public Integer call() { .isPresent(); val remoteRef = validateAndParseRemoteRef(); execOptionsBuilder.remoteRef(remoteRef); + val repositoryFileReader = createRepositoryFileReader(remoteRef); + execOptionsBuilder.repositoryFileReader(repositoryFileReader); if (StringUtils.equals(SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME, parentExecOptions.getConfigurationFileLocation()) && !configFileProvided) { - execOptionsBuilder.configurationFileLocation(constructRemoteConfigFileLocation(remoteRef)); + execOptionsBuilder.configurationFileLocation(repositoryFileReader.getAbsoluteLocation(SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME)); } - execOptionsBuilder.repositoryFileReader(createRepositoryFileReader(remoteRef)).build(); return parentCommand.call(execOptionsBuilder.build()); } - /** - * Construct the remote config file location - * - * @param remoteRef the remote reference - * @return the config file remote location - */ - protected abstract String constructRemoteConfigFileLocation(final RemoteRef remoteRef); - /** * Create the repository file reader based off the remote reference * @@ -76,11 +69,11 @@ public Integer call() { protected abstract RepositoryFileReader createRepositoryFileReader(final RemoteRef remoteRef); /** - * Get the remote reference descriptor + * Get the raw remote reference * - * @return the raw remote reference descriptor + * @return the raw remote reference */ - protected abstract Pair getRawRemoteReference(); + protected abstract Pair getRawRemoteReference(); /** * Parse the coordinates to github options diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/BitbucketScanCommand.java b/cli/src/main/java/com/optum/sourcehawk/cli/BitbucketScanCommand.java index 30c254b..c4caf44 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/BitbucketScanCommand.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/BitbucketScanCommand.java @@ -1,14 +1,15 @@ package com.optum.sourcehawk.cli; -import com.optum.sourcehawk.core.constants.SourcehawkConstants; import com.optum.sourcehawk.core.data.Pair; import com.optum.sourcehawk.core.data.RemoteRef; -import com.optum.sourcehawk.core.repository.BitbucketRepositoryFileReader; +import com.optum.sourcehawk.core.repository.RemoteRepositoryFileReader; import com.optum.sourcehawk.core.repository.RepositoryFileReader; import lombok.val; import picocli.CommandLine; -import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; import java.util.Optional; /** @@ -25,6 +26,9 @@ ) public class BitbucketScanCommand extends AbstractRemoteScanCommand { + private static final String DEFAULT_BASE_URL = "https://bitbucket.org"; + private static final String DEFAULT_REF = "main"; + /** * The Bitbucket options */ @@ -44,25 +48,24 @@ public static void main(final String... args) { /** {@inheritDoc} */ @Override protected RepositoryFileReader createRepositoryFileReader(final RemoteRef remoteRef) { - if (bitbucket.serverUrl != null) { - return new BitbucketRepositoryFileReader(bitbucket.token, bitbucket.serverUrl.toString(), remoteRef); + val rawFileUrlTemplate = Optional.ofNullable(bitbucket.serverUrl) + .map(bitbucketServerUrl -> String.format("%s/rest/api/1.0/projects/%s/repos/%s/raw/%%s?at=%s", + bitbucketServerUrl, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef())) + .orElseGet(() -> String.format("%s/api/2.0/repositories/%s/%s/src/%s/%%s", DEFAULT_BASE_URL, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef())); + val requestProperties = new HashMap(); + requestProperties.put("Accept", "text/plain"); + if (bitbucket.token != null) { + val authScheme = Optional.ofNullable(bitbucket.authScheme) + .orElse(CommandOptions.Bitbucket.DEFAULT_AUTH_SCHEME); + requestProperties.put("Authorization", String.format("%s %s", authScheme, bitbucket.token)); } - return new BitbucketRepositoryFileReader(bitbucket.token, remoteRef); - } - - /** {@inheritDoc} */ - @Override - protected Pair getRawRemoteReference() { - return Pair.of(RemoteRef.Type.BITBUCKET, bitbucket.remoteReference); + return new RemoteRepositoryFileReader(rawFileUrlTemplate, requestProperties); } /** {@inheritDoc} */ @Override - protected String constructRemoteConfigFileLocation(final RemoteRef remoteRef) { - val bitbucketBaseUrl = Optional.ofNullable(bitbucket.serverUrl) - .map(URL::toString) - .orElseGet(RemoteRef.Type.BITBUCKET::getBaseUrl); - return BitbucketRepositoryFileReader.constructBaseUrl(remoteRef, bitbucketBaseUrl) + SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME; + protected Pair getRawRemoteReference() { + return Pair.of(bitbucket.remoteReference, DEFAULT_REF); } } diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/CommandOptions.java b/cli/src/main/java/com/optum/sourcehawk/cli/CommandOptions.java index ceba6c5..d25f175 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/CommandOptions.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/CommandOptions.java @@ -152,6 +152,15 @@ static class Bitbucket { ) String token; + @CommandLine.Option( + names = {"-a", "--auth-scheme"}, + paramLabel = "auth-scheme", + defaultValue = DEFAULT_AUTH_SCHEME, + description = "The authorization scheme to use (either Bearer or Basic). If Basic, the provided token must be base64 encoded." + ) + String authScheme; + static final String DEFAULT_AUTH_SCHEME = "Bearer"; + @CommandLine.Option( names = {"-S", "--server-url"}, paramLabel = "bitbucket-server-url", @@ -162,7 +171,7 @@ static class Bitbucket { @CommandLine.Parameters( paramLabel = REMOTE_REFERENCE_LABEL, description = "The Bitbucket remote reference - project/repo@ref combination, " - + "i.e - project/repo, project/repo@master, project/repo@v1.4, or project/repo@a6de43fa51c", + + "i.e - project/repo, project/repo@main, project/repo@v1.4, or project/repo@a6de43fa51c", arity = "1" ) String remoteReference; diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/GithubScanCommand.java b/cli/src/main/java/com/optum/sourcehawk/cli/GithubScanCommand.java index 784cf1f..b233902 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/GithubScanCommand.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/GithubScanCommand.java @@ -1,14 +1,13 @@ package com.optum.sourcehawk.cli; -import com.optum.sourcehawk.core.constants.SourcehawkConstants; import com.optum.sourcehawk.core.data.Pair; import com.optum.sourcehawk.core.data.RemoteRef; -import com.optum.sourcehawk.core.repository.GithubRepositoryFileReader; +import com.optum.sourcehawk.core.repository.RemoteRepositoryFileReader; import com.optum.sourcehawk.core.repository.RepositoryFileReader; import lombok.val; import picocli.CommandLine; -import java.net.URL; +import java.util.HashMap; import java.util.Optional; /** @@ -25,6 +24,10 @@ ) public class GithubScanCommand extends AbstractRemoteScanCommand { + private static final String DEFAULT_BASE_URL = "raw.githubusercontent.com"; + private static final String DEFAULT_REF = "main"; + private static final String AUTHORIZATION_TOKEN_PREFIX = "Bearer"; + /** * The github options */ @@ -44,28 +47,22 @@ public static void main(final String... args) { /** {@inheritDoc} */ @Override protected RepositoryFileReader createRepositoryFileReader(final RemoteRef remoteRef) { - if (github.enterpriseUrl != null) { - return new GithubRepositoryFileReader(github.token, github.enterpriseUrl.toString(), remoteRef); + val baseUrl = Optional.ofNullable(github.enterpriseUrl) + .map(githubEnterpriseUrl -> String.format("%s/raw", github.enterpriseUrl)) + .orElse(DEFAULT_BASE_URL); + val rawFileUrlTemplate = String.format("%s/%s/%s/%s/%%s", baseUrl, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef()); + val requestProperties = new HashMap(); + requestProperties.put("Accept", "text/plain"); + if (github.token != null) { + requestProperties.put("Authorization", String.format("%s %s", AUTHORIZATION_TOKEN_PREFIX, github.token)); } - return new GithubRepositoryFileReader(github.token, remoteRef); - } - - /** {@inheritDoc} */ - @Override - protected Pair getRawRemoteReference() { - return Pair.of(RemoteRef.Type.GITHUB, github.remoteReference); + return new RemoteRepositoryFileReader(rawFileUrlTemplate, requestProperties); } /** {@inheritDoc} */ @Override - protected String constructRemoteConfigFileLocation(final RemoteRef remoteRef) { - val githubEnterpriseUrl = Optional.ofNullable(github.enterpriseUrl).map(URL::toString); - val githubRepoBaseUrl = GithubRepositoryFileReader.constructBaseUrl( - githubEnterpriseUrl.orElseGet(RemoteRef.Type.GITHUB::getBaseUrl), - githubEnterpriseUrl.isPresent(), - remoteRef - ); - return githubRepoBaseUrl + SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME; + protected Pair getRawRemoteReference() { + return Pair.of(github.remoteReference, DEFAULT_REF); } } diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/Sourcehawk.java b/cli/src/main/java/com/optum/sourcehawk/cli/Sourcehawk.java index 28f1e86..8840dab 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/Sourcehawk.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/Sourcehawk.java @@ -22,7 +22,7 @@ headerHeading = "@|fg(magenta) >_ S O U R C E H A W K|@", synopsisHeading = "%n", commandListHeading ="%nCommands:%n", - footer = "Copyright (c) 2020 Optum", + footer = "Copyright (c) 2022 Optum", versionProvider = Sourcehawk.VersionProvider.class, subcommands = { CommandLine.HelpCommand.class, diff --git a/cli/src/main/java/picocli/CommandLine.java b/cli/src/main/java/picocli/CommandLine.java index 636662b..07a29c9 100644 --- a/cli/src/main/java/picocli/CommandLine.java +++ b/cli/src/main/java/picocli/CommandLine.java @@ -72,7 +72,7 @@ * * // CheckSum implements Callable, so parsing, error handling and handling user * // requests for usage help or version help can be done with one line of code. - * public static void main(String[] args) throws Exception { + * public static void main(String[] args) { * int exitCode = new CommandLine(new CheckSum()).execute(args); * System.exit(exitCode); * } @@ -146,7 +146,7 @@ public class CommandLine { /** This is picocli version {@value}. */ - public static final String VERSION = "4.6.1"; + public static final String VERSION = "4.6.2"; private final Tracer tracer = new Tracer(); private CommandSpec commandSpec; @@ -1207,7 +1207,7 @@ public CommandLine setColorScheme(Help.ColorScheme colorScheme) { * help with a {@code --help} or similar option, the usage help message is printed to the standard output stream so that it can be easily searched and paged.

* @since 4.0 */ public PrintWriter getOut() { - if (out == null) { setOut(new PrintWriter(System.out, true)); } + if (out == null) { setOut(newPrintWriter(System.out, getStdoutEncoding())); } return out; } @@ -1234,7 +1234,7 @@ public CommandLine setOut(PrintWriter out) { * should use this writer to print error messages (which may include a usage help message) when an unexpected error occurs.

* @since 4.0 */ public PrintWriter getErr() { - if (err == null) { setErr(new PrintWriter(System.err, true)); } + if (err == null) { setErr(newPrintWriter(System.err, getStderrEncoding())); } return err; } @@ -1808,7 +1808,7 @@ protected R throwOrExit(ExecutionException ex) { * @since 2.0 */ @Deprecated public static class DefaultExceptionHandler extends AbstractHandler> implements IExceptionHandler, IExceptionHandler2 { public List handleException(ParameterException ex, PrintStream out, Help.Ansi ansi, String... args) { - internalHandleParseException(ex, new PrintWriter(out, true), Help.defaultColorScheme(ansi)); return Collections.emptyList(); } + internalHandleParseException(ex, newPrintWriter(out, getStdoutEncoding()), Help.defaultColorScheme(ansi)); return Collections.emptyList(); } /** Prints the message of the specified exception, followed by the usage message for the command or subcommand * whose input was invalid, to the stream returned by {@link #err()}. @@ -1818,7 +1818,7 @@ public List handleException(ParameterException ex, PrintStream out, Help * @return the empty list * @since 3.0 */ public R handleParseException(ParameterException ex, String[] args) { - internalHandleParseException(ex, new PrintWriter(err(), true), colorScheme()); return returnResultOrExit(null); } + internalHandleParseException(ex, newPrintWriter(err(), getStderrEncoding()), colorScheme()); return returnResultOrExit(null); } static void internalHandleParseException(ParameterException ex, PrintWriter writer, Help.ColorScheme colorScheme) { writer.println(colorScheme.errorText(ex.getMessage())); @@ -1879,7 +1879,7 @@ public static boolean printHelpIfRequested(ParseResult parseResult) { * @since 3.6 */ @Deprecated public static boolean printHelpIfRequested(List parsedCommands, PrintStream out, PrintStream err, Help.ColorScheme colorScheme) { // for backwards compatibility - for (CommandLine cmd : parsedCommands) { cmd.setOut(new PrintWriter(out, true)).setErr(new PrintWriter(err, true)).setColorScheme(colorScheme); } + for (CommandLine cmd : parsedCommands) { cmd.setOut(newPrintWriter(out, getStdoutEncoding())).setErr(newPrintWriter(err, getStderrEncoding())).setColorScheme(colorScheme); } return executeHelpRequest(parsedCommands) != null; } @@ -2135,8 +2135,8 @@ private T enrichForBackwardsCompatibility(T obj) { // and the application called #useOut, #useErr or #useAnsi on it if (obj instanceof AbstractHandler) { AbstractHandler handler = (AbstractHandler) obj; - if (handler.out() != System.out) { setOut(new PrintWriter(handler.out(), true)); } - if (handler.err() != System.err) { setErr(new PrintWriter(handler.err(), true)); } + if (handler.out() != System.out) { setOut(newPrintWriter(handler.out(), getStdoutEncoding())); } + if (handler.err() != System.err) { setErr(newPrintWriter(handler.err(), getStderrEncoding())); } if (handler.ansi() != Help.Ansi.AUTO) { setColorScheme(handler.colorScheme()); } } return obj; @@ -2229,6 +2229,9 @@ private int resolveExitCode(int exitCodeOnSuccess, R executionResult, List> implements IParseResultHandler { + /** {@inheritDoc} */ + public int execute(ParseResult parseResult) throws ExecutionException { return super.execute(parseResult); } + /** Prints help if requested, and otherwise executes the top-level {@code Runnable} or {@code Callable} command. * Finally, either a list of result objects is returned, or the JVM is terminated if an exit code {@linkplain #andExit(int) was set}. * If the top-level command does not implement either {@code Runnable} or {@code Callable}, an {@code ExecutionException} @@ -2310,6 +2313,9 @@ protected List extractExitCodeGenerators(ParseResult parseRe *

* @since 2.0 */ public static class RunLast extends AbstractParseResultHandler> implements IParseResultHandler { + /** {@inheritDoc} */ + public int execute(ParseResult parseResult) throws ExecutionException { return super.execute(parseResult); } + /** Prints help if requested, and otherwise executes the most specific {@code Runnable} or {@code Callable} subcommand. *

For {@linkplain Command#subcommandsRepeatable() repeatable subcommands}, this method * may execute multiple subcommands: the most deeply nested subcommands that have the same parent command.

@@ -2382,6 +2388,9 @@ protected List extractExitCodeGenerators(ParseResult parseRe * For use by the {@link #execute(String...) execute} method. * @since 2.0 */ public static class RunAll extends AbstractParseResultHandler> implements IParseResultHandler { + /** {@inheritDoc} */ + public int execute(ParseResult parseResult) throws ExecutionException { return super.execute(parseResult); } + /** Prints help if requested, and otherwise executes the top-level command and all subcommands as {@code Runnable}, * {@code Callable} or {@code Method}. Finally, either a list of result objects is returned, or the JVM is terminated if an exit * code {@linkplain #andExit(int) was set}. If any of the {@code CommandLine} commands does not implement either @@ -3648,22 +3657,33 @@ public enum ScopeType { boolean required() default false; /** - * Set {@code help=true} if this option should disable validation of the remaining arguments: - * If the {@code help} option is specified, no error message is generated for missing required options. - *

- * This attribute is useful for special options like help ({@code -h} and {@code --help} on unix, - * {@code -?} and {@code -Help} on Windows) or version ({@code -V} and {@code --version} on unix, - * {@code -Version} on Windows). + *

This should rarely be used: the recommended attributes are {@link #usageHelp() usageHelp} and {@link #versionHelp() versionHelp}. + *

+ * Only set {@code help=true} when this option should disable validation of the remaining + * arguments, and no error message should be generated for missing required options. + *

+ * This is useful for custom help options that are in addition to the standard help and + * version options. For example if your application has many hidden options or + * subcommands, and there is a custom help option like {@code --detailed-help} that prints + * the usage help message for these hidden options and subcommands. *

+ *

Note:

*

- * Note that the {@link #parse(String...)} method will not print help documentation. It will only set - * the value of the annotated field. It is the responsibility of the caller to inspect the annotated fields - * and take the appropriate action. + * Use the {@link #usageHelp() usageHelp} for "normal" help options (like {@code -h} and {@code --help} on unix, + * {@code -?} and {@code -Help} on Windows) + * and use {@link #versionHelp() versionHelp} for "normal" version help ({@code -V} and {@code --version} on unix, + * {@code -Version} on Windows): + * picocli has built-in logic so that options with {@code usageHelp=true} or {@code versionHelp=true} + * will automatically cause the requested help message to be printed in applications + * that use the {@link #execute(String...)} method, without any code in the application. + *

+ * Note that there is no such automatic help printing for options with {@code help=true}; + * applications need to check whether the end user specified this option and take appropriate action + * in the business logic of the application. *

* @return whether this option disables validation of the other arguments - * @deprecated Use {@link #usageHelp()} and {@link #versionHelp()} instead. See {@link #printHelpIfRequested(List, PrintStream, CommandLine.Help.Ansi)} */ - @Deprecated boolean help() default false; + boolean help() default false; /** * Set {@code usageHelp=true} for the {@code --help} option that triggers display of the usage help message. @@ -3735,7 +3755,7 @@ public enum ScopeType { * command line, a {@link MissingParameterException} is thrown by the {@link #parse(String...)} method. *

* In many cases picocli can deduce the number of required parameters from the field's type. - * By default, flags (boolean options) have arity zero, + * By default, flags (boolean options) have arity "0..1", * and single-valued type fields (String, int, Integer, double, Double, File, Date, etc) have arity one. * Generally, fields with types that cannot hold multiple values can omit the {@code arity} attribute. *

@@ -3747,7 +3767,8 @@ public enum ScopeType { *

* A note on boolean options *

- * By default picocli does not expect boolean options (also called "flags" or "switches") to have a parameter. + * By default picocli allows boolean options (also called "flags" or "switches") to have an optional parameter, + * which must be either "true" or "false" (lowercase, other values are rejected). * You can make a boolean option take a required parameter by annotating your field with {@code arity="1"}. * For example:

*
@Option(names = "-v", arity = "1") boolean verbose;
@@ -3757,12 +3778,11 @@ public enum ScopeType { * on the command line, or a {@link MissingParameterException} is thrown by the {@link #parse(String...)} * method. *

- * To make the boolean parameter possible but optional, define the field with {@code arity = "0..1"}. + * To remove the optional parameter, define the field with {@code arity = "0"}. * For example:

- *
@Option(names="-v", arity="0..1") boolean verbose;
- *

This will accept any of the below without throwing an exception:

+ *
@Option(names="-v", arity="0") boolean verbose;
+ *

This will reject any of the below:

*
-         * -v
          * -v true
          * -v false
          * 
@@ -4974,6 +4994,8 @@ private static class NoDefaultProvider implements IDefaultValueProvider { * } * } * } + *

If this interface does not meet your requirements, you may have a look at the more powerful + * and flexible {@link IParameterPreprocessor} interface introduced with picocli 4.6.

* @see Option#parameterConsumer() * @see Parameters#parameterConsumer() * @since 4.0 */ @@ -5602,9 +5624,14 @@ static Range adjustForType(Range result, IAnnotatedElement member) { return result.isUnspecified ? defaultArity(member) : result; } /** Returns the default arity {@code Range}: for interactive options/positional parameters, - * this is 0; for {@link Option options} this is 0 for booleans and 1 for - * other types, for {@link Parameters parameters} booleans have arity 0, arrays or Collections have + * this is 0; for {@link Option options} this is effectively "0..1" for booleans and 1 for + * other types, for {@link Parameters parameters} booleans have arity 1, arrays or Collections have * arity "0..*", and other types have arity 1. + *

Implementation Notes

+ *

The returned {@code Range} for boolean options has an effective arity of "0..1". + * This is implemented by returning a {@code Range} with arity "0", + * and its {@code unspecified} property set to {@code true}. + * This implementation may change in the future.

* @param field the field whose default arity to return * @return a new {@code Range} indicating the default arity of the specified field * @since 2.0 */ @@ -6386,7 +6413,7 @@ public CommandSpec addSubcommand(String name, CommandSpec subcommand) { */ public CommandSpec addSubcommand(String name, CommandLine subCommandLine) { CommandSpec subSpec = subCommandLine.getCommandSpec(); - String actualName = validateSubcommandName(name, subSpec); + String actualName = validateSubcommandName(interpolator.interpolateCommandName(name), subSpec); Tracer t = new Tracer(); if (t.isDebug()) {t.debug("Adding subcommand '%s' to '%s'%n", actualName, this.qualifiedName());} String previousName = commands.getCaseSensitiveKey(actualName); @@ -6396,7 +6423,7 @@ public CommandSpec addSubcommand(String name, CommandLine subCommandLine) { subSpec.parent(this); for (String alias : subSpec.aliases()) { if (t.isDebug()) {t.debug("Adding alias '%s' for '%s'%n", (parent == null ? "" : parent.qualifiedName() + " ") + alias, this.qualifiedName());} - previous = commands.put(alias, subCommandLine); + previous = commands.put(interpolator.interpolate(alias), subCommandLine); if (previous != null && previous != subCommandLine) { throw new DuplicateNameException("Alias '" + alias + "' for subcommand '" + actualName + "' is already used by another subcommand of '" + this.name() + "'"); } } subSpec.initCommandHierarchyWithResourceBundle(resourceBundleBaseName(), resourceBundle()); @@ -6424,8 +6451,8 @@ private void inheritAttributesFrom(CommandSpec root) { updatedSubcommandsToInheritFrom(root); } private void updatedSubcommandsToInheritFrom(CommandSpec root) { - if (root != this) { - mixinStandardHelpOptions(root.mixinStandardHelpOptions()); + if (root != this && root.mixinStandardHelpOptions()) { // #1331 only add, don't remove + mixinStandardHelpOptions(true); } Set subcommands = new HashSet(subcommands().values()); for (CommandLine sub : subcommands) { @@ -6702,6 +6729,7 @@ public CommandSpec remove(ArgSpec arg) { if (positionalParameters.remove(arg)) { removed++; } + args.remove(arg); if (removed == 0) { throw new NoSuchElementException(String.valueOf(arg)); } @@ -6936,7 +6964,8 @@ public Set names() { public List args() { return Collections.unmodifiableList(args); } Object[] commandMethodParamValues() { Object[] values = new Object[methodParams.length]; - int argIndex = mixins.containsKey(AutoHelpMixin.KEY) ? 2 : 0; + CommandSpec autoHelpMixin = mixins.get(AutoHelpMixin.KEY); + int argIndex = autoHelpMixin == null || autoHelpMixin.inherited() ? 0 : 2; for (int i = 0; i < methodParams.length; i++) { if (methodParams[i].isAnnotationPresent(Mixin.class)) { String name = methodParams[i].getAnnotation(Mixin.class).name(); @@ -7155,7 +7184,20 @@ public CommandSpec negatableOptionTransformer(INegatableOptionTransformer newVal public CommandSpec mixinStandardHelpOptions(boolean newValue) { if (newValue) { CommandSpec mixin = CommandSpec.forAnnotatedObject(new AutoHelpMixin(), new DefaultFactory()); - addMixin(AutoHelpMixin.KEY, mixin); + boolean overlap = false; + for (String key : mixin.optionsMap().keySet()) { + if (optionsMap().containsKey(key)) { overlap = true; break; } + } + if (!overlap) { // #1316, 1319 avoid DuplicateOptionAnnotationsException + mixin.inherited = this.inherited(); + addMixin(AutoHelpMixin.KEY, mixin); + } + // #1331 if inherit(ed) we also add to subcommands + if (scopeType() == ScopeType.INHERIT || inherited()) { + for (CommandLine sub : new HashSet(subcommands().values())) { + sub.getCommandSpec().mixinStandardHelpOptions(newValue); + } + } } else { CommandSpec helpMixin = mixins.remove(AutoHelpMixin.KEY); if (helpMixin != null) { @@ -7167,11 +7209,7 @@ public CommandSpec mixinStandardHelpOptions(boolean newValue) { } } } - } - if (scopeType() == ScopeType.INHERIT || inherited()) { - for (CommandLine sub : new HashSet(subcommands().values())) { - sub.getCommandSpec().mixinStandardHelpOptions(newValue); - } + // #1331 we don't remove StandardHelpOptions from subcommands, even if they inherit from us } return this; } @@ -8669,10 +8707,20 @@ public String[] description() { private String[] expandVariables(String[] desc) { if (desc.length == 0) { return desc; } StringBuilder candidates = new StringBuilder(); - if (completionCandidates() != null) { - for (String c : completionCandidates()) { - if (candidates.length() > 0) { candidates.append(", "); } - candidates.append(c); + boolean isCompletionCandidatesUsed = false; + for (String s: desc) { + if (s.contains(DESCRIPTION_VARIABLE_COMPLETION_CANDIDATES)) { + isCompletionCandidatesUsed = true; + break; + } + } + if (isCompletionCandidatesUsed) { + Iterable iter = completionCandidates(); + if (iter != null) { + for (String c : iter) { + if (candidates.length() > 0) { candidates.append(", "); } + candidates.append(c); + } } } String defaultValueString = defaultValueString(false); // interpolate later @@ -8740,7 +8788,7 @@ private String[] expandVariables(String[] desc) { /** Returns the root option or positional parameter (on the parent command), if this option or positional parameter was inherited; * or {@code null} if it was not. * @see Option#scope() - * @since 4.6.1 */ + * @since 4.6.2 */ public ArgSpec root() { return root; } /** Returns the type to convert the option or positional parameter to before {@linkplain #setValue(Object) setting} the value. @@ -8787,8 +8835,10 @@ public String mapFallbackValue() { public Object initialValue() { // not not initialize if already CACHED, or UNAVAILABLE, or if annotatedElement==null if (initialValueState == InitialValueState.POSTPONED && annotatedElement != null) { - try { initialValue = annotatedElement.getter().get(); } catch (Exception ex) { } - initialValueState = InitialValueState.CACHED; + try { + initialValue = annotatedElement.getter().get(); + initialValueState = InitialValueState.CACHED; // only if successfully initialized + } catch (Exception ex) { } // #1300 if error: keep initialValueState == POSTPONED } return initialValue; } @@ -9376,7 +9426,7 @@ private static String inferLabel(String label, String fieldName, ITypeInfo typeI /** Returns the root option or positional parameter (on the parent command), if this option or positional parameter was inherited; * or {@code null} if it was not. * @see Option#scope() - * @since 4.6.1 */ + * @since 4.6.2 */ public ArgSpec root() { return root; } /** Returns the type to convert the option or positional parameter to before {@linkplain #setValue(Object) setting} the value. @@ -9519,7 +9569,7 @@ private static String inferLabel(String label, String fieldName, ITypeInfo typeI /** * Sets the root object for this inherited option, and returns this builder. - * @since 4.6.1 */ + * @since 4.6.2 */ public T root(ArgSpec root) { this.root = root ; return self(); } /** Sets the type to convert the option or positional parameter to before {@linkplain #setValue(Object) setting} the value, and returns this builder. @@ -13264,12 +13314,14 @@ private void processArguments(List parsedCommands, if (commandSpec.parent() != null && commandSpec.parent().subcommandsRepeatable() && commandSpec.parent().subcommands().containsKey(arg)) { tracer.debug("'%s' is a repeatable subcommand of %s%n", arg, commandSpec.parent().qualifiedName());// #454 repeatable subcommands CommandLine subcommand = commandSpec.parent().subcommands().get(arg); + Set inheritedInitialized = initialized; if (subcommand.interpreter.parseResultBuilder != null) { tracer.debug("Subcommand '%s' has been matched before. Making a copy...%n", subcommand.getCommandName()); subcommand = subcommand.copy(); subcommand.getCommandSpec().parent(commandSpec.parent()); // hook it up with its parent + inheritedInitialized = new LinkedHashSet(inheritedInitialized); } - processSubcommand(subcommand, getParent().interpreter.parseResultBuilder, parsedCommands, args, required, initialized, originalArgs, nowProcessing, separator, arg); + processSubcommand(subcommand, getParent().interpreter.parseResultBuilder, parsedCommands, args, required, inheritedInitialized, originalArgs, nowProcessing, separator, arg); continue; } @@ -14387,7 +14439,9 @@ private static String optionDescription(String prefix, ArgSpec argSpec, int opti if (argSpec.arity().max > 1) { desc += " at index " + optionParamIndex; } - desc += " (" + argSpec.paramLabel() + ")"; + if (argSpec.arity().max > 0) { + desc += " (" + argSpec.paramLabel() + ")"; + } } } else { desc = prefix + "positional parameter at index " + ((PositionalParamSpec) argSpec).index() + " (" + argSpec.paramLabel() + ")"; @@ -14469,6 +14523,17 @@ static void close(Closeable closeable) { new Tracer().warn("Could not close " + closeable + ": " + ex.toString()); } } + static Charset getStdoutEncoding() { + String encoding = System.getProperty("sun.stdout.encoding"); + return encoding != null ? Charset.forName(encoding) : Charset.defaultCharset(); + } + static Charset getStderrEncoding() { + String encoding = System.getProperty("sun.stderr.encoding"); + return encoding != null ? Charset.forName(encoding) : Charset.defaultCharset(); + } + static PrintWriter newPrintWriter(OutputStream stream, Charset charset) { + return new PrintWriter(new BufferedWriter(new OutputStreamWriter(stream, charset)), true); + } static class PositionalParametersSorter implements Comparator { private static final Range OPTION_INDEX = new Range(0, 0, false, true, "0"); public int compare(ArgSpec p1, ArgSpec p2) { @@ -18008,7 +18073,7 @@ static List stripErrorMessage(List unmatched) { public boolean isUnknownOption() { return isUnknownOption(unmatched, getCommandLine()); } /** Returns {@code true} and prints suggested solutions to the specified stream if such solutions exist, otherwise returns {@code false}. * @since 3.3.0 */ - public boolean printSuggestions(PrintStream out) { return printSuggestions(new PrintWriter(out, true)); } + public boolean printSuggestions(PrintStream out) { return printSuggestions(newPrintWriter(out, getStdoutEncoding())); } /** Returns {@code true} and prints suggested solutions to the specified stream if such solutions exist, otherwise returns {@code false}. * @since 4.0 */ public boolean printSuggestions(PrintWriter writer) { diff --git a/cli/src/main/resources/META-INF/native-image/sourcehawk-cli/native-image.properties b/cli/src/main/resources/META-INF/native-image/sourcehawk-cli/native-image.properties index 10ea548..57ad3db 100644 --- a/cli/src/main/resources/META-INF/native-image/sourcehawk-cli/native-image.properties +++ b/cli/src/main/resources/META-INF/native-image/sourcehawk-cli/native-image.properties @@ -1,3 +1,2 @@ Args = -H:Class=${cli.class} \ - -H:Name=${cli.name} \ - --no-server \ No newline at end of file + -H:Name=${cli.name} \ No newline at end of file diff --git a/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanCommandSpec.groovy b/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanCommandSpec.groovy index 4bb99e7..249773b 100644 --- a/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanCommandSpec.groovy +++ b/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanCommandSpec.groovy @@ -10,11 +10,11 @@ class BitbucketScanCommandSpec extends CliBaseSpecification { def "getRawRemoteReference"() { when: - Pair rawRemoteReference = new BitbucketScanCommand(bitbucket: new CommandOptions.Bitbucket(remoteReference: "owner/repo@master")).getRawRemoteReference() + Pair rawRemoteReference = new BitbucketScanCommand(bitbucket: new CommandOptions.Bitbucket(remoteReference: "owner/repo@master")).getRawRemoteReference() then: - rawRemoteReference.left == RemoteRef.Type.BITBUCKET - rawRemoteReference.right == "owner/repo@master" + rawRemoteReference.left == "owner/repo@master" + rawRemoteReference.right == "main" } @Unroll @@ -63,16 +63,16 @@ class BitbucketScanCommandSpec extends CliBaseSpecification { def "createRepositoryFileReader"() { given: String rawReference = "project/repo@master" - BitbucketScanCommand githubScanCommand = new BitbucketScanCommand(bitbucket: new CommandOptions.Bitbucket(remoteReference: "owner/repo@main")) + BitbucketScanCommand bitbucketScanCommand = new BitbucketScanCommand(bitbucket: new CommandOptions.Bitbucket(remoteReference: "owner/repo@main")) when: - RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(RemoteRef.Type.BITBUCKET, rawReference)) + RepositoryFileReader repositoryFileReader = bitbucketScanCommand.createRepositoryFileReader(RemoteRef.parse(rawReference, "main")) then: repositoryFileReader } - def "createRepositoryFileReader - enterprise"() { + def "createRepositoryFileReader - server"() { given: String rawReference = "project/repo@master" BitbucketScanCommand githubScanCommand = new BitbucketScanCommand(bitbucket: @@ -80,7 +80,7 @@ class BitbucketScanCommandSpec extends CliBaseSpecification { ) when: - RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(RemoteRef.Type.BITBUCKET, rawReference)) + RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(rawReference, "main")) then: repositoryFileReader diff --git a/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanSubCommandSpec.groovy b/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanSubCommandSpec.groovy index 04f639d..b17a9d7 100644 --- a/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanSubCommandSpec.groovy +++ b/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanSubCommandSpec.groovy @@ -37,13 +37,15 @@ class BitbucketScanSubCommandSpec extends Specification { clientAndServer .when(HttpRequest.request() .withMethod("HEAD") - .withPath("/project/repo/raw/master/lombok.config"), + .withPath("/rest/api/1.0/projects/project/repos/repo/raw/lombok.config") + .withQueryStringParameter("at", "main"), Times.exactly(1)) .respond(HttpResponse.response().withStatusCode(200)) clientAndServer .when(HttpRequest.request() .withMethod("GET") - .withPath("/project/repo/raw/master/lombok.config") + .withPath("/rest/api/1.0/projects/project/repos/repo/raw/lombok.config") + .withQueryStringParameter("at", "main") .withHeader("Accept", "text/plain"), Times.exactly(2)) .respond(HttpResponse.response() @@ -65,13 +67,15 @@ class BitbucketScanSubCommandSpec extends Specification { clientAndServer .when(HttpRequest.request() .withMethod("HEAD") - .withPath("/project/repo/raw/develop/lombok2.config"), + .withPath("/rest/api/1.0/projects/project/repos/repo/raw/lombok2.config") + .withQueryStringParameter("at", "main"), Times.exactly(1)) .respond(HttpResponse.response().withStatusCode(200)) clientAndServer .when(HttpRequest.request() .withMethod("GET") - .withPath("/project/repo/raw/develop/lombok2.config") + .withPath("/rest/api/1.0/projects/project/repos/repo/raw/lombok2.config") + .withQueryStringParameter("at", "main") .withHeader("Accept", "text/plain"), Times.exactly(2)) .respond(HttpResponse.response() @@ -92,9 +96,9 @@ class BitbucketScanSubCommandSpec extends Specification { String[] args = ["bitbucket", "-S", bitbucketServerUrl, "project/repo@develop" ] clientAndServer .when(HttpRequest.request() - .withMethod("GET") - .withPath("/project/repo/raw/develop/sourcehawk.yml") - .withHeader("Accept", "text/plain"), + .withMethod("HEAD") + .withPath("/rest/api/1.0/projects/project/repos/repo/raw/lombok.config") + .withQueryStringParameter("at", "main"), Times.exactly(1)) .respond(HttpResponse.notFoundResponse()) diff --git a/cli/src/test/groovy/com/optum/sourcehawk/cli/CliBaseSpecification.groovy b/cli/src/test/groovy/com/optum/sourcehawk/cli/CliBaseSpecification.groovy index fafc98c..690f599 100644 --- a/cli/src/test/groovy/com/optum/sourcehawk/cli/CliBaseSpecification.groovy +++ b/cli/src/test/groovy/com/optum/sourcehawk/cli/CliBaseSpecification.groovy @@ -31,7 +31,7 @@ class CliBaseSpecification extends Specification { } def cleanupSpec() { - System.setSecurityManager(defaultSecurityManager) + System.setSecurityManager(defaultSecurityManager) // TODO: Deprecated starting in JDK 17 } protected void createParentDirectories(final File child) { diff --git a/cli/src/test/groovy/com/optum/sourcehawk/cli/GithubScanCommandSpec.groovy b/cli/src/test/groovy/com/optum/sourcehawk/cli/GithubScanCommandSpec.groovy index 87cfe86..6c84afe 100644 --- a/cli/src/test/groovy/com/optum/sourcehawk/cli/GithubScanCommandSpec.groovy +++ b/cli/src/test/groovy/com/optum/sourcehawk/cli/GithubScanCommandSpec.groovy @@ -10,11 +10,11 @@ class GithubScanCommandSpec extends CliBaseSpecification { def "getRawRemoteReference"() { when: - Pair rawRemoteReference = new GithubScanCommand(github: new CommandOptions.Github(remoteReference: "owner/repo@main")).getRawRemoteReference() + Pair rawRemoteReference = new GithubScanCommand(github: new CommandOptions.Github(remoteReference: "owner/repo@main")).getRawRemoteReference() then: - rawRemoteReference.left == RemoteRef.Type.GITHUB - rawRemoteReference.right == "owner/repo@main" + rawRemoteReference.left == "owner/repo@main" + rawRemoteReference.right == "main" } @Unroll @@ -66,7 +66,7 @@ class GithubScanCommandSpec extends CliBaseSpecification { GithubScanCommand githubScanCommand = new GithubScanCommand(github: new CommandOptions.Github(remoteReference: "owner/repo@main")) when: - RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(RemoteRef.Type.GITHUB, rawReference)) + RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(rawReference, "main")) then: repositoryFileReader @@ -80,7 +80,7 @@ class GithubScanCommandSpec extends CliBaseSpecification { ) when: - RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(RemoteRef.Type.GITHUB, rawReference)) + RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(rawReference, "main")) then: repositoryFileReader diff --git a/cli/src/test/groovy/com/optum/sourcehawk/cli/NativeImageConfigSpec.groovy b/cli/src/test/groovy/com/optum/sourcehawk/cli/NativeImageConfigSpec.groovy index e657673..85ad9bc 100644 --- a/cli/src/test/groovy/com/optum/sourcehawk/cli/NativeImageConfigSpec.groovy +++ b/cli/src/test/groovy/com/optum/sourcehawk/cli/NativeImageConfigSpec.groovy @@ -19,7 +19,6 @@ class NativeImageConfigSpec extends Specification { then: args.contains("-H:Class=${Sourcehawk.name}") args.contains("-H:Name=sourcehawk") - args.contains("--no-server") } def "all native image configs are on classpath"() { diff --git a/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml b/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml index b250224..2a183f9 100644 --- a/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml +++ b/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml @@ -53,4 +53,4 @@ file-protocols: enforcers: - enforcer: ".common.StringPropertyEquals" property-name: "distributionUrl" - expected-property-value: "https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip" + expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip" diff --git a/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml b/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml index 0f6a243..865aeb5 100644 --- a/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml +++ b/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml @@ -72,4 +72,4 @@ file-protocols: enforcers: - enforcer: ".common.StringPropertyEquals" property-name: "distributionUrl" - expected-property-value: "https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip" + expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip" diff --git a/cli/src/test/resources/sourcehawk-simple.yml b/cli/src/test/resources/sourcehawk-simple.yml index 908c760..8bf174b 100644 --- a/cli/src/test/resources/sourcehawk-simple.yml +++ b/cli/src/test/resources/sourcehawk-simple.yml @@ -8,28 +8,6 @@ file-protocols: repository-path: README.md required: true severity: WARNING - - name: Lombok - repository-path: lombok.config - required: true - severity: WARNING - enforcers: - - enforcer: .common.StringPropertyEquals - property-name: config.stopBubbling - expected-property-value: false - - enforcer: .common.StringPropertyEquals - property-name: lombok.addLombokGeneratedAnnotation - expected-property-value: false - - name: Lombok - repository-path: lombok.config - required: true - severity: WARNING - enforcers: - - enforcer: .common.StringPropertyEquals - property-name: config.stopBubbling - expected-property-value: false - - enforcer: .common.StringPropertyEquals - property-name: lombok.addLombokGeneratedAnnotation - expected-property-value: false - name: Lombok repository-path: lombok.config required: true diff --git a/core/pom.xml b/core/pom.xml index aa79050..0d73a61 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/core/src/main/java/com/optum/sourcehawk/core/data/RemoteRef.java b/core/src/main/java/com/optum/sourcehawk/core/data/RemoteRef.java index c17c32e..09f179e 100644 --- a/core/src/main/java/com/optum/sourcehawk/core/data/RemoteRef.java +++ b/core/src/main/java/com/optum/sourcehawk/core/data/RemoteRef.java @@ -25,13 +25,7 @@ public class RemoteRef { private static final String PARSE_ERROR_PREFIX = "Invalid remote reference"; /** - * The type of the remote reference - */ - @NonNull - Type type; - - /** - * The remote namespace, such as Github owner / organization, or Bitbucket project + * The remote namespace, such as Github owner / organization, or Bitbucket user / project */ @NonNull String namespace; @@ -51,20 +45,20 @@ public class RemoteRef { /** * Create the remote ref from the type and raw reference * - * @param type the type of the remote ref * @param rawRemoteRef the raw remote reference + * @param defaultRef the default ref to use if none provided * @return the remote reference */ - public static RemoteRef parse(final Type type, final String rawRemoteRef) { + public static RemoteRef parse(final String rawRemoteRef, final String defaultRef) { if (rawRemoteRef.indexOf(COORDINATES_DELIMITER) == -1) { - val message = String.format("%s, must contain '%s' separator between %s and repository", PARSE_ERROR_PREFIX, COORDINATES_DELIMITER, type.getNamespaceType()); + val message = String.format("%s, must contain '%s' separator between repository coordinates", PARSE_ERROR_PREFIX, COORDINATES_DELIMITER); throw new IllegalArgumentException(message); } val remoteRefBuilder = builder(); final String rawCoordinates; if (rawRemoteRef.indexOf(REF_DELIMITER) == -1) { rawCoordinates = rawRemoteRef; - remoteRefBuilder.ref(type.getDefaultBranch()); + remoteRefBuilder.ref(defaultRef); } else { val refDelimiterIndex= rawRemoteRef.indexOf(REF_DELIMITER); rawCoordinates = rawRemoteRef.substring(0, refDelimiterIndex); @@ -77,7 +71,7 @@ public static RemoteRef parse(final Type type, final String rawRemoteRef) { if (coordinates.length < 2) { throw new IllegalArgumentException(PARSE_ERROR_PREFIX + ", repository must not be empty"); } - return remoteRefBuilder.type(type) + return remoteRefBuilder .namespace(coordinates[0]) .repository(coordinates[1]) .build(); @@ -86,36 +80,7 @@ public static RemoteRef parse(final Type type, final String rawRemoteRef) { /** {@inheritDoc} */ @Override public String toString() { - return String.format("[%s] %s%s%s%s%s", type.name(), namespace, COORDINATES_DELIMITER, repository, REF_DELIMITER, ref); - } - - /** - * The type of the remote reference - * - * @author Brian Wyka - */ - @Getter - @AllArgsConstructor(access = AccessLevel.PRIVATE) - public enum Type { - - BITBUCKET("https://bitbucket.org", "master", "project"), - GITHUB("https://raw.githubusercontent.com", "main", "owner"); - - /** - * The base URL - */ - private final String baseUrl; - - /** - * The name of the default branch - */ - private final String defaultBranch; - - /** - * The namespace type - */ - private final String namespaceType; - + return String.format("%s%s%s%s%s", namespace, COORDINATES_DELIMITER, repository, REF_DELIMITER, ref); } } diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReader.java deleted file mode 100644 index ceb9ccf..0000000 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReader.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.optum.sourcehawk.core.repository; - -import com.optum.sourcehawk.core.data.RemoteRef; -import lombok.NonNull; -import lombok.val; - -/** - * A {@link RemoteRepositoryFileReader} implementation which reads files from remote Bitbucket repositories - * - * @author Brian Wyka - */ -public class BitbucketRepositoryFileReader extends RemoteRepositoryFileReader { - - /** - * The authorization token prefix - */ - private static final String AUTHORIZATION_TOKEN_PREFIX = "Bearer "; - - /** - * Constructs the Bitbucket repository file reader - * - * @param token the token - * @param bitbucketBaseUrl the Bitbucket base URL - * @param remoteRef the remote ref - */ - public BitbucketRepositoryFileReader(final String token, @NonNull final String bitbucketBaseUrl, final RemoteRef remoteRef) { - super(constructBaseUrl(remoteRef, bitbucketBaseUrl), constructRequestProperties(AUTHORIZATION_TOKEN_PREFIX, token)); - } - - /** - * Constructs an instance of this reader with the provided project, repo, and ref - * - * @param token the github token (optional) - * @param remoteRef the remote reference - */ - public BitbucketRepositoryFileReader(final String token, @NonNull final RemoteRef remoteRef) { - this(token, RemoteRef.Type.BITBUCKET.getBaseUrl(), remoteRef); - } - - /** - * Construct the base URL of the Bitbucket repository based on the provided project, repo, and ref - * - * @param remoteRef the remote reference - * @param bitbucketBaseUrl the base URL - * @return the constructed base URL with a trailing {@value #SEPARATOR} - */ - public static String constructBaseUrl(final RemoteRef remoteRef, final String bitbucketBaseUrl) { - val baseUrl = bitbucketBaseUrl.endsWith(SEPARATOR) ? bitbucketBaseUrl.substring(0, bitbucketBaseUrl.length() - 1) : bitbucketBaseUrl; - return String.format("%s/%s/%s/raw/%s/", baseUrl, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef()); - } - -} diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/GithubRepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/GithubRepositoryFileReader.java deleted file mode 100644 index dabcf8e..0000000 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/GithubRepositoryFileReader.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.optum.sourcehawk.core.repository; - -import com.optum.sourcehawk.core.data.RemoteRef; -import lombok.NonNull; -import lombok.val; - -/** - * A {@link RemoteRepositoryFileReader} implementation which reads files from remote Github repositories - * - * @author Brian Wyka - */ -public class GithubRepositoryFileReader extends RemoteRepositoryFileReader { - - /** - * The authorization token prefix - */ - private static final String AUTHORIZATION_TOKEN_PREFIX = "token "; - - /** - * Constructs an instance of this reader with the provided Github Enterprise URL - * - * @param token the github token (optional) - * @param githubEnterpriseUrl the Github enterprise URL - * @param remoteRef the remote reference - */ - public GithubRepositoryFileReader(final String token, @NonNull final String githubEnterpriseUrl, @NonNull final RemoteRef remoteRef) { - super(constructBaseUrl(githubEnterpriseUrl, true, remoteRef), constructRequestProperties(AUTHORIZATION_TOKEN_PREFIX, token)); - } - - /** - * Constructs an instance of this reader with the provided owner, repo, and ref - * - * @param token the github token (optional) - * @param remoteRef the remote reference - */ - public GithubRepositoryFileReader(final String token, @NonNull final RemoteRef remoteRef) { - super(constructBaseUrl(RemoteRef.Type.GITHUB.getBaseUrl(), false, remoteRef), constructRequestProperties(AUTHORIZATION_TOKEN_PREFIX, token)); - } - - /** - * Construct the base URL of the Github repository based on the provided owner, repo, and ref - * - * @param githubUrl the Github URL - * @param githubEnterprise true if Github Enterprise, false otherwise - * @param remoteRef the remote reference - * @return the constructed base URL with a trailing {@value #SEPARATOR} - */ - public static String constructBaseUrl(final String githubUrl, final boolean githubEnterprise, final RemoteRef remoteRef) { - val baseUrl = githubUrl.endsWith(SEPARATOR) ? githubUrl.substring(0, githubUrl.length() - 1) : githubUrl; - val githubBaseUrl = githubEnterprise ? baseUrl + "/raw" : baseUrl; - return String.format("%s/%s/%s/%s/", githubBaseUrl, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef()); - } - -} diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/LocalRepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/LocalRepositoryFileReader.java index 802d58d..c4f88c6 100644 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/LocalRepositoryFileReader.java +++ b/core/src/main/java/com/optum/sourcehawk/core/repository/LocalRepositoryFileReader.java @@ -59,6 +59,12 @@ public Optional read(@NonNull final String repositoryFilePath) thro return getInputStream(directory.resolve(Paths.get(repositoryFilePath))); } + /** {@inheritDoc} */ + @Override + public String getAbsoluteLocation(final String repositoryFilePath) { + return directory.resolve(Paths.get(repositoryFilePath)).toAbsolutePath().toString(); + } + /** * Get the {@link InputStream} from the {@link File} reference * diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java index 76bda54..20c49b8 100644 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java +++ b/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java @@ -1,36 +1,33 @@ package com.optum.sourcehawk.core.repository; -import com.optum.sourcehawk.core.utils.StringUtils; -import lombok.NonNull; -import lombok.val; - import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import lombok.NonNull; +import lombok.val; /** * A remote repository file reader which treats the repository file paths relative - * to the base URL provided during construction + * to the raw file URL template provided during construction * * @author Brian Wyka */ -abstract class RemoteRepositoryFileReader implements RepositoryFileReader { +public final class RemoteRepositoryFileReader implements RepositoryFileReader { /** * URL Separator */ - protected static final String SEPARATOR = "/"; + private static final String SEPARATOR = "/"; /** - * The base URL to read from + * The raw file URL template. Takes one parameter: The path of the file in the repository */ - private final String baseUrl; + private final String rawFileUrlTemplate; /** * The required request properties @@ -45,31 +42,18 @@ abstract class RemoteRepositoryFileReader implements RepositoryFileReader { /** * Constructs an instance of this reader with the provided base URL * - * @param baseUrl the base URL + * @param rawFileUrlTemplate the raw file URL template * @param requestProperties the request properties required for connection */ - protected RemoteRepositoryFileReader(@NonNull final String baseUrl, final Map requestProperties) { - if (baseUrl.endsWith(SEPARATOR)) { - this.baseUrl = baseUrl; - } else { - this.baseUrl = baseUrl + SEPARATOR; - } + public RemoteRepositoryFileReader(@NonNull final String rawFileUrlTemplate, @NonNull final Map requestProperties) { + this.rawFileUrlTemplate = rawFileUrlTemplate; this.requestProperties = requestProperties; } - /** - * Constructs an instance of this reader with the provided base URL - * - * @param baseUrl the base URL - */ - protected RemoteRepositoryFileReader(@NonNull final String baseUrl) { - this(baseUrl, Collections.emptyMap()); - } - /** {@inheritDoc} */ @Override public boolean exists(final String repositoryFilePath) throws IOException { - val absoluteUrl = constructAbsoluteUrl(baseUrl, repositoryFilePath); + val absoluteUrl = new URL(constructAbsoluteLocation(rawFileUrlTemplate, repositoryFilePath)); val absoluteUrlString = absoluteUrl.toString(); if (urlExistenceCache.containsKey(absoluteUrlString)) { return urlExistenceCache.get(absoluteUrlString); @@ -82,7 +66,7 @@ public boolean exists(final String repositoryFilePath) throws IOException { /** {@inheritDoc} */ @Override public Optional read(final String repositoryFilePath) throws IOException { - val absoluteUrl = constructAbsoluteUrl(baseUrl, repositoryFilePath); + val absoluteUrl = new URL(constructAbsoluteLocation(rawFileUrlTemplate, repositoryFilePath)); if (exists(repositoryFilePath)) { val httpUrlConnection = (HttpURLConnection) absoluteUrl.openConnection(); requestProperties.forEach(httpUrlConnection::setRequestProperty); @@ -91,6 +75,12 @@ public Optional read(final String repositoryFilePath) throws IOExce return Optional.empty(); } + /** {@inheritDoc} */ + @Override + public String getAbsoluteLocation(final String repositoryFilePath) { + return constructAbsoluteLocation(rawFileUrlTemplate, repositoryFilePath); + } + /** * Get the input stream from the {@link HttpURLConnection} * @@ -117,38 +107,25 @@ private boolean urlExists(final URL url) throws IOException { val httpUrlConnection = (HttpURLConnection) url.openConnection(); httpUrlConnection.setRequestMethod("HEAD"); requestProperties.forEach(httpUrlConnection::setRequestProperty); - return httpUrlConnection.getResponseCode() == HttpURLConnection.HTTP_OK; + val httpResponseCode = httpUrlConnection.getResponseCode(); + if (httpResponseCode != HttpURLConnection.HTTP_OK) { + System.err.println("HTTP Request to " + url + " returned response code " + httpResponseCode); // FIXME + } + return httpResponseCode == HttpURLConnection.HTTP_OK; } /** - * Construct the absolute URL to the remote file + * Construct the absolute location to the remote file * - * @param baseUrl the repository base URL + * @param rawFileUrlTemplate the raw file URL template * @param repositoryFilePath the repository file path - * @return the absolute URL to the file - * @throws IOException if the URL is malformed + * @return the absolute location to the repository file */ - private static URL constructAbsoluteUrl(final String baseUrl, final String repositoryFilePath) throws IOException { + private static String constructAbsoluteLocation(final String rawFileUrlTemplate, final String repositoryFilePath) { if (repositoryFilePath.startsWith(SEPARATOR)) { - return new URL(baseUrl + repositoryFilePath.substring(1)); - } - return new URL(baseUrl + repositoryFilePath); - } - - /** - * Construct the request properties for the provided github token - * - * @param authorizationPrefix the authorization request property prefix - * @param authorizationToken the authorization token - * @return the request properties - */ - protected static Map constructRequestProperties(final String authorizationPrefix, final String authorizationToken) { - val requestProperties = new HashMap(); - requestProperties.put("Accept", "text/plain"); - if (StringUtils.isNotBlankOrEmpty(authorizationToken)) { - requestProperties.put("Authorization", authorizationPrefix + authorizationToken); + return String.format(rawFileUrlTemplate, repositoryFilePath.substring(1)); } - return requestProperties; + return String.format(rawFileUrlTemplate, repositoryFilePath); } } diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/RepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/RepositoryFileReader.java index 1fabc10..ca4c43e 100644 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/RepositoryFileReader.java +++ b/core/src/main/java/com/optum/sourcehawk/core/repository/RepositoryFileReader.java @@ -38,4 +38,12 @@ default boolean supportsGlobPatterns() { */ Optional read(final String repositoryFilePath) throws IOException; + /** + * Get a string representation of the absolute location of {@code repositoryFilePath} + * + * @param repositoryFilePath the repository file path + * @return the absolute location + */ + String getAbsoluteLocation(final String repositoryFilePath); + } diff --git a/core/src/test/groovy/com/optum/sourcehawk/core/data/RemoteRefSpec.groovy b/core/src/test/groovy/com/optum/sourcehawk/core/data/RemoteRefSpec.groovy index 5d7e567..7b26ae4 100644 --- a/core/src/test/groovy/com/optum/sourcehawk/core/data/RemoteRefSpec.groovy +++ b/core/src/test/groovy/com/optum/sourcehawk/core/data/RemoteRefSpec.groovy @@ -7,14 +7,12 @@ class RemoteRefSpec extends Specification { def "builder and toString"() { given: - RemoteRef.Type type = RemoteRef.Type.GITHUB String namespace = "namespace" String repository = "repository" String ref = "ref" when: RemoteRef remoteRef = RemoteRef.builder() - .type(type) .namespace(namespace) .repository(repository) .ref(ref) @@ -22,32 +20,29 @@ class RemoteRefSpec extends Specification { then: remoteRef - remoteRef.type == type remoteRef.namespace == namespace remoteRef.repository == repository remoteRef.ref == ref and: - remoteRef.toString() == "[GITHUB] namespace/repository@ref" + remoteRef.toString() == "namespace/repository@ref" } def "parse"() { given: - RemoteRef.Type type = RemoteRef.Type.GITHUB String rawReference = "namespace/repository@ref" when: - RemoteRef remoteRef = RemoteRef.parse(type, rawReference) + RemoteRef remoteRef = RemoteRef.parse(rawReference, "main") then: remoteRef - remoteRef.type == type remoteRef.namespace == "namespace" remoteRef.repository == "repository" remoteRef.ref == "ref" and: - remoteRef.toString() == "[GITHUB] ${rawReference}" + remoteRef.toString() == rawReference } @Unroll @@ -56,31 +51,22 @@ class RemoteRefSpec extends Specification { String rawReference = "namespace/repository" when: - RemoteRef remoteRef = RemoteRef.parse(type, rawReference) + RemoteRef remoteRef = RemoteRef.parse(rawReference, "main") then: remoteRef - remoteRef.type == type remoteRef.namespace == "namespace" remoteRef.repository == "repository" - remoteRef.ref == expected + remoteRef.ref == "main" and: - remoteRef.toString() == "[${type.name()}] ${rawReference}@${expected}" - - where: - type | expected - RemoteRef.Type.GITHUB | "main" - RemoteRef.Type.BITBUCKET | "master" + remoteRef.toString() == "${rawReference}@main" } @Unroll def "parse - invalid (throws IllegalArgumentException)"() { - given: - RemoteRef.Type type = RemoteRef.Type.GITHUB - when: - RemoteRef.parse(type, rawReference) + RemoteRef.parse(rawReference, "main") then: thrown(IllegalArgumentException) diff --git a/core/src/test/groovy/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReaderSpec.groovy b/core/src/test/groovy/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReaderSpec.groovy deleted file mode 100644 index 6aef8f3..0000000 --- a/core/src/test/groovy/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReaderSpec.groovy +++ /dev/null @@ -1,184 +0,0 @@ -package com.optum.sourcehawk.core.repository - -import com.optum.sourcehawk.core.data.RemoteRef -import org.mockserver.configuration.ConfigurationProperties -import org.mockserver.integration.ClientAndServer -import org.mockserver.matchers.Times -import org.mockserver.model.HttpRequest -import org.mockserver.model.HttpResponse -import spock.lang.AutoCleanup -import spock.lang.Shared -import spock.lang.Specification - -class BitbucketRepositoryFileReaderSpec extends Specification { - - @Shared - @AutoCleanup - ClientAndServer clientAndServer - - @Shared - String baseUrl - - def setupSpec() { - clientAndServer = ClientAndServer.startClientAndServer("http://127.0.0.1", 8122) - ConfigurationProperties.logLevel("WARN") - baseUrl = "${clientAndServer.remoteAddress.hostString}:${clientAndServer.port}" - } - - def setup() { - clientAndServer.reset() - } - - def "constructors"() { - expect: - new BitbucketRepositoryFileReader(null, RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master")) - new BitbucketRepositoryFileReader("abc", "https://bitbucket.example.com/", RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master")) - } - - def "supportsGlobPatterns"() { - given: - BitbucketRepositoryFileReader reader = new BitbucketRepositoryFileReader(null, RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master")) - - when: - boolean supportsGlobPatterns = reader.supportsGlobPatterns() - - then: - !supportsGlobPatterns - } - - def "exists - found"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master") - BitbucketRepositoryFileReader reader = new BitbucketRepositoryFileReader(null, baseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/project/repo/raw/master/README.md"), - Times.exactly(1)) - .respond(HttpResponse.response().withStatusCode(200)) - - when: - boolean exists = reader.exists("README.md") - - then: - exists - } - - def "exists - not found"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master") - BitbucketRepositoryFileReader reader = new BitbucketRepositoryFileReader(null, baseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/project/repo/raw/master/README.md"), - Times.exactly(1)) - .respond(HttpResponse.notFoundResponse()) - - when: - boolean exists = reader.exists("README.md") - - then: - !exists - } - - def "read - found"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master") - BitbucketRepositoryFileReader reader = new BitbucketRepositoryFileReader(null, baseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/project/repo/raw/master/README.md"), - Times.exactly(1)) - .respond(HttpResponse.response().withStatusCode(200)) - clientAndServer - .when(HttpRequest.request() - .withMethod("GET") - .withPath("/project/repo/raw/master/README.md"), - Times.exactly(2)) - .respond(HttpResponse.response().withStatusCode(200).withBody("# Title".bytes)) - - when: - Optional inputStreamOptional = reader.read("README.md") - - then: - inputStreamOptional - inputStreamOptional.isPresent() - - when: - inputStreamOptional = reader.read("/README.md") - - then: - inputStreamOptional - inputStreamOptional.isPresent() - } - - def "read - not found"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master") - BitbucketRepositoryFileReader reader = new BitbucketRepositoryFileReader(null, baseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/project/repo/raw/master/README.md"), - Times.exactly(1)) - .respond(HttpResponse.notFoundResponse()) - - when: - Optional inputStreamOptional = reader.read("README.md") - - then: - !inputStreamOptional - } - - def "constructBaseUrl"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master") - - when: - String baseUrl = BitbucketRepositoryFileReader.constructBaseUrl(remoteRef, RemoteRef.Type.BITBUCKET.baseUrl) - - then: - baseUrl == "https://bitbucket.org/project/repo/raw/master/" - } - - def "constructor - null parameter"() { - when: - new BitbucketRepositoryFileReader("abc", null) - - then: - thrown(NullPointerException) - - when: - new BitbucketRepositoryFileReader(null, null) - - then: - thrown(NullPointerException) - - when: - new BitbucketRepositoryFileReader("abc", null, RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master")) - - then: - thrown(NullPointerException) - - when: - new BitbucketRepositoryFileReader(null, null, RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master")) - - then: - thrown(NullPointerException) - - when: - new BitbucketRepositoryFileReader("abc", null, null) - - then: - thrown(NullPointerException) - - when: - new BitbucketRepositoryFileReader(null, null, null) - - then: - thrown(NullPointerException) - } - -} diff --git a/core/src/test/groovy/com/optum/sourcehawk/core/repository/GithubRepositoryFileReaderSpec.groovy b/core/src/test/groovy/com/optum/sourcehawk/core/repository/GithubRepositoryFileReaderSpec.groovy deleted file mode 100644 index d2e3e37..0000000 --- a/core/src/test/groovy/com/optum/sourcehawk/core/repository/GithubRepositoryFileReaderSpec.groovy +++ /dev/null @@ -1,239 +0,0 @@ -package com.optum.sourcehawk.core.repository - -import com.optum.sourcehawk.core.data.RemoteRef -import org.mockserver.configuration.ConfigurationProperties -import org.mockserver.integration.ClientAndServer -import org.mockserver.matchers.Times -import org.mockserver.model.HttpRequest -import org.mockserver.model.HttpResponse -import spock.lang.AutoCleanup -import spock.lang.Shared -import spock.lang.Specification -import spock.lang.Unroll - -class GithubRepositoryFileReaderSpec extends Specification { - - @Shared - @AutoCleanup - ClientAndServer clientAndServer - - @Shared - String enterpriseUrl - - def setupSpec() { - clientAndServer = ClientAndServer.startClientAndServer("http://127.0.0.1", 8121) - ConfigurationProperties.logLevel("WARN") - enterpriseUrl = "${clientAndServer.remoteAddress.hostString}:${clientAndServer.port}" - } - - def setup() { - clientAndServer.reset() - } - - def "supportsGlobPatterns"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - GithubRepositoryFileReader githubRepositoryFileReader = new GithubRepositoryFileReader(null, enterpriseUrl, remoteRef) - - when: - boolean supportsGlobPatterns = githubRepositoryFileReader.supportsGlobPatterns() - - then: - !supportsGlobPatterns - } - - def "exists (found)"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - GithubRepositoryFileReader githubRepositoryFileReader = new GithubRepositoryFileReader(null, enterpriseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/raw/owner/repo/main/README.md"), - Times.exactly(1)) - .respond(HttpResponse.response().withStatusCode(200)) - - when: - boolean exists = githubRepositoryFileReader.exists("README.md") - - then: - exists - } - - def "exists (not found)"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@nope") - GithubRepositoryFileReader githubRepositoryFileReader = new GithubRepositoryFileReader(null, enterpriseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/raw/owner/repo/main/README.md"), - Times.exactly(1)) - .respond(HttpResponse.notFoundResponse()) - - when: - boolean exists = githubRepositoryFileReader.exists("README.md") - - then: - !exists - } - - def "read (found)"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - GithubRepositoryFileReader githubRepositoryFileReader = new GithubRepositoryFileReader(null, enterpriseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/raw/owner/repo/main/README.md"), - Times.exactly(1)) - .respond(HttpResponse.response().withStatusCode(200)) - clientAndServer - .when(HttpRequest.request() - .withMethod("GET") - .withPath("/raw/owner/repo/main/README.md"), - Times.exactly(2)) - .respond(HttpResponse.response().withStatusCode(200).withBody("# Title".bytes)) - - when: - Optional inputStream = githubRepositoryFileReader.read("README.md") - - then: - inputStream - inputStream.isPresent() - - when: - inputStream = githubRepositoryFileReader.read("/README.md") - - then: - inputStream - inputStream.isPresent() - } - - def "read (not found)"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@master") - GithubRepositoryFileReader githubRepositoryFileReader = new GithubRepositoryFileReader(null, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/raw/owner/repo/master/README.md"), - Times.exactly(1)) - .respond(HttpResponse.notFoundResponse()) - - when: - Optional inputStream = githubRepositoryFileReader.read("README.md") - - then: - !inputStream - !inputStream.isPresent() - } - - def "constructBaseUrl - public github"() { - given: - String githubUrl = "https://raw.githubusercontent.com" - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - - when: - String baseUrl = GithubRepositoryFileReader.constructBaseUrl(githubUrl, false, remoteRef) - - then: - baseUrl == "https://raw.githubusercontent.com/owner/repo/main/" - } - - @Unroll - def "constructBaseUrl - enterprise github"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - - when: - String baseUrl = GithubRepositoryFileReader.constructBaseUrl(githubUrl, true, remoteRef) - - then: - baseUrl == "https://github.example.com/raw/owner/repo/main/" - - where: - githubUrl << ["https://github.example.com", "https://github.example.com/"] - } - - def "constructRequestProperties"() { - when: - Map requestProperties = GithubRepositoryFileReader.constructRequestProperties("token ", "abc") - - then: - requestProperties - requestProperties.size() == 2 - requestProperties["Accept"] == "text/plain" - requestProperties["Authorization"] == "token abc" - } - - def "constructRequestProperties - null"() { - when: - Map requestProperties = GithubRepositoryFileReader.constructRequestProperties("token ", null) - - then: - requestProperties - requestProperties.size() == 1 - requestProperties["Accept"] == "text/plain" - } - - def "constructor - enterprise"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - - expect: - new GithubRepositoryFileReader(null, "https://github.example.com", remoteRef) - new GithubRepositoryFileReader("abc", "https://github.example.com", remoteRef) - } - - def "constructors - null parameter"() { - when: - new GithubRepositoryFileReader(null, null, null) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader("abc", null, null) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader("abc", null, RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main")) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader("abc", "https://github.example.com", null) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader(null, "https://github.example.com", null) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader(null, null, RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main")) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader("abc", null) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader(null, null) - - then: - thrown(NullPointerException) - } - -} diff --git a/core/src/test/groovy/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReaderSpec.groovy b/core/src/test/groovy/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReaderSpec.groovy index 51592b5..ef755dc 100644 --- a/core/src/test/groovy/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReaderSpec.groovy +++ b/core/src/test/groovy/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReaderSpec.groovy @@ -1,71 +1,168 @@ package com.optum.sourcehawk.core.repository -import spock.lang.Specification +import org.mockserver.configuration.ConfigurationProperties +import org.mockserver.integration.ClientAndServer +import org.mockserver.matchers.Times +import org.mockserver.model.HttpRequest +import org.mockserver.model.HttpResponse +import spock.lang.AutoCleanup +import spock.lang.Shared +import spock.lang.Specification class RemoteRepositoryFileReaderSpec extends Specification { - def "constructor"() { - expect: - new GenericRemoteRepositoryFileReader() - new TrailingSlashRemoteRepositoryFileReader() + @Shared + @AutoCleanup + ClientAndServer clientAndServer + + @Shared + String baseUrl + + def setupSpec() { + clientAndServer = ClientAndServer.startClientAndServer("http://127.0.0.1", 8122) + ConfigurationProperties.logLevel("WARN") + baseUrl = "${clientAndServer.remoteAddress.hostString}:${clientAndServer.port}" } - def "constructor - null argument"() { + def "supportsGlobPatterns"() { + given: + RepositoryFileReader reader = new RemoteRepositoryFileReader("rawFileUrlTemplate", Collections.emptyMap()) + when: - new InvalidRemoteRepositoryFileReader() + boolean supportsGlobPatterns = reader.supportsGlobPatterns() then: - thrown(NullPointerException) + !supportsGlobPatterns + } + + def "exists - found"() { + given: + String rawFileUrlTemplate = "$baseUrl/project/repo/raw/master/%s" + RepositoryFileReader reader = new RemoteRepositoryFileReader(rawFileUrlTemplate, Collections.emptyMap()) + clientAndServer + .when(HttpRequest.request() + .withMethod("HEAD") + .withPath("/project/repo/raw/master/README.md"), + Times.exactly(1)) + .respond(HttpResponse.response().withStatusCode(200)) when: - new InvalidRemoteRepositoryFileReader2() + boolean exists = reader.exists("README.md") then: - thrown(NullPointerException) + exists } - def "getInputStream - not found"() { + def "exists - not found"() { given: - HttpURLConnection mockHttpUrlConnection = Mock() + String rawFileUrlTemplate = "$baseUrl/project/repo/raw/master/%s" + RepositoryFileReader reader = new RemoteRepositoryFileReader(rawFileUrlTemplate, Collections.emptyMap()) + clientAndServer + .when(HttpRequest.request() + .withMethod("HEAD") + .withPath("/project/repo/raw/master/README.md"), + Times.exactly(1)) + .respond(HttpResponse.notFoundResponse()) when: - Optional inputStreamOptional = RemoteRepositoryFileReader.getInputStream(mockHttpUrlConnection) + boolean exists = reader.exists("README.md") then: - 1 * mockHttpUrlConnection.getInputStream() >> { throw new FileNotFoundException("404") } - 0 * _ - - and: - !inputStreamOptional.isPresent() + !exists } - private static class GenericRemoteRepositoryFileReader extends RemoteRepositoryFileReader { + def "read - found"() { + given: + String rawFileUrlTemplate = "$baseUrl/raw/project/repo/main/%s" + RepositoryFileReader reader = new RemoteRepositoryFileReader(rawFileUrlTemplate, Collections.emptyMap()) + clientAndServer + .when(HttpRequest.request() + .withMethod("HEAD") + .withPath("/raw/project/repo/main/README.md"), + Times.exactly(1)) + .respond(HttpResponse.response().withStatusCode(200)) + clientAndServer + .when(HttpRequest.request() + .withMethod("GET") + .withPath("/raw/project/repo/main/README.md"), + Times.exactly(2)) + .respond(HttpResponse.response().withStatusCode(200).withBody("# Title".bytes)) + + when: + Optional inputStreamOptional = reader.read("README.md") + + then: + inputStreamOptional + inputStreamOptional.isPresent() + + when: + inputStreamOptional = reader.read("/README.md") - protected GenericRemoteRepositoryFileReader() { - super("https://optum.github.io") - } + then: + inputStreamOptional + inputStreamOptional.isPresent() } - private static class TrailingSlashRemoteRepositoryFileReader extends RemoteRepositoryFileReader { + def "read - not found"() { + given: + String rawFileUrlTemplate = "$baseUrl/raw/project/repo/main/%s" + RepositoryFileReader reader = new RemoteRepositoryFileReader(rawFileUrlTemplate, Collections.emptyMap()) + clientAndServer + .when(HttpRequest.request() + .withMethod("HEAD") + .withPath("/raw/project/repo/main/README.md"), + Times.exactly(1)) + .respond(HttpResponse.notFoundResponse()) + + when: + Optional inputStreamOptional = reader.read("README.md") - protected TrailingSlashRemoteRepositoryFileReader() { - super("https://optum.github.io/") - } + then: + !inputStreamOptional.isPresent() } - private static class InvalidRemoteRepositoryFileReader extends RemoteRepositoryFileReader { + def "getAbsoluteLocation"() { + given: + String rawFileUrlTemplate = "$baseUrl/raw/project/repo/main/%s" + RepositoryFileReader reader = new RemoteRepositoryFileReader(rawFileUrlTemplate, Collections.emptyMap()) + String repositoryFilePath = "README.md" + + when: + String absoluteLocation = reader.getAbsoluteLocation(repositoryFilePath) + + then: + absoluteLocation + absoluteLocation == "$baseUrl/raw/project/repo/main/README.md" + + + when: + repositoryFilePath = "/path/to/file.txt" + absoluteLocation = reader.getAbsoluteLocation(repositoryFilePath) - protected InvalidRemoteRepositoryFileReader() { - super(null) - } + then: + absoluteLocation + absoluteLocation == "$baseUrl/raw/project/repo/main/path/to/file.txt" } - private static class InvalidRemoteRepositoryFileReader2 extends RemoteRepositoryFileReader { + def "constructor - null parameter"() { + when: + new RemoteRepositoryFileReader("abc", null) + + then: + thrown(NullPointerException) + + when: + new RemoteRepositoryFileReader(null, Collections.emptyMap()) + + then: + thrown(NullPointerException) + + when: + new RemoteRepositoryFileReader(null, null) - protected InvalidRemoteRepositoryFileReader2() { - super(null, Collections.emptyMap()) - } + then: + thrown(NullPointerException) } } \ No newline at end of file diff --git a/distributions/debian/pom.xml b/distributions/debian/pom.xml index 7c27965..717f22a 100644 --- a/distributions/debian/pom.xml +++ b/distributions/debian/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-dist - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml @@ -21,7 +21,7 @@ ${project.build.directory}/debian ${bintray.package} ${project.version} - ~${git.commit.time}+${git.commit.id.abbrev} + amd64 @@ -38,7 +38,7 @@ com.optum.sourcehawk sourcehawk-cli ${project.version} - bash-completion + completion sh provided @@ -49,15 +49,27 @@ - pl.project13.maven - git-commit-id-plugin - - ${maven.build.timestamp.format} - ${maven.multiModuleProjectDirectory}/.git - false - true - ${project.build.directory}/git.properties - + org.codehaus.gmavenplus + gmavenplus-plugin + + + save-git-information + validate + + execute + + + + + + + + @@ -124,7 +136,7 @@ com.optum.sourcehawk sourcehawk-cli - bash-completion + completion sh ${debian.build.directory}/usr/share/bash-completion/completions ${debian.package} @@ -280,95 +292,96 @@ - - - debian-package-release - - - ci.release - - - - - - - - debian-package-snapshot - - - ci.snapshot - - - - - - org.codehaus.mojo - exec-maven-plugin - - - dev-snapshots - - - - - - - - debian-package-publish - - - ci.deploy - - - - - - org.codehaus.mojo - exec-maven-plugin - - - ${bintray.organization} - ${debian.package} - ${debian.package.version}${debian.package.version.suffix} - ${debian.architecture},i386,x86_64 - - - - - publish-debian-buster-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${debian.package}-debian-buster.deb - deb - buster,jessie,stretch - - - - - publish-ubuntu-focal-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${debian.package}-ubuntu-focal.deb - ubuntu - focal - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/distributions/docker-builders/Dockerfile-nativeimage b/distributions/docker-builders/Dockerfile-nativeimage index d06225e..b0c52ca 100644 --- a/distributions/docker-builders/Dockerfile-nativeimage +++ b/distributions/docker-builders/Dockerfile-nativeimage @@ -1,6 +1,20 @@ -# Oracle GraalVM Java 8 Base Container -ARG FROM=ghcr.io/graalvm/graalvm-ce:java8-21.2.0 -FROM ${FROM} +# GraalVM Community Edition Base Container +ARG FROM_VERSION=java11-21.3.0 +FROM ghcr.io/graalvm/graalvm-ce:${FROM_VERSION} # Install native-image tool -RUN gu install --no-progress native-image \ No newline at end of file +RUN gu install --no-progress native-image + +# Install musl libc +RUN curl -ksL -o x86_64-linux-musl-native.tgz https://more.musl.cc/10.2.1/x86_64-linux-musl/x86_64-linux-musl-native.tgz \ + && tar -xf x86_64-linux-musl-native.tgz -C /opt \ + && export CC=/opt/x86_64-linux-musl-native/bin/gcc \ + && curl -ksL -o zlib.tar.gz https://zlib.net/zlib-1.2.11.tar.gz \ + && tar -xf zlib.tar.gz \ + && cd zlib-1.2.11 \ + && ./configure --prefix=/opt/x86_64-linux-musl-native --static \ + && make \ + && make install + +# Update PATH Environment Variable +ENV PATH="$PATH:/opt/x86_64-linux-musl-native/bin" diff --git a/distributions/docker/Dockerfile b/distributions/docker/Dockerfile index 8bb6827..168bf95 100644 --- a/distributions/docker/Dockerfile +++ b/distributions/docker/Dockerfile @@ -1,23 +1,17 @@ -# glibc is required for proper DNS resolution within app -FROM busybox:1.32.0-glibc +FROM scratch -# Dynamically pass in name ARG NAME="sourcehawk" -# Setup user and group -ENV GROUP=${NAME} USER=${NAME} -RUN addgroup ${GROUP} && adduser -h /home/${USER} -G ${NAME} -D ${USER} +# Copy Group and User Files +COPY "/etc" "/etc" # Copy the native image executable into the image -ARG NATIVE_IMAGE_PATH -COPY --chown=${GROUP}:${USER} ${NATIVE_IMAGE_PATH} /usr/bin/sourcehawk - -# Give the native image executable permissions -RUN chmod +x /usr/bin/${NAME} +COPY --chown="${NAME}":"${NAME}" "target/native-image" "/entrypoint" # Set the user and working directory -USER ${USER} -WORKDIR /home/${USER} +USER "${NAME}" +WORKDIR "/work" # Set the native image as the entrypoint -ENTRYPOINT ["/usr/bin/sourcehawk"] \ No newline at end of file +CMD ["--help"] +ENTRYPOINT ["/entrypoint"] \ No newline at end of file diff --git a/distributions/docker/README.md b/distributions/docker/README.md index ec3aa6d..7528041 100644 --- a/distributions/docker/README.md +++ b/distributions/docker/README.md @@ -1,15 +1,15 @@ -# Sourcehawk Alpine Docker Image +# Sourcehawk Docker Image ### Scanning from local directory ```shell script -docker run -v "$(pwd):/home/sourcehawk" optumopensource/sourcehawk:1.0.0-alpine +docker run --rm -v $PWD:/work optumopensource/sourcehawk ``` Or with a custom working directory: ```shell script -docker run -v "$(pwd):/tmp" -w /tmp optumopensource/sourcehawk:1.0.0-alpine +docker run --rm -v $PWD:/tmp -w /tmp optumopensource/sourcehawk ``` The volume mounting is necessary in order to give the container access to the files to scan. \ No newline at end of file diff --git a/distributions/docker/etc/group b/distributions/docker/etc/group new file mode 100644 index 0000000..0d9e16b --- /dev/null +++ b/distributions/docker/etc/group @@ -0,0 +1 @@ +sourcehawk:x:1000:sourcehawk \ No newline at end of file diff --git a/distributions/docker/etc/passwd b/distributions/docker/etc/passwd new file mode 100644 index 0000000..f96df68 --- /dev/null +++ b/distributions/docker/etc/passwd @@ -0,0 +1 @@ +sourcehawk:x:1000:1000:Sourcehawk User,,,:/home/sourcehawk:/entrypoint \ No newline at end of file diff --git a/distributions/docker/pom.xml b/distributions/docker/pom.xml index 4412275..2485517 100644 --- a/distributions/docker/pom.xml +++ b/distributions/docker/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-dist - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml @@ -19,6 +19,7 @@ ${project.version} + ghcr.io/optum/sourcehawk @@ -61,25 +62,68 @@ + + + org.codehaus.mojo + exec-maven-plugin + + + + + update-native-image-permissions + prepare-package + + exec + + + chmod + + +x + target/native-image + + + + + + com.spotify dockerfile-maven-plugin - build-and-tag + build prepare-package build + + + ${docker.repository} + ${docker.tag} + true + + + + tag-docker-hub + prepare-package + tag ${docker.repository} ${docker.tag} - - ${global.project.name} - target/native-image - + true + + + + tag-github-container-registry + prepare-package + + tag + + + ${ghcr.repository} + ${docker.tag} true @@ -103,6 +147,7 @@ docker run + --rm ${docker.repository}:${docker.tag} --version @@ -119,8 +164,9 @@ ${maven.multiModuleProjectDirectory} run + --rm -v - ${maven.multiModuleProjectDirectory}:/home/sourcehawk + ${maven.multiModuleProjectDirectory}:/work ${docker.repository}:${docker.tag} scan -f @@ -137,7 +183,7 @@ - docker-hub-push + docker-push ci.release @@ -150,18 +196,31 @@ exec-maven-plugin - deploy-docker-image + push-docker-image-to-docker-hub deploy exec - ${project.basedir}/scripts/push-docker-image-to-docker-hub.sh + ${project.basedir}/scripts/push-docker-image.sh ${docker.repository}:${docker.tag} + + push-docker-image-to-github-container-registry + deploy + + exec + + + ${project.basedir}/scripts/push-docker-image.sh + + ${ghcr.repository}:${docker.tag} + + + diff --git a/distributions/docker/scripts/push-docker-image-to-docker-hub.sh b/distributions/docker/scripts/push-docker-image.sh similarity index 100% rename from distributions/docker/scripts/push-docker-image-to-docker-hub.sh rename to distributions/docker/scripts/push-docker-image.sh diff --git a/distributions/linux/native-image-builder/Dockerfile b/distributions/linux/native-image-builder/Dockerfile index ce6898d..59aefa9 100644 --- a/distributions/linux/native-image-builder/Dockerfile +++ b/distributions/linux/native-image-builder/Dockerfile @@ -1,5 +1,5 @@ -ARG GRAALVM_VERSION=21.3.0-java8 -FROM ghcr.io/optum/sourcehawk-ci/nativeimage:graalvm-ce-${GRAALVM_VERSION} +ARG GRAALVM_VERSION=21.3.0-java11 +FROM ghcr.io/optum/ci/nativeimage:graalvm-ce-${GRAALVM_VERSION} # Build Arguments ARG NAME @@ -17,4 +17,5 @@ RUN native-image -cp native-image.jar \ -H:+ReportExceptionStackTraces \ --report-unsupported-elements-at-runtime \ --no-fallback \ + --libc=musl \ --static \ No newline at end of file diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index 561a2ca..b416378 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-dist - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml @@ -37,7 +37,7 @@ com.optum.sourcehawk sourcehawk-cli ${project.version} - bash-completion + completion sh provided @@ -89,7 +89,7 @@ - copy-native-image-bash-completion + copy-native-image-completion-script prepare-package copy @@ -99,9 +99,9 @@ com.optum.sourcehawk sourcehawk-cli - bash-completion + completion sh - ${project.build.directory}/dist/bash-completion + ${project.build.directory}/dist/completion ${global.project.name} @@ -193,12 +193,55 @@ + + + windows + + + windows + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + true + + + + + + + + + mac + + + mac + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + true + + + + + + linux-integration-tests unix + Linux @@ -229,20 +272,9 @@ - - - native-image-build-java8 - - 8 - - - 21.3.0-java8 - - - - native-image-build-java11 + java11 11 @@ -253,9 +285,9 @@ - native-image-build-java17 + java17 - 11 + 17 21.3.0-java17 diff --git a/distributions/pom.xml b/distributions/pom.xml index a5e8fb1..71a430b 100644 --- a/distributions/pom.xml +++ b/distributions/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml @@ -36,17 +36,6 @@ - - pl.project13.maven - git-commit-id-plugin - - ${maven.build.timestamp.format} - ${maven.multiModuleProjectDirectory}/.git - false - true - ${project.build.directory}/git.properties - - maven-enforcer-plugin diff --git a/distributions/rpm/Dockerfile b/distributions/rpm/Dockerfile index 7b8d31a..430d963 100644 --- a/distributions/rpm/Dockerfile +++ b/distributions/rpm/Dockerfile @@ -1,5 +1,5 @@ ARG FROM_TAG=centos7 -FROM ghcr.io/optum/sourcehawk-ci/rpmbuild:${FROM_TAG} +FROM ghcr.io/optum/ci/rpmbuild:${FROM_TAG} # Build Arguments ARG RPM_BUILD_DIRECTORY diff --git a/distributions/rpm/pom.xml b/distributions/rpm/pom.xml index 4a63270..6ae2df9 100644 --- a/distributions/rpm/pom.xml +++ b/distributions/rpm/pom.xml @@ -8,7 +8,7 @@ sourcehawk-dist com.optum.sourcehawk - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT sourcehawk-dist-rpm @@ -20,9 +20,10 @@ ${bintray.package} ${project.version} - a.${git.commit.time}~${git.commit.id.abbrev} + x86_64 ${project.build.directory}/rpmbuild + @@ -38,7 +39,7 @@ com.optum.sourcehawk sourcehawk-cli ${project.version} - bash-completion + completion sh provided @@ -51,8 +52,27 @@ - pl.project13.maven - git-commit-id-plugin + org.codehaus.gmavenplus + gmavenplus-plugin + + + save-git-information + validate + + execute + + + + + + + + @@ -137,7 +157,7 @@ com.optum.sourcehawk sourcehawk-cli - bash-completion + completion sh ${rpm.build.directory}/usr/share/bash-completion/completions ${rpm.package} @@ -193,23 +213,6 @@ - - build-and-tag-fedora-32 - prepare-package - - build - tag - - - ${project.artifactId}-builder-fedora-32 - ${project.version} - - fedora32 - /tmp/${rpm.package}-fedora-32.rpm - - true - - build-and-tag-fedora-33 prepare-package @@ -314,21 +317,6 @@ - - extract-fedora-32-package - package - - exec - - - ${project.parent.basedir}/scripts/extract-file-from-docker-container.sh - - ${project.artifactId}-builder-fedora-32:${project.version} - /tmp/${rpm.package}-fedora-32.rpm - ${project.build.directory} - - - extract-fedora-33-package package @@ -413,11 +401,6 @@ el8.${rpm.package.architecture} rpm - - ${project.build.directory}/${rpm.package}-fedora-32.rpm - fc32.${rpm.package.architecture} - rpm - ${project.build.directory}/${rpm.package}-fedora-33.rpm fc33.${rpm.package.architecture} @@ -442,156 +425,142 @@ - - - rpm-package-release - - - ci.release - - - - 1 - - - - rpm-package-snapshot - - - ci.snapshot - - - - - - org.codehaus.mojo - exec-maven-plugin - - - dev-snapshots - - - - - - - - rpm-package-publish - - - ci.deploy - - - - - - org.codehaus.mojo - exec-maven-plugin - - - ${bintray.organization} - ${rpm.package} - ${rpm.package.version} - ${rpm.package.release} - ${rpm.package.architecture} - - - - - publish-centos-7-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-centos-7.rpm - centos - el7 - - - - - publish-centos-8-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-centos-8.rpm - centos - el8 - - - - - publish-fedora-32-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-fedora-32.rpm - fedora - fc32 - - - - - publish-fedora-33-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-fedora-33.rpm - fedora - fc33 - - - - - publish-fedora-34-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-fedora-34.rpm - fedora - fc34 - - - - - publish-fedora-35-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-fedora-35.rpm - fedora - fc35 - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/distributions/scripts/build-and-push-docker-builders.sh b/distributions/scripts/build-and-push-docker-builders.sh deleted file mode 100755 index b0996ab..0000000 --- a/distributions/scripts/build-and-push-docker-builders.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -############################################################################################################## -# -# Push Docker Builders to Remote Registry -# -############################################################################################################## - -set -e - -ROOT_DIR="$( cd "$( dirname "$( dirname "$( dirname "${BASH_SOURCE[0]}" )")")" && pwd )" -DOCKER_BUILDERS_DIR="$ROOT_DIR/distributions/docker-builders" - -# Variables -DOCKER_ORG="optum/ci" -REGISTRY="ghcr.io" - -# Login to Registry -echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin $REGISTRY - -# Native Image -docker build -t $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.2.0-java8 -f "$DOCKER_BUILDERS_DIR/Dockerfile-nativeimage" --build-arg FROM=ghcr.io/graalvm/graalvm-ce:java8-21.2.0 . -docker build -t $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java11 -f "$DOCKER_BUILDERS_DIR/Dockerfile-nativeimage" --build-arg FROM=ghcr.io/graalvm/graalvm-ce:java11-21.3.0 . -docker build -t $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java17 -f "$DOCKER_BUILDERS_DIR/Dockerfile-nativeimage" --build-arg FROM=ghcr.io/graalvm/graalvm-ce:java17-21.3.0 . - -# RPM Build -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:centos7 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=centos:7 . -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:centos8 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=centos:8 . -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora32 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:32 . -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora33 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:33 . -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora34 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:34 . -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora35 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:35 . - -# Push All Builders to Remote Registry -docker push $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.2.0-java8 -docker push $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java11 -docker push $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java17 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:centos7 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:centos8 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora32 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora33 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora34 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora35 - -# Log out of registry -docker logout $REGISTRY \ No newline at end of file diff --git a/distributions/scripts/extract-file-from-docker-container.sh b/distributions/scripts/extract-file-from-docker-container.sh index e971015..347f34e 100755 --- a/distributions/scripts/extract-file-from-docker-container.sh +++ b/distributions/scripts/extract-file-from-docker-container.sh @@ -21,6 +21,7 @@ CONTAINER_ID=$(docker create "$DOCKER_IMAGE") # Copy the native images out of the docker container docker cp "$CONTAINER_ID:${DOCKER_PATH}" "$LOCAL_FILE_SYSTEM_PATH" +echo "Successfully copied [${DOCKER_PATH}] from container [$CONTAINER_ID] to [$LOCAL_FILE_SYSTEM_PATH]" # Remove the temporary container -docker rm -f -v "$CONTAINER_ID" +docker rm -f -v "$CONTAINER_ID" < /dev/null 2>&1 diff --git a/enforcer/core/pom.xml b/enforcer/core/pom.xml index 11ef76f..fb9c5d6 100644 --- a/enforcer/core/pom.xml +++ b/enforcer/core/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/aot/pom.xml b/enforcer/file/aot/pom.xml index 28d027c..43086e6 100644 --- a/enforcer/file/aot/pom.xml +++ b/enforcer/file/aot/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/aot/src/test/groovy/com/optum/sourcehawk/enforcer/file/aot/SourcehawkFileEnforcerRegistryProcessorSpec.groovy b/enforcer/file/aot/src/test/groovy/com/optum/sourcehawk/enforcer/file/aot/SourcehawkFileEnforcerRegistryProcessorSpec.groovy index 1aa5c6f..d657afd 100644 --- a/enforcer/file/aot/src/test/groovy/com/optum/sourcehawk/enforcer/file/aot/SourcehawkFileEnforcerRegistryProcessorSpec.groovy +++ b/enforcer/file/aot/src/test/groovy/com/optum/sourcehawk/enforcer/file/aot/SourcehawkFileEnforcerRegistryProcessorSpec.groovy @@ -154,7 +154,9 @@ class SourcehawkFileEnforcerRegistryProcessorSpec extends Specification { 1 * mockProcessingEnvironment.getFiler() >> mockFiler 1 * mockFiler.createSourceFile("com.optum.sourcehawk.enforcer.file.FileEnforcerRegistry", []) >> null 1 * mockProcessingEnvironment.getMessager() >> mockMessager - 1 * mockMessager.printMessage(Diagnostic.Kind.ERROR, 'Unable to generate file enforcer registry: java.lang.NullPointerException') + 1 * mockMessager.printMessage(Diagnostic.Kind.ERROR, _ as String) >> { kind, msg -> + msg == 'Unable to generate file enforcer registry: java.lang.NullPointerException' + } 0 * _ and: diff --git a/enforcer/file/common/pom.xml b/enforcer/file/common/pom.xml index aab36b7..8604ff5 100644 --- a/enforcer/file/common/pom.xml +++ b/enforcer/file/common/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/core/pom.xml b/enforcer/file/core/pom.xml index 082fa20..96c8013 100644 --- a/enforcer/file/core/pom.xml +++ b/enforcer/file/core/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/docker/pom.xml b/enforcer/file/docker/pom.xml index bda2632..6ec6ae9 100644 --- a/enforcer/file/docker/pom.xml +++ b/enforcer/file/docker/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/maven/pom.xml b/enforcer/file/maven/pom.xml index 06e1453..76dd2ef 100644 --- a/enforcer/file/maven/pom.xml +++ b/enforcer/file/maven/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml @@ -27,7 +27,7 @@ org.apache.maven maven-model - 3.6.3 + ${maven.version} compile diff --git a/enforcer/file/pom.xml b/enforcer/file/pom.xml index 5ef024b..bbb6251 100644 --- a/enforcer/file/pom.xml +++ b/enforcer/file/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/registry/pom.xml b/enforcer/file/registry/pom.xml index 71ac145..88c405e 100644 --- a/enforcer/file/registry/pom.xml +++ b/enforcer/file/registry/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml @@ -58,7 +58,7 @@ org.reflections reflections - 0.9.12 + 0.10.2 test diff --git a/enforcer/pom.xml b/enforcer/pom.xml index 2ecf225..ee974cc 100644 --- a/enforcer/pom.xml +++ b/enforcer/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/exec/pom.xml b/exec/pom.xml index a517011..a7bc559 100644 --- a/exec/pom.xml +++ b/exec/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml @@ -19,7 +19,7 @@ - 0.88 + 0.87 diff --git a/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanExecutor.java b/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanExecutor.java index edfbdce..7c8e96f 100644 --- a/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanExecutor.java +++ b/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanExecutor.java @@ -1,19 +1,15 @@ package com.optum.sourcehawk.exec.scan; import com.optum.sourcehawk.core.configuration.SourcehawkConfiguration; +import com.optum.sourcehawk.core.data.Severity; import com.optum.sourcehawk.core.protocol.file.FileProtocol; import com.optum.sourcehawk.core.result.ScanResult; -import com.optum.sourcehawk.core.data.Severity; import com.optum.sourcehawk.core.utils.CollectionUtils; import com.optum.sourcehawk.core.utils.FileUtils; import com.optum.sourcehawk.core.utils.Try; import com.optum.sourcehawk.enforcer.file.FileEnforcer; import com.optum.sourcehawk.exec.ConfigurationReader; import com.optum.sourcehawk.exec.ExecOptions; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.val; - import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; @@ -21,6 +17,9 @@ import java.util.Collection; import java.util.Collections; import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.val; /** * Entry point into executing scans @@ -117,7 +116,11 @@ private static ScanResult enforceFileProtocol(final ExecOptions execOptions, fin if (execOptions.getRepositoryFileReader().supportsGlobPatterns() && FileUtils.isGlobPattern(fileProtocol.getRepositoryPath())) { fileProtocolScanResults.addAll(executeFileEnforcerOnGlob(execOptions, fileProtocol, fileEnforcer)); } else { - fileProtocolScanResults.add(executeFileEnforcer(execOptions, fileProtocol.getRepositoryPath(), fileProtocol.getSeverity(), fileEnforcer)); + val scanResult = executeFileEnforcer(execOptions, fileProtocol.getRepositoryPath(), fileProtocol, fileEnforcer); + fileProtocolScanResults.add(scanResult); + if (isScanResultFileNotFound(scanResult)) { + break; + } } } return fileProtocolScanResults.stream() @@ -145,7 +148,7 @@ private static Collection executeFileEnforcerOnGlob(final ExecOption } val fileEnforcerScanResults = new ArrayList(repositoryPaths.size()); for (val repositoryPath : repositoryPaths) { - fileEnforcerScanResults.add(executeFileEnforcer(execOptions, repositoryPath, fileProtocol.getSeverity(), fileEnforcer)); + fileEnforcerScanResults.add(executeFileEnforcer(execOptions, repositoryPath, fileProtocol, fileEnforcer)); } return fileEnforcerScanResults; } @@ -154,19 +157,33 @@ private static Collection executeFileEnforcerOnGlob(final ExecOption * Execute the file enforcer to produce the scan result * * @param execOptions the exec options - * @param repositoryFilePath the repository file path - * @param severity the severity of the file protocol + * @param repositoryPath the repository path + * @param fileProtocol the file protocol * @param fileEnforcer the file enforcer to execute * @return the scan result * @throws IOException if any error occurs accessing the file or executing enforcer */ - private static ScanResult executeFileEnforcer(final ExecOptions execOptions, final String repositoryFilePath, final String severity, - final FileEnforcer fileEnforcer) throws IOException { - try (val fileInputStream = execOptions.getRepositoryFileReader().read(repositoryFilePath) - .orElseThrow(() -> new IOException(String.format("File not found: %s", repositoryFilePath)))) { + private static ScanResult executeFileEnforcer(final ExecOptions execOptions, final String repositoryPath, final FileProtocol fileProtocol, + final FileEnforcer fileEnforcer) throws IOException { + val fileInputStreamOptional = execOptions.getRepositoryFileReader().read(repositoryPath); + if (!fileInputStreamOptional.isPresent()) { + return ScanResultFactory.fileNotFound(execOptions, repositoryPath, fileProtocol.getSeverity()); + } + try (val fileInputStream = fileInputStreamOptional.get()) { val enforcerResult = fileEnforcer.enforce(fileInputStream); - return ScanResultFactory.enforcerResult(execOptions, repositoryFilePath, Severity.parse(severity), enforcerResult); + return ScanResultFactory.enforcerResult(execOptions, repositoryPath, Severity.parse(fileProtocol.getSeverity()), enforcerResult); } } + /** + * Determine if the {@link ScanResult} is because the file was not found + * + * @param scanResult the scan result + * @return true if because of file not being found, false otherwise + */ + private static boolean isScanResultFileNotFound(final ScanResult scanResult) { + return scanResult.getFormattedMessages().size() == 1 + && scanResult.getFormattedMessages().stream().anyMatch(message -> message.contains("File not found")); + } + } diff --git a/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanResultFactory.java b/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanResultFactory.java index 32c3245..9bf3099 100644 --- a/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanResultFactory.java +++ b/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanResultFactory.java @@ -5,13 +5,12 @@ import com.optum.sourcehawk.core.result.ScanResult; import com.optum.sourcehawk.enforcer.EnforcerResult; import com.optum.sourcehawk.exec.ExecOptions; -import lombok.experimental.UtilityClass; -import lombok.val; - import java.util.ArrayList; import java.util.Collections; import java.util.Optional; import java.util.function.IntConsumer; +import lombok.experimental.UtilityClass; +import lombok.val; /** * A factory for creating instances of {@link ScanResult} @@ -95,18 +94,31 @@ public ScanResult globalError(final Throwable throwable) { * @return the file not found scan result */ public ScanResult fileNotFound(final ExecOptions execOptions, final FileProtocol fileProtocol) { - val severity = Severity.parse(fileProtocol.getSeverity()); + return fileNotFound(execOptions, fileProtocol.getRepositoryPath(), fileProtocol.getSeverity()); + } + + /** + * Generate a scan result for situations where the file is not found + * + * @param execOptions the exec options + * @param repositoryPath the repository path + * @param fileProtocolSeverity the file protocol severity + * @return the file not found scan result + */ + public ScanResult fileNotFound(final ExecOptions execOptions, final String repositoryPath, final String fileProtocolSeverity) { + val severity = Severity.parse(fileProtocolSeverity); val messageDescriptor = ScanResult.MessageDescriptor.builder() - .severity(fileProtocol.getSeverity()) - .repositoryPath(fileProtocol.getRepositoryPath()) - .message("File not found") - .build(); + .severity(fileProtocolSeverity) + .repositoryPath(repositoryPath) + .message("File not found") + .build(); val scanResultBuilder = ScanResult.builder() - .passed(Severity.WARNING.equals(severity) && !execOptions.isFailOnWarnings()) - .messages(Collections.singletonMap(fileProtocol.getRepositoryPath(), Collections.singleton(messageDescriptor))) - .formattedMessages(Collections.singleton(messageDescriptor.toString())); - return acceptCount(scanResultBuilder, Severity.parse(fileProtocol.getSeverity()), 1) - .build(); + + .passed(Severity.WARNING.equals(severity) && !execOptions.isFailOnWarnings()) + .messages(Collections.singletonMap(repositoryPath, Collections.singleton(messageDescriptor))) + .formattedMessages(Collections.singleton(messageDescriptor.toString())); + return acceptCount(scanResultBuilder, severity, 1) + .build(); } /** diff --git a/exec/src/test/groovy/com/optum/sourcehawk/exec/ConfigurationReaderSpec.groovy b/exec/src/test/groovy/com/optum/sourcehawk/exec/ConfigurationReaderSpec.groovy index d4ee1fe..ba6c9cf 100644 --- a/exec/src/test/groovy/com/optum/sourcehawk/exec/ConfigurationReaderSpec.groovy +++ b/exec/src/test/groovy/com/optum/sourcehawk/exec/ConfigurationReaderSpec.groovy @@ -2,7 +2,6 @@ package com.optum.sourcehawk.exec import com.optum.sourcehawk.core.configuration.SourcehawkConfiguration import org.spockframework.util.IoUtil -import sun.nio.ch.ChannelInputStream class ConfigurationReaderSpec extends FileBaseSpecification { @@ -38,7 +37,7 @@ class ConfigurationReaderSpec extends FileBaseSpecification { InputStream inputStream = ConfigurationReader.obtainInputStream(repositoryRoot, configurationFileLocation).get() then: - inputStream instanceof ChannelInputStream + inputStream.class.name == 'sun.nio.ch.ChannelInputStream' } def "obtainInputStream - relative file"() { @@ -50,7 +49,7 @@ class ConfigurationReaderSpec extends FileBaseSpecification { then: inputStream - inputStream instanceof ChannelInputStream + inputStream.class.name == 'sun.nio.ch.ChannelInputStream' } def "merge"() { diff --git a/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy b/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy index 0ca74cc..20adfc2 100644 --- a/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy +++ b/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy @@ -1,9 +1,10 @@ package com.optum.sourcehawk.exec import com.optum.sourcehawk.core.data.RemoteRef -import com.optum.sourcehawk.core.repository.GithubRepositoryFileReader + import com.optum.sourcehawk.core.repository.LocalRepositoryFileReader import com.optum.sourcehawk.core.data.Verbosity +import com.optum.sourcehawk.core.repository.RemoteRepositoryFileReader import spock.lang.Specification import java.nio.file.Paths @@ -54,15 +55,15 @@ class ExecOptionsSpec extends Specification { !execOptions.remoteRef } - def "builder - github"() { + def "builder - remote"() { given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") + RemoteRef remoteRef = RemoteRef.parse("owner/repo", "main") ExecOptions.ExecOptionsBuilder builder = ExecOptions.builder() .repositoryRoot(Paths.get("/")) .configurationFileLocation("Sourcehawk") .verbosity(Verbosity.ZERO) .failOnWarnings(true) - .repositoryFileReader(new GithubRepositoryFileReader("token", remoteRef)) + .repositoryFileReader(new RemoteRepositoryFileReader("https://raw.githubusercontent.com/owner/repo/main/%s", Collections.emptyMap())) .remoteRef(remoteRef) when: @@ -75,7 +76,7 @@ class ExecOptionsSpec extends Specification { execOptions.verbosity == Verbosity.ZERO !execOptions.tags execOptions.failOnWarnings - execOptions.repositoryFileReader instanceof GithubRepositoryFileReader + execOptions.repositoryFileReader instanceof RemoteRepositoryFileReader execOptions.remoteRef == remoteRef } diff --git a/exec/src/test/resources/sourcehawk-flattened-base.yml b/exec/src/test/resources/sourcehawk-flattened-base.yml index b250224..2a183f9 100644 --- a/exec/src/test/resources/sourcehawk-flattened-base.yml +++ b/exec/src/test/resources/sourcehawk-flattened-base.yml @@ -53,4 +53,4 @@ file-protocols: enforcers: - enforcer: ".common.StringPropertyEquals" property-name: "distributionUrl" - expected-property-value: "https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip" + expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip" diff --git a/pom.xml b/pom.xml index 8ecf522..39b2b17 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ sourcehawk - 0.6.0-SNAPSHOT + 0.7.0-SNAPSHOT pom Sourcehawk @@ -28,7 +28,7 @@ 0.97 - 1.7.30 + 1.7.32 e1 @@ -108,13 +108,19 @@ org.spockframework spock-core - 2.0-M3-groovy-3.0 + 2.0-groovy-3.0 + test + + + org.codehaus.groovy + groovy + ${groovy.version} test junit junit - 4.13.1 + 4.13.2 test @@ -207,6 +213,7 @@ distributions-build + 11 ci.build