From faabc0d2a1bb8ae459f7d83fe9d8c184322e23af Mon Sep 17 00:00:00 2001 From: Gautier DI FOLCO Date: Thu, 28 Mar 2024 23:19:01 +0100 Subject: [PATCH] fix: simplify CI --- .github/workflows/call-check-advisories.yml | 75 ------------- .github/workflows/call-nix.yml | 37 ------- .github/workflows/check-advisories.yml | 65 ----------- .github/workflows/nix.yml | 117 +++++++++++++++----- 4 files changed, 91 insertions(+), 203 deletions(-) delete mode 100644 .github/workflows/call-check-advisories.yml delete mode 100644 .github/workflows/call-nix.yml delete mode 100644 .github/workflows/check-advisories.yml diff --git a/.github/workflows/call-check-advisories.yml b/.github/workflows/call-check-advisories.yml deleted file mode 100644 index a4548294..00000000 --- a/.github/workflows/call-check-advisories.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Check and publish security advisories -on: - workflow_call: - inputs: - fetch-key: - required: true - type: string - changed-advisories: - required: false - type: string - default: '[]' -jobs: - check-advisories: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - with: - path: source - # We need to retrieve full history to determine the correct - # `published` and `modified` timestamps - fetch-depth: 0 - - run: mkdir -p ~/.local/dockerImages - - name: Fetch cache - id: fetch-binaries - uses: actions/cache/restore@v3 - with: - key: ${{ inputs.fetch-key }} - path: ~/.local/dockerImages - fail-on-cache-miss: true - - run: docker load -i ~/.local/dockerImages/hsec-tools - - name: Run advisory syntax checks - env: - CHANGED_ADVISORIES_JSON: ${{ inputs.changed-advisories }} - run: | - CHANGED_ADVISORIES=( $(printenv CHANGED_ADVISORIES_JSON | jq -r '.[]') ) - cd source - RESULT=0 - # Remove the begining of the README to extract the example. - (echo '```toml'; sed -e '1,/```toml/d' README.md) > EXAMPLE_README.md - while read FILE ; do - [ "$(dirname "$FILE")" != advisories/reserved ] || continue - echo -n "$FILE: " - docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools check "$FILE" || RESULT=1 - done < <([ ${#CHANGED_ADVISORIES[@]} -gt 0 ] && printf "%s\n" "${CHANGED_ADVISORIES[@]}" || find advisories EXAMPLE_README.md EXAMPLE_ADVISORY.md -type f -name "*.md") - exit $RESULT - - name: Run advisory uniqueness checks - run: | - ! find source/advisories -type f -name '*.md' -print0 \ - | xargs -0n1 basename | sort | uniq -c | grep -E -v '[[:space:]]*1 ' - - name: Publish OSV data - if: ${{ github.event_name == 'push' && github.ref_name == 'main' && github.repository == 'haskell/security-advisories' }} - env: - GITHUB_SHA: ${{ github.sha }} - run: | - DATA_DIR=$PWD/osv - mkdir "$DATA_DIR" - cd source - while read FILE ; do - ID=$(basename "$FILE" .md) - YEAR=$(echo "$ID" | cut -d - -f 2) - mkdir -p $DATA_DIR/$YEAR - docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools osv "$FILE" > $DATA_DIR/$YEAR/$ID.json - done < <(find advisories -type f -name "*.md") - BRANCH=generated/osv-export - REF=refs/remotes/origin/$BRANCH - export GIT_WORK_TREE=$DATA_DIR - git read-tree "$REF" - git add --all --intent-to-add - git diff --quiet && exit - git add --all - TREE=$(git write-tree) - git config user.email security-advisories@haskell.org - git config user.name "Haskell Security Response Team" - COMMIT=$(git commit-tree "$TREE" -p "$REF" -m "$(date --utc --rfc-3339=seconds) ($GITHUB_SHA)") - git push origin $COMMIT:$BRANCH diff --git a/.github/workflows/call-nix.yml b/.github/workflows/call-nix.yml deleted file mode 100644 index b09e5c2a..00000000 --- a/.github/workflows/call-nix.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: nix build -on: - workflow_call: - inputs: - cache-key: - required: true - type: string -jobs: - check_nix: - name: Check nix build - runs-on: ubuntu-22.04 - steps: - - name: git checkout - uses: actions/checkout@v4 - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@main - with: - extra-conf: system-features = nixos-test benchmark big-parallel kvm - - uses: DeterminateSystems/magic-nix-cache-action@main - - name: Check Nix flake inputs - uses: DeterminateSystems/flake-checker-action@v4 - - name: Build executable - run: nix -L build - - name: Build docker image - run: nix build -L '.#packages.x86_64-linux.hsec-tools-image' - - run: mkdir -p ~/.local/dockerImages - - run: cp result ~/.local/dockerImages/hsec-tools - - uses: actions/cache/save@v3 - with: - key: ${{ inputs.cache-key }} - path: ~/.local/dockerImages - - name: upload executable - uses: actions/upload-artifact@v3 - if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} - with: - name: hsec-tools-${{ github.sha }} - path: ~/.local/dockerImages diff --git a/.github/workflows/check-advisories.yml b/.github/workflows/check-advisories.yml deleted file mode 100644 index 25ea428e..00000000 --- a/.github/workflows/check-advisories.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Check advisories -on: - - pull_request -jobs: - tools_changed: - continue-on-error: true - runs-on: ubuntu-22.04 - outputs: - should_skip: ${{ steps.skip_check.outputs.should_skip }} - steps: - - id: skip_check - uses: fkirc/skip-duplicate-actions@v5.3.1 - with: - concurrent_skipping: "never" - skip_after_successful_duplicate: "true" - paths: '["code/**"]' - do_not_skip: '["push", "workflow_dispatch", "schedule"]' - advisories_changed: - continue-on-error: true - runs-on: ubuntu-22.04 - outputs: - should_skip: ${{ steps.skip_check.outputs.should_skip }} - changed_files: ${{ steps.process-changed-files.outputs.out }} - steps: - - id: skip_check - uses: fkirc/skip-duplicate-actions@v5.3.1 - with: - concurrent_skipping: "never" - skip_after_successful_duplicate: "true" - paths: '["advisories/**", "EXAMPLE_ADVISORY.md"]' - do_not_skip: '["push", "workflow_dispatch", "schedule"]' - - id: process-changed-files - name: Extract matched files list - env: - PATHS_RESULT: ${{ steps.skip_check.outputs.paths_result }} - run: | - echo -n 'out=' >> "$GITHUB_OUTPUT" - # See https://github.com/fkirc/skip-duplicate-actions#paths_result - printenv PATHS_RESULT \ - | jq --compact-output .global.matched_files >> "$GITHUB_OUTPUT" - code_hash: - name: Compute code directory hash - runs-on: ubuntu-22.04 - outputs: - code_hash: ${{ steps.code-hash.outputs.code-hash }} - steps: - - name: git checkout - uses: actions/checkout@v4 - - id: code-hash - run: | - code_hash=$(git rev-parse HEAD:code) - echo "code-hash=$code_hash" >> "$GITHUB_OUTPUT" - populate_cache: - name: Populate cache - uses: ./.github/workflows/call-nix.yml - with: - cache-key: hsec-tools-${{ needs.code_hash.outputs.code_hash }} - check_advisories: - name: Invoke check-advisories workflow - needs: [tools_changed, advisories_changed, code_hash, populate_cache] - if: ${{ needs.tools_changed.outputs.should_skip == 'true' && needs.advisories_changed.outputs.should_skip != 'true' }} - uses: ./.github/workflows/call-check-advisories.yml - with: - fetch-key: hsec-tools-${{ needs.code_hash.outputs.code_hash }} - changed-advisories: ${{ needs.advisories_changed.outputs.changed_files }} diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index b316da03..2e2e9f31 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -1,35 +1,99 @@ -name: nix build +name: nix build and check advisories on: - push - pull_request jobs: - tools-changed: - continue-on-error: true + check-nix: + name: Check nix build runs-on: ubuntu-22.04 - outputs: - should_skip: ${{ steps.skip_check.outputs.should_skip }} steps: - - id: skip_check - uses: fkirc/skip-duplicate-actions@v5.3.1 + - name: git checkout + uses: actions/checkout@v4 + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@main with: - concurrent_skipping: "never" - skip_after_successful_duplicate: "true" - paths: '["code/**"]' - do_not_skip: '["push", "workflow_dispatch", "schedule"]' - check-nix: - name: Check nix build - needs: tools-changed - if: ${{ needs.tools_changed.outputs.should_skip != 'true' }} - uses: ./.github/workflows/call-nix.yml - with: - cache-key: hsec-tools-${{ github.sha }} + extra-conf: system-features = nixos-test benchmark big-parallel kvm + - uses: DeterminateSystems/magic-nix-cache-action@main + - name: Check Nix flake inputs + uses: DeterminateSystems/flake-checker-action@v4 + - name: Build executable (hsec-tools) + run: nix -L build + - name: Build docker image + run: nix build -L '.#packages.x86_64-linux.hsec-tools-image' + - run: mkdir -p ~/.local/dockerImages + - run: cp result ~/.local/dockerImages/hsec-tools + - name: Cache executable + uses: actions/cache/save@v4 + with: + key: hsec-tools-${{ github.sha }} + path: ~/.local/dockerImages + - name: upload executable + uses: actions/upload-artifact@v4 + if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} + with: + name: hsec-tools-${{ github.sha }} + path: ~/.local/dockerImages check-advisories: - name: Invoke check-advisories workflow - if: ${{ needs.tools_changed.outputs.should_skip != 'true' }} + name: Check advisories needs: check-nix - uses: ./.github/workflows/call-check-advisories.yml - with: - fetch-key: hsec-tools-${{ github.sha }} + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + path: source + # We need to retrieve full history to determine the correct + # `published` and `modified` timestamps + fetch-depth: 0 + - run: mkdir -p ~/.local/dockerImages + - name: Fetch binaries + uses: actions/cache/restore@v4 + with: + key: hsec-tools-${{ github.sha }} + path: ~/.local/dockerImages + fail-on-cache-miss: true + - run: docker load -i ~/.local/dockerImages/hsec-tools + - name: Run advisory syntax checks + run: | + cd source + RESULT=0 + # Remove the begining of the README to extract the example. + (echo '```toml'; sed -e '1,/```toml/d' README.md) > EXAMPLE_README.md + while read FILE ; do + [ "$(dirname "$FILE")" != advisories/reserved ] || continue + echo -n "$FILE: " + docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools check "$FILE" || RESULT=1 + done < <(find advisories EXAMPLE_README.md EXAMPLE_ADVISORY.md -type f -name "*.md") + exit $RESULT + - name: Run advisory uniqueness checks + run: | + ! find source/advisories -type f -name '*.md' -print0 \ + | xargs -0n1 basename | sort | uniq -c | grep -E -v '[[:space:]]*1 ' + - name: Publish OSV data + if: ${{ github.event_name == 'push' && github.ref_name == 'main' && github.repository == 'haskell/security-advisories' }} + env: + GITHUB_SHA: ${{ github.sha }} + run: | + DATA_DIR=$PWD/osv + mkdir "$DATA_DIR" + cd source + while read FILE ; do + ID=$(basename "$FILE" .md) + YEAR=$(echo "$ID" | cut -d - -f 2) + mkdir -p $DATA_DIR/$YEAR + docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools osv "$FILE" > $DATA_DIR/$YEAR/$ID.json + done < <(find advisories -type f -name "*.md") + BRANCH=generated/osv-export + REF=refs/remotes/origin/$BRANCH + export GIT_WORK_TREE=$DATA_DIR + git read-tree "$REF" + git add --all --intent-to-add + git diff --quiet && exit + git add --all + TREE=$(git write-tree) + git config user.email security-advisories@haskell.org + git config user.name "Haskell Security Response Team" + COMMIT=$(git commit-tree "$TREE" -p "$REF" -m "$(date --utc --rfc-3339=seconds) ($GITHUB_SHA)") + git push origin $COMMIT:$BRANCH generate-website: name: Generate advisories website if: ${{ github.ref == 'refs/heads/main' }} @@ -44,11 +108,12 @@ jobs: # `published` and `modified` timestamps fetch-depth: 0 - run: mkdir -p ~/.local/dockerImages - - name: Fetch artifact - uses: actions/download-artifact@v3 + - name: Fetch binaries + uses: actions/cache/restore@v4 with: - name: hsec-tools-${{ github.sha }} + key: hsec-tools-${{ github.sha }} path: ~/.local/dockerImages + fail-on-cache-miss: true - run: docker load -i ~/.local/dockerImages/hsec-tools - name: Generate the website run: |