diff --git a/.github/actions/build-cached-cves/action.yaml b/.github/actions/build-cached-cves/action.yaml new file mode 100644 index 0000000000..289dbc0fa2 --- /dev/null +++ b/.github/actions/build-cached-cves/action.yaml @@ -0,0 +1,46 @@ +name: "Build with cached CVEs" +inputs: + gh-token: + description: "GitHub Token for authentication" + required: true + +runs: + using: "composite" + steps: + - name: Install jq (JSON processor) if not found + run: | + if ! command -v jq &> /dev/null; then + sudo apt-get update + sudo apt-get install -y jq + else + echo "jq is already installed. Skipping install..." + fi + shell: bash + + - name: Download CVE Data + run: | + # Find the latest CVE upload workflow. + run_id=$(gh run list --workflow="post_release.yaml" --limit 1 --status=success --json databaseId | jq -r '.[0].databaseId') + echo 'Fetching artifacts from run $run_id' + # Remove any downloaded artifacts, should they exist. + rm -rf ./downloaded_artifacts + # Download the latest artifact to a new dir. + gh run download ${run_id} --name security-bulletins --dir ./downloaded_artifacts + shell: bash + env: + GH_TOKEN: ${{ inputs.gh-token }} + + - name: Unpack CVE data + run: | + # Ensure the correct folders exist. + mkdir -p .docusaurus/security-bulletins/default + # Move the files to their correct places in the checked out repository + mv downloaded_artifacts/data.json .docusaurus/security-bulletins/default/data.json + rm -rf downloaded_artifacts + shell: bash + + - name: Build + run: | + rm -rf build + npm run build + shell: bash diff --git a/.github/actions/build-cached-packs/action.yaml b/.github/actions/build-cached-packs/action.yaml index a3aa251cc2..c017dbd6c2 100644 --- a/.github/actions/build-cached-packs/action.yaml +++ b/.github/actions/build-cached-packs/action.yaml @@ -20,7 +20,7 @@ runs: - name: Download Packs Data run: | # Find the latest packs upload workflow. - run_id=$(gh run list --workflow="post_release.yaml" --limit 1 --json databaseId | jq -r '.[0].databaseId') + run_id=$(gh run list --workflow="post_release.yaml" --limit 1 --status=success --json databaseId | jq -r '.[0].databaseId') # Remove any downloaded artifacts, should they exist. rm -rf ./downloaded_artifacts # Download the latest artifact to a new dir. diff --git a/.github/workflows/dependabot.yaml b/.github/workflows/dependabot.yaml index 52e7f2c45d..87a010f9d7 100644 --- a/.github/workflows/dependabot.yaml +++ b/.github/workflows/dependabot.yaml @@ -53,3 +53,9 @@ jobs: uses: ./.github/actions/build-cached-packs with: gh-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build with cached CVEs + if: ${{ env.BUILD_EXIT_CODE == '7' }} + uses: ./.github/actions/build-cached-cves + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/nightly-docker-build.yaml b/.github/workflows/nightly-docker-build.yaml index 488a615bac..b3abdac225 100644 --- a/.github/workflows/nightly-docker-build.yaml +++ b/.github/workflows/nightly-docker-build.yaml @@ -42,6 +42,12 @@ jobs: uses: ./.github/actions/build-cached-packs with: gh-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build with cached CVEs + if: ${{ env.BUILD_EXIT_CODE == '7' }} + uses: ./.github/actions/build-cached-cves + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/post_release.yaml b/.github/workflows/post_release.yaml index 2e3be37168..9035299303 100644 --- a/.github/workflows/post_release.yaml +++ b/.github/workflows/post_release.yaml @@ -50,6 +50,12 @@ jobs: with: gh-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build with cached CVEs + if: ${{ env.BUILD_EXIT_CODE == '7' }} + uses: ./.github/actions/build-cached-cves + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Build Packs uses: actions/upload-artifact@v4 with: @@ -59,3 +65,11 @@ jobs: .docusaurus/packs-integrations if-no-files-found: error retention-days: 7 + + - name: Upload Built Security Bulletins + uses: actions/upload-artifact@v4 + with: + name: "security-bulletins" + path: .docusaurus/security-bulletins/default/data.json + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index df50bfe523..bdb7786297 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -23,7 +23,7 @@ env: PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }} DISABLE_PACKS_INTEGRATIONS: ${{ secrets.DISABLE_PACKS_INTEGRATIONS }} DISABLE_SECURITY_INTEGRATIONS: ${{ secrets.DISABLE_SECURITY_INTEGRATIONS }} - DSO_AUTH_TOKEN: ${{ secrets.DSO_AUTH_TOKEN }} + DSO_AUTH_TOKEN: ${{ secrets.DSO_AUTH_TOKEN }} jobs: run-ci: @@ -104,5 +104,11 @@ jobs: - name: Build with cached packs if: ${{ env.BUILD_EXIT_CODE == '5' }} uses: ./.github/actions/build-cached-packs + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build with cached CVEs + if: ${{ env.BUILD_EXIT_CODE == '7' }} + uses: ./.github/actions/build-cached-cves with: gh-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release-branch-pr.yaml b/.github/workflows/release-branch-pr.yaml index f429dbf51a..0cd295957f 100644 --- a/.github/workflows/release-branch-pr.yaml +++ b/.github/workflows/release-branch-pr.yaml @@ -73,3 +73,9 @@ jobs: uses: ./.github/actions/build-cached-packs with: gh-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build with cached CVEs + if: ${{ env.BUILD_EXIT_CODE == '7' }} + uses: ./.github/actions/build-cached-cves + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-preview.yaml b/.github/workflows/release-preview.yaml index 65ef0b126d..b72faaf46b 100644 --- a/.github/workflows/release-preview.yaml +++ b/.github/workflows/release-preview.yaml @@ -70,6 +70,12 @@ jobs: with: gh-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build with cached CVEs + if: ${{ env.BUILD_EXIT_CODE == '7' }} + uses: ./.github/actions/build-cached-cves + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} + - name: Deploy Preview run: | aws s3 sync --cache-control 'public, max-age=604800' --exclude '*.html' --exclude build/scripts/ build/ s3://docs-latest.spectrocloud.com --delete diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 53f582c5bf..de4399c5c5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -74,6 +74,12 @@ jobs: with: gh-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build with cached CVEs + if: ${{ env.BUILD_EXIT_CODE == '7' }} + uses: ./.github/actions/build-cached-cves + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} + - name: Upload to AWS run: | echo "CURRENT_STEP=Upload to AWS" >> $GITHUB_ENV diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index 23d94271d8..7d7046479f 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -57,6 +57,12 @@ jobs: with: gh-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build with cached CVEs + if: ${{ env.BUILD_EXIT_CODE == '7' }} + uses: ./.github/actions/build-cached-cves + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Build uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 71768d609b..88b8f83674 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -85,6 +85,12 @@ jobs: with: gh-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build with cached CVEs + if: ${{ env.BUILD_EXIT_CODE == '7' }} + uses: ./.github/actions/build-cached-cves + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Build uses: actions/upload-artifact@v4 with: diff --git a/Makefile b/Makefile index 075899d823..b7cd574b2b 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,20 @@ start-cached-packs: ## Start a local development server with cached packs retry. fi; \ } +start-cached-cves: ## Start a local development server with cached CVEs retry. + make generate-partials + @{ \ + npm run start; \ + exit_code=$$?; \ + if [ "$$exit_code" = "7" ]; then \ + echo "❌ Start has failed due to missing CVE data..."; \ + echo "ℹ️ Initializing fetch cached CVE data..."; \ + make get-cached-cves; \ + echo "ℹ️ Retrying start... "; \ + npm run start;\ + fi; \ + } + build: ## Run npm build @echo "building site" npm run clear @@ -128,6 +142,22 @@ build-cached-packs: ## Run npm build with cached packs retry fi; \ } +build-cached-cves: ## Run npm build with cached CVEs retry + @echo "building site" + npm run clear + rm -rf build + @{ \ + npm run build; \ + exit_code=$$?; \ + if [ "$$exit_code" = "7" ]; then \ + echo "❌ Build has failed due to missing CVE data..."; \ + echo "ℹ️ Initializing fetch cached CVE data..."; \ + make get-cached-cves; \ + echo "ℹ️ Retrying build... "; \ + npm run build;\ + fi; \ + } + build-ci: ## Run npm build in CI environment @echo "building site" npm run clear @@ -287,6 +317,10 @@ generate-partials: ## Generate get-cached-packs: ./scripts/get-cached-packs.sh + +###@ Fetch security bulletins +get-cached-cves: + ./scripts/get-cached-cves.sh ###@ Aloglia Indexing diff --git a/README.md b/README.md index ef48b39361..7c8788efe3 100644 --- a/README.md +++ b/README.md @@ -802,6 +802,31 @@ will stop the pre-build script from fetching the security bulletins. export DISABLE_SECURITY_INTEGRATIONS=true ``` +### Cached Security Bulletins + +All CVE related data is saved to a +[GitHub Workflow Artifact](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/storing-and-sharing-data-from-a-workflow#about-workflow-artifacts) +after every successful release to production. Check out the [post_release.yaml](.github/workflows/post_release.yaml) for +further details. The cached data enables us to build and start librarium without performing any CVE-related API queries. +All of our GitHub workflows will use this cached data as a fallback in the case of an API related build failure. Check +out the [build-cached-cves action.yaml](.github/actions/build-cached-cves/action.yaml) to learn how the cached data is +fetched and used. + +CVE data is saved locally in the `.docusaurus/security-bulletins/default/data.json` file. You can remove the data using +`make clean-security`. You can use the cached CVE artifact locally when you want to avoid the CVE download time. This +flow also helps you when you don't have any local CVE data and we are experiencing an API outage. + +librarium provides the following commands which fetch cached CVE data to your local environment. + +| **Command** | **Description** | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `make get-cached-cves` | Fetch the CVE data artifact and place files in the correct places. You can then execute `make start` or `make build` as usual. | +| `make start-cached-cves` | Attempt to start the local development server. If a CVE related outage is detected, fetch the CVE data artifact and retry the `start` command. | +| `make build-cached-cves` | Attempt to build the application. If a CVE related outage is detected, fetch the CVE data artifact and retry the `build` command. | + +These scripts will prompt you to install and authenticate the [GitHub CLI](https://cli.github.com/) before you can +proceed. + ## Packs Component The packs component is a custom component that displays all packs available in Palette SaaS by querying the Palette API @@ -870,7 +895,7 @@ Settting the `DISABLE_PACKS_PLUGIN` environment variable to `true` will also hav All pack related data is saved to a [GitHub Workflow Artifact](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/storing-and-sharing-data-from-a-workflow#about-workflow-artifacts) -after every succesful release to production. Check out the [post_release.yaml](.github/workflows/post_release.yaml) for +after every successful release to production. Check out the [post_release.yaml](.github/workflows/post_release.yaml) for further details. The cached data enables us to build and start librarium without performing any pack related API queries. All of our GitHub workflows will use this cached data as a fallback in the case of an API related build failure. Check out the [build-cached-packs action.yaml](.github/actions/build-cached-packs/action.yaml) to learn how the @@ -1166,8 +1191,9 @@ make clean-versions Librarium provides the following exit codes. These exit codes are returned by both the `npm run start` and `npm run build` commands. -| **Exit Code** | **Description** | -| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `0` | The command was executed successfully. | -| `5` | The command failed due to errors received from the API service. These requests are issued by the [Packs Component](#packs-component) and librarium cannot start without loading packs, either from the API service or the [cached packs data](#cached-packs-data) | -| Any other non-zero exit code. | The command failed due to another error. Check the command output. | +| **Exit Code** | **Description** | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `0` | The command was executed successfully. | +| `5` | The command failed due to errors received from the API service related to packs. These requests are issued by the [Packs Component](#packs-component) and librarium cannot start without loading packs, either from the API service or the [cached packs data](#cached-packs-data) | +| `7` | The command failed due to errors received from the API service related to security bulletins. These requests are issued by the [CVE script](./utils/cves/index.js) and librarium cannot start without loading the security bulletins. The [Build with Cached CVE](./.github/actions/build-cached-cves/action.yaml) action is built to handle this exit situation and build with cached CVEs. You can issue the command `make get-cached-cves` to fetch cached CVEs to build locally. | +| Any other non-zero exit code. | The command failed due to another error. Check the command output. | diff --git a/scripts/get-cached-cves.sh b/scripts/get-cached-cves.sh new file mode 100755 index 0000000000..db4eb8714e --- /dev/null +++ b/scripts/get-cached-cves.sh @@ -0,0 +1,47 @@ +#!/bin/bash + + +# Enable error handling +set -e + +echo "Starting fetch of cached cves..." + +if command -v gh &> /dev/null +then + echo "✅ GitHub CLI is installed." +else + echo "❌ GitHub CLI is not installed." + echo "ℹ️ Use 'brew install gh' to install it with Homebrew." + exit 1 +fi + +if gh auth status &> /dev/null +then + echo "✅ GitHub CLI is authenticated. " +else + echo "❌ GitHub CLI is not authenticated." + echo "ℹ️ Please log in with 'gh auth login'." + exit 1 +fi + # Find the latest cves upload workflow. + +run_id=$(gh run list --workflow="post_release.yaml" --status=success --limit 1 --json databaseId | jq -r '.[0].databaseId') + +# Remove any downloaded artifacts, should they exist. +rm -rf ./downloaded_artifacts + +# Download the latest artifact to a new dir. +gh run download ${run_id} --name security-bulletins --dir ./downloaded_artifacts +echo "✅ Cached CVEs artifact downloaded." + +# Ensure the correct folders exist. +mkdir -p .docusaurus/security-bulletins/default + +# Move the files to their correct places in the checked out repository +mv downloaded_artifacts/data.json .docusaurus/security-bulletins/default/data.json + +# Clean up. +rm -rf downloaded_artifacts + +echo "✅ Completed fetch of cached CVEs." +echo "⏭️ You can now execute 'make start' or 'make build'." \ No newline at end of file diff --git a/scripts/get-cached-packs.sh b/scripts/get-cached-packs.sh index e0e911641d..e2fec95733 100755 --- a/scripts/get-cached-packs.sh +++ b/scripts/get-cached-packs.sh @@ -24,7 +24,7 @@ else fi # Find the latest packs upload workflow. -run_id=$(gh run list --workflow="post_release.yaml" --limit 1 --json databaseId | jq -r '.[0].databaseId') +run_id=$(gh run list --workflow="post_release.yaml" --status=success --limit 1 --json databaseId | jq -r '.[0].databaseId') # Remove any downloaded artifacts, should they exist. rm -rf ./downloaded_artifacts diff --git a/utils/cves/index.js b/utils/cves/index.js index 541648bd72..ced233b0db 100644 --- a/utils/cves/index.js +++ b/utils/cves/index.js @@ -39,6 +39,8 @@ async function getSecurityBulletins(payload) { return { data: results }; } catch (error) { logger.error("Error:", error.response ? `${error.response.status} - ${error.response.data}` : error.message); + // Return exit code 7 to indicate that the script failed to fetch the security bulletins for GitHub Actions + process.exit(7); } } @@ -200,6 +202,8 @@ async function generateCVEs() { } catch (error) { logger.error(error); logger.error("Error:", error.response ? error.response.status : error.message); + // Return exit code 7 to indicate that the script failed to fetch the security bulletins for GitHub Actions + process.exit(7); } } @@ -313,5 +317,5 @@ ${revisionHistory ? revisionHistory : "No revision history available."} try { generateCVEs(); } catch (error) { - process.exit(5); + process.exit(7); }