chore: add caching for CVE data (#5281) (#5309)

* docs: add caching for CVE data * docs: modify folders * docs: clean up comments * chore: add additional try catch * docs: test with error * docs: remove error for new test * docs: modify unpack and add error back * ci: auto-formatting prettier issues * docs: add success status filter * chore: add status filter * docs: fix command error * docs: remove test error * docs: remove test error * docs: add filter to packs * docs: modify one more error code * docs: add to readme * test with dummy dso value * docs: change dso token back --------- Co-authored-by: lennessyy <[email protected]> (cherry picked from commit 472d925) Co-authored-by: Lenny Chen <[email protected]>
spectrocloud · Jan 7, 2025 · 3cd0b0f · 3cd0b0f
1 parent b95de7e
commit 3cd0b0f
Show file tree

Hide file tree

Showing 16 changed files with 229 additions and 10 deletions.
diff --git a/.github/actions/build-cached-cves/action.yaml b/.github/actions/build-cached-cves/action.yaml
@@ -0,0 +1,46 @@
+name: "Build with cached CVEs"
+inputs:
+  gh-token:
+    description: "GitHub Token for authentication"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install jq (JSON processor) if not found
+      run: |
+        if ! command -v jq &> /dev/null; then
+          sudo apt-get update
+          sudo apt-get install -y jq
+        else
+          echo "jq is already installed. Skipping install..."
+        fi
+      shell: bash
+
+    - name: Download CVE Data
+      run: |
+        # Find the latest CVE upload workflow.
+        run_id=$(gh run list --workflow="post_release.yaml" --limit 1 --status=success --json databaseId | jq -r '.[0].databaseId')
+        echo 'Fetching artifacts from run $run_id'
+        # Remove any downloaded artifacts, should they exist.
+        rm -rf ./downloaded_artifacts
+        # Download the latest artifact to a new dir.
+        gh run download ${run_id} --name security-bulletins --dir ./downloaded_artifacts
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.gh-token }}
+
+    - name: Unpack CVE data
+      run: |
+        # Ensure the correct folders exist.
+        mkdir -p .docusaurus/security-bulletins/default
+        # Move the files to their correct places in the checked out repository
+        mv downloaded_artifacts/data.json .docusaurus/security-bulletins/default/data.json
+        rm -rf downloaded_artifacts
+      shell: bash
+
+    - name: Build
+      run: |
+        rm -rf build
+        npm run build
+      shell: bash
diff --git a/.github/actions/build-cached-packs/action.yaml b/.github/actions/build-cached-packs/action.yaml
@@ -20,7 +20,7 @@ runs:
     - name: Download Packs Data
       run: |
         # Find the latest packs upload workflow.
-        run_id=$(gh run list --workflow="post_release.yaml" --limit 1 --json databaseId | jq -r '.[0].databaseId')
+        run_id=$(gh run list --workflow="post_release.yaml" --limit 1 --status=success --json databaseId | jq -r '.[0].databaseId')
         # Remove any downloaded artifacts, should they exist.
         rm -rf ./downloaded_artifacts
         # Download the latest artifact to a new dir.

diff --git a/.github/workflows/dependabot.yaml b/.github/workflows/dependabot.yaml
@@ -53,3 +53,9 @@ jobs:
         uses: ./.github/actions/build-cached-packs
         with:
           gh-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build with cached CVEs
+        if: ${{ env.BUILD_EXIT_CODE == '7' }}
+        uses: ./.github/actions/build-cached-cves
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/nightly-docker-build.yaml b/.github/workflows/nightly-docker-build.yaml
@@ -42,6 +42,12 @@ jobs:
         uses: ./.github/actions/build-cached-packs
         with:
           gh-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build with cached CVEs
+        if: ${{ env.BUILD_EXIT_CODE == '7' }}
+        uses: ./.github/actions/build-cached-cves
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3

diff --git a/.github/workflows/post_release.yaml b/.github/workflows/post_release.yaml
@@ -50,6 +50,12 @@ jobs:
         with:
           gh-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build with cached CVEs
+        if: ${{ env.BUILD_EXIT_CODE == '7' }}
+        uses: ./.github/actions/build-cached-cves
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Upload Build Packs
         uses: actions/upload-artifact@v4
         with:
@@ -59,3 +65,11 @@ jobs:
             .docusaurus/packs-integrations
           if-no-files-found: error
           retention-days: 7
+
+      - name: Upload Built Security Bulletins
+        uses: actions/upload-artifact@v4
+        with:
+          name: "security-bulletins"
+          path: .docusaurus/security-bulletins/default/data.json
+          if-no-files-found: error
+          retention-days: 7
diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
@@ -23,7 +23,7 @@ env:
   PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
   DISABLE_PACKS_INTEGRATIONS: ${{ secrets.DISABLE_PACKS_INTEGRATIONS }}
   DISABLE_SECURITY_INTEGRATIONS: ${{ secrets.DISABLE_SECURITY_INTEGRATIONS }}
-  DSO_AUTH_TOKEN: ${{ secrets.DSO_AUTH_TOKEN }}
+  DSO_AUTH_TOKEN:  ${{ secrets.DSO_AUTH_TOKEN }}
 
 jobs:
   run-ci:
@@ -104,5 +104,11 @@ jobs:
       - name: Build with cached packs
         if: ${{ env.BUILD_EXIT_CODE == '5' }}
         uses: ./.github/actions/build-cached-packs
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build with cached CVEs
+        if: ${{ env.BUILD_EXIT_CODE == '7' }}
+        uses: ./.github/actions/build-cached-cves
         with:
           gh-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/release-branch-pr.yaml b/.github/workflows/release-branch-pr.yaml
@@ -73,3 +73,9 @@ jobs:
       uses: ./.github/actions/build-cached-packs
       with:
         gh-token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build with cached CVEs
+      if: ${{ env.BUILD_EXIT_CODE == '7' }}
+      uses: ./.github/actions/build-cached-cves
+      with:
+        gh-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/release-preview.yaml b/.github/workflows/release-preview.yaml
@@ -70,6 +70,12 @@ jobs:
         with:
           gh-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build with cached CVEs
+        if: ${{ env.BUILD_EXIT_CODE == '7' }}
+        uses: ./.github/actions/build-cached-cves
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Deploy Preview
         run: |
           aws s3 sync --cache-control 'public, max-age=604800' --exclude '*.html' --exclude build/scripts/ build/ s3://docs-latest.spectrocloud.com --delete

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -74,6 +74,12 @@ jobs:
         with:
           gh-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build with cached CVEs
+        if: ${{ env.BUILD_EXIT_CODE == '7' }}
+        uses: ./.github/actions/build-cached-cves
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Upload to AWS
         run: |
           echo "CURRENT_STEP=Upload to AWS" >> $GITHUB_ENV

diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml
@@ -57,6 +57,12 @@ jobs:
         with:
           gh-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build with cached CVEs
+        if: ${{ env.BUILD_EXIT_CODE == '7' }}
+        uses: ./.github/actions/build-cached-cves
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Upload Build
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml
@@ -85,6 +85,12 @@ jobs:
         with:
           gh-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build with cached CVEs
+        if: ${{ env.BUILD_EXIT_CODE == '7' }}
+        uses: ./.github/actions/build-cached-cves
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Upload Build
         uses: actions/upload-artifact@v4
         with:

diff --git a/Makefile b/Makefile
@@ -106,6 +106,20 @@ start-cached-packs: ## Start a local development server with cached packs retry.
 		fi; \
 	}
 
+start-cached-cves: ## Start a local development server with cached CVEs retry.
+	make generate-partials
+	@{ \
+		npm run start; \
+		exit_code=$$?; \
+		if [ "$$exit_code" = "7" ]; then \
+			echo "❌ Start has failed due to missing CVE data..."; \
+			echo "ℹ️ Initializing fetch cached CVE data..."; \
+			make get-cached-cves; \
+			echo "ℹ️ Retrying start... "; \
+			npm run start;\
+		fi; \
+	}
+
 build: ## Run npm build
 	@echo "building site"
 	npm run clear
@@ -128,6 +142,22 @@ build-cached-packs: ## Run npm build with cached packs retry
 		fi; \
 	}
 
+build-cached-cves: ## Run npm build with cached CVEs retry
+	@echo "building site"
+	npm run clear
+	rm -rf build
+	@{ \
+		npm run build; \
+		exit_code=$$?; \
+		if [ "$$exit_code" = "7" ]; then \
+			echo "❌ Build has failed due to missing CVE data..."; \
+			echo "ℹ️ Initializing fetch cached CVE data..."; \
+			make get-cached-cves; \
+			echo "ℹ️ Retrying build... "; \
+			npm run build;\
+		fi; \
+	}
+
 build-ci: ## Run npm build in CI environment
 	@echo "building site"
 	npm run clear
@@ -287,6 +317,10 @@ generate-partials: ## Generate
 
 get-cached-packs:
 	./scripts/get-cached-packs.sh
+
+###@ Fetch security bulletins
+get-cached-cves: 
+	./scripts/get-cached-cves.sh
 
 ###@ Aloglia Indexing
 

diff --git a/README.md b/README.md
@@ -802,6 +802,31 @@ will stop the pre-build script from fetching the security bulletins.
 export DISABLE_SECURITY_INTEGRATIONS=true
 ```
 
+### Cached Security Bulletins
+
+All CVE related data is saved to a
+[GitHub Workflow Artifact](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/storing-and-sharing-data-from-a-workflow#about-workflow-artifacts)
+after every successful release to production. Check out the [post_release.yaml](.github/workflows/post_release.yaml) for
+further details. The cached data enables us to build and start librarium without performing any CVE-related API queries.
+All of our GitHub workflows will use this cached data as a fallback in the case of an API related build failure. Check
+out the [build-cached-cves action.yaml](.github/actions/build-cached-cves/action.yaml) to learn how the cached data is
+fetched and used.
+
+CVE data is saved locally in the `.docusaurus/security-bulletins/default/data.json` file. You can remove the data using
+`make clean-security`. You can use the cached CVE artifact locally when you want to avoid the CVE download time. This
+flow also helps you when you don't have any local CVE data and we are experiencing an API outage.
+
+librarium provides the following commands which fetch cached CVE data to your local environment.
+
+| **Command**              | **Description**                                                                                                                                |
+| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- |
+| `make get-cached-cves`   | Fetch the CVE data artifact and place files in the correct places. You can then execute `make start` or `make build` as usual.                 |
+| `make start-cached-cves` | Attempt to start the local development server. If a CVE related outage is detected, fetch the CVE data artifact and retry the `start` command. |
+| `make build-cached-cves` | Attempt to build the application. If a CVE related outage is detected, fetch the CVE data artifact and retry the `build` command.              |
+
+These scripts will prompt you to install and authenticate the [GitHub CLI](https://cli.github.com/) before you can
+proceed.
+
 ## Packs Component
 
 The packs component is a custom component that displays all packs available in Palette SaaS by querying the Palette API
@@ -870,7 +895,7 @@ Settting the `DISABLE_PACKS_PLUGIN` environment variable to `true` will also hav
 
 All pack related data is saved to a
 [GitHub Workflow Artifact](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/storing-and-sharing-data-from-a-workflow#about-workflow-artifacts)
-after every succesful release to production. Check out the [post_release.yaml](.github/workflows/post_release.yaml) for
+after every successful release to production. Check out the [post_release.yaml](.github/workflows/post_release.yaml) for
 further details. The cached data enables us to build and start librarium without performing any pack related API
 queries. All of our GitHub workflows will use this cached data as a fallback in the case of an API related build
 failure. Check out the [build-cached-packs action.yaml](.github/actions/build-cached-packs/action.yaml) to learn how the
@@ -1166,8 +1191,9 @@ make clean-versions
 Librarium provides the following exit codes. These exit codes are returned by both the `npm run start` and
 `npm run build` commands.
 
-| **Exit Code**                 | **Description**                                                                                                                                                                                                                                                   |
-| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `0`                           | The command was executed successfully.                                                                                                                                                                                                                            |
-| `5`                           | The command failed due to errors received from the API service. These requests are issued by the [Packs Component](#packs-component) and librarium cannot start without loading packs, either from the API service or the [cached packs data](#cached-packs-data) |
-| Any other non-zero exit code. | The command failed due to another error. Check the command output.                                                                                                                                                                                                |
+| **Exit Code**                 | **Description**                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `0`                           | The command was executed successfully.                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| `5`                           | The command failed due to errors received from the API service related to packs. These requests are issued by the [Packs Component](#packs-component) and librarium cannot start without loading packs, either from the API service or the [cached packs data](#cached-packs-data)                                                                                                                                                                                                   |
+| `7`                           | The command failed due to errors received from the API service related to security bulletins. These requests are issued by the [CVE script](./utils/cves/index.js) and librarium cannot start without loading the security bulletins. The [Build with Cached CVE](./.github/actions/build-cached-cves/action.yaml) action is built to handle this exit situation and build with cached CVEs. You can issue the command `make get-cached-cves` to fetch cached CVEs to build locally. |
+| Any other non-zero exit code. | The command failed due to another error. Check the command output.                                                                                                                                                                                                                                                                                                                                                                                                                   |
diff --git a/scripts/get-cached-cves.sh b/scripts/get-cached-cves.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+
+# Enable error handling
+set -e
+
+echo "Starting fetch of cached cves..."
+
+if command -v gh &> /dev/null
+then
+    echo "✅ GitHub CLI is installed."
+else
+    echo "❌ GitHub CLI is not installed."
+    echo "ℹ️ Use 'brew install gh' to install it with Homebrew."
+    exit 1
+fi
+
+if gh auth status &> /dev/null
+then
+    echo "✅ GitHub CLI is authenticated. "
+else
+    echo "❌ GitHub CLI is not authenticated."
+    echo "ℹ️ Please log in with 'gh auth login'."
+    exit 1
+fi
+ # Find the latest cves upload workflow.
+
+run_id=$(gh run list --workflow="post_release.yaml" --status=success --limit 1 --json databaseId | jq -r '.[0].databaseId')
+
+# Remove any downloaded artifacts, should they exist.
+rm -rf ./downloaded_artifacts
+
+# Download the latest artifact to a new dir.
+gh run download ${run_id} --name security-bulletins --dir ./downloaded_artifacts
+echo "✅ Cached CVEs artifact downloaded."
+
+# Ensure the correct folders exist.
+mkdir -p .docusaurus/security-bulletins/default
+
+# Move the files to their correct places in the checked out repository
+mv downloaded_artifacts/data.json .docusaurus/security-bulletins/default/data.json
+
+# Clean up. 
+rm -rf downloaded_artifacts
+
+echo "✅ Completed fetch of cached CVEs." 
+echo "⏭️ You can now execute 'make start' or 'make build'."
diff --git a/scripts/get-cached-packs.sh b/scripts/get-cached-packs.sh
@@ -24,7 +24,7 @@ else
 fi
 
  # Find the latest packs upload workflow.
-run_id=$(gh run list --workflow="post_release.yaml" --limit 1 --json databaseId | jq -r '.[0].databaseId')
+run_id=$(gh run list --workflow="post_release.yaml" --status=success --limit 1 --json databaseId | jq -r '.[0].databaseId')
 
 # Remove any downloaded artifacts, should they exist.
 rm -rf ./downloaded_artifacts

diff --git a/utils/cves/index.js b/utils/cves/index.js
@@ -39,6 +39,8 @@ async function getSecurityBulletins(payload) {
     return { data: results };
   } catch (error) {
     logger.error("Error:", error.response ? `${error.response.status} - ${error.response.data}` : error.message);
+    // Return exit code 7 to indicate that the script failed to fetch the security bulletins for GitHub Actions
+    process.exit(7);
   }
 }
 
@@ -200,6 +202,8 @@ async function generateCVEs() {
     } catch (error) {
       logger.error(error);
       logger.error("Error:", error.response ? error.response.status : error.message);
+      // Return exit code 7 to indicate that the script failed to fetch the security bulletins for GitHub Actions
+      process.exit(7);
     }
   }
 
@@ -313,5 +317,5 @@ ${revisionHistory ? revisionHistory : "No revision history available."}
 try {
   generateCVEs();
 } catch (error) {
-  process.exit(5);
+  process.exit(7);
 }