Merge branch 'version-3-4' into pause-3-4

.github/workflows/api_format.yaml

@@ -0,0 +1,87 @@
+name: API Format
+
+on:
+  pull_request_target:
+    types: ["labeled", "closed"]
+
+env:
+  GITHUB_BRANCH: ${{ github.ref_name }} 
+  NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+  NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+  FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
+  ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
+  ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
+  ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
+  ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
+
+jobs:
+  backport:
+    name: Format API PR
+    runs-on: ubuntu-latest
+    if: |
+      github.event.action == 'labeled'
+      && github.event.label.name == 'api-format'
+      && github.event.pull_request.draft == false
+  
+
+    steps:
+      - name: Retrieve Credentials
+        id: import-secrets
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.prism.spectrocloud.com
+          method: approle
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          secrets: /providers/github/organizations/spectrocloud/token?org_name=spectrocloud token | VAULT_GITHUB_TOKEN
+
+
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ steps.import-secrets.outputs.VAULT_GITHUB_TOKEN }}
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Determine branch name
+        id: extract_branch
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            echo "GITHUB_BRANCH=${{ github.head_ref }}" >> $GITHUB_ENV
+          else
+            echo "GITHUB_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+
+      - run: npm ci
+
+      - name: Format API
+        run: make api
+
+
+      - name: Commit Changes
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "ci: auto-formatting API changes"        
+
+
+      - name: Slack Notification
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_PRIVATE_TEAM_WEBHOOK }}
+          SLACK_USERNAME: "spectromate"
+          SLACK_ICON_EMOJI: ":robot_panic:"
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: ' The PR for branch ${{env.GITHUB_BRANCH}} failed when attempting to format the API. Review the GitHub Actions logs for more details.'
+
+      - name: Post Netlify progress
+        uses: mshick/add-pr-comment@v2
+        with:
+          message: |
+              🤖 The API has been formated and is ready for merging.
+          refresh-message-position: false

.github/workflows/url-checks.yaml

@@ -31,8 +31,8 @@ jobs:
       - name: URL Checker
         run: make verify-url-links-ci
 
-      - name: URL Security Bulletins Checker
-        run: make verify-security-bulletins-links-ci
+      - name: URL Rate Limit Checker
+        run: make verify-rate-limited-links-ci
 
       - name: Post Comment
         run: |

Makefile

@@ -13,8 +13,13 @@ CPUS := $(shell sysctl -n hw.ncpu | awk '{print int($$1 / 2)}')
 ALOGLIA_CONFIG=$(shell cat docsearch.dev.config.json | jq -r tostring)
 
 # Find all *.md files in docs, cut the prefix ./ 
-# Remove all security-bulletins and cve-reports.md
-VERIFY_URL_PATHS=$(shell find ./docs -name "*.md" | cut -c 3- | sed '/security-bulletins/d' | sed '/cve-reports/d' )
+# Remove all security-bulletins and cve-reports.md because they are rate limited by nvd.nist.gov
+# Remove oss-licenses.md because they are rate limited by npmjs.com
+VERIFY_URL_PATHS=$(shell find ./docs -name "*.md" | cut -c 3- | sed '/security-bulletins/d' | sed '/cve-reports/d' | sed '/oss-licenses/d')
+
+RATE_LIMITED_FILES_LIST:="docs/docs-content/security-bulletins/**/*.md" \
+	"docs/docs-content/security-bulletins/*.md" \
+	"docs/docs-content/unlisted/cve-reports.md"
 
 help: ## Display this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[0m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
@@ -171,78 +176,36 @@ pdf-local: ## Generate PDF from local docs
 verify-url-links:
 	@echo "Checking for broken external URLs in markdown files..."
 	rm link_report.csv || echo "No report exists. Proceeding to scan step"
-	@npx linkinator $(VERIFY_URL_PATHS) --concurrency 50 --markdown --recurse --timeout 100000 --retry --retry-errors-jitter --retry-errors-count 5 \
-		--skip "^https:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/docs\.spectrocloud\.com\/.*\/supplemental\-packs$$" \
-		--skip "^http:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/software-private\.spectrocloud\.com.*$$" \
-		--skip "^\/.*\.md$$" \
-		--skip "!\[.*\]\(.*\)$$" \
-		--skip "\.(jpg|jpeg|png|gif|webp)$$" \
-		--skip "https:\/\/linux\.die\.net\/man\/.*$$" \
-		--skip "https:\/\/mysql\.com\/.*\.*$$" \
-		--skip "https:\/\/dev\.mysql\.com\/doc\/.*$$" \
-		--format csv > temp_report.csv && sleep 2
+	@npx linkinator $(VERIFY_URL_PATHS) --config ./linkinator/linkinator.config.json > temp_report.csv && sleep 2
 	@grep -E 'https?://' temp_report.csv > filtered_report.csv
 	@grep -E ',[[:space:]]*([4-9][0-9]{2}|[0-9]{4,}),' filtered_report.csv > link_report.csv && rm temp_report.csv filtered_report.csv
 
-verify-security-bulletins-links:
-	@echo "Checking for broken URLs in security-bulletins markdown files..."
-	rm link_sec_bul_report.csv || echo "No security bulletins report exists. Proceeding to scan step"
-	@npx linkinator "docs/docs-content/security-bulletins/**/*.md" "docs/docs-content/security-bulletins/*.md" "docs/docs-content/unlisted/cve-reports.md" --concurrency 1 --markdown --recurse --timeout 100000 --retry --retry-errors-jitter --retry-errors-count 5 \
-		--skip "^https:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/docs\.spectrocloud\.com\/.*\/supplemental\-packs$$" \
-		--skip "^http:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/software-private\.spectrocloud\.com.*$$" \
-		--skip "^\/.*\.md$$" \
-		--skip "!\[.*\]\(.*\)$$" \
-		--skip "\.(jpg|jpeg|png|gif|webp)$$" \
-		--skip "https:\/\/linux\.die\.net\/man\/.*$$" \
-		--skip "https:\/\/mysql\.com\/.*\.*$$" \
-		--skip "https:\/\/dev\.mysql\.com\/doc\/.*$$" \
-		--format csv > temp_sec_bul_report.csv && sleep 2
-	@grep -E 'https?://' temp_sec_bul_report.csv > filtered_sec_bul_report.csv
-	@grep -E ',[[:space:]]*([4-9][0-9]{2}|[0-9]{4,}),' filtered_sec_bul_report.csv > link_sec_bul_report.csv && rm temp_sec_bul_report.csv filtered_sec_bul_report.csv
+verify-rate-limited-links:
+	@echo "Checking for broken URLs in security-bulletins and oss-licenses markdown files..."
+	@rm link_rate_limit_report.csv || echo "No rate limited report exists. Proceeding to scan step"
+	@echo "Checking the following paths: $(RATE_LIMITED_FILES_LIST)"
+	@npx linkinator $(RATE_LIMITED_FILES_LIST) --config ./linkinator/linkinator-rate-limit.config.json  > temp_rate_limit_report.csv && sleep 2
+	@grep -E 'https?://' temp_rate_limit_report.csv > filtered_rate_limit_report.csv
+	@grep -E ',[[:space:]]*([4-9][0-9]{2}|[0-9]{4,}),' filtered_rate_limit_report.csv > link_rate_limit_report.csv && rm temp_rate_limit_report.csv filtered_rate_limit_report.csv
 
 verify-url-links-ci: ## Check for broken URLs in production in a GitHub Actions CI environment
 	@echo "Checking for broken external URLs in CI environment..."
-	rm link_report.json || echo "No report exists. Proceeding to scan step"
-	@npx linkinator $(VERIFY_URL_PATHS) --concurrency 50 --markdown --recurse --timeout 100000 --retry --retry-errors-jitter --retry-errors-count 5 \
-		--skip "^https:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/docs\.spectrocloud\.com\/.*\/supplemental\-packs$$" \
-		--skip "^http:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/software-private\.spectrocloud\.com.*$$" \
-		--skip "^\/.*\.md$$" \
-		--skip "!\[.*\]\(.*\)$$" \
-		--skip "\.(jpg|jpeg|png|gif|webp)$$" \
-		--skip "https:\/\/linux\.die\.net\/man\/.*$$" \
-		--skip "https:\/\/mysql\.com\/.*\.*$$" \
-		--skip "https:\/\/dev\.mysql\.com\/doc\/.*$$" \
-		--format json > temp_report.json
+	@rm link_report.json || echo "No report exists. Proceeding to scan step"
+	@npx linkinator $(VERIFY_URL_PATHS) --config ./linkinator/linkinator-ci.config.json  > temp_report.json
 	@# Use jq to filter out links that do not start with http or https and keep only broken links
 	@jq '[.links[] | select(.url | test("^https?://")) | select(.status >= 400)]' temp_report.json > filtered_report.json
 	@rm temp_report.json
 	@mv filtered_report.json scripts/link_report.json
 
-verify-security-bulletins-links-ci: ## Check for broken URLs in production in a GitHub Actions CI environment
-	@echo "Checking for broken URLs in security-bulletins markdown files in CI environment..."
-	rm link_sec_bul_report.json || echo "No security bulletins report exists. Proceeding to scan step"
-	@npx linkinator "docs/docs-content/security-bulletins/**/*.md" "docs/docs-content/security-bulletins/*.md" "docs/docs-content/unlisted/cve-reports.md" --concurrency 1 --markdown --recurse --timeout 100000 --retry --retry-errors-jitter --retry-errors-count 5 \
-		--skip "^https:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/docs\.spectrocloud\.com\/.*\/supplemental\-packs$$" \
-		--skip "^http:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/software-private\.spectrocloud\.com.*$$" \
-		--skip "^\/.*\.md$$" \
-		--skip "!\[.*\]\(.*\)$$" \
-		--skip "\.(jpg|jpeg|png|gif|webp)$$" \
-		--skip "https:\/\/linux\.die\.net\/man\/.*$$" \
-		--skip "https:\/\/mysql\.com\/.*\.*$$" \
-		--skip "https:\/\/dev\.mysql\.com\/doc\/.*$$" \
-		--format json > temp_sec_bul_report.json
+verify-rate-limited-links-ci: ## Check for broken URLs in production in a GitHub Actions CI environment
+	@echo "Checking for broken URLs in security-bulletins and oss-licenses markdown files in CI environment..."
+	@rm link_rate_limit_report.json || echo "No rate limited report exists. Proceeding to scan step"
+	@echo "Checking the following paths: $(RATE_LIMITED_FILES_LIST)"
+	@npx linkinator $(RATE_LIMITED_FILES_LIST) --config ./linkinator/linkinator-rate-limit-ci.config.json  > temp_rate_limit_report.json
 	@# Use jq to filter out links that do not start with http or https and keep only broken links
-	@jq '[.links[] | select(.url | test("^https?://")) | select(.status >= 400)]' temp_sec_bul_report.json > filtered_sec_bul_report.json
-	@rm temp_sec_bul_report.json
-	@mv filtered_sec_bul_report.json scripts/link_sec_bul_report.json
+	@jq '[.links[] | select(.url | test("^https?://")) | select(.status >= 400)]' temp_rate_limit_report.json > filtered_rate_limit_report.json
+	@rm temp_rate_limit_report.json
+	@mv filtered_rate_limit_report.json scripts/link_rate_limit_report.json
 
 ###@ Image Formatting
 

linkinator/linkinator-ci.config.json

@@ -0,0 +1,25 @@
+{
+  "concurrency": 50,
+  "markdown": true,
+  "recurse": true,
+  "timeout": 100000,
+  "retry": true,
+  "retryErrors": true,
+  "retryErrorsJitter": 10000,
+  "retryErrorsCount": 5,
+  "format": "json",
+  "skip": [
+    "^https://docs.spectrocloud.com.*$$",
+    "^https://docs.spectrocloud.com/.*/supplemental-packs$$",
+    "^http://docs.spectrocloud.com.*$$",
+    "^https://software-private.spectrocloud.com.*$$",
+    "^/.*.md$$",
+    "![.*](.*)$$",
+    ".(jpg|jpeg|png|gif|webp)$$",
+    "https://linux.die.net/man/.*$$",
+    "https://mysql.com/.*.*$$",
+    "https://dev.mysql.com/doc/.*$$"
+  ],
+  "verbosity": "error",
+  "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"
+}

linkinator/linkinator-rate-limit-ci.config.json

@@ -0,0 +1,25 @@
+{
+  "concurrency": 1,
+  "markdown": true,
+  "recurse": false,
+  "timeout": 100000,
+  "retry": true,
+  "retryErrors": true,
+  "retryErrorsJitter": 10000,
+  "retryErrorsCount": 5,
+  "format": "json",
+  "skip": [
+    "^https://docs.spectrocloud.com.*$$",
+    "^https://docs.spectrocloud.com/.*/supplemental-packs$$",
+    "^http://docs.spectrocloud.com.*$$",
+    "^https://software-private.spectrocloud.com.*$$",
+    "^/.*.md$$",
+    "![.*](.*)$$",
+    ".(jpg|jpeg|png|gif|webp)$$",
+    "https://linux.die.net/man/.*$$",
+    "https://mysql.com/.*.*$$",
+    "https://dev.mysql.com/doc/.*$$"
+  ],
+  "verbosity": "error",
+  "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"
+}

linkinator/linkinator-rate-limit.config.json

@@ -0,0 +1,26 @@
+{
+  "concurrency": 1,
+  "markdown": true,
+  "recurse": false,
+  "timeout": 100000,
+  "retry": true,
+  "retryErrors": true,
+  "retryErrorsJitter": 10000,
+  "retryErrorsCount": 5,
+  "format": "csv",
+  "skip": [
+    "^https://docs.spectrocloud.com.*$$",
+    "^https://docs.spectrocloud.com/.*/supplemental-packs$$",
+    "^http://docs.spectrocloud.com.*$$",
+    "^https://software-private.spectrocloud.com.*$$",
+    "^/.*.md$$",
+    "^/.*.md#*$$",
+    "![.*](.*)$$",
+    ".(jpg|jpeg|png|gif|webp)$$",
+    "https://linux.die.net/man/.*$$",
+    "https://mysql.com/.*.*$$",
+    "https://dev.mysql.com/doc/.*$$"
+  ],
+  "verbosity": "error",
+  "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"
+}

linkinator/linkinator.config.json

@@ -0,0 +1,25 @@
+{
+  "concurrency": 50,
+  "markdown": true,
+  "recurse": true,
+  "timeout": 100000,
+  "retry": true,
+  "retryErrors": true,
+  "retryErrorsJitter": 10000,
+  "retryErrorsCount": 5,
+  "format": "csv",
+  "skip": [
+    "^https://docs.spectrocloud.com.*$$",
+    "^https://docs.spectrocloud.com/.*/supplemental-packs$$",
+    "^http://docs.spectrocloud.com.*$$",
+    "^https://software-private.spectrocloud.com.*$$",
+    "^/.*.md$$",
+    "![.*](.*)$$",
+    ".(jpg|jpeg|png|gif|webp)$$",
+    "https://linux.die.net/man/.*$$",
+    "https://mysql.com/.*.*$$",
+    "https://dev.mysql.com/doc/.*$$"
+  ],
+  "verbosity": "error",
+  "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"
+}