Merge branch 'master' into datadog2024

spectrocloud · Sep 20, 2024 · f7d0209 · f7d0209
2 parents cadad1c + 58bd446
commit f7d0209
Show file tree

Hide file tree

Showing 740 changed files with 71,584 additions and 48,299 deletions.
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -1,6 +1,6 @@
 module.exports = {
   env: { browser: true, es2015: true, node: true },
-  settings: { "import/resolver": "webpack" },
+  settings: { "import/resolver": "webpack", react: { version: "detect" } },
   extends: ["eslint:recommended", "plugin:react/recommended", "prettier"],
   overrides: [
     {

diff --git a/.github/workflows/api_format.yaml b/.github/workflows/api_format.yaml
@@ -0,0 +1,87 @@
+name: API Format
+
+on:
+  pull_request_target:
+    types: ["labeled", "closed"]
+
+env:
+  GITHUB_BRANCH: ${{ github.ref_name }} 
+  NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+  NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+  FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
+  ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
+  ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
+  ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
+  ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
+
+jobs:
+  backport:
+    name: Format API PR
+    runs-on: ubuntu-latest
+    if: |
+      github.event.action == 'labeled'
+      && github.event.label.name == 'api-format'
+      && github.event.pull_request.draft == false
+  
+
+    steps:
+      - name: Retrieve Credentials
+        id: import-secrets
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.prism.spectrocloud.com
+          method: approle
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          secrets: /providers/github/organizations/spectrocloud/token?org_name=spectrocloud token | VAULT_GITHUB_TOKEN
+
+
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ steps.import-secrets.outputs.VAULT_GITHUB_TOKEN }}
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Determine branch name
+        id: extract_branch
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            echo "GITHUB_BRANCH=${{ github.head_ref }}" >> $GITHUB_ENV
+          else
+            echo "GITHUB_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+
+      - run: npm ci
+
+      - name: Format API
+        run: make api
+
+
+      - name: Commit Changes
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "ci: auto-formatting API changes"        
+
+
+      - name: Slack Notification
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_PRIVATE_TEAM_WEBHOOK }}
+          SLACK_USERNAME: "spectromate"
+          SLACK_ICON_EMOJI: ":robot_panic:"
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: ' The PR for branch ${{env.GITHUB_BRANCH}} failed when attempting to format the API. Review the GitHub Actions logs for more details.'
+
+      - name: Post Netlify progress
+        uses: mshick/add-pr-comment@v2
+        with:
+          message: |
+              🤖 The API has been formated and is ready for merging.
+          refresh-message-position: false
diff --git a/.github/workflows/dependabot.yaml b/.github/workflows/dependabot.yaml
@@ -16,12 +16,12 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_KEY }}
   AWS_DEFAULT_REGION: us-east-1
   APPZI_TOKEN: ${{ secrets.APPZI_TOKEN }}
-  MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }}
   FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
   ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
   ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
   ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
   ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
 
 jobs:
   dependabot_build:

diff --git a/.github/workflows/nightly-docker-build.yaml b/.github/workflows/nightly-docker-build.yaml
@@ -12,6 +12,8 @@ env:
   ALGOLIA_ADMIN_KEY: "123456789"
   ALGOLIA_APP_ID: "123456789"
   ALGOLIA_SEARCH_KEY: "123456789"
+  ALGOLIA_INDEX_NAME: "madeup-index"
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
 
 jobs:
   build:
@@ -54,7 +56,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         id: build-and-push
         with:
           context: .
@@ -65,7 +67,7 @@ jobs:
           tags: ghcr.io/${{ github.repository }}:nightly
           labels: ${{ steps.meta.outputs.labels }}
 
-      - uses: sigstore/cosign-installer@v3.5.0
+      - uses: sigstore/cosign-installer@v3.6.0
 
       - name: Image Signing
         run: |

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
@@ -20,6 +20,7 @@ env:
   ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
   ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
   ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
 
 jobs:
   run-ci:
@@ -45,8 +46,20 @@ jobs:
     needs: [run-ci]
     runs-on: ubuntu-latest
     steps:
+      - name: Retrieve Credentials
+        id: import-secrets
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.prism.spectrocloud.com
+          method: approle
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          secrets: /providers/github/organizations/spectrocloud/token?org_name=spectrocloud token | VAULT_GITHUB_TOKEN
+
       - name: Checkout Repository
         uses: actions/checkout@v4
+        with:
+          token: ${{ steps.import-secrets.outputs.VAULT_GITHUB_TOKEN }}
 
       - name: Setup Node.js environment
         uses: actions/setup-node@v4
@@ -56,8 +69,13 @@ jobs:
 
       - run: npm ci
 
-      - name: Prettier Check
-        run: npx prettier . --check
+      - name: Auto Format (Prettier)
+        run: make format
+
+      - name: Commit Changes
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "ci: auto-formatting prettier issues"
 
   build:
     name: Build

diff --git a/.github/workflows/release-branch-pr.yaml b/.github/workflows/release-branch-pr.yaml
@@ -17,6 +17,7 @@ env:
   ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
   ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
   GITHUB_BRANCH: ${{ github.ref_name }} 
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
 
 
 concurrency:

diff --git a/.github/workflows/release-preview.yaml b/.github/workflows/release-preview.yaml
@@ -11,12 +11,12 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_KEY }}
   AWS_DEFAULT_REGION: us-east-1
   APPZI_TOKEN: ${{ secrets.APPZI_TOKEN }}
-  MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }}
   FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
   ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
   ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
   ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
   ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
 
 
 concurrency:

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -20,12 +20,12 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_KEY }}
   AWS_DEFAULT_REGION: us-east-1
   APPZI_TOKEN: ${{ secrets.APPZI_TOKEN }}
-  MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }}
   FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
   ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
   ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
   ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
   ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
 
 
 concurrency:

diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml
@@ -20,6 +20,7 @@ env:
   ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
   ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
   ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
 
 
 jobs:

diff --git a/.github/workflows/url-checks.yaml b/.github/workflows/url-checks.yaml
@@ -28,6 +28,9 @@ jobs:
 
       - name: URL Checker
         run: make verify-url-links-ci
+
+      - name: URL Rate Limit Checker
+        run: make verify-rate-limited-links-ci
 
       - name: Post Comment
         run: |

diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml
@@ -9,7 +9,7 @@ on:
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }}
+  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
   FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
   ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
   ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}

diff --git a/.gitignore b/.gitignore
@@ -64,14 +64,16 @@ test-results/
 playwright-report/
 artifact.zip
 
-# Netlify 
+# Netlify
 
 .netlify/
 # Ignore _partials/index.ts
 _partials/index.ts
 
+# Ignore statoc/img/packs
+static/img/packs
 
 .vale-config/
 vale/styles/spectrocloud/
 vale/styles/spectrocloud-docs-internal/
-vale/styles/config/vocabularies/spectrocloud-vocab
+vale/styles/config/vocabularies/spectrocloud-vocab
diff --git a/.gitleaksignore b/.gitleaksignore
@@ -15,6 +15,7 @@ d748cceb3859df5c354b96a0c3d115513f6c123e:docs/docs-content/devx/apps/deploy-app.
 404219eb2cad1fbd5e03d64783b1db285c8e08b3:docs/docs-content/integrations/portworx_operator.md:private-key:613
 504642da118b99f12e838c6ff9a1166cc8fcff61:docs/docs-content/integrations/portworx_operator.md:private-key:309
 86b8971f386ee3b00d2558a7fd6029f312b335cf:docs/docs-content/integrations/portworx.md:private-key:269
+1058de28bdc307a7b9a80ba6f1c26d2165d43414:docs/deprecated/integrations/portworx.md:private-key:263
 feaea7ee07b76653233cf4ff8376953bfb216c8d:docs/docs-content/integrations/ubuntu.md:generic-api-key:65
 261324e61a4dd4a3da73c8c03f45d3fcca826817:packages/docs/content/clusters/cluster-management/cluster-rbac.md:generic-api-key:112
 261324e61a4dd4a3da73c8c03f45d3fcca826817:packages/docs/content/clusters/cluster-management/cluster-rbac.md:generic-api-key:113
@@ -110,3 +111,7 @@ e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernet
 e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:1125
 e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:391
 87bf9f7353f94ac2ecf85da3be3412202de7b72f:docusaurus.config.js:generic-api-key:363
+969ac609f82bacb36093c429adfc096c5a97e10f:docs/docs-content/tutorials/cluster-deployment/pde/deploy-app.md:generic-api-key:1195
+969ac609f82bacb36093c429adfc096c5a97e10f:docs/docs-content/tutorials/cluster-deployment/pde/deploy-app.md:generic-api-key:1232
+969ac609f82bacb36093c429adfc096c5a97e10f:docs/docs-content/tutorials/edge/deploy-cluster.md:generic-api-key:240
+8f515d46ce2bb80b7173bf9684ed8e87cb96fd83:docs/docs-content/tutorials/edge/deploy-cluster-virtualbox.md:generic-api-key:229
diff --git a/.prettierignore b/.prettierignore
@@ -11,4 +11,8 @@ docs/api-content/**/*.json
 
 # Troublesome files
 tsconfig.json
-src/components/IconMapper/dynamicFontAwesomeImports.js
+src/components/IconMapper/dynamicFontAwesomeImports.*
+docs/docs-content/security-bulletins/cve-reports.md
+
+# Ignore partials
+_partials/
diff --git a/Makefile b/Makefile
@@ -12,6 +12,14 @@ CPUS := $(shell sysctl -n hw.ncpu | awk '{print int($$1 / 2)}')
 
 ALOGLIA_CONFIG=$(shell cat docsearch.dev.config.json | jq -r tostring)
 
+# Find all *.md files in docs, cut the prefix ./ 
+# Remove all security-bulletins and cve-reports.md because they are rate limited by nvd.nist.gov
+# Remove oss-licenses.md because they are rate limited by npmjs.com
+VERIFY_URL_PATHS=$(shell find ./docs -name "*.md" | cut -c 3- | sed '/security-bulletins/d' | sed '/cve-reports/d' | sed '/oss-licenses/d')
+
+RATE_LIMITED_FILES_LIST:="docs/docs-content/security-bulletins/**/*.md" \
+	"docs/docs-content/security-bulletins/*.md" \
+	"docs/docs-content/unlisted/cve-reports.md"
 
 help: ## Display this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[0m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
@@ -32,12 +40,21 @@ deep-clean: ## Clean all artifacts
 	rm -rf node_modules build public .cache .docusaurus
 	npm run clear && npm run clean-api-docs
 	docker image rm $(IMAGE) || echo "No image exists."
+	rm -rfv static/img/packs
+
+clean-logos: ## Clean logos
+	rm -rf static/img/packs
 
 clean-versions: ## Clean Docusarus content versions
 	@echo "cleaning versions"
 	rm -rf api_versions.json versions.json versioned_docs versioned_sidebars api_versioned_sidebars api_versioned_docs versioned_partials
 	git checkout -- docusaurus.config.js static/robots.txt
 
+
+clean-packs: ## Clean supplemental packs and pack images
+	rm -rf static/img/packs
+	rm -rf .docusaurus/packs-integrations/api_pack_response.json
+
 clean-api: ## Clean API docs
 	@echo "cleaning api docs"
 	npm run clean-api-docs
@@ -163,41 +180,37 @@ pdf-local: ## Generate PDF from local docs
 verify-url-links:
 	@echo "Checking for broken external URLs in markdown files..."
 	rm link_report.csv || echo "No report exists. Proceeding to scan step"
-	@npx linkinator "docs/**/*.md" --markdown --recurse --timeout 60000 --retry --retry-errors-jitter --retry-errors-count 3 \
-		--skip "^https:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/docs\.spectrocloud\.com\/.*\/supplemental\-packs$$" \
-		--skip "^http:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/software-private\.spectrocloud\.com.*$$" \
-		--skip "^\/.*\.md$$" \
-		--skip "!\[.*\]\(.*\)$$" \
-		--skip "\.(jpg|jpeg|png|gif|webp)$$" \
-		--skip "https:\/\/linux\.die\.net\/man\/.*$$" \
-		--skip "https:\/\/mysql\.com\/.*\.*$$" \
-		--skip "https:\/\/dev\.mysql\.com\/doc/\.*$$" \
-		--format csv > temp_report.csv && sleep 2
+	@npx linkinator $(VERIFY_URL_PATHS) --config ./linkinator/linkinator.config.json > temp_report.csv && sleep 2
 	@grep -E 'https?://' temp_report.csv > filtered_report.csv
 	@grep -E ',[[:space:]]*([4-9][0-9]{2}|[0-9]{4,}),' filtered_report.csv > link_report.csv && rm temp_report.csv filtered_report.csv
 
+verify-rate-limited-links:
+	@echo "Checking for broken URLs in security-bulletins and oss-licenses markdown files..."
+	@rm link_rate_limit_report.csv || echo "No rate limited report exists. Proceeding to scan step"
+	@echo "Checking the following paths: $(RATE_LIMITED_FILES_LIST)"
+	@npx linkinator $(RATE_LIMITED_FILES_LIST) --config ./linkinator/linkinator-rate-limit.config.json  > temp_rate_limit_report.csv && sleep 2
+	@grep -E 'https?://' temp_rate_limit_report.csv > filtered_rate_limit_report.csv
+	@grep -E ',[[:space:]]*([4-9][0-9]{2}|[0-9]{4,}),' filtered_rate_limit_report.csv > link_rate_limit_report.csv && rm temp_rate_limit_report.csv filtered_rate_limit_report.csv
+
 verify-url-links-ci: ## Check for broken URLs in production in a GitHub Actions CI environment
 	@echo "Checking for broken external URLs in CI environment..."
-	rm link_report.json || echo "No report exists. Proceeding to scan step"
-	@npx linkinator "docs/**/*.md" --markdown --recurse --timeout 60000 --retry --retry-errors-count 3 \
-		--skip "^https:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/docs\.spectrocloud\.com\/.*\/supplemental\-packs$$" \
-		--skip "^http:\/\/docs\.spectrocloud\.com.*$$" \
-		--skip "^https:\/\/software-private\.spectrocloud\.com.*$$" \
-		--skip "^\/.*\.md$$" \
-		--skip "!\[.*\]\(.*\)$$" \
-		--skip "\.(jpg|jpeg|png|gif|webp)$$" \
-		--skip "https:\/\/linux\.die\.net\/man\/.*$$" \
-		--skip "https:\/\/mysql\.com\/.*\.*$$" \
-		--skip "https:\/\/dev\.mysql\.com\/doc/\.*$$" \
-		--format json > temp_report.json
+	@rm link_report.json || echo "No report exists. Proceeding to scan step"
+	@npx linkinator $(VERIFY_URL_PATHS) --config ./linkinator/linkinator-ci.config.json  > temp_report.json
 	@# Use jq to filter out links that do not start with http or https and keep only broken links
 	@jq '[.links[] | select(.url | test("^https?://")) | select(.status >= 400)]' temp_report.json > filtered_report.json
 	@rm temp_report.json
 	@mv filtered_report.json scripts/link_report.json
 
+verify-rate-limited-links-ci: ## Check for broken URLs in production in a GitHub Actions CI environment
+	@echo "Checking for broken URLs in security-bulletins and oss-licenses markdown files in CI environment..."
+	@rm link_rate_limit_report.json || echo "No rate limited report exists. Proceeding to scan step"
+	@echo "Checking the following paths: $(RATE_LIMITED_FILES_LIST)"
+	@npx linkinator $(RATE_LIMITED_FILES_LIST) --config ./linkinator/linkinator-rate-limit-ci.config.json  > temp_rate_limit_report.json
+	@# Use jq to filter out links that do not start with http or https and keep only broken links
+	@jq '[.links[] | select(.url | test("^https?://")) | select(.status >= 400)]' temp_rate_limit_report.json > filtered_rate_limit_report.json
+	@rm temp_rate_limit_report.json
+	@mv filtered_rate_limit_report.json scripts/link_rate_limit_report.json
+
 ###@ Image Formatting
 
 format-images: ## Format images