From 449bc7d7ce98386696460fbd47cb8f8b945c960a Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 15 Apr 2024 13:54:12 -0700 Subject: [PATCH 1/9] chore: added dev aloglia config --- Makefile | 15 ++++++++++++++- docsearch.config.json | 3 +-- docsearch.dev.config.json | 28 ++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 docsearch.dev.config.json diff --git a/Makefile b/Makefile index 8e4067a769..c9efe65211 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ .PHONY: initialize start commit build +include .env + IMAGE:=spectrocloud/librarium # Retrieve all modified files in the content folder and compare the difference between the master branch git tree blob AND this commit's git tree blob CHANGED_FILE=$(shell git diff-tree -r --no-commit-id --name-only master HEAD | grep content) @@ -8,6 +10,7 @@ TEMP_DIR=$(shell $TMPDIR) CPUS := $(shell sysctl -n hw.ncpu | awk '{print int($$1 / 2)}') +ALOGLIA_CONFIG=$(shell cat docsearch.dev.config.json | jq -r tostring) help: ## Display this help @@ -198,4 +201,14 @@ verify-url-links-ci: ## Check for broken URLs in production in a GitHub Actions format-images: ## Format images @echo "formatting images in /static/assets/docs/images/ folder" - ./scripts/compress-convert-images.sh \ No newline at end of file + ./scripts/compress-convert-images.sh + +###@ Aloglia Indexing + +update-dev-index: ## Update the Algolia index for the dev environment + @echo "updating Algolia index for dev environment" + docker run -e APPLICATION_ID=${ALGOLIA_APP_ID} -e API_KEY=${ALGOLIA_ADMIN_KEY} -e CONFIG='${ALOGLIA_CONFIG}' algolia/docsearch-scraper + +remove-dev-index: ## Remove the Algolia index for the dev environment + @echo "removing Algolia index for dev environment" + algolia index delete dev-docusaurus-librarium --confirm \ No newline at end of file diff --git a/docsearch.config.json b/docsearch.config.json index 6f9feba013..7d62f011c6 100644 --- a/docsearch.config.json +++ b/docsearch.config.json @@ -3,10 +3,9 @@ "start_urls": ["https://docs.spectrocloud.com/"], "sitemap_urls": ["https://docs.spectrocloud.com/sitemap.xml"], "sitemap_alternate_links": true, - "stop_urls": ["^/api/(?!.*\\d+\\.\\d+\\.x/v1/)"], "selectors": { "lvl0": { - "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]/text())[last()]", + "selector": "(//ul[contains(@class,\"menu__list\")]//a[contains(@class, \"menu__link menu__link--sublist menu__link--active\")]/text() | //nav[contains(@class, \"navbar\")]//a[contains(@class, \"navbar__link--active\")]/text())[last()]", "type": "xpath", "global": true, "default_value": "Documentation" diff --git a/docsearch.dev.config.json b/docsearch.dev.config.json new file mode 100644 index 0000000000..bc259e741f --- /dev/null +++ b/docsearch.dev.config.json @@ -0,0 +1,28 @@ +{ + "index_name": "dev-docusaurus-librarium", + "start_urls": ["https://docs.spectrocloud.com/"], + "sitemap_urls": ["https://docs.spectrocloud.com/sitemap.xml"], + "sitemap_alternate_links": true, + "stop_urls": ["/api/"], + "selectors": { + "lvl0": { + "selector": "(//ul[contains(@class,\"menu__list\")]//a[contains(@class, \"menu__link menu__link--sublist menu__link--active\")]/text() | //nav[contains(@class, \"navbar\")]//a[contains(@class, \"navbar__link--active\")]/text())[last()]", + "type": "xpath", + "global": true, + "default_value": "Documentation" + }, + "lvl1": "article h1", + "lvl2": "article h2", + "lvl3": "article h3", + "lvl4": "article h4", + "lvl5": "article h5", + "lvl6": "article h6", + "text": "article p, article li, article td, article tr" + }, + "strip_chars": " .,;:#", + "custom_settings": { + "separatorsToIndex": "_", + "attributesForFaceting": ["language", "version", "type", "docusaurus_tag"], + "attributesToRetrieve": ["hierarchy", "content", "anchor", "url", "url_without_anchor", "type"] + } +} From bed805eed3bbbbd61fa80d7744effcfb287dfc85 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 15 Apr 2024 16:01:20 -0700 Subject: [PATCH 2/9] chore: add trailing slash --- docusaurus.config.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index a0c91d6825..6b90a12dfd 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -24,6 +24,7 @@ const config = { onBrokenLinks: "throw", onBrokenAnchors: "throw", onBrokenMarkdownLinks: "throw", + trailingSlash: true, // Even if you don't use internalization, you can use this field to set useful // metadata like html lang. For example, if your site is Chinese, you may want // to replace "en" with "zh-Hans". @@ -127,7 +128,7 @@ const config = { editUrl: "https://github.com/spectrocloud/librarium/blob/master", }, sitemap: { - changefreq: "weekly", + changefreq: "daily", priority: 0.5, ignorePatterns: ["/tags/**"], filename: "sitemap.xml", From 87bf9f7353f94ac2ecf85da3be3412202de7b72f Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Tue, 16 Apr 2024 15:44:45 -0700 Subject: [PATCH 3/9] save --- docusaurus.config.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index 6b90a12dfd..867439cb69 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -358,10 +358,10 @@ const config = { ], algolia: { // The application ID provided by Algolia - appId: process.env.ALGOLIA_APP_ID, + appId: "9AJ6SKEWEN", // Public API key: it is safe to commit it - apiKey: process.env.ALGOLIA_SEARCH_KEY, - indexName: "prod-docusaurus-librarium", + apiKey: "ce2b4db7872279451d8b8964a2239e28", + indexName: "spectrocloud", // Optional: see doc section below contextualSearch: true, // Optional: Specify domains where the navigation should occur through window.location instead on history.push. Useful when our Algolia config crawls multiple documentation sites and we want to navigate with window.location.href to them. @@ -376,6 +376,7 @@ const config = { searchParameters: {}, // Optional: path for search page that enabled by default (`false` to disable it) searchPagePath: "search", + debug: true, }, sidebar: { hideable: true, From b96384e629d61238c835feed25df22f7f366361d Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Wed, 17 Apr 2024 07:34:11 -0700 Subject: [PATCH 4/9] chore: updated aloglia index --- docusaurus.config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index 867439cb69..5e3852a7ab 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -358,9 +358,9 @@ const config = { ], algolia: { // The application ID provided by Algolia - appId: "9AJ6SKEWEN", + appId: process.env.ALGOLIA_APP_ID, // Public API key: it is safe to commit it - apiKey: "ce2b4db7872279451d8b8964a2239e28", + apiKey: process.env.ALGOLIA_SEARCH_KEY, indexName: "spectrocloud", // Optional: see doc section below contextualSearch: true, From 5859d1d1ad1cef3f9b726708586c1ea3314ddb83 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Wed, 17 Apr 2024 08:18:25 -0700 Subject: [PATCH 5/9] chore: test --- .github/workflows/aloglia_crawler.yaml | 53 +++++++++++++++++--------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/.github/workflows/aloglia_crawler.yaml b/.github/workflows/aloglia_crawler.yaml index a188a88890..e3f49ed027 100644 --- a/.github/workflows/aloglia_crawler.yaml +++ b/.github/workflows/aloglia_crawler.yaml @@ -1,10 +1,20 @@ name: Algolia Crawler +# on: +# workflow_run: +# workflows: ["Release to Production"] +# types: [completed] +# workflow_dispatch: + on: - workflow_run: - workflows: ["Release to Production"] - types: [completed] - workflow_dispatch: + push: + branches: + - aloglia-index-research + +env: + ALGOLIA_CRAWLER_USER_ID: ${{ secrets.ALGOLIA_CRAWLER_V3_USER_ID }} + ALGOLIA_CRAWLER_API_KEY: ${{ secrets.ALGOLIA_CRAWLER_V3_API_KEY }} + CRAWLER_ID: ${{ secrets.ALGOLIA_CRAWLER_V3_ID }} jobs: docsearch: @@ -14,18 +24,23 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - - name: Run scraper - env: - APPLICATION_ID: ${{ secrets.ALGOLIA_APP_ID }} - API_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} - run: docker run -e APPLICATION_ID=$APPLICATION_ID -e API_KEY=$API_KEY -e "CONFIG=$(cat ./docsearch.config.json | jq -r tostring)" algolia/docsearch-scraper - - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - SLACK_USERNAME: "spectromate" - SLACK_ICON_EMOJI: ":robot_panic:" - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: 'The Docs Algolia crawler job failed. Please check the GitHub Actions logs for more details.' \ No newline at end of file + # - name: Run scraper + # env: + # APPLICATION_ID: ${{ secrets.ALGOLIA_APP_ID }} + # API_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} + # run: docker run -e APPLICATION_ID=$APPLICATION_ID -e API_KEY=$API_KEY -e "CONFIG=$(cat ./docsearch.config.json | jq -r tostring)" algolia/docsearch-scraper + + - name: Trigger DocSearch v3 Crawler + run: | + curl -H "Content-Type: application/json" -X POST --user ${ALGOLIA_CRAWLER_USER_ID}:${ALGOLIA_CRAWLER_API_KEY} "https://crawler.algolia.com/api/1/crawlers/${CRAWLER_ID}/reindex" + + + # - name: Slack Notification + # if: ${{ failure() }} + # uses: rtCamp/action-slack-notify@v2 + # env: + # SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + # SLACK_USERNAME: "spectromate" + # SLACK_ICON_EMOJI: ":robot_panic:" + # SLACK_COLOR: ${{ job.status }} + # SLACK_MESSAGE: 'The Docs Algolia crawler job failed. Please check the GitHub Actions logs for more details.' \ No newline at end of file From 48009060050d44eae7eaa36c51b9da37e792b579 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Wed, 17 Apr 2024 08:19:09 -0700 Subject: [PATCH 6/9] chore: test --- .github/workflows/aloglia_crawler.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aloglia_crawler.yaml b/.github/workflows/aloglia_crawler.yaml index e3f49ed027..79b249c369 100644 --- a/.github/workflows/aloglia_crawler.yaml +++ b/.github/workflows/aloglia_crawler.yaml @@ -19,7 +19,7 @@ env: jobs: docsearch: runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + # if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} steps: - name: Checkout repo uses: actions/checkout@v4 From 2451fe275995b65a94e6a4d8ce69b034948574f2 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Wed, 17 Apr 2024 08:44:16 -0700 Subject: [PATCH 7/9] chore: update config --- .github/workflows/aloglia_crawler.yaml | 41 +++++++++++--------------- docusaurus.config.js | 1 - 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/.github/workflows/aloglia_crawler.yaml b/.github/workflows/aloglia_crawler.yaml index 79b249c369..c429f18c91 100644 --- a/.github/workflows/aloglia_crawler.yaml +++ b/.github/workflows/aloglia_crawler.yaml @@ -1,15 +1,10 @@ name: Algolia Crawler -# on: -# workflow_run: -# workflows: ["Release to Production"] -# types: [completed] -# workflow_dispatch: - on: - push: - branches: - - aloglia-index-research + workflow_run: + workflows: ["Release to Production"] + types: [completed] + workflow_dispatch: env: ALGOLIA_CRAWLER_USER_ID: ${{ secrets.ALGOLIA_CRAWLER_V3_USER_ID }} @@ -24,23 +19,23 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - # - name: Run scraper - # env: - # APPLICATION_ID: ${{ secrets.ALGOLIA_APP_ID }} - # API_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} - # run: docker run -e APPLICATION_ID=$APPLICATION_ID -e API_KEY=$API_KEY -e "CONFIG=$(cat ./docsearch.config.json | jq -r tostring)" algolia/docsearch-scraper + - name: Run scraper + env: + APPLICATION_ID: ${{ secrets.ALGOLIA_APP_ID }} + API_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} + run: docker run -e APPLICATION_ID=$APPLICATION_ID -e API_KEY=$API_KEY -e "CONFIG=$(cat ./docsearch.config.json | jq -r tostring)" algolia/docsearch-scraper - name: Trigger DocSearch v3 Crawler run: | curl -H "Content-Type: application/json" -X POST --user ${ALGOLIA_CRAWLER_USER_ID}:${ALGOLIA_CRAWLER_API_KEY} "https://crawler.algolia.com/api/1/crawlers/${CRAWLER_ID}/reindex" - # - name: Slack Notification - # if: ${{ failure() }} - # uses: rtCamp/action-slack-notify@v2 - # env: - # SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - # SLACK_USERNAME: "spectromate" - # SLACK_ICON_EMOJI: ":robot_panic:" - # SLACK_COLOR: ${{ job.status }} - # SLACK_MESSAGE: 'The Docs Algolia crawler job failed. Please check the GitHub Actions logs for more details.' \ No newline at end of file + - name: Slack Notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_USERNAME: "spectromate" + SLACK_ICON_EMOJI: ":robot_panic:" + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: 'The Docs Algolia crawler job failed. Please check the GitHub Actions logs for more details.' \ No newline at end of file diff --git a/docusaurus.config.js b/docusaurus.config.js index 5e3852a7ab..c98289f04e 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -376,7 +376,6 @@ const config = { searchParameters: {}, // Optional: path for search page that enabled by default (`false` to disable it) searchPagePath: "search", - debug: true, }, sidebar: { hideable: true, From 6cd0523b8a69676ab741d4f9a1e8f572a5372cf0 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Wed, 17 Apr 2024 13:20:39 -0700 Subject: [PATCH 8/9] chore: reverted index name --- .github/workflows/aloglia_crawler.yaml | 2 +- docusaurus.config.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/aloglia_crawler.yaml b/.github/workflows/aloglia_crawler.yaml index c429f18c91..a199c8d82c 100644 --- a/.github/workflows/aloglia_crawler.yaml +++ b/.github/workflows/aloglia_crawler.yaml @@ -14,7 +14,7 @@ env: jobs: docsearch: runs-on: ubuntu-latest - # if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} steps: - name: Checkout repo uses: actions/checkout@v4 diff --git a/docusaurus.config.js b/docusaurus.config.js index c98289f04e..99402a88e4 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -361,7 +361,7 @@ const config = { appId: process.env.ALGOLIA_APP_ID, // Public API key: it is safe to commit it apiKey: process.env.ALGOLIA_SEARCH_KEY, - indexName: "spectrocloud", + indexName: "prod-docusaurus-librarium", // Optional: see doc section below contextualSearch: true, // Optional: Specify domains where the navigation should occur through window.location instead on history.push. Useful when our Algolia config crawls multiple documentation sites and we want to navigate with window.location.href to them. @@ -376,6 +376,7 @@ const config = { searchParameters: {}, // Optional: path for search page that enabled by default (`false` to disable it) searchPagePath: "search", + maxResultsPerGroup: 7, }, sidebar: { hideable: true, From ebe63248c7d239cb6f0742cff87d207f11778409 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Wed, 17 Apr 2024 14:29:49 -0700 Subject: [PATCH 9/9] chore: gitleaks fix --- .gitleaksignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitleaksignore b/.gitleaksignore index 60479efc5e..882d679bec 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -109,3 +109,4 @@ fd60bdc4fdfe8b66925db07865cb530eab4978df:docs/docs-content/integrations/kubernet e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:759 e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:1125 e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:391 +87bf9f7353f94ac2ecf85da3be3412202de7b72f:docusaurus.config.js:generic-api-key:363