From 6c564b766942b4c05896e84c11878cf019b52d6b Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Wed, 17 Apr 2024 14:56:25 -0700 Subject: [PATCH] chore: added dev aloglia config (#2654) * chore: added dev aloglia config * chore: add trailing slash * save * chore: updated aloglia index * chore: test * chore: test * chore: update config * chore: reverted index name * chore: gitleaks fix (cherry picked from commit 604123e54d9e38324375860be42da7613ce21975) --- .github/workflows/aloglia_crawler.yaml | 10 +++++++++ .gitleaksignore | 1 + Makefile | 15 +++++++++++++- docsearch.config.json | 3 +-- docsearch.dev.config.json | 28 ++++++++++++++++++++++++++ docusaurus.config.js | 3 ++- 6 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 docsearch.dev.config.json diff --git a/.github/workflows/aloglia_crawler.yaml b/.github/workflows/aloglia_crawler.yaml index a188a88890..a199c8d82c 100644 --- a/.github/workflows/aloglia_crawler.yaml +++ b/.github/workflows/aloglia_crawler.yaml @@ -6,6 +6,11 @@ on: types: [completed] workflow_dispatch: +env: + ALGOLIA_CRAWLER_USER_ID: ${{ secrets.ALGOLIA_CRAWLER_V3_USER_ID }} + ALGOLIA_CRAWLER_API_KEY: ${{ secrets.ALGOLIA_CRAWLER_V3_API_KEY }} + CRAWLER_ID: ${{ secrets.ALGOLIA_CRAWLER_V3_ID }} + jobs: docsearch: runs-on: ubuntu-latest @@ -20,6 +25,11 @@ jobs: API_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} run: docker run -e APPLICATION_ID=$APPLICATION_ID -e API_KEY=$API_KEY -e "CONFIG=$(cat ./docsearch.config.json | jq -r tostring)" algolia/docsearch-scraper + - name: Trigger DocSearch v3 Crawler + run: | + curl -H "Content-Type: application/json" -X POST --user ${ALGOLIA_CRAWLER_USER_ID}:${ALGOLIA_CRAWLER_API_KEY} "https://crawler.algolia.com/api/1/crawlers/${CRAWLER_ID}/reindex" + + - name: Slack Notification if: ${{ failure() }} uses: rtCamp/action-slack-notify@v2 diff --git a/.gitleaksignore b/.gitleaksignore index 60479efc5e..882d679bec 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -109,3 +109,4 @@ fd60bdc4fdfe8b66925db07865cb530eab4978df:docs/docs-content/integrations/kubernet e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:759 e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:1125 e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:391 +87bf9f7353f94ac2ecf85da3be3412202de7b72f:docusaurus.config.js:generic-api-key:363 diff --git a/Makefile b/Makefile index 8e4067a769..c9efe65211 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ .PHONY: initialize start commit build +include .env + IMAGE:=spectrocloud/librarium # Retrieve all modified files in the content folder and compare the difference between the master branch git tree blob AND this commit's git tree blob CHANGED_FILE=$(shell git diff-tree -r --no-commit-id --name-only master HEAD | grep content) @@ -8,6 +10,7 @@ TEMP_DIR=$(shell $TMPDIR) CPUS := $(shell sysctl -n hw.ncpu | awk '{print int($$1 / 2)}') +ALOGLIA_CONFIG=$(shell cat docsearch.dev.config.json | jq -r tostring) help: ## Display this help @@ -198,4 +201,14 @@ verify-url-links-ci: ## Check for broken URLs in production in a GitHub Actions format-images: ## Format images @echo "formatting images in /static/assets/docs/images/ folder" - ./scripts/compress-convert-images.sh \ No newline at end of file + ./scripts/compress-convert-images.sh + +###@ Aloglia Indexing + +update-dev-index: ## Update the Algolia index for the dev environment + @echo "updating Algolia index for dev environment" + docker run -e APPLICATION_ID=${ALGOLIA_APP_ID} -e API_KEY=${ALGOLIA_ADMIN_KEY} -e CONFIG='${ALOGLIA_CONFIG}' algolia/docsearch-scraper + +remove-dev-index: ## Remove the Algolia index for the dev environment + @echo "removing Algolia index for dev environment" + algolia index delete dev-docusaurus-librarium --confirm \ No newline at end of file diff --git a/docsearch.config.json b/docsearch.config.json index 6f9feba013..7d62f011c6 100644 --- a/docsearch.config.json +++ b/docsearch.config.json @@ -3,10 +3,9 @@ "start_urls": ["https://docs.spectrocloud.com/"], "sitemap_urls": ["https://docs.spectrocloud.com/sitemap.xml"], "sitemap_alternate_links": true, - "stop_urls": ["^/api/(?!.*\\d+\\.\\d+\\.x/v1/)"], "selectors": { "lvl0": { - "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]/text())[last()]", + "selector": "(//ul[contains(@class,\"menu__list\")]//a[contains(@class, \"menu__link menu__link--sublist menu__link--active\")]/text() | //nav[contains(@class, \"navbar\")]//a[contains(@class, \"navbar__link--active\")]/text())[last()]", "type": "xpath", "global": true, "default_value": "Documentation" diff --git a/docsearch.dev.config.json b/docsearch.dev.config.json new file mode 100644 index 0000000000..bc259e741f --- /dev/null +++ b/docsearch.dev.config.json @@ -0,0 +1,28 @@ +{ + "index_name": "dev-docusaurus-librarium", + "start_urls": ["https://docs.spectrocloud.com/"], + "sitemap_urls": ["https://docs.spectrocloud.com/sitemap.xml"], + "sitemap_alternate_links": true, + "stop_urls": ["/api/"], + "selectors": { + "lvl0": { + "selector": "(//ul[contains(@class,\"menu__list\")]//a[contains(@class, \"menu__link menu__link--sublist menu__link--active\")]/text() | //nav[contains(@class, \"navbar\")]//a[contains(@class, \"navbar__link--active\")]/text())[last()]", + "type": "xpath", + "global": true, + "default_value": "Documentation" + }, + "lvl1": "article h1", + "lvl2": "article h2", + "lvl3": "article h3", + "lvl4": "article h4", + "lvl5": "article h5", + "lvl6": "article h6", + "text": "article p, article li, article td, article tr" + }, + "strip_chars": " .,;:#", + "custom_settings": { + "separatorsToIndex": "_", + "attributesForFaceting": ["language", "version", "type", "docusaurus_tag"], + "attributesToRetrieve": ["hierarchy", "content", "anchor", "url", "url_without_anchor", "type"] + } +} diff --git a/docusaurus.config.js b/docusaurus.config.js index c9b7869c95..99402a88e4 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -128,7 +128,7 @@ const config = { editUrl: "https://github.com/spectrocloud/librarium/blob/master", }, sitemap: { - changefreq: "weekly", + changefreq: "daily", priority: 0.5, ignorePatterns: ["/tags/**"], filename: "sitemap.xml", @@ -376,6 +376,7 @@ const config = { searchParameters: {}, // Optional: path for search page that enabled by default (`false` to disable it) searchPagePath: "search", + maxResultsPerGroup: 7, }, sidebar: { hideable: true,