[version-4-1] chore: added dev aloglia config (#2654)

.github/workflows/aloglia_crawler.yaml

@@ -6,6 +6,11 @@ on:
     types: [completed]
   workflow_dispatch:
 
+env:
+  ALGOLIA_CRAWLER_USER_ID: ${{ secrets.ALGOLIA_CRAWLER_V3_USER_ID }}
+  ALGOLIA_CRAWLER_API_KEY: ${{ secrets.ALGOLIA_CRAWLER_V3_API_KEY }}
+  CRAWLER_ID: ${{ secrets.ALGOLIA_CRAWLER_V3_ID }}
+
 jobs:
   docsearch:
     runs-on: ubuntu-latest
@@ -20,6 +25,11 @@ jobs:
           API_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
         run: docker run  -e APPLICATION_ID=$APPLICATION_ID  -e API_KEY=$API_KEY  -e "CONFIG=$(cat ./docsearch.config.json | jq -r tostring)" algolia/docsearch-scraper
 
+      - name: Trigger DocSearch v3 Crawler
+        run: |
+          curl -H "Content-Type: application/json" -X POST --user ${ALGOLIA_CRAWLER_USER_ID}:${ALGOLIA_CRAWLER_API_KEY} "https://crawler.algolia.com/api/1/crawlers/${CRAWLER_ID}/reindex"
+
+
       - name: Slack Notification
         if: ${{ failure() }}
         uses: rtCamp/action-slack-notify@v2

.gitleaksignore

@@ -115,3 +115,16 @@ bee050efeb29aba56698a812cd87a44c757cf058:docs/docs-content/enterprise-version/sy
 bee050efeb29aba56698a812cd87a44c757cf058:docs/docs-content/vertex/system-management/reverse-proxy.md:private-key:163
 bee050efeb29aba56698a812cd87a44c757cf058:docs/docs-content/vertex/system-management/reverse-proxy.md:private-key:194
 bee050efeb29aba56698a812cd87a44c757cf058:docs/docs-content/vertex/system-management/reverse-proxy.md:private-key:275
+511e735952ff4babb08c522a7febdd856740c3f9:docs/docs-content/enterprise-version/system-management/reverse-proxy.md:private-key:141
+511e735952ff4babb08c522a7febdd856740c3f9:docs/docs-content/enterprise-version/system-management/reverse-proxy.md:private-key:167
+511e735952ff4babb08c522a7febdd856740c3f9:docs/docs-content/enterprise-version/system-management/reverse-proxy.md:private-key:239
+511e735952ff4babb08c522a7febdd856740c3f9:docs/docs-content/vertex/system-management/reverse-proxy.md:private-key:141
+511e735952ff4babb08c522a7febdd856740c3f9:docs/docs-content/vertex/system-management/reverse-proxy.md:private-key:167
+511e735952ff4babb08c522a7febdd856740c3f9:docs/docs-content/vertex/system-management/reverse-proxy.md:private-key:239
+9e62b4b635976b0ab93d4dddcf29d33365664091:docs/docs-content/integrations/kubernetes.md:generic-api-key:391
+9e62b4b635976b0ab93d4dddcf29d33365664091:docs/docs-content/integrations/kubernetes.md:generic-api-key:759
+9e62b4b635976b0ab93d4dddcf29d33365664091:docs/docs-content/integrations/kubernetes.md:generic-api-key:1125
+e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:759
+e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:1125
+e4040084011d4d7935a589959b96ebc5cfba7a94:docs/docs-content/integrations/kubernetes.md:generic-api-key:391
+87bf9f7353f94ac2ecf85da3be3412202de7b72f:docusaurus.config.js:generic-api-key:363

Makefile

@@ -1,5 +1,7 @@
 .PHONY: initialize start commit build
 
+include .env
+
 IMAGE:=spectrocloud/librarium
 # Retrieve all modified files in the content folder and compare the difference between the master branch git tree blob AND this commit's git tree blob
 CHANGED_FILE=$(shell git diff-tree -r --no-commit-id --name-only master HEAD | grep content)
@@ -8,6 +10,7 @@ TEMP_DIR=$(shell $TMPDIR)
 
 CPUS := $(shell sysctl -n hw.ncpu | awk '{print int($$1 / 2)}')
 
+ALOGLIA_CONFIG=$(shell cat docsearch.dev.config.json | jq -r tostring)
 
 
 help: ## Display this help
@@ -198,4 +201,14 @@ verify-url-links-ci: ## Check for broken URLs in production in a GitHub Actions
 
 format-images: ## Format images
 	@echo "formatting images in /static/assets/docs/images/ folder"
-	./scripts/compress-convert-images.sh
+	./scripts/compress-convert-images.sh
+
+###@ Aloglia Indexing
+
+update-dev-index: ## Update the Algolia index for the dev environment
+	@echo "updating Algolia index for dev environment"
+	docker run  -e APPLICATION_ID=${ALGOLIA_APP_ID}  -e API_KEY=${ALGOLIA_ADMIN_KEY} -e CONFIG='${ALOGLIA_CONFIG}'  algolia/docsearch-scraper
+
+remove-dev-index: ## Remove the Algolia index for the dev environment
+	@echo "removing Algolia index for dev environment"
+	algolia index delete dev-docusaurus-librarium --confirm

docsearch.config.json

@@ -3,10 +3,9 @@
   "start_urls": ["https://docs.spectrocloud.com/"],
   "sitemap_urls": ["https://docs.spectrocloud.com/sitemap.xml"],
   "sitemap_alternate_links": true,
-  "stop_urls": ["^/api/(?!.*\\d+\\.\\d+\\.x/v1/)"],
   "selectors": {
     "lvl0": {
-      "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]/text())[last()]",
+      "selector": "(//ul[contains(@class,\"menu__list\")]//a[contains(@class, \"menu__link menu__link--sublist menu__link--active\")]/text() | //nav[contains(@class, \"navbar\")]//a[contains(@class, \"navbar__link--active\")]/text())[last()]",
       "type": "xpath",
       "global": true,
       "default_value": "Documentation"

docsearch.dev.config.json

@@ -0,0 +1,28 @@
+{
+  "index_name": "dev-docusaurus-librarium",
+  "start_urls": ["https://docs.spectrocloud.com/"],
+  "sitemap_urls": ["https://docs.spectrocloud.com/sitemap.xml"],
+  "sitemap_alternate_links": true,
+  "stop_urls": ["/api/"],
+  "selectors": {
+    "lvl0": {
+      "selector": "(//ul[contains(@class,\"menu__list\")]//a[contains(@class, \"menu__link menu__link--sublist menu__link--active\")]/text() | //nav[contains(@class, \"navbar\")]//a[contains(@class, \"navbar__link--active\")]/text())[last()]",
+      "type": "xpath",
+      "global": true,
+      "default_value": "Documentation"
+    },
+    "lvl1": "article h1",
+    "lvl2": "article h2",
+    "lvl3": "article h3",
+    "lvl4": "article h4",
+    "lvl5": "article h5",
+    "lvl6": "article h6",
+    "text": "article p, article li, article td, article tr"
+  },
+  "strip_chars": " .,;:#",
+  "custom_settings": {
+    "separatorsToIndex": "_",
+    "attributesForFaceting": ["language", "version", "type", "docusaurus_tag"],
+    "attributesToRetrieve": ["hierarchy", "content", "anchor", "url", "url_without_anchor", "type"]
+  }
+}

docusaurus.config.js

@@ -123,7 +123,7 @@ const config = {
           editUrl: "https://github.com/spectrocloud/librarium/blob/master",
         },
         sitemap: {
-          changefreq: "weekly",
+          changefreq: "daily",
           priority: 0.5,
           ignorePatterns: ["/tags/**"],
           filename: "sitemap.xml",
@@ -342,6 +342,7 @@ const config = {
         searchParameters: {},
         // Optional: path for search page that enabled by default (`false` to disable it)
         searchPagePath: "search",
+        maxResultsPerGroup: 7,
       },
       sidebar: {
         hideable: true,