DEX-413 feat: added SDK references generation code

.github/workflows/update-refs-in-docs.yml

@@ -0,0 +1,85 @@
+name: Update references in Docsite
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  generate-node-sdk-refs:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        ref: master
+        fetch-depth: 0
+        persist-credentials: false
+
+    - name: install modules
+      run: |
+        npm install
+
+    - name: Generate Node SDK references
+      run: |
+        npm run docs-gen
+        cd ../..
+        mkdir refs
+        cp -r skyflow-node/skyflow-node/docs/* refs/
+        cd skyflow-node/skyflow-node/
+        rm -r docs/
+        git checkout -- package-lock.json
+        echo "SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_ENV
+
+    - name: Create a branch in skyflow-docs
+      env:
+        TOKEN: ${{ secrets.PAT_ACTIONS }}
+        REPO_OWNER: skyflowapi
+        REPO_NAME: skyflow-docs
+        ACTOR: ${{ github.actor }}
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        # Create a new branch in skyflow-docs
+        cd $GITHUB_WORKSPACE
+        git remote add skyflow-docs https://${TOKEN}@github.com/${REPO_OWNER}/${REPO_NAME}.git
+        git fetch skyflow-docs main
+        BRANCH_NAME="SDK/node/${{ env.SHORT_SHA }}"
+        git remote set-url --push skyflow-docs https://${ACTOR}:${TOKEN}@github.com/${REPO_OWNER}/${REPO_NAME}.git
+        git checkout -b $BRANCH_NAME skyflow-docs/main
+        cp -r ../../refs/* src/pages/content/docs/sdks/skyflow-node/
+
+    - name: Push files and raise a PR
+      env:
+        TOKEN: ${{ secrets.PAT_ACTIONS }}
+        REPO_OWNER: skyflowapi
+        REPO_NAME: skyflow-docs
+        ACTOR: ${{ github.actor }}
+      run: |
+        git config user.name ${{ github.actor }}
+        git config user.email ${{ github.actor }}@users.noreply.github.com
+        BRANCH_NAME="SDK/node/${{ env.SHORT_SHA }}"
+        git add .
+
+        # Check if there are changes to commit
+        if [[ -n "$(git status --porcelain)" ]]; then
+          git commit -m "SDK-${{ env.SHORT_SHA }} Updated Node SDK references"
+          git push skyflow-docs $BRANCH_NAME
+          # Raise a pull request
+          BASE_BRANCH="main"
+          BRANCH_NAME="SDK/node/${{ env.SHORT_SHA }}"
+          TITLE="SDK-${{ env.SHORT_SHA }}: Updated Node SDK references"
+          BODY="This pull request adds the latest Node SDK references. Commit id for reference: $GITHUB_SHA"
+          API_URL="https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls"
+          echo "API URL: $API_URL"       
+          RESPONSE=$(curl -X POST \
+            -H "Accept: application/vnd.github.v3+json" \
+            -H "Authorization: token $TOKEN" \
+            -d "{\"title\":\"$TITLE\",\"body\":\"$BODY\",\"head\":\"${BRANCH_NAME}\",\"base\":\"$BASE_BRANCH\"}" \
+            "$API_URL")
+          echo "Response Body: $RESPONSE"
+          PR_URL=$(echo "$RESPONSE" | jq -r '.html_url')
+          echo "Pull Request URL: $PR_URL"
+        else
+          echo "No changes to commit. Skipping push files and raise a PR."
+          exit 0
+        fi       

package.json

@@ -10,7 +10,8 @@
     "eslint": "eslint '**/*.js' --max-warnings 0",
     "prettier": "prettier --list-different '**/*.{js,ts}'",
     "lint": "npm run prettier && npm run eslint",
-    "lint-fix": "prettier --write '**/*.{js,ts}' && eslint --fix '**/*.js'"
+    "lint-fix": "prettier --write '**/*.{js,ts}' && eslint --fix '**/*.js'",
+    "docs-gen": "typedoc && node scripts/docs-script/markdown-gen.js && npx ts-node scripts/docs-script/processMarkdown.ts"
   },
   "repository": {
     "type": "git",
@@ -76,6 +77,8 @@
     "null-loader": "^0.1.1",
     "nyc": "^15.1.0",
     "prettier": "^1.13.7",
-    "typescript": "^4.4.4"
+    "typescript": "^4.4.4",
+    "typedoc": "^0.24.4",
+    "typedoc-plugin-markdown": "^3.15.1"
   }
 }

scripts/docs-script/markdown-gen.js

@@ -0,0 +1,43 @@
+const fs = require('fs');
+const path = require('path');
+
+function getModuleName(filePath) {
+  const parsedPath = path.parse(filePath);
+  let moduleName = parsedPath.name;
+
+  if (moduleName.endsWith('.default')) {
+    moduleName = moduleName.slice(0, -8);
+  }
+
+  return moduleName;
+}
+
+const directoryPath = path.join(__dirname, '../../docs/classes');
+
+fs.readdir(directoryPath, (err, files) => {
+  if (err) {
+    console.error(`Error reading directory: ${err}`);
+    return;
+  }
+  files.forEach(file => {
+    if (path.extname(file) === '.md') {
+      const filePath = path.join(directoryPath, file);
+      fs.readFile(filePath, 'utf8', (err, data) => {
+        if (err) {
+          console.error(`Error reading file: ${err}`);
+          return;
+        }
+        const lines = data.split('\n');
+        lines[0] = `# Class: ${getModuleName(path.parse(file).name)}`;
+        const updatedContent = lines.join('\n');
+        fs.writeFile(filePath, updatedContent, err => {
+          if (err) {
+            console.error(`Error writing file: ${err}`);
+          } else {
+            console.log(`Updated file: ${filePath}`);
+          }
+        });
+      });
+    }
+  });
+});

scripts/docs-script/processMarkdown.ts

@@ -0,0 +1,144 @@
+import { readdirSync, readFileSync, statSync, writeFileSync, unlinkSync } from "fs";
+import path from "path";
+
+function removeListCharacters(markdown: string): string {
+  // Match list items with bullets, numbered lists, or other list-type characters
+  // const listRegex = /^(\s*[\*\-\+\•\▸]\s*|\s*\d+\.\s*|\s*\w\.\s*)/;
+  const listRegex = /^(\s*[\*\-\+\•\▸]\s|^\s*\d+\.\s|^\s*\w\.\s)/;
+
+  // Split the Markdown file into lines
+  const lines = markdown.split('\n');
+
+  // Track whether a list is currently in progress
+  let inList = false;
+
+  // Iterate through each line and remove list characters if necessary
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const match = line.match(listRegex);
+
+    if (match) {
+      // Check if the list is continuing or if it's a new list
+      if (!inList && !lines[i+1].match(listRegex)) {
+        // Remove list characters
+        lines[i] = line.replace(listRegex, '');
+      }
+      inList = true;
+    } else {
+      inList = false;
+    }
+  }
+
+  // Join the lines back into a single string
+  const result = lines.join('\n');
+
+  return result;
+}
+
+function processUrls(markdown: string, itemPath: string): string {
+  // remove extension .md from urls
+  const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
+  const editedMarkdown = markdown.replace(linkRegex, (match, text, url) => {
+    if(url.split('.').pop().startsWith('md'))
+    {
+      const hash = url.split('.').pop().replace('md', '')
+      let editedUrl = (url.split('.').slice(0, -1)).join('.')+hash;
+      if(editedUrl.startsWith('../') && editedUrl.split('/').length > 1)
+      {
+        editedUrl = `/sdks/skyflow-node/${editedUrl.split('/').slice(1).join('/')}`;
+      }
+      else
+      {
+        editedUrl = `/sdks/skyflow-node/${itemPath.split('/')[1]}/${editedUrl}`;
+      }
+      return `[${text}](${editedUrl})`;
+    }
+    return `[${text}](${url})`;
+  });
+  return editedMarkdown;
+}
+
+function createEnumerationTable(markdown) {
+  const enumMembersRegex = /### (.+)[\s\S]*?\*\*(.+)\*\* = `(.+)`/g;
+  const matches = [...markdown.matchAll(enumMembersRegex)];
+  const lines = markdown.split('\n');
+
+  if (matches.length === 0) {
+    return ''; // No enumeration members found
+  }
+
+  const index = lines.indexOf('## Enumeration Members')
+
+  let editedMarkdown = `${lines.slice(0, index+1). join('\n')}\n\n| Member | Value |\n| --- | --- |`;
+
+  for (const match of matches) {
+    const member = match[1];
+    const value = match[3];
+    editedMarkdown += `\n| ${member} | ${value} |`
+  }
+  editedMarkdown += '\n';
+  return editedMarkdown;
+}
+
+
+function processMarkdown(markdown: string, isEnum: boolean, itemPath: string): string {
+  const processLists = removeListCharacters(markdown);
+
+  const editUrls = processUrls(processLists, itemPath);
+
+  let processedMarkdown = editUrls;
+
+  if(isEnum)
+  {
+    const processEnums = createEnumerationTable(editUrls);
+    processedMarkdown = processEnums;
+  }
+
+  return processedMarkdown;
+}
+
+// Example usage
+// const markdown = `# Class: Skyflow
+// ...
+// ### init
+// * \`Static\` **init**(\`config\`): [\`default\`](Skyflow.default.md)
+
+// * \`Static\` **init**(\`config\`): [\`IGetByIdInput\`](../interfaces/utils_common.IGetByIdInput.md)
+
+// * \`Static\` **init**(\`config\`): [\`IGetByIdInput\`](../interfaces/utils_common.IGetByIdInput.txt)
+// * \`Static\` **init**(\`config\`): [\`IGetByIdInput\`](../interfaces/utils_common.IGetByIdInput)
+// ...
+// `;
+
+function readFolderStructure(folderPath: string, isEnum: boolean) {
+  const folderContents = readdirSync(folderPath);
+  folderContents.forEach((item) => {
+    const itemPath = path.join(folderPath, item);
+    const isDirectory = statSync(itemPath).isDirectory();
+
+    if (isDirectory) {
+      if(item == 'enums') {
+        readFolderStructure(itemPath, true);
+      }
+      else{
+        readFolderStructure(itemPath, false);
+      }
+    } else {
+      const markdown = readFileSync(itemPath, 'utf8');
+      const transformedMarkdown = processMarkdown(markdown, isEnum, itemPath);
+      const flaggedMarkdown = '{% env enable="nodeJsSdkRef" %}\n\n' + transformedMarkdown + '\n{% /env %}'
+      writeFileSync(itemPath, flaggedMarkdown, 'utf-8');
+    }
+  });
+}
+
+const folderPath = './docs/';
+readFolderStructure(folderPath, false);
+// const transformedMarkdown = processMarkdown(markdown, true);
+// console.log(transformedMarkdown);
+const readmePath = path.join(folderPath, 'README.md');
+try {
+    unlinkSync(readmePath);
+} catch (error: any) {
+    console.error(`Error updating file: ${error.message}`);
+}