From 2da8f5b0edf33ed93cad49f7e1170f952ab10d3b Mon Sep 17 00:00:00 2001 From: kamal-skyflow Date: Tue, 21 Nov 2023 16:17:05 +0530 Subject: [PATCH] DEX-416 feat: added script and workflow for refs generation --- .github/workflows/update-refs-in-docs.yml | 83 +++++++++++++ package.json | 7 +- scripts/docs-script/markdown-gen.js | 43 +++++++ scripts/docs-script/processMarkdown.ts | 139 ++++++++++++++++++++++ typedoc.json | 28 +++++ 5 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/update-refs-in-docs.yml create mode 100644 scripts/docs-script/markdown-gen.js create mode 100644 scripts/docs-script/processMarkdown.ts create mode 100644 typedoc.json diff --git a/.github/workflows/update-refs-in-docs.yml b/.github/workflows/update-refs-in-docs.yml new file mode 100644 index 0000000..fdda474 --- /dev/null +++ b/.github/workflows/update-refs-in-docs.yml @@ -0,0 +1,83 @@ +name: Update references in Docsite + +on: + push: + branches: + - DEX-416/test-docs-workflow + +jobs: + create-branch: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: master + fetch-depth: 0 + persist-credentials: false + + - name: install modules + run: npm install + + - name: Generate Node SDK references + run: | + npm run docs-gen + cd .. + mkdir refs + cp skyflow-node/docs/* refs/ + + - name: Remove specific file + run: rm -r docs/ + + - name: Create a branch in skyflow-docs + env: + TOKEN: ${{ secrets.PAT_ACTIONS }} + REPO_OWNER: skyflowapi + REPO_NAME: skyflow-docs + ACTOR: ${{ github.actor }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Create a new branch in skyflow-docs + cd $GITHUB_WORKSPACE + git remote add skyflow-docs https://${TOKEN}@github.com/${REPO_OWNER}/${REPO_NAME}.git + git fetch skyflow-docs main + BRANCH_NAME="feature-branch-${{ github.run_number }}" + git remote set-url --push skyflow-docs https://${ACTOR}:${TOKEN}@github.com/${REPO_OWNER}/${REPO_NAME}.git + git checkout -b $BRANCH_NAME skyflow-docs/main + cp ../refs/* src/pages/content/docs/sdks/skyflow-node/ + + - name: Push files and raise a PR + env: + TOKEN: ${{ secrets.PAT_ACTIONS }} + REPO_OWNER: skyflowapi + REPO_NAME: skyflow-docs + ACTOR: ${{ github.actor }} + run: | + git config user.name ${{ github.actor }} + git config user.email ${{ github.actor }}@users.noreply.github.com + BRANCH_NAME="feature-branch-${{ github.run_number }}" + git add . + + # Check if there are changes to commit + if [[ -n "$(git status --porcelain)" ]]; then + git commit -m "Added Node SDK references" + git push skyflow-docs $BRANCH_NAME + # Raise a pull request + BASE_BRANCH="main" + BRANCH_NAME="feature-branch-${{ github.run_number }}" + TITLE="Updated Node SDK references" + BODY="This pull request adds the latest Node SDK references." + API_URL="https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls" + echo "API URL: $API_URL" + RESPONSE=$(curl -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $TOKEN" \ + -d "{\"title\":\"$TITLE\",\"body\":\"$BODY\",\"head\":\"${BRANCH_NAME}\",\"base\":\"$BASE_BRANCH\"}" \ + "$API_URL") + echo "Response Body: $RESPONSE" + PR_URL=$(echo "$RESPONSE" | jq -r '.html_url') + echo "Pull Request URL: $PR_URL" + else + echo "No changes to commit. Skipping push files and raise a PR." + exit 0 + fi diff --git a/package.json b/package.json index d59c283..a3087cc 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "eslint": "eslint '**/*.js' --max-warnings 0", "prettier": "prettier --list-different '**/*.{js,ts}'", "lint": "npm run prettier && npm run eslint", - "lint-fix": "prettier --write '**/*.{js,ts}' && eslint --fix '**/*.js'" + "lint-fix": "prettier --write '**/*.{js,ts}' && eslint --fix '**/*.js'", + "docs-gen": "typedoc && node scripts/docs-script/markdown-gen.js && npx ts-node scripts/docs-script/processMarkdown.ts" }, "repository": { "type": "git", @@ -76,6 +77,8 @@ "null-loader": "^0.1.1", "nyc": "^15.1.0", "prettier": "^1.13.7", - "typescript": "^4.4.4" + "typescript": "^4.7.3", + "typedoc": "^0.24.4", + "typedoc-plugin-markdown": "^3.15.1" } } diff --git a/scripts/docs-script/markdown-gen.js b/scripts/docs-script/markdown-gen.js new file mode 100644 index 0000000..8449d0c --- /dev/null +++ b/scripts/docs-script/markdown-gen.js @@ -0,0 +1,43 @@ +const fs = require('fs'); +const path = require('path'); + +function getModuleName(filePath) { + const parsedPath = path.parse(filePath); + let moduleName = parsedPath.name; + + if (moduleName.endsWith('.default')) { + moduleName = moduleName.slice(0, -8); + } + + return moduleName; +} + +const directoryPath = path.join(__dirname, '../../docs/classes'); + +fs.readdir(directoryPath, (err, files) => { + if (err) { + console.error(`Error reading directory: ${err}`); + return; + } + files.forEach(file => { + if (path.extname(file) === '.md') { + const filePath = path.join(directoryPath, file); + fs.readFile(filePath, 'utf8', (err, data) => { + if (err) { + console.error(`Error reading file: ${err}`); + return; + } + const lines = data.split('\n'); + lines[0] = `# Class: ${getModuleName(path.parse(file).name)}`; + const updatedContent = lines.join('\n'); + fs.writeFile(filePath, updatedContent, err => { + if (err) { + console.error(`Error writing file: ${err}`); + } else { + console.log(`Updated file: ${filePath}`); + } + }); + }); + } + }); +}); \ No newline at end of file diff --git a/scripts/docs-script/processMarkdown.ts b/scripts/docs-script/processMarkdown.ts new file mode 100644 index 0000000..caaa2dc --- /dev/null +++ b/scripts/docs-script/processMarkdown.ts @@ -0,0 +1,139 @@ +import { readdirSync, readFileSync, statSync, writeFileSync } from "fs"; +import path from "path"; + +function removeListCharacters(markdown: string): string { + // Match list items with bullets, numbered lists, or other list-type characters + // const listRegex = /^(\s*[\*\-\+\•\▸]\s*|\s*\d+\.\s*|\s*\w\.\s*)/; + const listRegex = /^(\s*[\*\-\+\•\▸]\s|^\s*\d+\.\s|^\s*\w\.\s)/; + + // Split the Markdown file into lines + const lines = markdown.split('\n'); + + // Track whether a list is currently in progress + let inList = false; + + // Iterate through each line and remove list characters if necessary + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const match = line.match(listRegex); + + if (match) { + // Check if the list is continuing or if it's a new list + if (!inList && !lines[i+1].match(listRegex)) { + // Remove list characters + lines[i] = line.replace(listRegex, ''); + } + inList = true; + } else { + inList = false; + } + } + + // Join the lines back into a single string + const result = lines.join('\n'); + + return result; +} + +function processUrls(markdown: string, itemPath: string): string { + // remove extension .md from urls + const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g; + const editedMarkdown = markdown.replace(linkRegex, (match, text, url) => { + if(url.split('.').pop().startsWith('md')) + { + const hash = url.split('.').pop().replace('md', '') + let editedUrl = (url.split('.').slice(0, -1)).join('.')+hash; + if(editedUrl.startsWith('../') && editedUrl.split('/').length > 1) + { + editedUrl = `/sdks/skyflow-node/${editedUrl.split('/').slice(1).join('/')}`; + } + else + { + editedUrl = `/sdks/skyflow-node/${itemPath.split('/')[1]}/${editedUrl}`; + } + return `[${text}](${editedUrl})`; + } + return `[${text}](${url})`; + }); + return editedMarkdown; +} + +function createEnumerationTable(markdown) { + const enumMembersRegex = /### (.+)[\s\S]*?\*\*(.+)\*\* = `(.+)`/g; + const matches = [...markdown.matchAll(enumMembersRegex)]; + const lines = markdown.split('\n'); + + if (matches.length === 0) { + return ''; // No enumeration members found + } + + const index = lines.indexOf('## Enumeration Members') + + let editedMarkdown = `${lines.slice(0, index+1). join('\n')}\n\n| Member | Value |\n| --- | --- |`; + + for (const match of matches) { + const member = match[1]; + const value = match[3]; + editedMarkdown += `\n| ${member} | ${value} |` + } + editedMarkdown += '\n'; + return editedMarkdown; +} + + +function processMarkdown(markdown: string, isEnum: boolean, itemPath: string): string { + const processLists = removeListCharacters(markdown); + + const editUrls = processUrls(processLists, itemPath); + + let processedMarkdown = editUrls; + + if(isEnum) + { + const processEnums = createEnumerationTable(editUrls); + processedMarkdown = processEnums; + } + + return processedMarkdown; +} + +// Example usage +// const markdown = `# Class: Skyflow +// ... +// ### init +// * \`Static\` **init**(\`config\`): [\`default\`](Skyflow.default.md) + +// * \`Static\` **init**(\`config\`): [\`IGetByIdInput\`](../interfaces/utils_common.IGetByIdInput.md) + +// * \`Static\` **init**(\`config\`): [\`IGetByIdInput\`](../interfaces/utils_common.IGetByIdInput.txt) +// * \`Static\` **init**(\`config\`): [\`IGetByIdInput\`](../interfaces/utils_common.IGetByIdInput) +// ... +// `; + +function readFolderStructure(folderPath: string, isEnum: boolean) { + const folderContents = readdirSync(folderPath); + folderContents.forEach((item) => { + const itemPath = path.join(folderPath, item); + const isDirectory = statSync(itemPath).isDirectory(); + + if (isDirectory) { + if(item == 'enums') { + readFolderStructure(itemPath, true); + } + else{ + readFolderStructure(itemPath, false); + } + } else { + const markdown = readFileSync(itemPath, 'utf8'); + const transformedMarkdown = processMarkdown(markdown, isEnum, itemPath); + const flaggedMarkdown = '{% env enable="nodeJsSdkRef" %}\n\n' + transformedMarkdown + '\n{% /env %}' + writeFileSync(itemPath, flaggedMarkdown, 'utf-8'); + } + }); +} + +const folderPath = './docs/'; +readFolderStructure(folderPath, false); +// const transformedMarkdown = processMarkdown(markdown, true); +// console.log(transformedMarkdown); + \ No newline at end of file diff --git a/typedoc.json b/typedoc.json new file mode 100644 index 0000000..0a6483c --- /dev/null +++ b/typedoc.json @@ -0,0 +1,28 @@ +{ + "out": "docs", + "exclude": "**/node_modules/**", + "excludeExternals": false, + "excludePrivate": true, + "excludeProtected": true, + "excludeInternal": true, + "name": "My Project", + "theme": "default", + "plugin": ["typedoc-plugin-markdown"], + "entryPoints": [ + "src/vault-api/Skyflow.ts", + "src/vault-api/utils/common/index.ts" + ], + "entryPointStrategy": "expand", + "githubPages": false, + "readme": "none", + "hideGenerator": true, + "hideInPageTOC": true, + "disableSources": true, + "hideBreadcrumbs": true, + "markedOptions": { + "sanitize": true, + "tables": true, + "smartypants": true + }, + "sort": ["source-order", "alphabetical"] +} \ No newline at end of file