From add91250e6ca684a53e5b83cbf0e430f9f783aeb Mon Sep 17 00:00:00 2001 From: kamal-skyflow Date: Wed, 13 Dec 2023 14:06:59 +0530 Subject: [PATCH] DEX-391 feat: added automatic updates of node sdk ref support --- .github/workflows/update-refs-in-docs.yml | 85 +++++++++++++ package.json | 7 +- scripts/docs-script/markdown-gen.js | 43 +++++++ scripts/docs-script/processMarkdown.ts | 144 ++++++++++++++++++++++ src/vault-api/Skyflow.ts | 83 +++++++++++++ src/vault-api/utils/common/index.ts | 118 +++++++++++++++++- typedoc.json | 28 +++++ 7 files changed, 505 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/update-refs-in-docs.yml create mode 100644 scripts/docs-script/markdown-gen.js create mode 100644 scripts/docs-script/processMarkdown.ts create mode 100644 typedoc.json diff --git a/.github/workflows/update-refs-in-docs.yml b/.github/workflows/update-refs-in-docs.yml new file mode 100644 index 0000000..4237ad0 --- /dev/null +++ b/.github/workflows/update-refs-in-docs.yml @@ -0,0 +1,85 @@ +name: Update references in Docsite + +on: + push: + branches: + - master + +jobs: + generate-node-sdk-refs: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: master + fetch-depth: 0 + persist-credentials: false + + - name: install modules + run: | + npm install + + - name: Generate Node SDK references + run: | + npm run docs-gen + cd ../.. + mkdir refs + cp -r skyflow-node/skyflow-node/docs/* refs/ + cd skyflow-node/skyflow-node/ + rm -r docs/ + git checkout -- package-lock.json + echo "SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_ENV + + - name: Create a branch in skyflow-docs + env: + TOKEN: ${{ secrets.PAT_ACTIONS }} + REPO_OWNER: skyflowapi + REPO_NAME: skyflow-docs + ACTOR: ${{ github.actor }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Create a new branch in skyflow-docs + cd $GITHUB_WORKSPACE + git remote add skyflow-docs https://${TOKEN}@github.com/${REPO_OWNER}/${REPO_NAME}.git + git fetch skyflow-docs main + BRANCH_NAME="SDK/node/${{ env.SHORT_SHA }}" + git remote set-url --push skyflow-docs https://${ACTOR}:${TOKEN}@github.com/${REPO_OWNER}/${REPO_NAME}.git + git checkout -b $BRANCH_NAME skyflow-docs/main + cp -r ../../refs/* src/pages/content/docs/sdks/skyflow-node/ + + - name: Push files and raise a PR + env: + TOKEN: ${{ secrets.PAT_ACTIONS }} + REPO_OWNER: skyflowapi + REPO_NAME: skyflow-docs + ACTOR: ${{ github.actor }} + run: | + git config user.name ${{ github.actor }} + git config user.email ${{ github.actor }}@users.noreply.github.com + BRANCH_NAME="SDK/node/${{ env.SHORT_SHA }}" + git add . + + # Check if there are changes to commit + if [[ -n "$(git status --porcelain)" ]]; then + git commit -m "SDK-${{ env.SHORT_SHA }} Updated Node SDK references" + git push skyflow-docs $BRANCH_NAME + # Raise a pull request + BASE_BRANCH="main" + BRANCH_NAME="SDK/node/${{ env.SHORT_SHA }}" + TITLE="SDK-${{ env.SHORT_SHA }}: Updated Node SDK references" + BODY="This pull request adds the latest Node SDK references. Commit id for reference: $GITHUB_SHA" + API_URL="https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls" + echo "API URL: $API_URL" + RESPONSE=$(curl -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $TOKEN" \ + -d "{\"title\":\"$TITLE\",\"body\":\"$BODY\",\"head\":\"${BRANCH_NAME}\",\"base\":\"$BASE_BRANCH\"}" \ + "$API_URL") + echo "Response Body: $RESPONSE" + PR_URL=$(echo "$RESPONSE" | jq -r '.html_url') + echo "Pull Request URL: $PR_URL" + else + echo "No changes to commit. Skipping push files and raise a PR." + exit 0 + fi \ No newline at end of file diff --git a/package.json b/package.json index d59c283..03a3ddc 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "eslint": "eslint '**/*.js' --max-warnings 0", "prettier": "prettier --list-different '**/*.{js,ts}'", "lint": "npm run prettier && npm run eslint", - "lint-fix": "prettier --write '**/*.{js,ts}' && eslint --fix '**/*.js'" + "lint-fix": "prettier --write '**/*.{js,ts}' && eslint --fix '**/*.js'", + "docs-gen": "typedoc && node scripts/docs-script/markdown-gen.js && npx ts-node scripts/docs-script/processMarkdown.ts" }, "repository": { "type": "git", @@ -76,6 +77,8 @@ "null-loader": "^0.1.1", "nyc": "^15.1.0", "prettier": "^1.13.7", - "typescript": "^4.4.4" + "typescript": "^4.4.4", + "typedoc": "^0.24.4", + "typedoc-plugin-markdown": "^3.15.1" } } diff --git a/scripts/docs-script/markdown-gen.js b/scripts/docs-script/markdown-gen.js new file mode 100644 index 0000000..8449d0c --- /dev/null +++ b/scripts/docs-script/markdown-gen.js @@ -0,0 +1,43 @@ +const fs = require('fs'); +const path = require('path'); + +function getModuleName(filePath) { + const parsedPath = path.parse(filePath); + let moduleName = parsedPath.name; + + if (moduleName.endsWith('.default')) { + moduleName = moduleName.slice(0, -8); + } + + return moduleName; +} + +const directoryPath = path.join(__dirname, '../../docs/classes'); + +fs.readdir(directoryPath, (err, files) => { + if (err) { + console.error(`Error reading directory: ${err}`); + return; + } + files.forEach(file => { + if (path.extname(file) === '.md') { + const filePath = path.join(directoryPath, file); + fs.readFile(filePath, 'utf8', (err, data) => { + if (err) { + console.error(`Error reading file: ${err}`); + return; + } + const lines = data.split('\n'); + lines[0] = `# Class: ${getModuleName(path.parse(file).name)}`; + const updatedContent = lines.join('\n'); + fs.writeFile(filePath, updatedContent, err => { + if (err) { + console.error(`Error writing file: ${err}`); + } else { + console.log(`Updated file: ${filePath}`); + } + }); + }); + } + }); +}); \ No newline at end of file diff --git a/scripts/docs-script/processMarkdown.ts b/scripts/docs-script/processMarkdown.ts new file mode 100644 index 0000000..4b75878 --- /dev/null +++ b/scripts/docs-script/processMarkdown.ts @@ -0,0 +1,144 @@ +import { readdirSync, readFileSync, statSync, writeFileSync, unlinkSync } from "fs"; +import path from "path"; + +function removeListCharacters(markdown: string): string { + // Match list items with bullets, numbered lists, or other list-type characters + // const listRegex = /^(\s*[\*\-\+\•\▸]\s*|\s*\d+\.\s*|\s*\w\.\s*)/; + const listRegex = /^(\s*[\*\-\+\•\▸]\s|^\s*\d+\.\s|^\s*\w\.\s)/; + + // Split the Markdown file into lines + const lines = markdown.split('\n'); + + // Track whether a list is currently in progress + let inList = false; + + // Iterate through each line and remove list characters if necessary + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const match = line.match(listRegex); + + if (match) { + // Check if the list is continuing or if it's a new list + if (!inList && !lines[i+1].match(listRegex)) { + // Remove list characters + lines[i] = line.replace(listRegex, ''); + } + inList = true; + } else { + inList = false; + } + } + + // Join the lines back into a single string + const result = lines.join('\n'); + + return result; +} + +function processUrls(markdown: string, itemPath: string): string { + // remove extension .md from urls + const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g; + const editedMarkdown = markdown.replace(linkRegex, (match, text, url) => { + if(url.split('.').pop().startsWith('md')) + { + const hash = url.split('.').pop().replace('md', '') + let editedUrl = (url.split('.').slice(0, -1)).join('.')+hash; + if(editedUrl.startsWith('../') && editedUrl.split('/').length > 1) + { + editedUrl = `/sdks/skyflow-node/${editedUrl.split('/').slice(1).join('/')}`; + } + else + { + editedUrl = `/sdks/skyflow-node/${itemPath.split('/')[1]}/${editedUrl}`; + } + return `[${text}](${editedUrl})`; + } + return `[${text}](${url})`; + }); + return editedMarkdown; +} + +function createEnumerationTable(markdown) { + const enumMembersRegex = /### (.+)[\s\S]*?\*\*(.+)\*\* = `(.+)`/g; + const matches = [...markdown.matchAll(enumMembersRegex)]; + const lines = markdown.split('\n'); + + if (matches.length === 0) { + return ''; // No enumeration members found + } + + const index = lines.indexOf('## Enumeration Members') + + let editedMarkdown = `${lines.slice(0, index+1). join('\n')}\n\n| Member | Value |\n| --- | --- |`; + + for (const match of matches) { + const member = match[1]; + const value = match[3]; + editedMarkdown += `\n| ${member} | ${value} |` + } + editedMarkdown += '\n'; + return editedMarkdown; +} + + +function processMarkdown(markdown: string, isEnum: boolean, itemPath: string): string { + const processLists = removeListCharacters(markdown); + + const editUrls = processUrls(processLists, itemPath); + + let processedMarkdown = editUrls; + + if(isEnum) + { + const processEnums = createEnumerationTable(editUrls); + processedMarkdown = processEnums; + } + + return processedMarkdown; +} + +// Example usage +// const markdown = `# Class: Skyflow +// ... +// ### init +// * \`Static\` **init**(\`config\`): [\`default\`](Skyflow.default.md) + +// * \`Static\` **init**(\`config\`): [\`IGetByIdInput\`](../interfaces/utils_common.IGetByIdInput.md) + +// * \`Static\` **init**(\`config\`): [\`IGetByIdInput\`](../interfaces/utils_common.IGetByIdInput.txt) +// * \`Static\` **init**(\`config\`): [\`IGetByIdInput\`](../interfaces/utils_common.IGetByIdInput) +// ... +// `; + +function readFolderStructure(folderPath: string, isEnum: boolean) { + const folderContents = readdirSync(folderPath); + folderContents.forEach((item) => { + const itemPath = path.join(folderPath, item); + const isDirectory = statSync(itemPath).isDirectory(); + + if (isDirectory) { + if(item == 'enums') { + readFolderStructure(itemPath, true); + } + else{ + readFolderStructure(itemPath, false); + } + } else { + const markdown = readFileSync(itemPath, 'utf8'); + const transformedMarkdown = processMarkdown(markdown, isEnum, itemPath); + const flaggedMarkdown = '{% env enable="nodeJsSdkRef" %}\n\n' + transformedMarkdown + '\n{% /env %}' + writeFileSync(itemPath, flaggedMarkdown, 'utf-8'); + } + }); +} + +const folderPath = './docs/'; +readFolderStructure(folderPath, false); +// const transformedMarkdown = processMarkdown(markdown, true); +// console.log(transformedMarkdown); +const readmePath = path.join(folderPath, 'README.md'); +try { + unlinkSync(readmePath); +} catch (error: any) { + console.error(`Error updating file: ${error.message}`); +} \ No newline at end of file diff --git a/src/vault-api/Skyflow.ts b/src/vault-api/Skyflow.ts index 2fef186..e3a6f0c 100644 --- a/src/vault-api/Skyflow.ts +++ b/src/vault-api/Skyflow.ts @@ -2,6 +2,10 @@ Copyright (c) 2022 Skyflow, Inc. */ +/** + * @module Skyflow + */ + import Client from './client'; import { printLog } from './utils/logs-helper'; import logs from './utils/logs'; @@ -25,6 +29,13 @@ import { } from './utils/common'; import { formatVaultURL } from './utils/helpers'; +/** + * Parameters for the Skyflow client. + * @property vaultID ID of the vault to connect to. + * @property vaultURL URL of the vault to connect to. + * @property getBearerToken Function that retrieves a Skyflow bearer token from your backend. + * @property options Additional configuration options. + */ export interface ISkyflow { vaultID?: string; vaultURL?: string; @@ -32,11 +43,29 @@ export interface ISkyflow { options?: Record; } +/** +* Parent Skyflow class that consists of all the methods exposed to the client. +* @class Skyflow +*/ class Skyflow { + /** + * @internal + */ #client: Client; + + /** + * @internal + */ #metadata = { }; + + /** + * @internal + */ #Controller: Controller; + /** + * @internal + */ constructor(config: ISkyflow) { this.#client = new Client( { @@ -50,6 +79,12 @@ class Skyflow { printLog(logs.infoLogs.BEARER_TOKEN_LISTENER, MessageType.LOG); } + /** + * Initializes the Skyflow client. + * @public + * @param config Configuration for the Skyflow client. + * @returns Returns an instance of the Skyflow client. + */ static init(config: ISkyflow): Skyflow { printLog(logs.infoLogs.INITIALIZE_CLIENT, MessageType.LOG); config.vaultURL = formatVaultURL(config.vaultURL) @@ -58,6 +93,13 @@ class Skyflow { return skyflow; } + /** + * Inserts data into the vault. + * @public + * @param records Records to insert. + * @param options Options for the insertion. + * @returns Returns the insert response. + */ insert( records: IInsertRecordInput, options?: IInsertOptions, @@ -66,24 +108,51 @@ class Skyflow { return this.#Controller.insert(records, options); } + /** + * Returns values that correspond to the specified tokens. + * @public + * @param detokenizeInput Tokens to return values for. + * @returns Tokens to return values for. + */ detokenize(detokenizeInput: IDetokenizeInput): Promise { printLog(logs.infoLogs.DETOKENIZE_TRIGGERED, MessageType.LOG); return this.#Controller.detokenize(detokenizeInput); } + /** + * Returns records by Skyflow ID. + * @public + * @deprecated Use {@link get} instead. + * @param getByIdInput Skyflow IDs. + * @returns Returns the specified records and any errors. + * @public + */ getById(getByIdInput: IGetByIdInput) { printLog(logs.infoLogs.GET_BY_ID_TRIGGERED, MessageType.LOG); return this.#Controller.getById(getByIdInput); } + /** + * Returns records by Skyflow IDs or column values. + * @public + * @param getInput Identifiers for the records. + * @param options options for getting the records. + * @returns Returns the specified records and any errors. + */ get(getInput: IGetInput,options?:IGetOptions) { printLog(logs.infoLogs.GET_CALL_TRIGGERED, MessageType.LOG); return this.#Controller.get(getInput,options); } + /** + * Invokes a connection to a third-party service. + * @public + * @param config Configuration for the connection. + * @returns Returns the connection response. + */ invokeConnection(config: IConnectionConfig) { printLog(logs.infoLogs.INVOKE_CONNECTION_TRIGGERED, MessageType.LOG); @@ -91,12 +160,26 @@ class Skyflow { return this.#Controller.invokeConnection(config); } + /** + * Updates the configuration of elements inside the composable container. + * @public + * @param updateInput Input data for the update operation. + * @param options Options for the container update. + * @returns Returns the response for the update operation. + */ update(updateInput: IUpdateInput,options?:IUpdateOptions){ printLog(logs.infoLogs.UPDATE_TRIGGERED, MessageType.LOG); return this.#Controller.update(updateInput,options); } + /** + * Deletes record from vault + * @public + * @param deleteInput Input data for the delete operation. + * @param options Options for the deletion. + * @returns Returns the response for the delete operation. + */ delete(deleteInput: IDeleteInput, options?: IDeleteOptions) { printLog(logs.infoLogs.UPDATE_TRIGGERED, MessageType.LOG); return this.#Controller.delete(deleteInput, options) diff --git a/src/vault-api/utils/common/index.ts b/src/vault-api/utils/common/index.ts index c5cf834..e72103c 100644 --- a/src/vault-api/utils/common/index.ts +++ b/src/vault-api/utils/common/index.ts @@ -1,6 +1,14 @@ /* Copyright (c) 2022 Skyflow, Inc. */ + +/** + * @module Utils + */ + +/** + * Supported redaction types. + */ export enum RedactionType { DEFAULT = 'DEFAULT', PLAIN_TEXT = 'PLAIN_TEXT', @@ -8,6 +16,9 @@ export enum RedactionType { REDACTED = 'REDACTED', } +/** + * Supported request methods. + */ export enum RequestMethod { GET = 'GET', POST = 'POST', @@ -16,6 +27,9 @@ export enum RequestMethod { DELETE = 'DELETE', } +/** + * Supported log levels. + */ export enum LogLevel { WARN = 'WARN', INFO = 'INFO', @@ -24,36 +38,69 @@ export enum LogLevel { OFF = 'OFF' } - +/** + * Supported message types. + */ export enum MessageType { LOG = 'LOG', WARN = 'WARN', ERROR = 'ERROR', } +/** + * Parameters for the insert record input. + * @property records An array of insert records. + */ export interface IInsertRecordInput { records: IInsertRecord[]; } +/** + * Parameters for inserting a record. + * @property table Table that the data belongs to. + * @property fields Fields to insert data into. + */ export interface IInsertRecord { table: string; fields: Record; } +/** + * Parameters by the Reveal record. + * @property redaction Redaction type applied to the data. Defaults to `RedactionType.PLAIN_TEXT`. + * @property token Token of the revealed data. + */ export interface IRevealRecord { token: string; redaction?: RedactionType; } +/** + * Parameters by the reveal response. + * @property records Records revealed, if any. + * @property errors Errors, if any. + */ export interface IRevealResponseType { records?: Record[]; errors?: Record[]; } +/** + * Parameters for detokenizing input. + * @property records Revealed records. + */ export interface IDetokenizeInput { records: IRevealRecord[]; } +/** + * Parameters for Skyflow ID record. + * @property ids Skyflow IDs of the records to get. + * @property redaction Type of redaction for values. + * @property table Type of redaction for values. + * @property columnName Column the data belongs to. + * @property columnValues Values of the records. + */ export interface ISkyflowIdRecord { ids?: string[]; redaction?: RedactionType; @@ -62,16 +109,30 @@ export interface ISkyflowIdRecord { columnValues?: string[]; } +/** + * Parameters by Skyflow record. + * @property ids Skyflow IDs of the records to get. + * @property redaction Type of redaction for values. + * @property table Type of redaction for values. + */ export interface ISkyflowRecord { ids: string[]; redaction: RedactionType; table: string; } +/** + * Parameters by the getbyid input. + * @property records Records to get. + */ export interface IGetByIdInput { records: ISkyflowRecord[]; } +/** + * Parameters to retrieve input. + * @property records Records to retrieve. + */ export interface IGetInput { records: ISkyflowIdRecord[]; } @@ -80,6 +141,15 @@ export interface IGetInput { // logLevel:LogLevel // } +/** + * Configuration to establish a connection. + * @property connectionURL URL of the outbound/inbound connection. + * @property methodName The HTTP request method to be used. + * @property pathParams Parameters to be included in the URL path. + * @property queryParams Query parameters to be included in the URL. + * @property requestBody Data to be included in the request body. + * @property requestHeader Headers to be included in the request. + */ export interface IConnectionConfig { connectionURL: string; methodName: RequestMethod; @@ -97,6 +167,9 @@ export const TYPES = { INVOKE_CONNECTION: 'INVOKE_CONNECTION', }; +/** + * Supported content types. + */ export enum ContentType { APPLICATIONORJSON = 'application/json', TEXTORPLAIN = 'text/plain', @@ -105,42 +178,85 @@ export enum ContentType { FORMDATA = 'multipart/form-data', } +/** + * Parameters by upsert option. + * @property table Table that the data belongs to. + * @property column Name of the unique column. + */ export interface IUpsertOption { table: string; column: string; } +/** + * Parameters by insert options. + * @property tokens If `true`, returns tokens for the collected data. Defaults to `false`. + * @property upsert If specified, upserts data. If not specified, inserts data. + */ export interface IInsertOptions { tokens?: boolean; upsert?: IUpsertOption[]; } +/** + * Parameters for updating a record. + * @property id Skyflow ID of the record to update. + * @property table Table that the data belongs to. + * @property fields Fields to update data into. + */ export interface IUpdateRecord{ id: string, table: string, fields: Record } + +/** + * Parameters for updating a record. + * @property records An array of update records. + */ export interface IUpdateInput{ records: IUpdateRecord[]; } +/** + * Parameters by update options. + * @property tokens If `true`, returns tokens for the collected data. Defaults to `false`. + */ export interface IUpdateOptions{ tokens: boolean } +/** + * Parameters by get records options. + * @property tokens If `true`, returns tokens for the collected data. Defaults to `false`. + * @property encodeURI If `true`, encoded column values will be sent in API call. Defaults to `true`. + */ export interface IGetOptions{ tokens?: boolean encodeURI?: boolean } + +/** + * Parameters for deleting a record. + * @property id Skyflow ID of the record to be deleted. + * @property table Table name from which the record has to be deleted. + */ export interface IDeleteRecord { id: string; table: string; } +/** + * Parameters for deleting a record. + * @property records An array of delete records. + */ export interface IDeleteInput { records: IDeleteRecord[]; } +/** + * Parameters by delete options. + */ export interface IDeleteOptions { } diff --git a/typedoc.json b/typedoc.json new file mode 100644 index 0000000..0a6483c --- /dev/null +++ b/typedoc.json @@ -0,0 +1,28 @@ +{ + "out": "docs", + "exclude": "**/node_modules/**", + "excludeExternals": false, + "excludePrivate": true, + "excludeProtected": true, + "excludeInternal": true, + "name": "My Project", + "theme": "default", + "plugin": ["typedoc-plugin-markdown"], + "entryPoints": [ + "src/vault-api/Skyflow.ts", + "src/vault-api/utils/common/index.ts" + ], + "entryPointStrategy": "expand", + "githubPages": false, + "readme": "none", + "hideGenerator": true, + "hideInPageTOC": true, + "disableSources": true, + "hideBreadcrumbs": true, + "markedOptions": { + "sanitize": true, + "tables": true, + "smartypants": true + }, + "sort": ["source-order", "alphabetical"] +} \ No newline at end of file