From f2ee73e84d82a099f180329cf877d402eea7debb Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Tue, 16 Apr 2024 16:43:16 +1000 Subject: [PATCH 01/14] Upgrade to Magda minion SDK v3 --- .gitignore | 3 +- .mocharc.json | 5 - CHANGES.md | 21 +- Dockerfile | 2 +- package.json | 46 +- prettier.config.js => prettier.config.cjs | 0 src/HttpRequests.ts | 146 ++- src/index.ts | 91 +- src/onRecordFound.ts | 571 +++++---- src/request.ts | 23 + src/test/HttpRequests.spec.ts | 319 ++--- src/test/arbitraries.ts | 151 ++- src/test/getUrlWaitTime.spec.ts | 380 +++--- src/test/onRecordFound.spec.ts | 1312 ++++++++++----------- src/test/wait.spec.ts | 57 +- tsconfig-global.json | 127 +- yarn.lock | 1040 ++++++++++++---- 17 files changed, 2431 insertions(+), 1863 deletions(-) delete mode 100644 .mocharc.json rename prettier.config.js => prettier.config.cjs (100%) create mode 100644 src/request.ts diff --git a/.gitignore b/.gitignore index 14b3f53..db5bc90 100644 --- a/.gitignore +++ b/.gitignore @@ -51,4 +51,5 @@ magda-tenant-api/lib/ #typescript tsconfig.tsbuildinfo -deploy/*/charts \ No newline at end of file +deploy/*/charts +coverage/ \ No newline at end of file diff --git a/.mocharc.json b/.mocharc.json deleted file mode 100644 index be36d84..0000000 --- a/.mocharc.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "extension": ["ts"], - "spec": "src/test/**/*.spec.ts", - "require": "ts-node/register" -} diff --git a/CHANGES.md b/CHANGES.md index 01012ed..a53e8ff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,13 +1,18 @@ +# 3.0.0 + +- Upgrade nodejs to version 18 +- Upgrade to Magda minion SDK v3 + # 2.0.0 -- Upgrade nodejs to version 14 -- Upgrade other dependencies -- Release all artifacts to GitHub Container Registry (instead of docker.io & https://charts.magda.io) -- Upgrade magda-common chart version -- Upgrade api to batch/v1 to be compatible with k8s 1.25 (now requires >=1.21) +- Upgrade nodejs to version 14 +- Upgrade other dependencies +- Release all artifacts to GitHub Container Registry (instead of docker.io & https://charts.magda.io) +- Upgrade magda-common chart version +- Upgrade api to batch/v1 to be compatible with k8s 1.25 (now requires >=1.21) # 1.0.0 -- Upgrade dependencies -- Upgrade CI scripts -- Related to https://github.com/magda-io/magda/issues/3229, Use magda-common for docker image related logic +- Upgrade dependencies +- Upgrade CI scripts +- Related to https://github.com/magda-io/magda/issues/3229, Use magda-common for docker image related logic diff --git a/Dockerfile b/Dockerfile index 832169e..1c4d1c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:14-alpine +FROM node:18-alpine RUN mkdir -p /usr/src/app COPY . /usr/src/app diff --git a/package.json b/package.json index 4ddf6b7..0e9881e 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "@magda/minion-broken-link", "description": "MAGDA Broken Link Minion", "version": "2.0.0", + "type": "module", "scripts": { "prebuild": "rimraf dist tsconfig.tsbuildinfo", "build": "yarn run compile", @@ -11,7 +12,7 @@ "dev": "run-typescript-in-nodemon src/index.ts", "docker-build-local": "create-docker-context-for-node-component --build --push --tag auto --local", "docker-build-prod": "create-docker-context-for-node-component --build --push --tag auto", - "test": "mocha", + "test": "c8 mocha", "helm-lint": "helm lint deploy/magda-minion-broken-link -f deploy/test-deploy.yaml", "retag-and-push": "retag-and-push", "helm-docs": "helm-docs -t ./README.md.gotmpl -o ../../README.md", @@ -23,39 +24,59 @@ "author": "", "license": "Apache-2.0", "devDependencies": { - "@magda/arbitraries": "^2.1.1", + "@magda/arbitraries": "^3.0.2-alpha.0", "@magda/ci-utils": "^1.0.5", - "@magda/docker-utils": "^2.1.1", + "@magda/docker-utils": "^3.0.2-alpha.0", "@types/chai": "^4.3.3", "@types/ftp": "^0.3.29", "@types/lodash": "^4.14.185", "@types/lru-cache": "4.0.0", "@types/mocha": "^9.1.1", "@types/nock": "^11.1.0", + "@types/node": "^18.19.31", "@types/read-pkg-up": "^3.0.1", "@types/request": "^2.48.1", "@types/sinon": "^7.5.1", "@types/urijs": "^1.19.19", "ajv": "^6.10.2", - "chai": "^4.2.0", + "c8": "^9.0.0", + "chai": "^5.0.0-rc.0", "husky": "^3.1.0", "jsverify": "^0.8.2", - "mocha": "^10.0.0", - "nock": "^13.2.9", + "mocha": "^10.2.0", + "nock": "^13.4.0", "prettier": "^1.19.1", "pretty-quick": "^2.0.1", "rimraf": "^3.0.0", "sinon": "^8.1.1", - "ts-node": "^10.9.1", - "typescript": "^4.2.4" + "tsx": "^4.7.0", + "typescript": "^5.3.3" + }, + "mocha": { + "import": "tsx/esm", + "spec": [ + "src/test/**/*.spec.ts" + ] + }, + "c8": { + "all": true, + "clean": true, + "src": [ + "./src" + ], + "exclude": [ + "src/test/**" + ] }, "dependencies": { - "@magda/minion-sdk": "^2.1.1", - "@magda/registry-aspects": "^2.1.1", - "@magda/utils": "^2.1.1", + "@magda/minion-sdk": "^3.0.2-alpha.0", + "@magda/registry-aspects": "^3.0.2-alpha.0", + "@magda/utils": "^3.0.2-alpha.0", "ftp": "^0.3.10", "lodash": "^4.17.4", "lru-cache": "4.0.2", + "read-pkg-up": "^3.0.0", + "request": "^2.88.2", "urijs": "^1.19.11" }, "config": { @@ -77,5 +98,8 @@ "hooks": { "pre-commit": "pretty-quick --staged" } + }, + "engines": { + "node": ">=18.19.0" } } diff --git a/prettier.config.js b/prettier.config.cjs similarity index 100% rename from prettier.config.js rename to prettier.config.cjs diff --git a/src/HttpRequests.ts b/src/HttpRequests.ts index 72bc225..4d2cd04 100644 --- a/src/HttpRequests.ts +++ b/src/HttpRequests.ts @@ -1,25 +1,25 @@ import { CoreOptions } from "request"; -import { request } from "@magda/utils"; +import request from "./request.js"; import http from "http"; -import DevNull from "./DevNull"; +import DevNull from "./DevNull.js"; /** * Depends on statusCode, determine a request is failed or not * @param response http.IncomingMessage */ function processResponse(response: http.IncomingMessage) { - if ( - (response.statusCode >= 200 && response.statusCode <= 299) || - response.statusCode === 429 - ) { - return response.statusCode; - } else { - throw new BadHttpResponseError( - response.statusMessage, - response, - response.statusCode - ); - } + if ( + (response.statusCode >= 200 && response.statusCode <= 299) || + response.statusCode === 429 + ) { + return response.statusCode; + } else { + throw new BadHttpResponseError( + response.statusMessage, + response, + response.statusCode + ); + } } /** @@ -28,10 +28,10 @@ function processResponse(response: http.IncomingMessage) { * @param url String: url to be tested */ export async function headRequest( - url: string, - requestOpts: CoreOptions = {} + url: string, + requestOpts: CoreOptions = {} ): Promise { - return doRequest(url, "head", requestOpts); + return doRequest(url, "head", requestOpts); } /** @@ -40,15 +40,15 @@ export async function headRequest( * @param url String: url to be tested */ export async function getRequest( - url: string, - requestOpts: CoreOptions = {} + url: string, + requestOpts: CoreOptions = {} ): Promise { - return doRequest(url, "get", { - ...requestOpts, - headers: { - Range: "bytes=0-50" - } - }); + return doRequest(url, "get", { + ...requestOpts, + headers: { + Range: "bytes=0-50" + } + }); } /** @@ -57,66 +57,64 @@ export async function getRequest( * @param url String: url to be tested */ export async function doRequest( - url: string, - method: "get" | "head", - requestOpts: CoreOptions = {} + url: string, + method: "get" | "head", + requestOpts: CoreOptions = {} ): Promise { - const devnull = new DevNull(); - console.info(`${method} ${url}`); + const devnull = new DevNull(); + console.info(`${method} ${url}`); - let resolveResponse: (number: number) => void; - let resolveStreamEnd: (v?: any) => void; - let rejectResponse: (error: Error) => void; - let rejectStreamEnd: (error: Error) => void; + let resolveResponse: (number: number) => void; + let resolveStreamEnd: (v?: any) => void; + let rejectResponse: (error: Error) => void; + let rejectStreamEnd: (error: Error) => void; - const reqPromise: Promise = new Promise((resolve, reject) => { - resolveResponse = resolve; - rejectResponse = reject; - }); + const reqPromise: Promise = new Promise((resolve, reject) => { + resolveResponse = resolve; + rejectResponse = reject; + }); - const streamPromise = new Promise((resolve, reject) => { - rejectStreamEnd = reject; - resolveStreamEnd = resolve; - }); + const streamPromise = new Promise((resolve, reject) => { + rejectStreamEnd = reject; + resolveStreamEnd = resolve; + }); - const req = request[method](url, requestOpts) - .on("error", err => rejectResponse(err)) - .on("response", (response: http.IncomingMessage) => { - try { - console.info( - `Got ${response.statusCode} from ${method} ${url}` - ); + const req = request[method](url, requestOpts) + .on("error", (err: Error) => rejectResponse(err)) + .on("response", (response: http.IncomingMessage) => { + try { + console.info(`Got ${response.statusCode} from ${method} ${url}`); - resolveResponse(processResponse(response)); - } catch (e) { - rejectResponse(e as Error); - } - }) - .on("end", () => { - resolveStreamEnd(); - }); + resolveResponse(processResponse(response)); + } catch (e) { + rejectResponse(e as Error); + } + }) + .on("end", () => { + resolveStreamEnd(); + }); - req.pipe(devnull).on("error", rejectStreamEnd); - req.on("error", rejectStreamEnd); + req.pipe(devnull).on("error", rejectStreamEnd); + req.on("error", rejectStreamEnd); - const [responseCode] = await Promise.all([reqPromise, streamPromise]); + const [responseCode] = await Promise.all([reqPromise, streamPromise]); - return responseCode as number; + return responseCode as number; } export class BadHttpResponseError extends Error { - public response: http.IncomingMessage; - public httpStatusCode: number; + public response: http.IncomingMessage; + public httpStatusCode: number; - constructor( - message?: string, - response?: http.IncomingMessage, - httpStatusCode?: number - ) { - super(message); - this.message = message; - this.response = response; - this.httpStatusCode = httpStatusCode; - this.stack = new Error().stack; - } + constructor( + message?: string, + response?: http.IncomingMessage, + httpStatusCode?: number + ) { + super(message); + this.message = message; + this.response = response; + this.httpStatusCode = httpStatusCode; + this.stack = new Error().stack; + } } diff --git a/src/index.ts b/src/index.ts index 0f4b0ba..3f453a3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,63 +1,62 @@ import minion, { commonYargs } from "@magda/minion-sdk"; -import onRecordFound from "./onRecordFound"; -import brokenLinkAspectDef from "./brokenLinkAspectDef"; +import onRecordFound from "./onRecordFound.js"; +import brokenLinkAspectDef from "./brokenLinkAspectDef.js"; import { CoreOptions } from "request"; import { coerceJson } from "@magda/utils"; const ID = "minion-broken-link"; const argv = commonYargs(6111, "http://localhost:6111", argv => - argv - .option("externalRetries", { - describe: - "Number of times to retry external links when checking whether they're broken", - type: "number", - default: 1 - }) - .option("domainWaitTimeConfig", { - describe: - "A object that defines wait time for each of domain. " + - "Echo property name of the object would be the domain name and property value is the wait time in seconds", - coerce: coerceJson("domainWaitTimeConfig"), - default: process.env.DOMAIN_WAIT_TIME_CONFIG || JSON.stringify({}) - }) - .option("requestOpts", { - describe: - "The default options to use for the JS request library when making HTTP HEAD/GET requests", - type: "string", - coerce: coerceJson("requestOpts"), - default: - process.env.REQUEST_OPTS || JSON.stringify({ timeout: 20000 }) - }) + argv + .option("externalRetries", { + describe: + "Number of times to retry external links when checking whether they're broken", + type: "number", + default: 1 + }) + .option("domainWaitTimeConfig", { + describe: + "A object that defines wait time for each of domain. " + + "Echo property name of the object would be the domain name and property value is the wait time in seconds", + coerce: coerceJson("domainWaitTimeConfig"), + default: process.env.DOMAIN_WAIT_TIME_CONFIG || JSON.stringify({}) + }) + .option("requestOpts", { + describe: + "The default options to use for the JS request library when making HTTP HEAD/GET requests", + type: "string", + coerce: coerceJson("requestOpts"), + default: process.env.REQUEST_OPTS || JSON.stringify({ timeout: 20000 }) + }) ); console.log( - "domainWaitTimeConfig: ", - JSON.stringify(argv.domainWaitTimeConfig as any, null, 2) + "domainWaitTimeConfig: ", + JSON.stringify(argv.domainWaitTimeConfig as any, null, 2) ); function sleuthBrokenLinks() { - return minion({ - argv, - id: ID, - aspects: ["dataset-distributions"], - optionalAspects: [], - async: true, - writeAspectDefs: [brokenLinkAspectDef], - dereference: true, - onRecordFound: (record, registry) => - onRecordFound( - record, - registry, - argv.externalRetries, - 1, - argv.domainWaitTimeConfig as any, - argv.requestOpts as CoreOptions - ) - }); + return minion({ + argv, + id: ID, + aspects: ["dataset-distributions"], + optionalAspects: [], + async: true, + writeAspectDefs: [brokenLinkAspectDef], + dereference: true, + onRecordFound: (record, registry) => + onRecordFound( + record, + registry, + argv.externalRetries, + 1, + argv.domainWaitTimeConfig as any, + argv.requestOpts as CoreOptions + ) + }); } sleuthBrokenLinks().catch(e => { - console.error("Error: " + e.message, e); - process.exit(1); + console.error("Error: " + e.message, e); + process.exit(1); }); diff --git a/src/onRecordFound.ts b/src/onRecordFound.ts index 3022aca..665330e 100644 --- a/src/onRecordFound.ts +++ b/src/onRecordFound.ts @@ -2,144 +2,145 @@ import _ from "lodash"; import { CoreOptions } from "request"; import { retryBackoff, unionToThrowable } from "@magda/utils"; import { - AuthorizedRegistryClient as Registry, - Record + AuthorizedRegistryClient as Registry, + Record } from "@magda/minion-sdk"; -import { BrokenLinkAspect, RetrieveResult } from "./brokenLinkAspectDef"; -import FTPHandler from "./FtpHandler"; -import parseUriSafe from "./parseUriSafe"; -import { headRequest, getRequest, BadHttpResponseError } from "./HttpRequests"; -import getUrlWaitTime from "./getUrlWaitTime"; -import wait from "./wait"; +import { BrokenLinkAspect, RetrieveResult } from "./brokenLinkAspectDef.js"; +import FTPHandler from "./FtpHandler.js"; +import parseUriSafe from "./parseUriSafe.js"; +import { + headRequest, + getRequest, + BadHttpResponseError +} from "./HttpRequests.js"; +import getUrlWaitTime from "./getUrlWaitTime.js"; +import wait from "./wait.js"; export default async function onRecordFound( - record: Record, - registry: Registry, - retries: number = 1, - baseRetryDelaySeconds: number = 1, - domainWaitTimeConfig: { [domain: string]: number } = {}, - requestOpts: CoreOptions = {}, - ftpHandler: FTPHandler = new FTPHandler() + record: Record, + registry: Registry, + retries: number = 1, + baseRetryDelaySeconds: number = 1, + domainWaitTimeConfig: { [domain: string]: number } = {}, + requestOpts: CoreOptions = {}, + ftpHandler: FTPHandler = new FTPHandler() ) { - const distributions: Record[] = - record.aspects["dataset-distributions"] && - record.aspects["dataset-distributions"].distributions; + const distributions: Record[] = + record.aspects["dataset-distributions"] && + record.aspects["dataset-distributions"].distributions; - if (!distributions || distributions.length === 0) { - return Promise.resolve(); - } + if (!distributions || distributions.length === 0) { + return Promise.resolve(); + } - // Check each link - const linkChecks: DistributionLinkCheck[] = _.flatMap( - distributions, - (distribution: Record) => - checkDistributionLink( - distribution, - distribution.aspects["dcat-distribution-strings"], - baseRetryDelaySeconds, - retries, - ftpHandler, - _.partialRight(getUrlWaitTime, domainWaitTimeConfig), - requestOpts - ) - ); + // Check each link + const linkChecks: DistributionLinkCheck[] = _.flatMap( + distributions, + (distribution: Record) => + checkDistributionLink( + distribution, + distribution.aspects["dcat-distribution-strings"], + baseRetryDelaySeconds, + retries, + ftpHandler, + _.partialRight(getUrlWaitTime, domainWaitTimeConfig), + requestOpts + ) + ); - // Group the checks against their host so that we're only making one request per site simultaneously. - const brokenLinkChecksByHost: Promise[] = _( - linkChecks - ) - .groupBy(check => check.host) - .values() - .map((checks: DistributionLinkCheck[]) => checks.map(check => check.op)) - .map(checksForHost => - // Make the checks for this host run one after the other but return their results as an array. - checksForHost.reduce( - ( - megaPromise: Promise, - promiseLambda: () => Promise - ) => - megaPromise.then( - (megaResult: BrokenLinkSleuthingResult[]) => - promiseLambda().then(promiseResult => - megaResult.concat([promiseResult]) - ) - ), - Promise.resolve([]) + // Group the checks against their host so that we're only making one request per site simultaneously. + const brokenLinkChecksByHost: Promise[] = _( + linkChecks + ) + .groupBy(check => check.host) + .values() + .map((checks: DistributionLinkCheck[]) => checks.map(check => check.op)) + .map(checksForHost => + // Make the checks for this host run one after the other but return their results as an array. + checksForHost.reduce( + ( + megaPromise: Promise, + promiseLambda: () => Promise + ) => + megaPromise.then((megaResult: BrokenLinkSleuthingResult[]) => + promiseLambda().then(promiseResult => + megaResult.concat([promiseResult]) ) - ) - .value(); - - const checkResultsPerHost: BrokenLinkSleuthingResult[][] = await Promise.all( - brokenLinkChecksByHost - ); + ), + Promise.resolve([]) + ) + ) + .value(); - const allResults = _.flatten(checkResultsPerHost); + const checkResultsPerHost: BrokenLinkSleuthingResult[][] = await Promise.all( + brokenLinkChecksByHost + ); - const bestResultPerDistribution = _(allResults) - .groupBy(result => result.distribution.id) - .values() - .map((results: BrokenLinkSleuthingResult[]) => - _(results) - .sortBy(result => { - return ( - { none: 1, downloadURL: 2, accessURL: 3 }[ - result.urlType - ] || Number.MAX_VALUE - ); - }) - .sortBy(result => { - return ( - { active: 1, unknown: 2, broken: 3 }[ - result.aspect.status - ] || Number.MAX_VALUE - ); - }) - .head() - ) - .value(); + const allResults = _.flatten(checkResultsPerHost); - // Record a broken links aspect for each distribution. - const brokenLinksAspectPromise = Promise.all( - bestResultPerDistribution.map((result: BrokenLinkSleuthingResult) => { - return recordBrokenLinkAspect(registry, record, result); + const bestResultPerDistribution = _(allResults) + .groupBy(result => result.distribution.id) + .values() + .map((results: BrokenLinkSleuthingResult[]) => + _(results) + .sortBy(result => { + return ( + { none: 1, downloadURL: 2, accessURL: 3 }[result.urlType] || + Number.MAX_VALUE + ); }) - ); + .sortBy(result => { + return ( + { active: 1, unknown: 2, broken: 3 }[result.aspect.status] || + Number.MAX_VALUE + ); + }) + .head() + ) + .value(); + + // Record a broken links aspect for each distribution. + const brokenLinksAspectPromise = Promise.all( + bestResultPerDistribution.map((result: BrokenLinkSleuthingResult) => { + return recordBrokenLinkAspect(registry, record, result); + }) + ); - await brokenLinksAspectPromise; + await brokenLinksAspectPromise; } function recordBrokenLinkAspect( - registry: Registry, - record: Record, - result: BrokenLinkSleuthingResult + registry: Registry, + record: Record, + result: BrokenLinkSleuthingResult ): Promise { - const theTenantId = record.tenantId; - const { errorDetails, ...aspectDataWithNoErrorDetails } = result.aspect; - const aspectData = errorDetails - ? { - ...aspectDataWithNoErrorDetails, - errorDetails: `${ - errorDetails.httpStatusCode - ? `Http Status Code: ${errorDetails.httpStatusCode}: ` - : "" - }${errorDetails}` - } - : aspectDataWithNoErrorDetails; + const theTenantId = record.tenantId; + const { errorDetails, ...aspectDataWithNoErrorDetails } = result.aspect; + const aspectData = errorDetails + ? { + ...aspectDataWithNoErrorDetails, + errorDetails: `${ + errorDetails.httpStatusCode + ? `Http Status Code: ${errorDetails.httpStatusCode}: ` + : "" + }${errorDetails}` + } + : aspectDataWithNoErrorDetails; - return registry - .putRecordAspect( - result.distribution.id, - "source-link-status", - aspectData, - true, - theTenantId - ) - .then(unionToThrowable); + return registry + .putRecordAspect( + result.distribution.id, + "source-link-status", + aspectData, + true, + theTenantId + ) + .then(unionToThrowable); } type DistributionLinkCheck = { - host?: string; - op: () => Promise; + host?: string; + op: () => Promise; }; /** @@ -154,137 +155,133 @@ type DistributionLinkCheck = { * @param requestOpts The base options to use for the request library (e.g. timeouts, headers etc) */ function checkDistributionLink( - distribution: Record, - distStringsAspect: any, - baseRetryDelay: number, - retries: number, - ftpHandler: FTPHandler, - getUrlWaitTime: (url: string) => number, - requestOpts: CoreOptions + distribution: Record, + distStringsAspect: any, + baseRetryDelay: number, + retries: number, + ftpHandler: FTPHandler, + getUrlWaitTime: (url: string) => number, + requestOpts: CoreOptions ): DistributionLinkCheck[] { - type DistURL = { - url?: URI; - type: "downloadURL" | "accessURL"; - }; + type DistURL = { + url?: URI; + type: "downloadURL" | "accessURL"; + }; - const urls: DistURL[] = [ - { - url: distStringsAspect.downloadURL as string, - type: "downloadURL" as "downloadURL" - }, - { - url: distStringsAspect.accessURL as string, - type: "accessURL" as "accessURL" - } - ] - .map(urlObj => ({ ...urlObj, url: parseUriSafe(urlObj.url) })) - .filter(x => x.url && x.url.protocol().length > 0); + const urls: DistURL[] = [ + { + url: distStringsAspect.downloadURL as string, + type: "downloadURL" as "downloadURL" + }, + { + url: distStringsAspect.accessURL as string, + type: "accessURL" as "accessURL" + } + ] + .map(urlObj => ({ ...urlObj, url: parseUriSafe(urlObj.url) })) + .filter(x => x.url && x.url.protocol().length > 0); - if (urls.length === 0) { - return [ - { - op: () => - Promise.resolve({ - distribution, - urlType: "none" as "none", - aspect: { - status: "broken" as RetrieveResult, - errorDetails: new Error( - "No distribution urls to check." - ) - } - }) + if (urls.length === 0) { + return [ + { + op: () => + Promise.resolve({ + distribution, + urlType: "none" as "none", + aspect: { + status: "broken" as RetrieveResult, + errorDetails: new Error("No distribution urls to check.") } - ]; - } + }) + } + ]; + } - return urls.map(({ type, url: parsedURL }) => { - return { - host: (parsedURL && parsedURL.host()) as string, - op: () => { - console.info("Retrieving " + parsedURL); + return urls.map(({ type, url: parsedURL }) => { + return { + host: (parsedURL && parsedURL.host()) as string, + op: () => { + console.info("Retrieving " + parsedURL); - return retrieve( - parsedURL, - baseRetryDelay, - retries, - ftpHandler, - getUrlWaitTime, - requestOpts - ) - .then(aspect => { - console.info("Finished retrieving " + parsedURL); - return aspect; - }) - .then(aspect => ({ - distribution, - urlType: type, - aspect - })) - .catch(err => ({ - distribution, - urlType: type, - aspect: { - status: "broken" as RetrieveResult, - errorDetails: err - } - })) as Promise; + return retrieve( + parsedURL, + baseRetryDelay, + retries, + ftpHandler, + getUrlWaitTime, + requestOpts + ) + .then(aspect => { + console.info("Finished retrieving " + parsedURL); + return aspect; + }) + .then(aspect => ({ + distribution, + urlType: type, + aspect + })) + .catch(err => ({ + distribution, + urlType: type, + aspect: { + status: "broken" as RetrieveResult, + errorDetails: err } - }; - }); + })) as Promise; + } + }; + }); } function retrieve( - parsedURL: URI, - baseRetryDelay: number, - retries: number, - ftpHandler: FTPHandler, - getUrlWaitTime: (url: string) => number, - requestOpts: CoreOptions + parsedURL: URI, + baseRetryDelay: number, + retries: number, + ftpHandler: FTPHandler, + getUrlWaitTime: (url: string) => number, + requestOpts: CoreOptions ): Promise { - if (parsedURL.protocol() === "http" || parsedURL.protocol() === "https") { - return retrieveHttp( - parsedURL.toString(), - baseRetryDelay, - retries, - getUrlWaitTime, - requestOpts - ); - } else if (parsedURL.protocol() === "ftp") { - return retrieveFtp(parsedURL, ftpHandler); - } else { - console.info(`Unrecognised URL: ${parsedURL.toString()}`); - return Promise.resolve({ - status: "unknown" as "unknown", - errorDetails: new Error( - "Could not check protocol " + parsedURL.protocol() - ) - }); - } + if (parsedURL.protocol() === "http" || parsedURL.protocol() === "https") { + return retrieveHttp( + parsedURL.toString(), + baseRetryDelay, + retries, + getUrlWaitTime, + requestOpts + ); + } else if (parsedURL.protocol() === "ftp") { + return retrieveFtp(parsedURL, ftpHandler); + } else { + console.info(`Unrecognised URL: ${parsedURL.toString()}`); + return Promise.resolve({ + status: "unknown" as "unknown", + errorDetails: new Error( + "Could not check protocol " + parsedURL.protocol() + ) + }); + } } function retrieveFtp( - parsedURL: URI, - ftpHandler: FTPHandler + parsedURL: URI, + ftpHandler: FTPHandler ): Promise { - const port = +(parsedURL.port() || 21); - const pClient = ftpHandler.getClient(parsedURL.hostname(), port); + const port = +(parsedURL.port() || 21); + const pClient = ftpHandler.getClient(parsedURL.hostname(), port); - return pClient.then(client => { - return new Promise((resolve, reject) => { - client.list(parsedURL.path(), (err, list) => { - if (err) { - reject(err); - } else if (list.length === 0) { - reject( - new Error(`File "${parsedURL.toString()}" not found`) - ); - } else { - resolve({ status: "active" as "active" }); - } - }); - }); + return pClient.then(client => { + return new Promise((resolve, reject) => { + client.list(parsedURL.path(), (err, list) => { + if (err) { + reject(err); + } else if (list.length === 0) { + reject(new Error(`File "${parsedURL.toString()}" not found`)); + } else { + resolve({ status: "active" as "active" }); + } + }); }); + }); } /** @@ -293,69 +290,69 @@ function retrieveFtp( * @param url The url to retrieve */ async function retrieveHttp( - url: string, - baseRetryDelay: number, - retries: number, - getUrlWaitTime: (url: string) => number, - requestOpts: CoreOptions + url: string, + baseRetryDelay: number, + retries: number, + getUrlWaitTime: (url: string) => number, + requestOpts: CoreOptions ): Promise { - async function operation() { - try { - await wait(getUrlWaitTime(url)); - return await headRequest(url, requestOpts); - } catch (e) { - // --- HEAD Method not allowed - await wait(getUrlWaitTime(url)); - return await getRequest(url, requestOpts); - } + async function operation() { + try { + await wait(getUrlWaitTime(url)); + return await headRequest(url, requestOpts); + } catch (e) { + // --- HEAD Method not allowed + await wait(getUrlWaitTime(url)); + return await getRequest(url, requestOpts); } + } - const onRetry = (err: BadHttpResponseError, retries: number) => { - console.info( - `Downloading ${url} failed: ${err.httpStatusCode || - err} (${retries} retries remaining)` - ); - }; + const onRetry = (err: BadHttpResponseError, retries: number) => { + console.info( + `Downloading ${url} failed: ${err.httpStatusCode || + err} (${retries} retries remaining)` + ); + }; - const innerOp = () => - retryBackoff(operation, baseRetryDelay, retries, onRetry); + const innerOp = () => + retryBackoff(operation, baseRetryDelay, retries, onRetry); - const outerOp: () => Promise = () => - innerOp().then( - code => { - if (code === 429) { - throw { message: "429 encountered", httpStatusCode: 429 }; - } else { - return { - status: "active" as "active", - httpStatusCode: code - }; - } - }, - error => { - return { - status: "broken" as "broken", - httpStatusCode: error.httpStatusCode, - errorDetails: error - }; - } - ); + const outerOp: () => Promise = () => + innerOp().then( + code => { + if (code === 429) { + throw { message: "429 encountered", httpStatusCode: 429 }; + } else { + return { + status: "active" as "active", + httpStatusCode: code + }; + } + }, + error => { + return { + status: "broken" as "broken", + httpStatusCode: error.httpStatusCode, + errorDetails: error + }; + } + ); - return retryBackoff( - outerOp, - baseRetryDelay, - retries, - onRetry, - (x: number) => x * 5 - ).catch(err => ({ - status: "unknown" as "unknown", - errorDetails: err, - httpStatusCode: 429 - })); + return retryBackoff( + outerOp, + baseRetryDelay, + retries, + onRetry, + (x: number) => x * 5 + ).catch(err => ({ + status: "unknown" as "unknown", + errorDetails: err, + httpStatusCode: 429 + })); } interface BrokenLinkSleuthingResult { - distribution: Record; - aspect?: BrokenLinkAspect; - urlType: "downloadURL" | "accessURL" | "none"; + distribution: Record; + aspect?: BrokenLinkAspect; + urlType: "downloadURL" | "accessURL" | "none"; } diff --git a/src/request.ts b/src/request.ts new file mode 100644 index 0000000..eada498 --- /dev/null +++ b/src/request.ts @@ -0,0 +1,23 @@ +const requestOriginal = require("request"); +import { RequestAPI, Request, CoreOptions, RequiredUriUrl } from "request"; +import readPkgUp from "read-pkg-up"; +const pkg = readPkgUp.sync().pkg; + +if (!pkg || !pkg.name) { + throw new Error( + "magda-minion-broken-link: Can't locate package.json in current working directory or `name` field is empty." + ); +} + +// include user agent derived from package.json in all http requests +const request = requestOriginal.defaults({ + headers: { + "User-Agent": "".concat( + pkg.name.replace("/", "-").replace("@", ""), + "/", + pkg.version + ) + } +}) as RequestAPI; + +export default request; diff --git a/src/test/HttpRequests.spec.ts b/src/test/HttpRequests.spec.ts index c11beb1..07c43e2 100644 --- a/src/test/HttpRequests.spec.ts +++ b/src/test/HttpRequests.spec.ts @@ -2,178 +2,179 @@ import {} from "mocha"; import nock from "nock"; import jsc from "jsverify"; import { expect } from "chai"; -import { headRequest, getRequest, BadHttpResponseError } from "../HttpRequests"; -import RandomStream from "./RandomStream"; +import { + headRequest, + getRequest, + BadHttpResponseError +} from "../HttpRequests.js"; +import RandomStream from "./RandomStream.js"; const onMatchFail = (req: any, interceptor: any) => { - console.error( - `Match failure: ${req.method ? req.method : interceptor.method} ${ - req.host ? req.host : interceptor.host - }${req.path}` - ); + console.error( + `Match failure: ${req.method ? req.method : interceptor.method} ${ + req.host ? req.host : interceptor.host + }${req.path}` + ); }; const errorCodeArb = jsc.oneof([jsc.integer(300, 428), jsc.integer(430, 600)]); describe("Test HttpRequests.ts", () => { - before(() => { - nock.disableNetConnect(); - nock.emitter.on("no match", onMatchFail); + before(() => { + nock.disableNetConnect(); + nock.emitter.on("no match", onMatchFail); + }); + + after(() => { + nock.emitter.removeListener("no match", onMatchFail); + nock.cleanAll(); + nock.abortPendingRequests(); + }); + + describe("headRequest", () => { + it("should return status code when response status is between 200 to 299", async function() { + return jsc.assert( + jsc.forall(jsc.integer(200, 299), async function(statusCode) { + const url = "http://example.com"; + const path = "/xx"; + nock(url) + .head(path) + .reply(statusCode); + + const resStatusCode = await headRequest(`${url}${path}`); + expect(resStatusCode).to.equal(statusCode); + return true; + }) + ); }); - after(() => { - nock.emitter.removeListener("no match", onMatchFail); - nock.cleanAll(); - nock.abortPendingRequests(); + it("should return status code when response status is 429", async function() { + const url = "http://example.com"; + const path = "/xx"; + nock(url) + .head(path) + .reply(429); + + const resStatusCode = await headRequest(`${url}${path}`); + expect(resStatusCode).to.equal(429); + return true; }); - describe("headRequest", () => { - it("should return status code when response status is between 200 to 299", async function() { - return jsc.assert( - jsc.forall(jsc.integer(200, 299), async function(statusCode) { - const url = "http://example.com"; - const path = "/xx"; - nock(url) - .head(path) - .reply(statusCode); - - const resStatusCode = await headRequest(`${url}${path}`); - expect(resStatusCode).to.equal(statusCode); - return true; - }) - ); - }); - - it("should return status code when response status is 429", async function() { - const url = "http://example.com"; - const path = "/xx"; - nock(url) - .head(path) - .reply(429); - - const resStatusCode = await headRequest(`${url}${path}`); - expect(resStatusCode).to.equal(429); - return true; - }); - - it("should throw `BadHttpResponseError` when response status is not between 200 to 299 or 429", async function() { - return jsc.assert( - jsc.forall(errorCodeArb, async function(statusCode) { - const url = "http://example.com"; - const path = "/xx"; - nock(url) - .head(path) - .reply(statusCode); - - let error: any = null; - try { - await headRequest(`${url}${path}`); - } catch (e) { - error = e; - } - expect(error).to.be.an.instanceof(BadHttpResponseError); - expect(error.httpStatusCode).to.equal(statusCode); - return true; - }) - ); - }); + it("should throw `BadHttpResponseError` when response status is not between 200 to 299 or 429", async function() { + return jsc.assert( + jsc.forall(errorCodeArb, async function(statusCode) { + const url = "http://example.com"; + const path = "/xx"; + nock(url) + .head(path) + .reply(statusCode); + + let error: any = null; + try { + await headRequest(`${url}${path}`); + } catch (e) { + error = e; + } + expect(error).to.be.an.instanceof(BadHttpResponseError); + expect(error.httpStatusCode).to.equal(statusCode); + return true; + }) + ); }); + }); - describe("getRequest", () => { - afterEach(() => { - nock.cleanAll(); - nock.abortPendingRequests(); - }); - it("should return status code when response status is between 200 to 299", async function(this: Mocha.Context) { - this.timeout(5000); - return jsc.assert( - jsc.forall( - jsc.integer(200, 299), - jsc.integer(0, 10), - async function(statusCode, streamWaitTime) { - const url = "http://example.com"; - const path = "/xx"; - const scope = nock(url) - .get(path) - .reply(statusCode, () => { - return new RandomStream(streamWaitTime); - }); - - const resStatusCode = await getRequest(`${url}${path}`); - scope.done(); - expect(resStatusCode).to.equal(statusCode); - return true; - } - ) - ); - }); - - it("should wait until stream completes", async function(this: Mocha.Context) { - this.timeout(30000); - return ( - jsc.assert( - jsc.forall( - jsc.integer(200, 299), - jsc.integer(1500, 3000), - async function(statusCode, streamWaitTime) { - const url = "http://example.com"; - const path = "/xx"; - nock(url) - .get(path) - .reply(statusCode, () => { - return new RandomStream(streamWaitTime); - }); - - const now = new Date().getTime(); - const resStatusCode = await getRequest( - `${url}${path}` - ); - const newTime = new Date().getTime(); - const diff = newTime - now; - expect(resStatusCode).to.equal(statusCode); - expect(diff).to.closeTo(streamWaitTime, 50); - return true; - } - ) - ), - { - times: 3 - } - ); - }); - - it("should return status code when response status is 429", async function() { - const url = "http://example.com"; - const path = "/xx"; - nock(url) + describe("getRequest", () => { + afterEach(() => { + nock.cleanAll(); + nock.abortPendingRequests(); + }); + it("should return status code when response status is between 200 to 299", async function(this: Mocha.Context) { + this.timeout(5000); + return jsc.assert( + jsc.forall(jsc.integer(200, 299), jsc.integer(0, 10), async function( + statusCode, + streamWaitTime + ) { + const url = "http://example.com"; + const path = "/xx"; + const scope = nock(url) + .get(path) + .reply(statusCode, () => { + return new RandomStream(streamWaitTime); + }); + + const resStatusCode = await getRequest(`${url}${path}`); + scope.done(); + expect(resStatusCode).to.equal(statusCode); + return true; + }) + ); + }); + + it("should wait until stream completes", async function(this: Mocha.Context) { + this.timeout(30000); + return ( + jsc.assert( + jsc.forall( + jsc.integer(200, 299), + jsc.integer(1500, 3000), + async function(statusCode, streamWaitTime) { + const url = "http://example.com"; + const path = "/xx"; + nock(url) .get(path) - .reply(429); - - const resStatusCode = await getRequest(`${url}${path}`); - expect(resStatusCode).to.equal(429); - return true; - }); - - it("should throw `BadHttpResponseError` when response status is not between 200 to 299 or 429", async function() { - return jsc.assert( - jsc.forall(errorCodeArb, async function(statusCode) { - const url = "http://example.com"; - const path = "/xx"; - nock(url) - .get(path) - .reply(statusCode); - - let error: any = null; - try { - await getRequest(`${url}${path}`); - } catch (e) { - error = e; - } - expect(error).to.be.an.instanceof(BadHttpResponseError); - expect(error.httpStatusCode).to.equal(statusCode); - return true; - }) - ); - }); + .reply(statusCode, () => { + return new RandomStream(streamWaitTime); + }); + + const now = new Date().getTime(); + const resStatusCode = await getRequest(`${url}${path}`); + const newTime = new Date().getTime(); + const diff = newTime - now; + expect(resStatusCode).to.equal(statusCode); + expect(diff).to.closeTo(streamWaitTime, 50); + return true; + } + ) + ), + { + times: 3 + } + ); + }); + + it("should return status code when response status is 429", async function() { + const url = "http://example.com"; + const path = "/xx"; + nock(url) + .get(path) + .reply(429); + + const resStatusCode = await getRequest(`${url}${path}`); + expect(resStatusCode).to.equal(429); + return true; + }); + + it("should throw `BadHttpResponseError` when response status is not between 200 to 299 or 429", async function() { + return jsc.assert( + jsc.forall(errorCodeArb, async function(statusCode) { + const url = "http://example.com"; + const path = "/xx"; + nock(url) + .get(path) + .reply(statusCode); + + let error: any = null; + try { + await getRequest(`${url}${path}`); + } catch (e) { + error = e; + } + expect(error).to.be.an.instanceof(BadHttpResponseError); + expect(error.httpStatusCode).to.equal(statusCode); + return true; + }) + ); }); + }); }); diff --git a/src/test/arbitraries.ts b/src/test/arbitraries.ts index 4f9b46a..56a2c64 100644 --- a/src/test/arbitraries.ts +++ b/src/test/arbitraries.ts @@ -1,20 +1,20 @@ import jsc from "jsverify"; import { Record } from "@magda/minion-sdk"; import { - distUrlArb, - arrayOfSizeArb, - arbFlatMap, - recordArbWithDistArbs, - stringArb + distUrlArb, + arrayOfSizeArb, + arbFlatMap, + recordArbWithDistArbs, + stringArb } from "@magda/arbitraries"; -import urlsFromDataSet from "./urlsFromDataSet"; +import urlsFromDataSet from "./urlsFromDataSet.js"; import _ from "lodash"; import URI from "urijs"; export const KNOWN_PROTOCOLS = ["https", "http", "ftp"]; const defaultRecordArb = recordArbWithDistArbs({ - url: jsc.oneof([distUrlArb(), stringArb]) + url: jsc.oneof([distUrlArb(), stringArb]) }); /** @@ -22,64 +22,62 @@ const defaultRecordArb = recordArbWithDistArbs({ * or not it should successfully return. */ export const recordArbWithSuccesses = arbFlatMap( - defaultRecordArb, - (record: Record) => { - const knownProtocolUrls = getKnownProtocolUrls(record); + defaultRecordArb, + (record: Record) => { + const knownProtocolUrls = getKnownProtocolUrls(record); - const urlWithSuccessArb: jsc.Arbitrary = arrayOfSizeArb( - knownProtocolUrls.length, - checkResultArb - ); - - return urlWithSuccessArb.smap( - resultArr => { - const successLookup = knownProtocolUrls.reduce( - (soFar, current, index) => { - soFar[current] = resultArr[index]; - return soFar; - }, - {} as { [a: string]: CheckResult } - ); - // some server configurations will disallow HEAD - // method requests. When that fails, we try - // to make a get request to verify the link - const disallowHead = jsc.bool.generator(0); + const urlWithSuccessArb: jsc.Arbitrary = arrayOfSizeArb( + knownProtocolUrls.length, + checkResultArb + ); - return { record, successLookup, disallowHead }; - }, - ({ record, successLookup }) => { - return getKnownProtocolUrls(record).map( - url => successLookup[url] - ); - } + return urlWithSuccessArb.smap( + resultArr => { + const successLookup = knownProtocolUrls.reduce( + (soFar, current, index) => { + soFar[current] = resultArr[index]; + return soFar; + }, + {} as { [a: string]: CheckResult } ); - }, - ({ record, successLookup }) => record + // some server configurations will disallow HEAD + // method requests. When that fails, we try + // to make a get request to verify the link + const disallowHead = jsc.bool.generator(0); + + return { record, successLookup, disallowHead }; + }, + ({ record, successLookup }) => { + return getKnownProtocolUrls(record).map(url => successLookup[url]); + } + ); + }, + ({ record, successLookup }) => record ); /** * Gets all the urls for distributions in this dataset record that have known protocols (http etc.). */ function getKnownProtocolUrls(record: Record) { - return _(urlsFromDataSet(record)) - .filter(url => { - let uri; - try { - uri = URI(url); - } catch (e) { - return false; - } - return KNOWN_PROTOCOLS.indexOf(uri.scheme()) >= 0; - }) - .uniq() - .value(); + return _(urlsFromDataSet(record)) + .filter(url => { + let uri; + try { + uri = URI(url); + } catch (e) { + return false; + } + return KNOWN_PROTOCOLS.indexOf(uri.scheme()) >= 0; + }) + .uniq() + .value(); } export type CheckResult = "success" | "error" | "notfound"; export const checkResultArb: jsc.Arbitrary = jsc.oneof( - ["success" as "success", "error" as "error", "notfound" as "notfound"].map( - jsc.constant - ) as jsc.Arbitrary[] + ["success" as "success", "error" as "error", "notfound" as "notfound"].map( + jsc.constant + ) as jsc.Arbitrary[] ); /** @@ -88,34 +86,27 @@ export const checkResultArb: jsc.Arbitrary = jsc.oneof( * distribution, for testing retries. */ export const httpOnlyRecordArb = jsc.suchthat( - recordArbWithDistArbs({ - url: jsc.oneof([ - jsc.constant(undefined), - distUrlArb({ - schemeArb: jsc.oneof([ - jsc.constant("http"), - jsc.constant("https") - ]) - }) - ]) - }), - record => - record.aspects["dataset-distributions"].distributions.length > 1 && - record.aspects["dataset-distributions"].distributions.every( - (dist: any) => { - const aspect = dist.aspects["dcat-distribution-strings"]; + recordArbWithDistArbs({ + url: jsc.oneof([ + jsc.constant(undefined), + distUrlArb({ + schemeArb: jsc.oneof([jsc.constant("http"), jsc.constant("https")]) + }) + ]) + }), + record => + record.aspects["dataset-distributions"].distributions.length > 1 && + record.aspects["dataset-distributions"].distributions.every((dist: any) => { + const aspect = dist.aspects["dcat-distribution-strings"]; - const definedURLs = [ - aspect.accessURL, - aspect.downloadURL - ].filter(x => !!x); + const definedURLs = [aspect.accessURL, aspect.downloadURL].filter( + x => !!x + ); - return ( - definedURLs.length > 0 && - definedURLs.every(x => x.startsWith("http")) - ); - } - ) + return ( + definedURLs.length > 0 && definedURLs.every(x => x.startsWith("http")) + ); + }) ); /** @@ -123,6 +114,6 @@ export const httpOnlyRecordArb = jsc.suchthat( * triggers different behaviour. */ export const failureCodeArb = jsc.suchthat( - jsc.integer(300, 600), - int => int !== 429 + jsc.integer(300, 600), + int => int !== 429 ); diff --git a/src/test/getUrlWaitTime.spec.ts b/src/test/getUrlWaitTime.spec.ts index 01252eb..d16e02a 100644 --- a/src/test/getUrlWaitTime.spec.ts +++ b/src/test/getUrlWaitTime.spec.ts @@ -1,218 +1,194 @@ import {} from "mocha"; import jsc from "jsverify"; import getUrlWaitTime, { - getHostWaitTime, - defaultDomainWaitTime, - clearDomainAccessTimeStore -} from "../getUrlWaitTime"; + getHostWaitTime, + defaultDomainWaitTime, + clearDomainAccessTimeStore +} from "../getUrlWaitTime.js"; import sinon from "sinon"; import URI from "urijs"; import { expect } from "chai"; import { distUrlArb } from "@magda/arbitraries"; function randomInt(min: number, max: number) { - return Math.floor(Math.random() * (max - min + 1)) + min; + return Math.floor(Math.random() * (max - min + 1)) + min; } describe("Test getUrlWaitTime.ts", () => { - describe("getHostWaitTime", () => { - it("should return waitTime set for a domain if available", async () => { - return jsc.assert( - jsc.forall(jsc.nestring, jsc.integer(1), (host, waitTime) => { - const domainWaitTimeConfig = { - [host]: waitTime - }; - expect( - getHostWaitTime(host, domainWaitTimeConfig) - ).to.equal(waitTime); - return true; - }) - ); - }); - - it("should return global default waitTime for a domain if not set", async () => { - return jsc.assert( - jsc.forall(jsc.nestring, jsc.integer(1), (host, waitTime) => { - const domainWaitTimeConfig = { - [host]: waitTime - }; - expect( - getHostWaitTime( - `${host}_extra_suffix`, - domainWaitTimeConfig - ) - ).to.equal(defaultDomainWaitTime); - return true; - }) - ); - }); + describe("getHostWaitTime", () => { + it("should return waitTime set for a domain if available", async () => { + return jsc.assert( + jsc.forall(jsc.nestring, jsc.integer(1), (host, waitTime) => { + const domainWaitTimeConfig = { + [host]: waitTime + }; + expect(getHostWaitTime(host, domainWaitTimeConfig)).to.equal( + waitTime + ); + return true; + }) + ); }); - describe("getUrlWaitTime", () => { - let clock: sinon.SinonFakeTimers = null; - - const urlArb = (jsc as any).nonshrink( - distUrlArb({ - schemeArb: jsc.elements(["http", "https"]), - hostArb: jsc.elements(["example1", "example2", "example3"]) - }) - ); - - before(() => { - clock = sinon.useFakeTimers(); - }); - - after(() => { - clock.reset(); - clock.restore(); - }); - - it("should return 0 if it's the first time for the domain", async () => { - return jsc.assert( - jsc.forall( - urlArb, - jsc.integer(1), - (url: string, waitTime: number) => { - clearDomainAccessTimeStore(); - - const uri = new URI(url); - const host = uri.hostname(); - const domainWaitTimeConfig = { - [host]: waitTime - }; - - expect( - getUrlWaitTime(url, domainWaitTimeConfig) - ).to.be.equal(0); - - return true; - } - ) - ); - }); - - it("should return waitTime x 1 set for a domain if call the second time immediately", async () => { - return jsc.assert( - jsc.forall(urlArb, jsc.integer(1, 99999), async function( - url: string, - domainWaitTime: number //--- seconds - ) { - clearDomainAccessTimeStore(); - - const uri = new URI(url); - const host = uri.hostname(); - const domainWaitTimeConfig = { - [host]: domainWaitTime - }; - - expect( - getUrlWaitTime(url, domainWaitTimeConfig) - ).to.be.equal(0); - - expect( - getUrlWaitTime(url, domainWaitTimeConfig) - ).to.be.equal(domainWaitTime * 1000); - - return true; - }) - ); - }); - - it("should return 0 after delayed domainWaitTime before call the second time", async () => { - return jsc.assert( - jsc.forall(urlArb, jsc.integer(1, 200), async function( - url: string, - domainWaitTime: number //--- seconds - ) { - clearDomainAccessTimeStore(); - - const uri = new URI(url); - const host = uri.hostname(); - const domainWaitTimeConfig = { - [host]: domainWaitTime - }; - - expect( - getUrlWaitTime(url, domainWaitTimeConfig) - ).to.be.equal(0); - - clock.tick(domainWaitTime * 1000); - - expect( - getUrlWaitTime(url, domainWaitTimeConfig) - ).to.be.equal(0); - - return true; - }) - ); - }); - - it("should return (domainWaitTime - `Random Delayed Time`) after delayed `Random Delayed Time` (lower than `domainWaitTime`) before call the second time", async () => { - return jsc.assert( - jsc.forall(urlArb, jsc.integer(1, 200), async function( - url: string, - domainWaitTime: number //--- seconds - ) { - clearDomainAccessTimeStore(); - - const uri = new URI(url); - const host = uri.hostname(); - const domainWaitTimeConfig = { - [host]: domainWaitTime - }; - - expect( - getUrlWaitTime(url, domainWaitTimeConfig) - ).to.be.equal(0); - - const randomWaitTime = randomInt( - 1, - domainWaitTime * 1000 - 1 - ); - - clock.tick(randomWaitTime); - - expect( - getUrlWaitTime(url, domainWaitTimeConfig) - ).to.be.equal(domainWaitTime * 1000 - randomWaitTime); - - return true; - }) - ); - }); - - it("should return 0 after delayed `Random Delayed Time` (higher than `domainWaitTime`) before call the second time", async () => { - return jsc.assert( - jsc.forall(urlArb, jsc.integer(1, 200), async function( - url: string, - domainWaitTime: number //--- seconds - ) { - clearDomainAccessTimeStore(); - - const uri = new URI(url); - const host = uri.hostname(); - const domainWaitTimeConfig = { - [host]: domainWaitTime - }; - - expect( - getUrlWaitTime(url, domainWaitTimeConfig) - ).to.be.equal(0); - - const randomWaitTime = randomInt( - domainWaitTime * 1000 + 1, - domainWaitTime * 1000 * 2 - ); - - clock.tick(randomWaitTime); - - expect( - getUrlWaitTime(url, domainWaitTimeConfig) - ).to.be.equal(0); - - return true; - }) - ); - }); + it("should return global default waitTime for a domain if not set", async () => { + return jsc.assert( + jsc.forall(jsc.nestring, jsc.integer(1), (host, waitTime) => { + const domainWaitTimeConfig = { + [host]: waitTime + }; + expect( + getHostWaitTime(`${host}_extra_suffix`, domainWaitTimeConfig) + ).to.equal(defaultDomainWaitTime); + return true; + }) + ); }); + }); + + describe("getUrlWaitTime", () => { + let clock: sinon.SinonFakeTimers = null; + + const urlArb = (jsc as any).nonshrink( + distUrlArb({ + schemeArb: jsc.elements(["http", "https"]), + hostArb: jsc.elements(["example1", "example2", "example3"]) + }) + ); + + before(() => { + clock = sinon.useFakeTimers(); + }); + + after(() => { + clock.reset(); + clock.restore(); + }); + + it("should return 0 if it's the first time for the domain", async () => { + return jsc.assert( + jsc.forall(urlArb, jsc.integer(1), (url: string, waitTime: number) => { + clearDomainAccessTimeStore(); + + const uri = new URI(url); + const host = uri.hostname(); + const domainWaitTimeConfig = { + [host]: waitTime + }; + + expect(getUrlWaitTime(url, domainWaitTimeConfig)).to.be.equal(0); + + return true; + }) + ); + }); + + it("should return waitTime x 1 set for a domain if call the second time immediately", async () => { + return jsc.assert( + jsc.forall(urlArb, jsc.integer(1, 99999), async function( + url: string, + domainWaitTime: number //--- seconds + ) { + clearDomainAccessTimeStore(); + + const uri = new URI(url); + const host = uri.hostname(); + const domainWaitTimeConfig = { + [host]: domainWaitTime + }; + + expect(getUrlWaitTime(url, domainWaitTimeConfig)).to.be.equal(0); + + expect(getUrlWaitTime(url, domainWaitTimeConfig)).to.be.equal( + domainWaitTime * 1000 + ); + + return true; + }) + ); + }); + + it("should return 0 after delayed domainWaitTime before call the second time", async () => { + return jsc.assert( + jsc.forall(urlArb, jsc.integer(1, 200), async function( + url: string, + domainWaitTime: number //--- seconds + ) { + clearDomainAccessTimeStore(); + + const uri = new URI(url); + const host = uri.hostname(); + const domainWaitTimeConfig = { + [host]: domainWaitTime + }; + + expect(getUrlWaitTime(url, domainWaitTimeConfig)).to.be.equal(0); + + clock.tick(domainWaitTime * 1000); + + expect(getUrlWaitTime(url, domainWaitTimeConfig)).to.be.equal(0); + + return true; + }) + ); + }); + + it("should return (domainWaitTime - `Random Delayed Time`) after delayed `Random Delayed Time` (lower than `domainWaitTime`) before call the second time", async () => { + return jsc.assert( + jsc.forall(urlArb, jsc.integer(1, 200), async function( + url: string, + domainWaitTime: number //--- seconds + ) { + clearDomainAccessTimeStore(); + + const uri = new URI(url); + const host = uri.hostname(); + const domainWaitTimeConfig = { + [host]: domainWaitTime + }; + + expect(getUrlWaitTime(url, domainWaitTimeConfig)).to.be.equal(0); + + const randomWaitTime = randomInt(1, domainWaitTime * 1000 - 1); + + clock.tick(randomWaitTime); + + expect(getUrlWaitTime(url, domainWaitTimeConfig)).to.be.equal( + domainWaitTime * 1000 - randomWaitTime + ); + + return true; + }) + ); + }); + + it("should return 0 after delayed `Random Delayed Time` (higher than `domainWaitTime`) before call the second time", async () => { + return jsc.assert( + jsc.forall(urlArb, jsc.integer(1, 200), async function( + url: string, + domainWaitTime: number //--- seconds + ) { + clearDomainAccessTimeStore(); + + const uri = new URI(url); + const host = uri.hostname(); + const domainWaitTimeConfig = { + [host]: domainWaitTime + }; + + expect(getUrlWaitTime(url, domainWaitTimeConfig)).to.be.equal(0); + + const randomWaitTime = randomInt( + domainWaitTime * 1000 + 1, + domainWaitTime * 1000 * 2 + ); + + clock.tick(randomWaitTime); + + expect(getUrlWaitTime(url, domainWaitTimeConfig)).to.be.equal(0); + + return true; + }) + ); + }); + }); }); diff --git a/src/test/onRecordFound.spec.ts b/src/test/onRecordFound.spec.ts index ec8856f..db47889 100644 --- a/src/test/onRecordFound.spec.ts +++ b/src/test/onRecordFound.spec.ts @@ -11,751 +11,669 @@ import Ajv from "ajv"; import { Record, AuthorizedRegistryClient } from "@magda/minion-sdk"; import { encodeURIComponentWithApost } from "@magda/utils"; import { - specificRecordArb, - distUrlArb, - arrayOfSizeArb, - arbFlatMap, - recordArbWithDistArbs + specificRecordArb, + distUrlArb, + arrayOfSizeArb, + arbFlatMap, + recordArbWithDistArbs } from "@magda/arbitraries"; -import onRecordFound from "../onRecordFound"; -import { BrokenLinkAspect } from "../brokenLinkAspectDef"; -import urlsFromDataSet from "./urlsFromDataSet"; +import onRecordFound from "../onRecordFound.js"; +import { BrokenLinkAspect } from "../brokenLinkAspectDef.js"; +import urlsFromDataSet from "./urlsFromDataSet.js"; import { - CheckResult, - recordArbWithSuccesses, - KNOWN_PROTOCOLS, - httpOnlyRecordArb, - failureCodeArb -} from "./arbitraries"; -import FtpHandler from "../FtpHandler"; -import parseUriSafe from "../parseUriSafe"; -import RandomStream from "./RandomStream"; + CheckResult, + recordArbWithSuccesses, + KNOWN_PROTOCOLS, + httpOnlyRecordArb, + failureCodeArb +} from "./arbitraries.js"; +import FtpHandler from "../FtpHandler.js"; +import parseUriSafe from "../parseUriSafe.js"; +import RandomStream from "./RandomStream.js"; import { - setDefaultDomainWaitTime, - getDefaultDomainWaitTime -} from "../getUrlWaitTime"; + setDefaultDomainWaitTime, + getDefaultDomainWaitTime +} from "../getUrlWaitTime.js"; const schema = require("@magda/registry-aspects/source-link-status.schema.json"); describe("onRecordFound", function(this: Mocha.Suite) { - this.timeout(20000); + this.timeout(20000); + nock.disableNetConnect(); + const registryUrl = "http://example.com"; + const secret = "secret!"; + const registry = new AuthorizedRegistryClient({ + baseUrl: registryUrl, + jwtSecret: secret, + userId: "1", + maxRetries: 0, + tenantId: 1 + }); + let registryScope: nock.Scope; + let clients: { [s: string]: Client[] }; + let ftpSuccesses: { [url: string]: CheckResult }; + const orignalDefaultDomainWaitTime: number = getDefaultDomainWaitTime(); + + before(() => { + // --- set default domain wait time to 0 second (i.e. for any domains that has no specific setting) + // --- Otherwise, it will take too long to complete property tests + setDefaultDomainWaitTime(0); + + sinon.stub(console, "info"); nock.disableNetConnect(); - const registryUrl = "http://example.com"; - const secret = "secret!"; - const registry = new AuthorizedRegistryClient({ - baseUrl: registryUrl, - jwtSecret: secret, - userId: "1", - maxRetries: 0, - tenantId: 1 - }); - let registryScope: nock.Scope; - let clients: { [s: string]: Client[] }; - let ftpSuccesses: { [url: string]: CheckResult }; - const orignalDefaultDomainWaitTime: number = getDefaultDomainWaitTime(); - - before(() => { - // --- set default domain wait time to 0 second (i.e. for any domains that has no specific setting) - // --- Otherwise, it will take too long to complete property tests - setDefaultDomainWaitTime(0); - - sinon.stub(console, "info"); - nock.disableNetConnect(); - - nock.emitter.on("no match", onMatchFail); - }); - - const onMatchFail = (req: any, interceptor: any) => { - console.error( - `Match failure: ${req.method ? req.method : interceptor.method} ${ - req.host ? req.host : interceptor.host - }${req.path}` - ); - }; - - after(() => { - setDefaultDomainWaitTime(orignalDefaultDomainWaitTime); - (console.info as any).restore(); + nock.emitter.on("no match", onMatchFail); + }); - nock.emitter.removeListener("no match", onMatchFail); - }); + const onMatchFail = (req: any, interceptor: any) => { + console.error( + `Match failure: ${req.method ? req.method : interceptor.method} ${ + req.host ? req.host : interceptor.host + }${req.path}` + ); + }; + + after(() => { + setDefaultDomainWaitTime(orignalDefaultDomainWaitTime); + + (console.info as any).restore(); + + nock.emitter.removeListener("no match", onMatchFail); + }); + + const beforeEachProperty = () => { + registryScope = nock(registryUrl); //.log(console.log); + clients = {}; + ftpSuccesses = {}; + }; + + const afterEachProperty = () => { + nock.cleanAll(); + }; + + /** + * Builds FTP clients that have all their important methods stubbed out - these + * will respond based on the current content of ftpSuccesses. + */ + const clientFactory = () => { + const client = new Client(); + let readyCallback: () => void; + let key: string; + sinon + .stub(client, "connect") + .callsFake(({ host, port }: Client.Options) => { + const keyPort = port !== 21 ? `:${port}` : ""; + key = `${host}${keyPort}`; + if (!clients[key]) { + clients[key] = []; + } + clients[key].push(client); + readyCallback(); + }); + sinon + .stub(client, "on") + .callsFake((event: string | symbol, callback: () => void) => { + if (event === "ready") { + readyCallback = callback; + } + return client; + }); + sinon.stub(client, "list").callsFake((( + path: string, + callback: (err: Error, list: string[]) => void + ) => { + try { + expect(key).not.to.be.undefined; + const url = `ftp://${key}${path}`; + + const success = ftpSuccesses[url]; + expect(success).not.to.be.undefined; + + if (success === "success") { + callback(null, ["file"]); + } else if (success === "notfound") { + callback(null, []); + } else { + callback(new Error("Fake error!"), null); + } + } catch (e) { + console.error(e); + callback(e as Error, null); + } + }) as any); + return client; + }; + + const fakeFtpHandler = new FtpHandler(clientFactory); + + /** + * Generator-driven super-test: generates records and runs them through the + * onRecordFound function, listening for HTTP and FTP calls made and returning + * success or failure based on generated outcomes, then checks that they're + * recorded on a by-distribution basis as link status as well as on a by-record + * basis as a part of dataset quality. + */ + it("Should correctly record link statuses", function() { + const ajv = new Ajv(); + const validate = ajv.compile(schema); + return jsc.assert( + jsc.forall(recordArbWithSuccesses, jsc.integer(1, 100), function( + { record, successLookup, disallowHead }, + streamWaitTime + ) { + beforeEachProperty(); + + // Tell the FTP server to return success/failure for the various FTP + // paths with this dodgy method. Note that because the FTP server can + // only see paths and not host, we only send it the path of the req. + ftpSuccesses = _.pickBy(successLookup, (value, url) => + hasProtocol(url, "ftp") + ); - const beforeEachProperty = () => { - registryScope = nock(registryUrl); //.log(console.log); - clients = {}; - ftpSuccesses = {}; - }; + const allDists = record.aspects["dataset-distributions"].distributions; + + const httpDistUrls = _(urlsFromDataSet(record)) + .filter((url: string) => hasProtocol(url, "http")) + .map((url: string) => ({ + url, + success: successLookup[url] + })) + .value(); + + // Set up a nock scope for every HTTP URL - the minion will actually + // attempt to download these but it'll be intercepted by nock. + const distScopes = httpDistUrls.map( + ({ url, success }: { url: string; success: CheckResult }) => { + const scope = nock(url, { + reqheaders: { "User-Agent": /magda.*/ } + }); - const afterEachProperty = () => { - nock.cleanAll(); - }; + const scopeHead = scope.head(url.endsWith("/") ? "/" : ""); + const scopeGet = scope.get(url.endsWith("/") ? "/" : ""); - /** - * Builds FTP clients that have all their important methods stubbed out - these - * will respond based on the current content of ftpSuccesses. - */ - const clientFactory = () => { - const client = new Client(); - let readyCallback: () => void; - let key: string; - sinon - .stub(client, "connect") - .callsFake(({ host, port }: Client.Options) => { - const keyPort = port !== 21 ? `:${port}` : ""; - key = `${host}${keyPort}`; - if (!clients[key]) { - clients[key] = []; - } - clients[key].push(client); - readyCallback(); - }); - sinon - .stub(client, "on") - .callsFake((event: string | symbol, callback: () => void) => { - if (event === "ready") { - readyCallback = callback; + if (success !== "error") { + if (!disallowHead && success === "success") { + scopeHead.reply(200); + } else { + if (disallowHead) { + // Not everything returns a 405 for HEAD not allowed :() + scopeHead.reply(success === "success" ? 405 : 400); + } else { + scopeHead.replyWithError("fail"); } - return client; - }); - sinon.stub(client, "list").callsFake((( - path: string, - callback: (err: Error, list: string[]) => void - ) => { - try { - expect(key).not.to.be.undefined; - const url = `ftp://${key}${path}`; - - const success = ftpSuccesses[url]; - expect(success).not.to.be.undefined; if (success === "success") { - callback(null, ["file"]); - } else if (success === "notfound") { - callback(null, []); + scopeGet.reply(200, () => { + const s = new RandomStream(streamWaitTime); + + return s; + }); } else { - callback(new Error("Fake error!"), null); + scopeGet.reply(404); } - } catch (e) { - console.error(e); - callback(e as Error, null); + } + } else { + scopeHead.replyWithError("fail"); + scopeGet.replyWithError("fail"); } - }) as any); - return client; - }; - const fakeFtpHandler = new FtpHandler(clientFactory); + return scope; + } + ); - /** - * Generator-driven super-test: generates records and runs them through the - * onRecordFound function, listening for HTTP and FTP calls made and returning - * success or failure based on generated outcomes, then checks that they're - * recorded on a by-distribution basis as link status as well as on a by-record - * basis as a part of dataset quality. - */ - it("Should correctly record link statuses", function() { - const ajv = new Ajv(); - const validate = ajv.compile(schema); - return jsc.assert( - jsc.forall(recordArbWithSuccesses, jsc.integer(1, 100), function( - { record, successLookup, disallowHead }, - streamWaitTime - ) { - beforeEachProperty(); - - // Tell the FTP server to return success/failure for the various FTP - // paths with this dodgy method. Note that because the FTP server can - // only see paths and not host, we only send it the path of the req. - ftpSuccesses = _.pickBy(successLookup, (value, url) => - hasProtocol(url, "ftp") - ); + allDists.forEach((dist: Record) => { + const { downloadURL, accessURL } = dist.aspects[ + "dcat-distribution-strings" + ]; + const success = + successLookup[downloadURL] === "success" + ? "success" + : successLookup[accessURL]; + + const isUnknownProtocol = (url: string) => { + if (!url) { + return false; + } + const protocol = new URI(url).protocol(); + return ( + protocol && + protocol.length > 0 && + KNOWN_PROTOCOLS.indexOf(protocol) === -1 + ); + }; + + const downloadUnknown = isUnknownProtocol(downloadURL); + const accessUnknown = isUnknownProtocol(accessURL); + + const result = + success === "success" + ? "active" + : downloadUnknown || accessUnknown + ? "unknown" + : "broken"; + + registryScope + .put( + `/records/${encodeURIComponentWithApost( + dist.id + )}/aspects/source-link-status?merge=true`, + (body: BrokenLinkAspect) => { + const validationResult = validate(body); + if (!validationResult) { + throw new Error( + "Json schema validation error: \n" + + validate.errors + .map(error => `${error.dataPath}: ${error.message}`) + .join("\n") + ); + } - const allDists = - record.aspects["dataset-distributions"].distributions; - - const httpDistUrls = _(urlsFromDataSet(record)) - .filter((url: string) => hasProtocol(url, "http")) - .map((url: string) => ({ - url, - success: successLookup[url] - })) - .value(); - - // Set up a nock scope for every HTTP URL - the minion will actually - // attempt to download these but it'll be intercepted by nock. - const distScopes = httpDistUrls.map( - ({ - url, - success - }: { - url: string; - success: CheckResult; - }) => { - const scope = nock(url, { - reqheaders: { "User-Agent": /magda.*/ } - }); - - const scopeHead = scope.head( - url.endsWith("/") ? "/" : "" - ); - const scopeGet = scope.get( - url.endsWith("/") ? "/" : "" - ); - - if (success !== "error") { - if (!disallowHead && success === "success") { - scopeHead.reply(200); - } else { - if (disallowHead) { - // Not everything returns a 405 for HEAD not allowed :() - scopeHead.reply( - success === "success" ? 405 : 400 - ); - } else { - scopeHead.replyWithError("fail"); - } - - if (success === "success") { - scopeGet.reply(200, () => { - const s = new RandomStream( - streamWaitTime - ); - - return s; - }); - } else { - scopeGet.reply(404); - } - } - } else { - scopeHead.replyWithError("fail"); - scopeGet.replyWithError("fail"); - } - - return scope; - } + const doesStatusMatch = body.status === result; + + const isDownloadUrlHttp = hasProtocol(downloadURL, "http"); + const isAccessUrlHttp = hasProtocol(accessURL, "http"); + + const isDownloadUrlHttpSuccess = + isDownloadUrlHttp && successLookup[downloadURL] === "success"; + + const isDownloadUrlFtpSuccess = + !isDownloadUrlHttp && + successLookup[downloadURL] === "success"; + + const isAccessURLHttpSuccess = + isAccessUrlHttp && successLookup[accessURL] === "success"; + + const isHttpSuccess: boolean = + isDownloadUrlHttpSuccess || + (!isDownloadUrlFtpSuccess && isAccessURLHttpSuccess); + + const downloadUri = parseUriSafe(downloadURL); + const isDownloadUrlDefined = + _.isUndefined(downloadUri) || + !downloadUri.scheme() || + parseUriSafe(downloadURL).scheme().length === 0; + + const is404: boolean = + result === "broken" && + ((isDownloadUrlHttp && + successLookup[downloadURL] === "notfound") || + (isDownloadUrlDefined && + isAccessUrlHttp && + successLookup[accessURL] === "notfound")); + + const doesResponseCodeMatch = ((code?: number) => { + if (isHttpSuccess) { + return code === 200; + } else if (is404) { + return code === 404; + } else { + return _.isUndefined(code); + } + })(body.httpStatusCode); + + const doesErrorMatch = ((arg?: Error) => + success === "success" + ? _.isUndefined(arg) + : !_.isUndefined(arg))(body.errorDetails); + + // console.log( + // `${ + // dist.id + // }: ${doesStatusMatch} && ${doesResponseCodeMatch} && ${doesErrorMatch} ` + // ); + + return ( + doesStatusMatch && doesResponseCodeMatch && doesErrorMatch ); + } + ) + .reply(201); + }); + + return onRecordFound(record, registry, 0, 0, {}, {}, fakeFtpHandler) + .then(() => { + distScopes.forEach(scope => scope.done()); + registryScope.done(); + }) + .then(() => { + afterEachProperty(); + return true; + }) + .catch(e => { + afterEachProperty(); + throw e; + }); + }), + { + tests: 500 + } + ); + }); - allDists.forEach((dist: Record) => { - const { downloadURL, accessURL } = dist.aspects[ - "dcat-distribution-strings" - ]; - const success = - successLookup[downloadURL] === "success" - ? "success" - : successLookup[accessURL]; - - const isUnknownProtocol = (url: string) => { - if (!url) { - return false; - } - const protocol = new URI(url).protocol(); - return ( - protocol && - protocol.length > 0 && - KNOWN_PROTOCOLS.indexOf(protocol) === -1 - ); - }; - - const downloadUnknown = isUnknownProtocol(downloadURL); - const accessUnknown = isUnknownProtocol(accessURL); - - const result = - success === "success" - ? "active" - : downloadUnknown || accessUnknown - ? "unknown" - : "broken"; - - registryScope - .put( - `/records/${encodeURIComponentWithApost( - dist.id - )}/aspects/source-link-status?merge=true`, - (body: BrokenLinkAspect) => { - const validationResult = validate(body); - if (!validationResult) { - throw new Error( - "Json schema validation error: \n" + - validate.errors - .map( - error => - `${error.dataPath}: ${error.message}` - ) - .join("\n") - ); - } - - const doesStatusMatch = body.status === result; - - const isDownloadUrlHttp = hasProtocol( - downloadURL, - "http" - ); - const isAccessUrlHttp = hasProtocol( - accessURL, - "http" - ); - - const isDownloadUrlHttpSuccess = - isDownloadUrlHttp && - successLookup[downloadURL] === "success"; - - const isDownloadUrlFtpSuccess = - !isDownloadUrlHttp && - successLookup[downloadURL] === "success"; - - const isAccessURLHttpSuccess = - isAccessUrlHttp && - successLookup[accessURL] === "success"; - - const isHttpSuccess: boolean = - isDownloadUrlHttpSuccess || - (!isDownloadUrlFtpSuccess && - isAccessURLHttpSuccess); - - const downloadUri = parseUriSafe(downloadURL); - const isDownloadUrlDefined = - _.isUndefined(downloadUri) || - !downloadUri.scheme() || - parseUriSafe(downloadURL).scheme() - .length === 0; - - const is404: boolean = - result === "broken" && - ((isDownloadUrlHttp && - successLookup[downloadURL] === - "notfound") || - (isDownloadUrlDefined && - isAccessUrlHttp && - successLookup[accessURL] === - "notfound")); - - const doesResponseCodeMatch = (( - code?: number - ) => { - if (isHttpSuccess) { - return code === 200; - } else if (is404) { - return code === 404; - } else { - return _.isUndefined(code); - } - })(body.httpStatusCode); - - const doesErrorMatch = ((arg?: Error) => - success === "success" - ? _.isUndefined(arg) - : !_.isUndefined(arg))( - body.errorDetails - ); - - // console.log( - // `${ - // dist.id - // }: ${doesStatusMatch} && ${doesResponseCodeMatch} && ${doesErrorMatch} ` - // ); - - return ( - doesStatusMatch && - doesResponseCodeMatch && - doesErrorMatch - ); - } - ) - .reply(201); - }); - - return onRecordFound( - record, - registry, - 0, - 0, - {}, - {}, - fakeFtpHandler - ) - .then(() => { - distScopes.forEach(scope => scope.done()); - registryScope.done(); - }) - .then(() => { - afterEachProperty(); - return true; - }) - .catch(e => { - afterEachProperty(); - throw e; - }); - }), - { - tests: 500 - } - ); - }); + describe("retrying", () => { + /** + * Runs onRecordFound with a number of failing codes, testing whether the + * minion retries the correct number of times, and whether it correctly + * records a success after retries or a failure after the retries run out. + * + * This tests both 429 retries and other retries - this involves different + * behaviour as the retry for 429 (which indicates rate limiting) require + * a much longer cool-off time and hence are done differently. + * + * @param caption The caption to use for the mocha "it" call. + * @param result Whether to test for a number of retries then a success, a + * number of retries then a failure because of too many 429s, + * or a number of retries then a failure because of too many + * non-429 failures (e.g. 404s) + */ + const retrySpec = ( + caption: string, + result: "success" | "fail429" | "failNormal" + ) => { + it(caption, function() { + const retryCountArb = jsc.integer(0, 5); + + type FailuresArbResult = { + retryCount: number; + allResults: number[][]; + }; - describe("retrying", () => { /** - * Runs onRecordFound with a number of failing codes, testing whether the - * minion retries the correct number of times, and whether it correctly - * records a success after retries or a failure after the retries run out. - * - * This tests both 429 retries and other retries - this involves different - * behaviour as the retry for 429 (which indicates rate limiting) require - * a much longer cool-off time and hence are done differently. - * - * @param caption The caption to use for the mocha "it" call. - * @param result Whether to test for a number of retries then a success, a - * number of retries then a failure because of too many 429s, - * or a number of retries then a failure because of too many - * non-429 failures (e.g. 404s) + * Generates a retryCount and a nested array of results to return to the + * minion - the inner arrays are status codes to be returned (in order), + * after each inner array is finished a 429 will be returned, then the + * next array of error codes will be returned. */ - const retrySpec = ( - caption: string, - result: "success" | "fail429" | "failNormal" - ) => { - it(caption, function() { - const retryCountArb = jsc.integer(0, 5); - - type FailuresArbResult = { - retryCount: number; - allResults: number[][]; - }; - - /** - * Generates a retryCount and a nested array of results to return to the - * minion - the inner arrays are status codes to be returned (in order), - * after each inner array is finished a 429 will be returned, then the - * next array of error codes will be returned. - */ - const failuresArb: jsc.Arbitrary = arbFlatMap< - number, - FailuresArbResult - >( - retryCountArb, - (retryCount: number) => { - /** Generates how many 429 codes will be returned */ - const count429Arb = - result === "fail429" - ? jsc.constant(retryCount) - : jsc.integer(0, retryCount); - - /** Generates how long the array of non-429 failures should be. */ - const failureCodeLengthArb = jsc.integer(0, retryCount); - - const allResultsArb = arbFlatMap( - count429Arb, - count429s => - arrayOfSizeArb( - count429s + 1, - failureCodeLengthArb - ), - (failureCodeArr: number[]) => failureCodeArr.length - ).flatMap( - (failureCodeArrSizes: number[]) => { - const failureCodeArbs = failureCodeArrSizes.map( - size => arrayOfSizeArb(size, failureCodeArb) - ); - - if (result === "failNormal") { - failureCodeArbs[ - failureCodeArbs.length - 1 - ] = arrayOfSizeArb( - retryCount + 1, - failureCodeArb - ); - } - - return failureCodeArrSizes.length > 0 - ? jsc.tuple(failureCodeArbs) - : jsc.constant([]); - }, - failures => failures.map(inner => inner.length) - ); - - const combined = jsc.record({ - retryCount: jsc.constant(retryCount), - allResults: allResultsArb - }); - - return combined; - }, - ({ retryCount }: FailuresArbResult) => { - return retryCount; - } + const failuresArb: jsc.Arbitrary = arbFlatMap< + number, + FailuresArbResult + >( + retryCountArb, + (retryCount: number) => { + /** Generates how many 429 codes will be returned */ + const count429Arb = + result === "fail429" + ? jsc.constant(retryCount) + : jsc.integer(0, retryCount); + + /** Generates how long the array of non-429 failures should be. */ + const failureCodeLengthArb = jsc.integer(0, retryCount); + + const allResultsArb = arbFlatMap( + count429Arb, + count429s => arrayOfSizeArb(count429s + 1, failureCodeLengthArb), + (failureCodeArr: number[]) => failureCodeArr.length + ).flatMap( + (failureCodeArrSizes: number[]) => { + const failureCodeArbs = failureCodeArrSizes.map(size => + arrayOfSizeArb(size, failureCodeArb) ); - return jsc.assert( - jsc.forall( - httpOnlyRecordArb, - failuresArb, - ( - record: Record, - { retryCount, allResults }: FailuresArbResult - ) => { - beforeEachProperty(); - - const distScopes = urlsFromDataSet(record).map( - url => { - const scope = nock(url); //.log(console.log); - - allResults.forEach((failureCodes, i) => { - failureCodes.forEach(failureCode => { - scope - .head( - url.endsWith("/") ? "/" : "" - ) - .reply(failureCode); - - scope - .get( - url.endsWith("/") ? "/" : "" - ) - .reply(failureCode); - }); - if ( - i < allResults.length - 1 || - result === "fail429" - ) { - scope - .head( - url.endsWith("/") ? "/" : "" - ) - .reply(429); - } - }); - - if (result === "success") { - scope - .head(url.endsWith("/") ? "/" : "") - .reply(200); - } - - return scope; - } - ); - - const allDists = - record.aspects["dataset-distributions"] - .distributions; - - allDists.forEach((dist: Record) => { - registryScope - .put( - `/records/${encodeURIComponentWithApost( - dist.id - )}/aspects/source-link-status?merge=true`, - (response: any) => { - const statusMatch = - response.status === - { - success: "active", - failNormal: "broken", - fail429: "unknown" - }[result]; - const codeMatch = - !_.isUndefined( - response.httpStatusCode - ) && - response.httpStatusCode === - { - success: 200, - failNormal: _.last( - _.last(allResults) - ), - fail429: 429 - }[result]; - - return statusMatch && codeMatch; - } - ) - .reply(201); - }); - - return onRecordFound( - record, - registry, - retryCount, - 0 - ) - .then(() => { - registryScope.done(); - distScopes.forEach(scope => scope.done()); - }) - .then(() => { - afterEachProperty(); - return true; - }) - .catch(e => { - afterEachProperty(); - throw e; - }); - } - ), - { - tests: 50 - } - ); + if (result === "failNormal") { + failureCodeArbs[failureCodeArbs.length - 1] = arrayOfSizeArb( + retryCount + 1, + failureCodeArb + ); + } + + return failureCodeArrSizes.length > 0 + ? jsc.tuple(failureCodeArbs) + : jsc.constant([]); + }, + failures => failures.map(inner => inner.length) + ); + + const combined = jsc.record({ + retryCount: jsc.constant(retryCount), + allResults: allResultsArb }); - }; - retrySpec( - "Should result in success if the last retry is successful", - "success" + return combined; + }, + ({ retryCount }: FailuresArbResult) => { + return retryCount; + } ); - retrySpec( - "Should result in failures if the max number of retries is exceeded", - "failNormal" - ); - retrySpec( - "Should result in failures if the max number of 429s is exceeded", - "fail429" - ); - }); - it("Should only try to make one request per host at a time", function() { - const urlArb = (jsc as any).nonshrink( - distUrlArb({ - schemeArb: jsc.elements(["http", "https"]), - hostArb: jsc.elements(["example1", "example2", "example3"]) - }) - ); + return jsc.assert( + jsc.forall( + httpOnlyRecordArb, + failuresArb, + (record: Record, { retryCount, allResults }: FailuresArbResult) => { + beforeEachProperty(); + + const distScopes = urlsFromDataSet(record).map(url => { + const scope = nock(url); //.log(console.log); + + allResults.forEach((failureCodes, i) => { + failureCodes.forEach(failureCode => { + scope.head(url.endsWith("/") ? "/" : "").reply(failureCode); + + scope.get(url.endsWith("/") ? "/" : "").reply(failureCode); + }); + if (i < allResults.length - 1 || result === "fail429") { + scope.head(url.endsWith("/") ? "/" : "").reply(429); + } + }); - const thisRecordArb = jsc.suchthat( - recordArbWithDistArbs({ url: urlArb }), - record => { - const urls: string[] = urlsFromDataSet(record); - const hosts: string[] = urls.map(url => { - const uri = new URI(url); + if (result === "success") { + scope.head(url.endsWith("/") ? "/" : "").reply(200); + } - return uri.scheme() + "://" + uri.host(); + return scope; + }); + + const allDists = + record.aspects["dataset-distributions"].distributions; + + allDists.forEach((dist: Record) => { + registryScope + .put( + `/records/${encodeURIComponentWithApost( + dist.id + )}/aspects/source-link-status?merge=true`, + (response: any) => { + const statusMatch = + response.status === + { + success: "active", + failNormal: "broken", + fail429: "unknown" + }[result]; + const codeMatch = + !_.isUndefined(response.httpStatusCode) && + response.httpStatusCode === + { + success: 200, + failNormal: _.last(_.last(allResults)), + fail429: 429 + }[result]; + + return statusMatch && codeMatch; + } + ) + .reply(201); + }); + + return onRecordFound(record, registry, retryCount, 0) + .then(() => { + registryScope.done(); + distScopes.forEach(scope => scope.done()); + }) + .then(() => { + afterEachProperty(); + return true; + }) + .catch(e => { + afterEachProperty(); + throw e; }); - - return !_.isEqual(_.uniq(hosts), hosts); } + ), + { + tests: 50 + } ); + }); + }; - return jsc.assert( - jsc.forall( - thisRecordArb, - jsc.nearray(failureCodeArb), - jsc.integer(0, 25), - (record: Record, failures: number[], delayMs: number) => { - beforeEachProperty(); - - const delayConfig = {} as any; - - const distScopes = urlsFromDataSet(record).reduce( - (scopeLookup, url) => { - const uri = new URI(url); - const base = uri.scheme() + "://" + uri.host(); - delayConfig[uri.hostname()] = (delayMs + 10) / 1000; - - if (!scopeLookup[base]) { - scopeLookup[base] = nock(base); - } - - const scope = scopeLookup[base]; - - failures.forEach(failureCode => { - scope - .head(uri.path()) - .delay(delayMs) - .reply(failureCode); - - scope - .get(uri.path()) - .delay(delayMs) - .reply(failureCode); - }); - - scope - .head(uri.path()) - .delay(delayMs) - .reply(200); - return scopeLookup; - }, - {} as { [host: string]: nock.Scope } - ); - - _.forEach(distScopes, (scope: nock.Scope, host: string) => { - let countForThisScope = 0; - - scope.on("request", () => { - countForThisScope++; - expect(countForThisScope).to.equal(1); - }); - - scope.on("replied", () => { - countForThisScope--; - expect(countForThisScope).to.equal(0); - }); - }); - - const allDists = - record.aspects["dataset-distributions"].distributions; - - registryScope - .put(/.*/) - .times(allDists.length) - .reply(201); - - return onRecordFound( - record, - registry, - failures.length, - 0, - delayConfig - ) - .then(() => { - _.values(distScopes).forEach(scope => scope.done()); - }) - .then(() => { - afterEachProperty(); - return true; - }) - .catch(e => { - afterEachProperty(); - throw e; - }); - } - ), - { - tests: 10 - } - ); - }); - - const emptyRecordArb = jsc.oneof([ - specificRecordArb({ - "dataset-distributions": jsc.constant(undefined) - }), - specificRecordArb({ - "dataset-distributions": jsc.record({ - distributions: jsc.constant([]) - }) - }) - ]); + retrySpec( + "Should result in success if the last retry is successful", + "success" + ); + retrySpec( + "Should result in failures if the max number of retries is exceeded", + "failNormal" + ); + retrySpec( + "Should result in failures if the max number of 429s is exceeded", + "fail429" + ); + }); + + it("Should only try to make one request per host at a time", function() { + const urlArb = (jsc as any).nonshrink( + distUrlArb({ + schemeArb: jsc.elements(["http", "https"]), + hostArb: jsc.elements(["example1", "example2", "example3"]) + }) + ); + + const thisRecordArb = jsc.suchthat( + recordArbWithDistArbs({ url: urlArb }), + record => { + const urls: string[] = urlsFromDataSet(record); + const hosts: string[] = urls.map(url => { + const uri = new URI(url); - jsc.property( - "Should do nothing if no distributions", - emptyRecordArb, - record => { - beforeEachProperty(); + return uri.scheme() + "://" + uri.host(); + }); + + return !_.isEqual(_.uniq(hosts), hosts); + } + ); - return onRecordFound(record, registry).then(() => { - afterEachProperty(); + return jsc.assert( + jsc.forall( + thisRecordArb, + jsc.nearray(failureCodeArb), + jsc.integer(0, 25), + (record: Record, failures: number[], delayMs: number) => { + beforeEachProperty(); + + const delayConfig = {} as any; + + const distScopes = urlsFromDataSet(record).reduce( + (scopeLookup, url) => { + const uri = new URI(url); + const base = uri.scheme() + "://" + uri.host(); + delayConfig[uri.hostname()] = (delayMs + 10) / 1000; + + if (!scopeLookup[base]) { + scopeLookup[base] = nock(base); + } + + const scope = scopeLookup[base]; + + failures.forEach(failureCode => { + scope + .head(uri.path()) + .delay(delayMs) + .reply(failureCode); + + scope + .get(uri.path()) + .delay(delayMs) + .reply(failureCode); + }); + + scope + .head(uri.path()) + .delay(delayMs) + .reply(200); + return scopeLookup; + }, + {} as { [host: string]: nock.Scope } + ); + + _.forEach(distScopes, (scope: nock.Scope, host: string) => { + let countForThisScope = 0; + + scope.on("request", () => { + countForThisScope++; + expect(countForThisScope).to.equal(1); + }); - registryScope.done(); - return true; + scope.on("replied", () => { + countForThisScope--; + expect(countForThisScope).to.equal(0); + }); + }); + + const allDists = + record.aspects["dataset-distributions"].distributions; + + registryScope + .put(/.*/) + .times(allDists.length) + .reply(201); + + return onRecordFound( + record, + registry, + failures.length, + 0, + delayConfig + ) + .then(() => { + _.values(distScopes).forEach(scope => scope.done()); + }) + .then(() => { + afterEachProperty(); + return true; + }) + .catch(e => { + afterEachProperty(); + throw e; }); } + ), + { + tests: 10 + } ); + }); + + const emptyRecordArb = jsc.oneof([ + specificRecordArb({ + "dataset-distributions": jsc.constant(undefined) + }), + specificRecordArb({ + "dataset-distributions": jsc.record({ + distributions: jsc.constant([]) + }) + }) + ]); + + jsc.property( + "Should do nothing if no distributions", + emptyRecordArb, + record => { + beforeEachProperty(); + + return onRecordFound(record, registry).then(() => { + afterEachProperty(); + + registryScope.done(); + return true; + }); + } + ); }); function hasProtocol(url: string, protocol: string) { - const uri = parseUriSafe(url); + const uri = parseUriSafe(url); - return uri && uri.protocol().startsWith(protocol); + return uri && uri.protocol().startsWith(protocol); } diff --git a/src/test/wait.spec.ts b/src/test/wait.spec.ts index 709f955..4459444 100644 --- a/src/test/wait.spec.ts +++ b/src/test/wait.spec.ts @@ -2,36 +2,37 @@ import {} from "mocha"; import _ from "lodash"; import jsc from "jsverify"; import { expect } from "chai"; -import wait from "../wait"; -import { shrink } from "jsverify"; +import wait from "../wait.js"; +import { default as jsverify } from "jsverify"; +const { shrink } = jsverify; describe("wait function", () => { - it("should return resolved promise if passed 0 parameter", done => { - const promise = wait(0); - let isResolved = false; - promise.then(() => { - isResolved = true; - }); - _.defer(() => { - expect(isResolved).to.equal(true); - done(); - }); + it("should return resolved promise if passed 0 parameter", done => { + const promise = wait(0); + let isResolved = false; + promise.then(() => { + isResolved = true; }); - - it("should wait around `waitTime` milliseconds", async function(this: Mocha.Context) { - this.timeout(10000); - return jsc.assert( - jsc.forall( - { ...jsc.integer(1, 2000), shrink: shrink.noop }, - async function(waitTime: number) { - const now = new Date().getTime(); - await wait(waitTime); - const newTime = new Date().getTime(); - expect(newTime).to.be.closeTo(now + waitTime, 100); - return true; - } - ), - { tests: 3 } - ); + _.defer(() => { + expect(isResolved).to.equal(true); + done(); }); + }); + + it("should wait around `waitTime` milliseconds", async function(this: Mocha.Context) { + this.timeout(10000); + return jsc.assert( + jsc.forall( + { ...jsc.integer(1, 2000), shrink: shrink.noop }, + async function(waitTime: number) { + const now = new Date().getTime(); + await wait(waitTime); + const newTime = new Date().getTime(); + expect(newTime).to.be.closeTo(now + waitTime, 100); + return true; + } + ), + { tests: 3 } + ); + }); }); diff --git a/tsconfig-global.json b/tsconfig-global.json index 50bff40..170e2b6 100644 --- a/tsconfig-global.json +++ b/tsconfig-global.json @@ -1,72 +1,71 @@ { - "compilerOptions": { - /* Basic Options */ - // "incremental": true, /* Enable incremental compilation */ - "target": "ES2015" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */, - "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */, - "lib": [ - "es2015", - "es2017", - "dom", - "ScriptHost" - ] /* Specify library files to be included in the compilation. */, - "allowJs": true /* Allow javascript files to be compiled. */, - "resolveJsonModule": true, - // "checkJs": true, /* Report errors in .js files. */ - // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ - "declaration": true /* Generates corresponding '.d.ts' file. */, - // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ - "sourceMap": true /* Generates corresponding '.map' file. */, - // "outFile": "./", /* Concatenate and emit output to single file. */ - // "outDir": "./", /* Redirect output structure to the directory. */ - // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ - "composite": true /* Enable project compilation */, - // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ - // "removeComments": true, /* Do not emit comments to output. */ - // "noEmit": true, /* Do not emit outputs. */ - // "importHelpers": true, /* Import emit helpers from 'tslib'. */ - // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ - // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ + "compilerOptions": { + /* Basic Options */ + // "incremental": true, /* Enable incremental compilation */ + "target": "ES2022" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */, + "module": "Node16" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */, + "lib": [ + "es2015", + "es2017", + "dom" + ] /* Specify library files to be included in the compilation. */, + "allowJs": true /* Allow javascript files to be compiled. */, + "resolveJsonModule": true, + // "checkJs": true, /* Report errors in .js files. */ + // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ + "declaration": true /* Generates corresponding '.d.ts' file. */, + // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ + "sourceMap": true /* Generates corresponding '.map' file. */, + // "outFile": "./", /* Concatenate and emit output to single file. */ + // "outDir": "./", /* Redirect output structure to the directory. */ + // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ + "composite": true /* Enable project compilation */, + // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ + // "removeComments": true, /* Do not emit comments to output. */ + // "noEmit": true, /* Do not emit outputs. */ + // "importHelpers": true, /* Import emit helpers from 'tslib'. */ + // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ + // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ - /* Strict Type-Checking Options */ - "strict": true /* Enable all strict type-checking options. */, - "noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */, - "strictNullChecks": false /* Enable strict null checks. */, - // "strictFunctionTypes": true, /* Enable strict checking of function types. */ - // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ - // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ - "noImplicitThis": true /* Raise error on 'this' expressions with an implied 'any' type. */, - "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */, + /* Strict Type-Checking Options */ + "strict": true /* Enable all strict type-checking options. */, + "noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */, + // "strictNullChecks": true, /* Enable strict null checks. */ + // "strictFunctionTypes": true, /* Enable strict checking of function types. */ + // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ + // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ + "noImplicitThis": true /* Raise error on 'this' expressions with an implied 'any' type. */, + "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */, - /* Additional Checks */ - "noUnusedLocals": true /* Report errors on unused locals. */, - // "noUnusedParameters": true, /* Report errors on unused parameters. */ - "noImplicitReturns": true /* Report error when not all code paths in function return a value. */, - // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ + /* Additional Checks */ + "noUnusedLocals": true /* Report errors on unused locals. */, + // "noUnusedParameters": true, /* Report errors on unused parameters. */ + "noImplicitReturns": true /* Report error when not all code paths in function return a value. */, + // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ - /* Module Resolution Options */ - "moduleResolution": "node" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */, - "baseUrl": "./" /* Base directory to resolve non-absolute module names. */, - // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ - // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ - // "typeRoots": [], /* List of folders to include type definitions from. */ - // "types": [], /* Type declaration files to be included in compilation. */ - // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ - "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, - // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ - // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + /* Module Resolution Options */ + "moduleResolution": "Node16" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */, + "baseUrl": "./" /* Base directory to resolve non-absolute module names. */, + // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ + // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ + // "typeRoots": [], /* List of folders to include type definitions from. */ + // "types": [], /* Type declaration files to be included in compilation. */ + // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ + "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, + // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ - /* Source Map Options */ - // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ - // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ - // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ - // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ + /* Source Map Options */ + // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ + // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ - /* Experimental Options */ - // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ - // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + /* Experimental Options */ + // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ + // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ - /* Advanced Options */ - "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ - } + /* Advanced Options */ + "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ + } } diff --git a/yarn.lock b/yarn.lock index 5b34fab..7700b88 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23,35 +23,153 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@cspotcode/source-map-support@^0.8.0": - version "0.8.1" - resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" - integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw== - dependencies: - "@jridgewell/trace-mapping" "0.3.9" +"@bcoe/v8-coverage@^0.2.3": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" + integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== + +"@esbuild/aix-ppc64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.19.12.tgz#d1bc06aedb6936b3b6d313bf809a5a40387d2b7f" + integrity sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA== + +"@esbuild/android-arm64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.19.12.tgz#7ad65a36cfdb7e0d429c353e00f680d737c2aed4" + integrity sha512-P0UVNGIienjZv3f5zq0DP3Nt2IE/3plFzuaS96vihvD0Hd6H/q4WXUGpCxD/E8YrSXfNyRPbpTq+T8ZQioSuPA== + +"@esbuild/android-arm@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.19.12.tgz#b0c26536f37776162ca8bde25e42040c203f2824" + integrity sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w== + +"@esbuild/android-x64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.19.12.tgz#cb13e2211282012194d89bf3bfe7721273473b3d" + integrity sha512-3k7ZoUW6Q6YqhdhIaq/WZ7HwBpnFBlW905Fa4s4qWJyiNOgT1dOqDiVAQFwBH7gBRZr17gLrlFCRzF6jFh7Kew== + +"@esbuild/darwin-arm64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.19.12.tgz#cbee41e988020d4b516e9d9e44dd29200996275e" + integrity sha512-B6IeSgZgtEzGC42jsI+YYu9Z3HKRxp8ZT3cqhvliEHovq8HSX2YX8lNocDn79gCKJXOSaEot9MVYky7AKjCs8g== + +"@esbuild/darwin-x64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.19.12.tgz#e37d9633246d52aecf491ee916ece709f9d5f4cd" + integrity sha512-hKoVkKzFiToTgn+41qGhsUJXFlIjxI/jSYeZf3ugemDYZldIXIxhvwN6erJGlX4t5h417iFuheZ7l+YVn05N3A== + +"@esbuild/freebsd-arm64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.12.tgz#1ee4d8b682ed363b08af74d1ea2b2b4dbba76487" + integrity sha512-4aRvFIXmwAcDBw9AueDQ2YnGmz5L6obe5kmPT8Vd+/+x/JMVKCgdcRwH6APrbpNXsPz+K653Qg8HB/oXvXVukA== + +"@esbuild/freebsd-x64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.19.12.tgz#37a693553d42ff77cd7126764b535fb6cc28a11c" + integrity sha512-EYoXZ4d8xtBoVN7CEwWY2IN4ho76xjYXqSXMNccFSx2lgqOG/1TBPW0yPx1bJZk94qu3tX0fycJeeQsKovA8gg== + +"@esbuild/linux-arm64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.19.12.tgz#be9b145985ec6c57470e0e051d887b09dddb2d4b" + integrity sha512-EoTjyYyLuVPfdPLsGVVVC8a0p1BFFvtpQDB/YLEhaXyf/5bczaGeN15QkR+O4S5LeJ92Tqotve7i1jn35qwvdA== + +"@esbuild/linux-arm@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.19.12.tgz#207ecd982a8db95f7b5279207d0ff2331acf5eef" + integrity sha512-J5jPms//KhSNv+LO1S1TX1UWp1ucM6N6XuL6ITdKWElCu8wXP72l9MM0zDTzzeikVyqFE6U8YAV9/tFyj0ti+w== + +"@esbuild/linux-ia32@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.19.12.tgz#d0d86b5ca1562523dc284a6723293a52d5860601" + integrity sha512-Thsa42rrP1+UIGaWz47uydHSBOgTUnwBwNq59khgIwktK6x60Hivfbux9iNR0eHCHzOLjLMLfUMLCypBkZXMHA== + +"@esbuild/linux-loong64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.19.12.tgz#9a37f87fec4b8408e682b528391fa22afd952299" + integrity sha512-LiXdXA0s3IqRRjm6rV6XaWATScKAXjI4R4LoDlvO7+yQqFdlr1Bax62sRwkVvRIrwXxvtYEHHI4dm50jAXkuAA== + +"@esbuild/linux-mips64el@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.19.12.tgz#4ddebd4e6eeba20b509d8e74c8e30d8ace0b89ec" + integrity sha512-fEnAuj5VGTanfJ07ff0gOA6IPsvrVHLVb6Lyd1g2/ed67oU1eFzL0r9WL7ZzscD+/N6i3dWumGE1Un4f7Amf+w== + +"@esbuild/linux-ppc64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.19.12.tgz#adb67dadb73656849f63cd522f5ecb351dd8dee8" + integrity sha512-nYJA2/QPimDQOh1rKWedNOe3Gfc8PabU7HT3iXWtNUbRzXS9+vgB0Fjaqr//XNbd82mCxHzik2qotuI89cfixg== + +"@esbuild/linux-riscv64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.19.12.tgz#11bc0698bf0a2abf8727f1c7ace2112612c15adf" + integrity sha512-2MueBrlPQCw5dVJJpQdUYgeqIzDQgw3QtiAHUC4RBz9FXPrskyyU3VI1hw7C0BSKB9OduwSJ79FTCqtGMWqJHg== + +"@esbuild/linux-s390x@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.19.12.tgz#e86fb8ffba7c5c92ba91fc3b27ed5a70196c3cc8" + integrity sha512-+Pil1Nv3Umes4m3AZKqA2anfhJiVmNCYkPchwFJNEJN5QxmTs1uzyy4TvmDrCRNT2ApwSari7ZIgrPeUx4UZDg== + +"@esbuild/linux-x64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.19.12.tgz#5f37cfdc705aea687dfe5dfbec086a05acfe9c78" + integrity sha512-B71g1QpxfwBvNrfyJdVDexenDIt1CiDN1TIXLbhOw0KhJzE78KIFGX6OJ9MrtC0oOqMWf+0xop4qEU8JrJTwCg== + +"@esbuild/netbsd-x64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.19.12.tgz#29da566a75324e0d0dd7e47519ba2f7ef168657b" + integrity sha512-3ltjQ7n1owJgFbuC61Oj++XhtzmymoCihNFgT84UAmJnxJfm4sYCiSLTXZtE00VWYpPMYc+ZQmB6xbSdVh0JWA== + +"@esbuild/openbsd-x64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.19.12.tgz#306c0acbdb5a99c95be98bdd1d47c916e7dc3ff0" + integrity sha512-RbrfTB9SWsr0kWmb9srfF+L933uMDdu9BIzdA7os2t0TXhCRjrQyCeOt6wVxr79CKD4c+p+YhCj31HBkYcXebw== + +"@esbuild/sunos-x64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.19.12.tgz#0933eaab9af8b9b2c930236f62aae3fc593faf30" + integrity sha512-HKjJwRrW8uWtCQnQOz9qcU3mUZhTUQvi56Q8DPTLLB+DawoiQdjsYq+j+D3s9I8VFtDr+F9CjgXKKC4ss89IeA== + +"@esbuild/win32-arm64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.19.12.tgz#773bdbaa1971b36db2f6560088639ccd1e6773ae" + integrity sha512-URgtR1dJnmGvX864pn1B2YUYNzjmXkuJOIqG2HdU62MVS4EHpU2946OZoTMnRUHklGtJdJZ33QfzdjGACXhn1A== + +"@esbuild/win32-ia32@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.19.12.tgz#000516cad06354cc84a73f0943a4aa690ef6fd67" + integrity sha512-+ZOE6pUkMOJfmxmBZElNOx72NKpIa/HFOMGzu8fqzQJ5kgf6aTGrcJaFsNiVMH4JKpMipyK+7k0n2UXN7a8YKQ== + +"@esbuild/win32-x64@0.19.12": + version "0.19.12" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.19.12.tgz#c57c8afbb4054a3ab8317591a0b7320360b444ae" + integrity sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA== + +"@istanbuljs/schema@^0.1.2", "@istanbuljs/schema@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.3.tgz#e45e384e4b8ec16bce2fd903af78450f6bf7ec98" + integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA== -"@jridgewell/resolve-uri@^3.0.3": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" - integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== -"@jridgewell/sourcemap-codec@^1.4.10": - version "1.4.14" - resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" - integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== +"@jridgewell/sourcemap-codec@^1.4.14": + version "1.4.15" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" + integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== -"@jridgewell/trace-mapping@0.3.9": - version "0.3.9" - resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9" - integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ== +"@jridgewell/trace-mapping@^0.3.12": + version "0.3.25" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" + integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== dependencies: - "@jridgewell/resolve-uri" "^3.0.3" - "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" -"@magda/arbitraries@^2.1.1": - version "2.1.1" - resolved "https://registry.yarnpkg.com/@magda/arbitraries/-/arbitraries-2.1.1.tgz#5e599d157dd494b547b0b0350d61f5428de25e6b" - integrity sha512-+Y4ij/VikXQ3jG4PjSxzSQAJcCe0K9J/t8qQ0JWSaczTm9mJmc2cDCsQfcnE4IXnBakq98G3Hljsvguu0aJmjg== +"@magda/arbitraries@^3.0.2-alpha.0": + version "3.0.2-alpha.0" + resolved "https://registry.yarnpkg.com/@magda/arbitraries/-/arbitraries-3.0.2-alpha.0.tgz#14602378199107deac505354b5c861b990411eae" + integrity sha512-oXZ2CSy8NNjig05obngSTaiIvoSNNRVGDMBJeOcfClid7KLURI6lSnH+RYTGiJrJI2aA8+DgttX+9HZxgl3qOA== dependencies: jsverify "^0.8.2" @@ -64,34 +182,41 @@ recursive-readdir "^2.2.2" yaml "^1.10.2" -"@magda/docker-utils@^2.1.1": - version "2.1.1" - resolved "https://registry.yarnpkg.com/@magda/docker-utils/-/docker-utils-2.1.1.tgz#7534986693dfa17bfcb4ef54e40328c9ffabdfbb" - integrity sha512-ohfEHWVRrt/hUn+fZDP5C1hDT8SZea8NRXqvl7JUh3k4TgRohGYUqPtn5ze/3YOvaJIG0pmoxkp6WEFCJf+rUQ== +"@magda/docker-utils@^3.0.2-alpha.0": + version "3.0.2-alpha.0" + resolved "https://registry.yarnpkg.com/@magda/docker-utils/-/docker-utils-3.0.2-alpha.0.tgz#d700d5f5efd096091757d83f0cd35bba03a01b1a" + integrity sha512-Oynq+q+SKKhMUpsYtR8pDAVyWPvVYTcJPE7xHxCq7RuflTj67zh/11qCyDluT2doknhzbHzCMX1DmealF7S6+g== dependencies: - fs-extra "^2.1.2" + fs-extra "^11.2.0" is-subdir "^1.0.2" lodash "^4.17.5" yargs "^12.0.5" -"@magda/minion-sdk@^2.1.1": - version "2.1.1" - resolved "https://registry.yarnpkg.com/@magda/minion-sdk/-/minion-sdk-2.1.1.tgz#1fc2040a402d72def5ba1ed1916dd5c67a98ffbf" - integrity sha512-0QvSAtuRCFT2uLMqaWgIt2flFG7q7RR53Je1cYlhKx9fphUVNtS5gYbSfdcqbRjihipSac2AMZCfvFK5jgixDA== +"@magda/esm-utils@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@magda/esm-utils/-/esm-utils-1.0.1.tgz#1d5e9882958aefa879106e55d32421874baae559" + integrity sha512-A300Mds4wl41axrqcrqrw4y1EjUIcrVWipaZpTwlf6ChDLnI57bEqL9EVQLm8muW/qMnY3QzNjxg7EEYly18yA== + +"@magda/minion-sdk@^3.0.2-alpha.0": + version "3.0.2-alpha.0" + resolved "https://registry.yarnpkg.com/@magda/minion-sdk/-/minion-sdk-3.0.2-alpha.0.tgz#2483f1c5350ebcb9ddfbc754fec766161c8fd1f8" + integrity sha512-/4fBJrxm8mSEwOA1qAWgn5CeH2Kh+0qsmt9GyKLSnvVfBE4JGSBYZC0inaoHEK8c6ynJ4BLg/MmAboRabIeTDA== dependencies: - "@types/express" "^4.17.6" + "@types/express" "^4.17.21" "@types/urijs" "^1.19.19" "@types/yargs" "^12.0.8" -"@magda/registry-aspects@^2.1.1": - version "2.1.1" - resolved "https://registry.yarnpkg.com/@magda/registry-aspects/-/registry-aspects-2.1.1.tgz#0e06085de6307c6a368483d186323f3df1395717" - integrity sha512-wiAX2FTap6Nt0RxhPFZOmWtKaoUQmeF6GOYcHOhfRsHIc8aJ1UqYdWo52X5sf+FQWuIXrQNzpsnJumNr942T7w== +"@magda/registry-aspects@^3.0.2-alpha.0": + version "3.0.2-alpha.0" + resolved "https://registry.yarnpkg.com/@magda/registry-aspects/-/registry-aspects-3.0.2-alpha.0.tgz#55f1da0285e0001debc241ad15e8ef7aa803a80f" + integrity sha512-dQU++0a2G1Nljdx+duNvWODOUv5jLzhn9z1f//Bp7EZjFrkdIqagd+zJw6oSDDYl0wD+PEBGX1ek4NaEHhvgxw== + dependencies: + "@magda/esm-utils" "^1.0.1" -"@magda/utils@^2.1.1": - version "2.1.1" - resolved "https://registry.yarnpkg.com/@magda/utils/-/utils-2.1.1.tgz#40864414bd5a7bc8f30e2606f659904552ce8953" - integrity sha512-Ezgn8xofgIIwXyziDimNAqXFm51F1df8bMISj3Pz0VennbDFiBT6iTYE7Ml2x/Sdl0JEliJRtCGIRM64WW1vXA== +"@magda/utils@^3.0.2-alpha.0": + version "3.0.2-alpha.0" + resolved "https://registry.yarnpkg.com/@magda/utils/-/utils-3.0.2-alpha.0.tgz#cf0a1a2a235c5595710295410bf25b88bad9c350" + integrity sha512-grZrczKax+L+07dZ24OKTxVu2bO0Po/LSVcapP6Ts6wqdyi/1yldCub8qV5fi4uyGIV8Ke2F4fVMWgBgOusM+A== dependencies: "@types/request" "^2.48.1" @@ -124,26 +249,6 @@ resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz#8da5c6530915653f3a1f38fd5f101d8c3f8079c5" integrity sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ== -"@tsconfig/node10@^1.0.7": - version "1.0.9" - resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2" - integrity sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA== - -"@tsconfig/node12@^1.0.7": - version "1.0.11" - resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.11.tgz#ee3def1f27d9ed66dac6e46a295cffb0152e058d" - integrity sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag== - -"@tsconfig/node14@^1.0.0": - version "1.0.3" - resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.3.tgz#e4386316284f00b98435bf40f72f75a09dabf6c1" - integrity sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow== - -"@tsconfig/node16@^1.0.2": - version "1.0.3" - resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.3.tgz#472eaab5f15c1ffdd7f8628bd4c4f753995ec79e" - integrity sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ== - "@types/body-parser@*": version "1.19.1" resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.1.tgz#0c0174c42a7d017b818303d4b5d969cb0b75929c" @@ -169,22 +274,23 @@ dependencies: "@types/node" "*" -"@types/express-serve-static-core@^4.17.18": - version "4.17.24" - resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.24.tgz#ea41f93bf7e0d59cd5a76665068ed6aab6815c07" - integrity sha512-3UJuW+Qxhzwjq3xhwXm2onQcFHn76frIYVbTu+kn24LFxI+dEhdfISDFovPB8VpEgW8oQCTpRuCe+0zJxB7NEA== +"@types/express-serve-static-core@^4.17.33": + version "4.19.0" + resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.19.0.tgz#3ae8ab3767d98d0b682cda063c3339e1e86ccfaa" + integrity sha512-bGyep3JqPCRry1wq+O5n7oiBgGWmeIJXPjXXCo8EK0u8duZGSYar7cGqd3ML2JUsLGeB7fmc06KYo9fLGWqPvQ== dependencies: "@types/node" "*" "@types/qs" "*" "@types/range-parser" "*" + "@types/send" "*" -"@types/express@^4.17.6": - version "4.17.13" - resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.13.tgz#a76e2995728999bab51a33fabce1d705a3709034" - integrity sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA== +"@types/express@^4.17.21": + version "4.17.21" + resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.21.tgz#c26d4a151e60efe0084b23dc3369ebc631ed192d" + integrity sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ== dependencies: "@types/body-parser" "*" - "@types/express-serve-static-core" "^4.17.18" + "@types/express-serve-static-core" "^4.17.33" "@types/qs" "*" "@types/serve-static" "*" @@ -195,6 +301,11 @@ dependencies: "@types/node" "*" +"@types/istanbul-lib-coverage@^2.0.1": + version "2.0.6" + resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz#7739c232a1fee9b4d3ce8985f314c0c6d33549d7" + integrity sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w== + "@types/lodash@^4.14.185": version "4.14.186" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.186.tgz#862e5514dd7bd66ada6c70ee5fce844b06c8ee97" @@ -232,6 +343,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.10.3.tgz#7a8f2838603ea314d1d22bb3171d899e15c57bd5" integrity sha512-ho3Ruq+fFnBrZhUYI46n/bV2GjwzSkwuT4dTf0GkuNFmnb8nq4ny2z9JEVemFi6bdEJanHLlYfy9c6FN9B9McQ== +"@types/node@^18.19.31": + version "18.19.31" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.31.tgz#b7d4a00f7cb826b60a543cebdbda5d189aaecdcd" + integrity sha512-ArgCD39YpyyrtFKIqMDvjz79jto5fcI/SVUs2HwB+f0dAzq68yqOdyaSivLiLugSziTpNXLQrVb7RZFmdZzbhA== + dependencies: + undici-types "~5.26.4" + "@types/normalize-package-data@*", "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" @@ -264,6 +382,14 @@ "@types/tough-cookie" "*" form-data "^2.5.0" +"@types/send@*": + version "0.17.4" + resolved "https://registry.yarnpkg.com/@types/send/-/send-0.17.4.tgz#6619cd24e7270793702e4e6a4b958a9010cfc57a" + integrity sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA== + dependencies: + "@types/mime" "^1" + "@types/node" "*" + "@types/serve-static@*": version "1.13.10" resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.10.tgz#f5e0ce8797d2d7cc5ebeda48a52c96c4fa47a8d9" @@ -292,22 +418,7 @@ resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-12.0.20.tgz#50c390e95b8dd32105560e08ea0571dde99d2d8a" integrity sha512-MjOKUoDmNattFOBJvAZng7X9KXIKSGy6XHoXY9mASkKwCn35X4Ckh+Ugv1DewXZXrWYXMNtLiXhlCfWlpcAV+Q== -"@ungap/promise-all-settled@1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz#aa58042711d6e3275dd37dc597e5d31e8c290a44" - integrity sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q== - -acorn-walk@^8.1.1: - version "8.2.0" - resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" - integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== - -acorn@^8.4.1: - version "8.8.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.0.tgz#88c0187620435c7f6015803f5539dae05a9dbea8" - integrity sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w== - -ajv@^6.10.2: +ajv@^6.10.2, ajv@^6.12.3: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -359,11 +470,6 @@ anymatch@~3.1.2: normalize-path "^3.0.0" picomatch "^2.0.4" -arg@^4.1.0: - version "4.1.3" - resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" - integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== - argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -391,10 +497,22 @@ arrify@^2.0.1: resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== -assertion-error@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b" - integrity sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw== +asn1@~0.2.3: + version "0.2.6" + resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" + integrity sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ== + dependencies: + safer-buffer "~2.1.0" + +assert-plus@1.0.0, assert-plus@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" + integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== + +assertion-error@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-2.0.1.tgz#f641a196b335690b1070bf00b6e7593fec190bf7" + integrity sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA== asynckit@^0.4.0: version "0.4.0" @@ -406,11 +524,28 @@ at-least-node@^1.0.0: resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2" integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + integrity sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA== + +aws4@^1.8.0: + version "1.12.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.12.0.tgz#ce1c9d143389679e253b314241ea9aa5cec980d3" + integrity sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg== + balanced-match@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +bcrypt-pbkdf@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" + integrity sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w== + dependencies: + tweetnacl "^0.14.3" + better-path-resolve@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/better-path-resolve/-/better-path-resolve-1.0.0.tgz#13a35a1104cdd48a7b74bf8758f96a1ee613f99d" @@ -450,6 +585,23 @@ browser-stdout@1.3.1: resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw== +c8@^9.0.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/c8/-/c8-9.1.0.tgz#0e57ba3ab9e5960ab1d650b4a86f71e53cb68112" + integrity sha512-mBWcT5iqNir1zIkzSPyI3NCR9EZCVI3WUD+AVO17MVWTSFNyUueXE82qTeampNtTr+ilN/5Ua3j24LgbCKjDVg== + dependencies: + "@bcoe/v8-coverage" "^0.2.3" + "@istanbuljs/schema" "^0.1.3" + find-up "^5.0.0" + foreground-child "^3.1.1" + istanbul-lib-coverage "^3.2.0" + istanbul-lib-report "^3.0.1" + istanbul-reports "^3.1.6" + test-exclude "^6.0.0" + v8-to-istanbul "^9.0.0" + yargs "^17.7.2" + yargs-parser "^21.1.1" + caller-callsite@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/caller-callsite/-/caller-callsite-2.0.0.tgz#847e0fce0a223750a9a027c54b33731ad3154134" @@ -479,17 +631,21 @@ camelcase@^6.0.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -chai@^4.2.0: - version "4.3.4" - resolved "https://registry.yarnpkg.com/chai/-/chai-4.3.4.tgz#b55e655b31e1eac7099be4c08c21964fce2e6c49" - integrity sha512-yS5H68VYOCtN1cjfwumDSuzn/9c+yza4f3reKXlE5rUg7SFcCEy90gJvydNgOYtblyf4Zi6jIWRnXOgErta0KA== +caseless@~0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" + integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== + +chai@^5.0.0-rc.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/chai/-/chai-5.1.0.tgz#648cf2d8b5d16f32646612e22ffd12dc617ef960" + integrity sha512-kDZ7MZyM6Q1DhR9jy7dalKohXQ2yrlXkk59CR52aRKxJrobmlBNqnFQxX9xOX8w+4mz8SYlKJa/7D7ddltFXCw== dependencies: - assertion-error "^1.1.0" - check-error "^1.0.2" - deep-eql "^3.0.1" - get-func-name "^2.0.0" - pathval "^1.1.1" - type-detect "^4.0.5" + assertion-error "^2.0.1" + check-error "^2.0.0" + deep-eql "^5.0.1" + loupe "^3.1.0" + pathval "^2.0.0" chalk@^2.0.0, chalk@^2.4.2: version "2.4.2" @@ -508,10 +664,10 @@ chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -check-error@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82" - integrity sha1-V00xLt2Iu13YkS6Sht1sCu1KrII= +check-error@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/check-error/-/check-error-2.0.0.tgz#589a4f201b6256fd93a2d165089fe43d2676d8c6" + integrity sha512-tjLAOBHKVxtPoHe/SA7kNOMvhCRdCJ3vETdeY0RuAc9popf+hyaSV6ZEg9hr4cpWF7jmo/JSWEnLDrnijS9Tog== chokidar@3.5.3: version "3.5.3" @@ -551,6 +707,15 @@ cliui@^7.0.2: strip-ansi "^6.0.0" wrap-ansi "^7.0.0" +cliui@^8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-8.0.1.tgz#0c04b075db02cbfe60dc8e6cf2f5486b1a3608aa" + integrity sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ== + dependencies: + string-width "^4.2.0" + strip-ansi "^6.0.1" + wrap-ansi "^7.0.0" + code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" @@ -580,7 +745,7 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -combined-stream@^1.0.6: +combined-stream@^1.0.6, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== @@ -592,6 +757,16 @@ concat-map@0.0.1: resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= +convert-source-map@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a" + integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg== + +core-util-is@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ== + core-util-is@~1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" @@ -607,11 +782,6 @@ cosmiconfig@^5.2.1: js-yaml "^3.13.1" parse-json "^4.0.0" -create-require@^1.1.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" - integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== - cross-spawn@^6.0.0: version "6.0.5" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" @@ -632,6 +802,13 @@ cross-spawn@^7.0.0: shebang-command "^2.0.0" which "^2.0.1" +dashdash@^1.12.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" + integrity sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g== + dependencies: + assert-plus "^1.0.0" + debug@4.3.4, debug@^4.1.0: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" @@ -649,12 +826,10 @@ decamelize@^4.0.0: resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-4.0.0.tgz#aa472d7bf660eb15f3494efd531cab7f2a709837" integrity sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ== -deep-eql@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df" - integrity sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw== - dependencies: - type-detect "^4.0.0" +deep-eql@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-5.0.1.tgz#21ea2c0d561a4d08cdd99c417ac584e0fb121385" + integrity sha512-nwQCf6ne2gez3o1MxWifqkciwt0zhl0LO1/UwVu4uMBuPmflWM4oQ70XMqHqnBJA+nhzncaqL9HVL6KkHJ28lw== delayed-stream@~1.0.0: version "1.0.0" @@ -666,11 +841,19 @@ diff@5.0.0: resolved "https://registry.yarnpkg.com/diff/-/diff-5.0.0.tgz#7ed6ad76d859d030787ec35855f5b1daf31d852b" integrity sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w== -diff@^4.0.1, diff@^4.0.2: +diff@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== +ecc-jsbn@~0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" + integrity sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw== + dependencies: + jsbn "~0.1.0" + safer-buffer "^2.1.0" + emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" @@ -690,6 +873,35 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" +esbuild@~0.19.10: + version "0.19.12" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.19.12.tgz#dc82ee5dc79e82f5a5c3b4323a2a641827db3e04" + integrity sha512-aARqgq8roFBj054KvQr5f1sFu0D65G+miZRCuJyJ0G13Zwx7vRar5Zhn2tkQNzIXcBrNVsv/8stehpj+GAjgbg== + optionalDependencies: + "@esbuild/aix-ppc64" "0.19.12" + "@esbuild/android-arm" "0.19.12" + "@esbuild/android-arm64" "0.19.12" + "@esbuild/android-x64" "0.19.12" + "@esbuild/darwin-arm64" "0.19.12" + "@esbuild/darwin-x64" "0.19.12" + "@esbuild/freebsd-arm64" "0.19.12" + "@esbuild/freebsd-x64" "0.19.12" + "@esbuild/linux-arm" "0.19.12" + "@esbuild/linux-arm64" "0.19.12" + "@esbuild/linux-ia32" "0.19.12" + "@esbuild/linux-loong64" "0.19.12" + "@esbuild/linux-mips64el" "0.19.12" + "@esbuild/linux-ppc64" "0.19.12" + "@esbuild/linux-riscv64" "0.19.12" + "@esbuild/linux-s390x" "0.19.12" + "@esbuild/linux-x64" "0.19.12" + "@esbuild/netbsd-x64" "0.19.12" + "@esbuild/openbsd-x64" "0.19.12" + "@esbuild/sunos-x64" "0.19.12" + "@esbuild/win32-arm64" "0.19.12" + "@esbuild/win32-ia32" "0.19.12" + "@esbuild/win32-x64" "0.19.12" + escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" @@ -738,6 +950,21 @@ execa@^2.1.0: signal-exit "^3.0.2" strip-final-newline "^2.0.0" +extend@~3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + +extsprintf@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + integrity sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g== + +extsprintf@^1.2.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" + integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== + fast-deep-equal@^3.1.1: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -755,7 +982,7 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" -find-up@5.0.0: +find-up@5.0.0, find-up@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== @@ -763,6 +990,13 @@ find-up@5.0.0: locate-path "^6.0.0" path-exists "^4.0.0" +find-up@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7" + integrity sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ== + dependencies: + locate-path "^2.0.0" + find-up@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73" @@ -783,6 +1017,19 @@ flat@^5.0.2: resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== +foreground-child@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.1.1.tgz#1d173e776d75d2772fed08efe4a0de1ea1b12d0d" + integrity sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg== + dependencies: + cross-spawn "^7.0.0" + signal-exit "^4.0.1" + +forever-agent@~0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" + integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== + form-data@^2.5.0: version "2.5.1" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4" @@ -792,13 +1039,23 @@ form-data@^2.5.0: combined-stream "^1.0.6" mime-types "^2.1.12" -fs-extra@^2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-2.1.2.tgz#046c70163cef9aad46b0e4a7fa467fb22d71de35" - integrity sha1-BGxwFjzvmq1GsOSn+kZ/si1x3jU= +form-data@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== dependencies: - graceful-fs "^4.1.2" - jsonfile "^2.1.0" + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + +fs-extra@^11.2.0: + version "11.2.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.2.0.tgz#e70e17dfad64232287d01929399e0ea7c86b0e5b" + integrity sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" fs-extra@^9.1.0: version "9.1.0" @@ -820,6 +1077,11 @@ fsevents@~2.3.2: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== +fsevents@~2.3.3: + version "2.3.3" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" + integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== + ftp@^0.3.10: version "0.3.10" resolved "https://registry.yarnpkg.com/ftp/-/ftp-0.3.10.tgz#9197d861ad8142f3e63d5a83bfe4c59f7330885d" @@ -843,10 +1105,10 @@ get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-func-name@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.0.tgz#ead774abee72e20409433a066366023dd6887a41" - integrity sha1-6td0q+5y4gQJQzoGY2YCPdaIekE= +get-func-name@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.2.tgz#0d7cf20cd13fda808669ffa88f4ffc7a3943fc41" + integrity sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ== get-stdin@^7.0.0: version "7.0.0" @@ -867,6 +1129,20 @@ get-stream@^5.0.0: dependencies: pump "^3.0.0" +get-tsconfig@^4.7.2: + version "4.7.3" + resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.7.3.tgz#0498163d98f7b58484dd4906999c0c9d5f103f83" + integrity sha512-ZvkrzoUA0PQZM6fy6+/Hce561s+faD1rsNwhnO5FelNjyy7EMGJ3Rz1AQ8GYDWjhRs/7dBLOEJvhK8MiEJOAFg== + dependencies: + resolve-pkg-maps "^1.0.0" + +getpass@^0.1.1: + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" + integrity sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng== + dependencies: + assert-plus "^1.0.0" + glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -874,7 +1150,18 @@ glob-parent@~5.1.2: dependencies: is-glob "^4.0.1" -glob@7.2.0, glob@^7.1.3: +glob@8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-8.1.0.tgz#d388f656593ef708ee3e34640fdfb99a9fd1c33e" + integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^5.0.1" + once "^1.3.0" + +glob@^7.1.3: version "7.2.0" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== @@ -886,11 +1173,41 @@ glob@7.2.0, glob@^7.1.3: once "^1.3.0" path-is-absolute "^1.0.0" -graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0: +glob@^7.1.4: + version "7.2.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" + integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.1.1" + once "^1.3.0" + path-is-absolute "^1.0.0" + +graceful-fs@^4.1.2: + version "4.2.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" + integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== + +graceful-fs@^4.1.6, graceful-fs@^4.2.0: version "4.2.8" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a" integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + integrity sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q== + +har-validator@~5.1.3: + version "5.1.5" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" + integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== + dependencies: + ajv "^6.12.3" + har-schema "^2.0.0" + has-flag@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" @@ -918,6 +1235,20 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== +html-escaper@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" + integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== + +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + integrity sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ== + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + husky@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/husky/-/husky-3.1.0.tgz#5faad520ab860582ed94f0c1a77f0f04c90b57c0" @@ -1046,6 +1377,11 @@ is-subdir@^1.0.2: dependencies: better-path-resolve "1.0.0" +is-typedarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== + is-unicode-supported@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" @@ -1066,6 +1402,33 @@ isexe@^2.0.0: resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= +isstream@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" + integrity sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g== + +istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.2.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz#2d166c4b0644d43a39f04bf6c2edd1e585f31756" + integrity sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg== + +istanbul-lib-report@^3.0.0, istanbul-lib-report@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz#908305bac9a5bd175ac6a74489eafd0fc2445a7d" + integrity sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw== + dependencies: + istanbul-lib-coverage "^3.0.0" + make-dir "^4.0.0" + supports-color "^7.1.0" + +istanbul-reports@^3.1.6: + version "3.1.7" + resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-3.1.7.tgz#daed12b9e1dca518e15c056e1e537e741280fa0b" + integrity sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g== + dependencies: + html-escaper "^2.0.0" + istanbul-lib-report "^3.0.0" + js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -1086,6 +1449,11 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" +jsbn@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" + integrity sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg== + json-parse-better-errors@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" @@ -1101,18 +1469,16 @@ json-schema-traverse@^0.4.1: resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== -json-stringify-safe@^5.0.1: +json-schema@0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" + integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== + +json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= -jsonfile@^2.1.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-2.4.0.tgz#3736a2b428b87bbda0cc83b53fa3d633a35c2ae8" - integrity sha1-NzaitCi4e72gzIO1P6PWM6NcKug= - optionalDependencies: - graceful-fs "^4.1.6" - jsonfile@^6.0.1: version "6.1.0" resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" @@ -1122,6 +1488,16 @@ jsonfile@^6.0.1: optionalDependencies: graceful-fs "^4.1.6" +jsprim@^1.2.2: + version "1.4.2" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.2.tgz#712c65533a15c878ba59e9ed5f0e26d5b77c5feb" + integrity sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw== + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.4.0" + verror "1.10.0" + jsverify@^0.8.2: version "0.8.4" resolved "https://registry.yarnpkg.com/jsverify/-/jsverify-0.8.4.tgz#b914ccc024103d946b503f18ee780ec43febeece" @@ -1154,6 +1530,24 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA= +load-json-file@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-4.0.0.tgz#2f5f45ab91e33216234fd53adab668eb4ec0993b" + integrity sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw== + dependencies: + graceful-fs "^4.1.2" + parse-json "^4.0.0" + pify "^3.0.0" + strip-bom "^3.0.0" + +locate-path@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e" + integrity sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA== + dependencies: + p-locate "^2.0.0" + path-exists "^3.0.0" + locate-path@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-3.0.0.tgz#dbec3b3ab759758071b58fe59fc41871af21400e" @@ -1201,6 +1595,13 @@ lolex@^5.0.1, lolex@^5.1.2: dependencies: "@sinonjs/commons" "^1.7.0" +loupe@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/loupe/-/loupe-3.1.0.tgz#46ef1a4ffee73145f5c0a627536d754787c1ea2a" + integrity sha512-qKl+FrLXUhFuHUoDJG7f8P8gEMHq9NFS0c6ghXG1J0rldmZFQZoNVv/vyirE9qwCIhWZDsvEFd1sbFu3GvRQFg== + dependencies: + get-func-name "^2.0.1" + lru-cache@4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.0.2.tgz#1d17679c069cda5d040991a09dbc2c0db377e55e" @@ -1209,10 +1610,19 @@ lru-cache@4.0.2: pseudomap "^1.0.1" yallist "^2.0.0" -make-error@^1.1.1: - version "1.3.6" - resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" - integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== +lru-cache@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" + integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== + dependencies: + yallist "^4.0.0" + +make-dir@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-4.0.0.tgz#c3c2307a771277cd9638305f915c29ae741b614e" + integrity sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw== + dependencies: + semver "^7.5.3" map-age-cleaner@^0.1.1: version "0.1.3" @@ -1240,6 +1650,11 @@ mime-db@1.50.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.50.0.tgz#abd4ac94e98d3c0e185016c67ab45d5fde40c11f" integrity sha512-9tMZCDlYHqeERXEHO9f/hKfNXhre5dK2eE/krIvUjZbS2KPcqGDfNShIWS1uW9XOTKQKqK6qbeOci18rbfW77A== +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + mime-types@^2.1.12: version "2.1.33" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.33.tgz#1fa12a904472fafd068e48d9e8401f74d3f70edb" @@ -1247,6 +1662,13 @@ mime-types@^2.1.12: dependencies: mime-db "1.50.0" +mime-types@~2.1.19: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mimic-fn@^2.0.0, mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" @@ -1266,12 +1688,25 @@ minimatch@5.0.1: dependencies: brace-expansion "^2.0.1" -mocha@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.0.0.tgz#205447d8993ec755335c4b13deba3d3a13c4def9" - integrity sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA== +minimatch@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== + dependencies: + brace-expansion "^1.1.7" + +minimatch@^5.0.1: + version "5.1.6" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.6.tgz#1cfcb8cf5522ea69952cd2af95ae09477f122a96" + integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g== + dependencies: + brace-expansion "^2.0.1" + +mocha@^10.2.0: + version "10.4.0" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.4.0.tgz#ed03db96ee9cfc6d20c56f8e2af07b961dbae261" + integrity sha512-eqhGB8JKapEYcC4ytX/xrzKforgEc3j1pGlAXVy3eRwrtAy5/nIfT1SvgGzfN0XZZxeLq0aQWkOUAmqIJiv+bA== dependencies: - "@ungap/promise-all-settled" "1.1.2" ansi-colors "4.1.1" browser-stdout "1.3.1" chokidar "3.5.3" @@ -1279,13 +1714,12 @@ mocha@^10.0.0: diff "5.0.0" escape-string-regexp "4.0.0" find-up "5.0.0" - glob "7.2.0" + glob "8.1.0" he "1.2.0" js-yaml "4.1.0" log-symbols "4.1.0" minimatch "5.0.1" ms "2.1.3" - nanoid "3.3.3" serialize-javascript "6.0.0" strip-json-comments "3.1.1" supports-color "8.1.1" @@ -1320,11 +1754,6 @@ multimatch@^4.0.0: arrify "^2.0.1" minimatch "^3.0.4" -nanoid@3.3.3: - version "3.3.3" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.3.tgz#fd8e8b7aa761fe807dba2d1b98fb7241bb724a25" - integrity sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w== - nice-try@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" @@ -1342,7 +1771,7 @@ nise@^3.0.1: lolex "^5.0.1" path-to-regexp "^1.7.0" -nock@*, nock@^13.2.9: +nock@*: version "13.2.9" resolved "https://registry.yarnpkg.com/nock/-/nock-13.2.9.tgz#4faf6c28175d36044da4cfa68e33e5a15086ad4c" integrity sha512-1+XfJNYF1cjGB+TKMWi29eZ0b82QOvQs2YoLNzbpWGqFMtRQHTa57osqdGj4FrFPgkO4D4AZinzUJR9VvW3QUA== @@ -1352,7 +1781,16 @@ nock@*, nock@^13.2.9: lodash "^4.17.21" propagate "^2.0.0" -normalize-package-data@^2.5.0: +nock@^13.4.0: + version "13.5.4" + resolved "https://registry.yarnpkg.com/nock/-/nock-13.5.4.tgz#8918f0addc70a63736170fef7106a9721e0dc479" + integrity sha512-yAyTfdeNJGGBFxWdzSKCBYxs5FxLbCg5X5Q4ets974hcQzG1+qCxvIyOo4j2Ry6MUlhWVMX4OoYDefAIIwupjw== + dependencies: + debug "^4.1.0" + json-stringify-safe "^5.0.1" + propagate "^2.0.0" + +normalize-package-data@^2.3.2, normalize-package-data@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA== @@ -1386,6 +1824,11 @@ number-is-nan@^1.0.0: resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= +oauth-sign@~0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" + integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== + once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -1434,6 +1877,13 @@ p-is-promise@^2.0.0: resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-2.1.0.tgz#918cebaea248a62cf7ffab8e3bca8c5f882fc42e" integrity sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg== +p-limit@^1.1.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8" + integrity sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q== + dependencies: + p-try "^1.0.0" + p-limit@^2.0.0, p-limit@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" @@ -1448,6 +1898,13 @@ p-limit@^3.0.2: dependencies: yocto-queue "^0.1.0" +p-locate@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" + integrity sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg== + dependencies: + p-limit "^1.1.0" + p-locate@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4" @@ -1469,6 +1926,11 @@ p-locate@^5.0.0: dependencies: p-limit "^3.0.2" +p-try@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" + integrity sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww== + p-try@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" @@ -1529,10 +1991,22 @@ path-to-regexp@^1.7.0: dependencies: isarray "0.0.1" -pathval@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.1.tgz#8534e77a77ce7ac5a2512ea21e0fdb8fcf6c3d8d" - integrity sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ== +path-type@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-3.0.0.tgz#cef31dc8e0a1a3bb0d105c0cd97cf3bf47f4e36f" + integrity sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg== + dependencies: + pify "^3.0.0" + +pathval@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/pathval/-/pathval-2.0.0.tgz#7e2550b422601d4f6b8e26f1301bc8f15a741a25" + integrity sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA== + +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== picomatch@^2.0.4: version "2.3.0" @@ -1544,6 +2018,11 @@ picomatch@^2.2.1: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== +pify@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176" + integrity sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg== + pkg-dir@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" @@ -1585,6 +2064,11 @@ pseudomap@^1.0.1: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM= +psl@^1.1.28: + version "1.9.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" + integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== + pump@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" @@ -1598,6 +2082,16 @@ punycode@^2.1.0: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== +punycode@^2.1.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" + integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== + +qs@~6.5.2: + version "6.5.3" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" + integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== + randombytes@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -1610,6 +2104,23 @@ rc4@~0.1.5: resolved "https://registry.yarnpkg.com/rc4/-/rc4-0.1.5.tgz#08c6e04a0168f6eb621c22ab6cb1151bd9f4a64d" integrity sha1-CMbgSgFo9utiHCKrbLEVG9n0pk0= +read-pkg-up@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-3.0.0.tgz#3ed496685dba0f8fe118d0691dc51f4a1ff96f07" + integrity sha512-YFzFrVvpC6frF1sz8psoHDBGF7fLPc+llq/8NB43oagqWkx8ar5zYtsTORtOjw9W2RHLpWP+zTWwBvf1bCmcSw== + dependencies: + find-up "^2.0.0" + read-pkg "^3.0.0" + +read-pkg@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-3.0.0.tgz#9cbc686978fee65d16c00e2b19c237fcf6e38389" + integrity sha512-BLq/cCO9two+lBgiTYNqD6GdtK8s4NpaWrl6/rCO9w0TUS8oJl7cmToOZfRYllKTISY6nt1U7jQ53brmKqY6BA== + dependencies: + load-json-file "^4.0.0" + normalize-package-data "^2.3.2" + path-type "^3.0.0" + read-pkg@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-5.2.0.tgz#7bf295438ca5a33e56cd30e053b34ee7250c93cc" @@ -1644,6 +2155,32 @@ recursive-readdir@^2.2.2: dependencies: minimatch "3.0.4" +request@^2.88.2: + version "2.88.2" + resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" + integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.8.0" + caseless "~0.12.0" + combined-stream "~1.0.6" + extend "~3.0.2" + forever-agent "~0.6.1" + form-data "~2.3.2" + har-validator "~5.1.3" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.19" + oauth-sign "~0.9.0" + performance-now "^2.1.0" + qs "~6.5.2" + safe-buffer "^5.1.2" + tough-cookie "~2.5.0" + tunnel-agent "^0.6.0" + uuid "^3.3.2" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -1659,6 +2196,11 @@ resolve-from@^3.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-3.0.0.tgz#b22c7af7d9d6881bc8b6e653335eebcb0a188748" integrity sha1-six699nWiBvItuZTM17rywoYh0g= +resolve-pkg-maps@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz#616b3dc2c57056b5588c31cdf4b3d64db133720f" + integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw== + resolve@^1.10.0: version "1.20.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" @@ -1679,11 +2221,16 @@ run-node@^1.0.0: resolved "https://registry.yarnpkg.com/run-node/-/run-node-1.0.0.tgz#46b50b946a2aa2d4947ae1d886e9856fd9cabe5e" integrity sha512-kc120TBlQ3mih1LSzdAJXo4xn/GWS2ec0l3S+syHDXP9uRr0JAT8Qd3mdMuyjqCzeZktgP3try92cEgf9Nks8A== -safe-buffer@^5.1.0: +safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.2: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== +safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + semver-compare@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" @@ -1694,6 +2241,13 @@ semver-compare@^1.0.0: resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== +semver@^7.5.3: + version "7.6.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" + integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== + dependencies: + lru-cache "^6.0.0" + serialize-javascript@6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.0.tgz#efae5d88f45d7924141da8b5c3a7a7e663fefeb8" @@ -1735,6 +2289,11 @@ signal-exit@^3.0.0, signal-exit@^3.0.2: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.5.tgz#9e3e8cc0c75a99472b44321033a7702e7738252f" integrity sha512-KWcOiKeQj6ZyXx7zq4YxSMgHRlod4czeBQZrPb8OKcohcqAXShm7E20kEMle9WBt26hFcAf0qLOcp5zmY7kOqQ== +signal-exit@^4.0.1: + version "4.1.0" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" + integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== + sinon@^8.1.1: version "8.1.1" resolved "https://registry.yarnpkg.com/sinon/-/sinon-8.1.1.tgz#21fffd5ad0a2d072a8aa7f8a3cf7ed2ced497497" @@ -1784,6 +2343,21 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= +sshpk@^1.7.0: + version "1.18.0" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.18.0.tgz#1663e55cddf4d688b86a46b77f0d5fe363aba028" + integrity sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ== + dependencies: + asn1 "~0.2.3" + assert-plus "^1.0.0" + bcrypt-pbkdf "^1.0.0" + dashdash "^1.12.0" + ecc-jsbn "~0.1.1" + getpass "^0.1.1" + jsbn "~0.1.0" + safer-buffer "^2.0.2" + tweetnacl "~0.14.0" + string-width@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" @@ -1801,7 +2375,7 @@ string-width@^2.0.0, string-width@^2.1.1: is-fullwidth-code-point "^2.0.0" strip-ansi "^4.0.0" -string-width@^4.1.0, string-width@^4.2.0: +string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -1836,6 +2410,11 @@ strip-ansi@^6.0.0, strip-ansi@^6.0.1: dependencies: ansi-regex "^5.0.1" +strip-bom@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" + integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA== + strip-eof@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf" @@ -1872,6 +2451,15 @@ supports-color@^7.1.0: dependencies: has-flag "^4.0.0" +test-exclude@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-6.0.0.tgz#04a8698661d805ea6fa293b6cb9e63ac044ef15e" + integrity sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w== + dependencies: + "@istanbuljs/schema" "^0.1.2" + glob "^7.1.4" + minimatch "^3.0.4" + to-regex-range@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" @@ -1879,31 +2467,42 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + trampa@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/trampa/-/trampa-1.0.1.tgz#b866bc192331f64b4f57a8acf4ab71d0e900b692" integrity sha512-93WeyHNuRggPEsfCe+yHxCgM2s6H3Q8Wmlt6b6ObJL8qc7eZlRaFjQxwTrB+zbvGtlDRnAkMoYYo3+2uH/fEwA== -ts-node@^10.9.1: - version "10.9.1" - resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b" - integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw== - dependencies: - "@cspotcode/source-map-support" "^0.8.0" - "@tsconfig/node10" "^1.0.7" - "@tsconfig/node12" "^1.0.7" - "@tsconfig/node14" "^1.0.0" - "@tsconfig/node16" "^1.0.2" - acorn "^8.4.1" - acorn-walk "^8.1.1" - arg "^4.1.0" - create-require "^1.1.0" - diff "^4.0.1" - make-error "^1.1.1" - v8-compile-cache-lib "^3.0.1" - yn "3.1.1" - -type-detect@4.0.8, type-detect@^4.0.0, type-detect@^4.0.5, type-detect@^4.0.8: +tsx@^4.7.0: + version "4.7.2" + resolved "https://registry.yarnpkg.com/tsx/-/tsx-4.7.2.tgz#a108b1a6e16876cd4c9a4b4ba263f2a07f9cf562" + integrity sha512-BCNd4kz6fz12fyrgCTEdZHGJ9fWTGeUzXmQysh0RVocDY3h4frk05ZNCXSy4kIenF7y/QnrdiVpTsyNRn6vlAw== + dependencies: + esbuild "~0.19.10" + get-tsconfig "^4.7.2" + optionalDependencies: + fsevents "~2.3.3" + +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== + dependencies: + safe-buffer "^5.0.1" + +tweetnacl@^0.14.3, tweetnacl@~0.14.0: + version "0.14.5" + resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" + integrity sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA== + +type-detect@4.0.8, type-detect@^4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== @@ -1913,16 +2512,21 @@ type-fest@^0.6.0: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b" integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg== -typescript@^4.2.4: - version "4.8.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.8.4.tgz#c464abca159669597be5f96b8943500b238e60e6" - integrity sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ== +typescript@^5.3.3: + version "5.4.5" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.4.5.tgz#42ccef2c571fdbd0f6718b1d1f5e6e5ef006f611" + integrity sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ== typify-parser@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/typify-parser/-/typify-parser-1.1.0.tgz#ac73bfa5f25343468e2d0f3346c6117bc03d3c99" integrity sha1-rHO/pfJTQ0aOLQ8zRsYRe8A9PJk= +undici-types@~5.26.4: + version "5.26.5" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" + integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== + universalify@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" @@ -1940,10 +2544,19 @@ urijs@^1.19.11: resolved "https://registry.yarnpkg.com/urijs/-/urijs-1.19.11.tgz#204b0d6b605ae80bea54bea39280cdb7c9f923cc" integrity sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ== -v8-compile-cache-lib@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" - integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== +uuid@^3.3.2: + version "3.4.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== + +v8-to-istanbul@^9.0.0: + version "9.2.0" + resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-9.2.0.tgz#2ed7644a245cddd83d4e087b9b33b3e62dfd10ad" + integrity sha512-/EH/sDgxU2eGxajKdwLCDmQ4FWq+kpi3uCmBGpw1xJtnAxEjlD8j8PEiGWpCIMIs3ciNAgH0d3TTJiUkYzyZjA== + dependencies: + "@jridgewell/trace-mapping" "^0.3.12" + "@types/istanbul-lib-coverage" "^2.0.1" + convert-source-map "^2.0.0" validate-npm-package-license@^3.0.1: version "3.0.4" @@ -1953,6 +2566,15 @@ validate-npm-package-license@^3.0.1: spdx-correct "^3.0.0" spdx-expression-parse "^3.0.0" +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" + integrity sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw== + dependencies: + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" + which-module@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" @@ -2019,6 +2641,11 @@ yallist@^2.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI= +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yaml@^1.10.2: version "1.10.2" resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" @@ -2042,6 +2669,11 @@ yargs-parser@^20.2.2: resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== +yargs-parser@^21.1.1: + version "21.1.1" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35" + integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== + yargs-unparser@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/yargs-unparser/-/yargs-unparser-2.0.0.tgz#f131f9226911ae5d9ad38c432fe809366c2325eb" @@ -2083,10 +2715,18 @@ yargs@^12.0.5: y18n "^3.2.1 || ^4.0.0" yargs-parser "^11.1.1" -yn@3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" - integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== +yargs@^17.7.2: + version "17.7.2" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" + integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== + dependencies: + cliui "^8.0.1" + escalade "^3.1.1" + get-caller-file "^2.0.5" + require-directory "^2.1.1" + string-width "^4.2.3" + y18n "^5.0.5" + yargs-parser "^21.1.1" yocto-queue@^0.1.0: version "0.1.0" From d80b378b499442af0ce8886cb53725d4dc47b1d5 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Tue, 16 Apr 2024 17:33:49 +1000 Subject: [PATCH 02/14] upgrade node to 18 in CI --- .github/workflows/main.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 98753fc..b593b44 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Use Node.js 14 + - name: Use Node.js 18 uses: actions/setup-node@v1 with: - node-version: 14 + node-version: 18 - name: Login to GitHub Container Registry run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8bcab9a..dd91207 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,10 +14,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Use Node.js 14 + - name: Use Node.js 18 uses: actions/setup-node@v1 with: - node-version: 14 + node-version: 18 - name: Login to GitHub Container Registry run: | From 7d52360b95f9442bc2d50eb7cda03469422334f2 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 00:23:46 +1000 Subject: [PATCH 03/14] - add `storageApiBaseUrl` & `datasetBucketName` config option to helm charts - upgrade prettier --- .../templates/deployment.yaml | 4 + deploy/magda-minion-broken-link/values.yaml | 8 + package.json | 14 +- prettier.config.cjs => prettier.config.mjs | 4 +- src/index.ts | 103 +++++----- yarn.lock | 186 ++++++++---------- 6 files changed, 167 insertions(+), 152 deletions(-) rename prettier.config.cjs => prettier.config.mjs (92%) diff --git a/deploy/magda-minion-broken-link/templates/deployment.yaml b/deploy/magda-minion-broken-link/templates/deployment.yaml index 3fd3212..7b55167 100644 --- a/deploy/magda-minion-broken-link/templates/deployment.yaml +++ b/deploy/magda-minion-broken-link/templates/deployment.yaml @@ -48,6 +48,10 @@ spec: value: {{ .Values.global.defaultAdminUserId | default .Values.defaultAdminUserId }} - name: INTERNAL_URL value: "http://minion-broken-link" + - name: STORAGE_API_BASE_URL + value: {{ .Values.storageApiBaseUrl | quote }} + - name: DATASET_BUCKET_NAME + value: {{ .Values.datasetBucketName | default .Values.global.defaultDatasetBucket | quote }} {{- if .Values.domainWaitTimeConfig }} - name: DOMAIN_WAIT_TIME_CONFIG value: '{{ toJson .Values.domainWaitTimeConfig | indent 2 }}' diff --git a/deploy/magda-minion-broken-link/values.yaml b/deploy/magda-minion-broken-link/values.yaml index fcec8a0..97fee63 100644 --- a/deploy/magda-minion-broken-link/values.yaml +++ b/deploy/magda-minion-broken-link/values.yaml @@ -32,6 +32,14 @@ schedule: "0 0 14,28 * *" defaultAdminUserId: "00000000-0000-4000-8000-000000000000" +# -- The base URL of the storage API to use when generating access URLs for MAGDA internal stored resources. +storageApiBaseUrl: "http://storage-api/v0" + +# -- The name of the storage bucket where all dataset files are stored. +# Should match storage API config. +# By default, it will use the value of `global.defaultDatasetBucket` (defined in `magda-core` chart) unless you specify a different value here. +datasetBucketName: "" + # Setup Domain Wait Time # domainWaitTimeConfig: # data.csiro.au: 5 diff --git a/package.json b/package.json index 0e9881e..4c1104c 100644 --- a/package.json +++ b/package.json @@ -24,9 +24,9 @@ "author": "", "license": "Apache-2.0", "devDependencies": { - "@magda/arbitraries": "^3.0.2-alpha.0", + "@magda/arbitraries": "^3.0.2-alpha.1", "@magda/ci-utils": "^1.0.5", - "@magda/docker-utils": "^3.0.2-alpha.0", + "@magda/docker-utils": "^3.0.2-alpha.1", "@types/chai": "^4.3.3", "@types/ftp": "^0.3.29", "@types/lodash": "^4.14.185", @@ -45,8 +45,8 @@ "jsverify": "^0.8.2", "mocha": "^10.2.0", "nock": "^13.4.0", - "prettier": "^1.19.1", - "pretty-quick": "^2.0.1", + "prettier": "^3.2.5", + "pretty-quick": "^4.0.0", "rimraf": "^3.0.0", "sinon": "^8.1.1", "tsx": "^4.7.0", @@ -69,9 +69,9 @@ ] }, "dependencies": { - "@magda/minion-sdk": "^3.0.2-alpha.0", - "@magda/registry-aspects": "^3.0.2-alpha.0", - "@magda/utils": "^3.0.2-alpha.0", + "@magda/minion-sdk": "^3.0.2-alpha.1", + "@magda/registry-aspects": "^3.0.2-alpha.1", + "@magda/utils": "^3.0.2-alpha.1", "ftp": "^0.3.10", "lodash": "^4.17.4", "lru-cache": "4.0.2", diff --git a/prettier.config.cjs b/prettier.config.mjs similarity index 92% rename from prettier.config.cjs rename to prettier.config.mjs index 3cc2ec5..d116864 100644 --- a/prettier.config.cjs +++ b/prettier.config.mjs @@ -1,4 +1,4 @@ -module.exports = { +const config = { tabWidth: 4, singleQuote: false, printWidth: 80, @@ -25,3 +25,5 @@ module.exports = { } ] }; + +export default config; diff --git a/src/index.ts b/src/index.ts index 3f453a3..c935602 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,57 +6,70 @@ import { coerceJson } from "@magda/utils"; const ID = "minion-broken-link"; -const argv = commonYargs(6111, "http://localhost:6111", argv => - argv - .option("externalRetries", { - describe: - "Number of times to retry external links when checking whether they're broken", - type: "number", - default: 1 - }) - .option("domainWaitTimeConfig", { - describe: - "A object that defines wait time for each of domain. " + - "Echo property name of the object would be the domain name and property value is the wait time in seconds", - coerce: coerceJson("domainWaitTimeConfig"), - default: process.env.DOMAIN_WAIT_TIME_CONFIG || JSON.stringify({}) - }) - .option("requestOpts", { - describe: - "The default options to use for the JS request library when making HTTP HEAD/GET requests", - type: "string", - coerce: coerceJson("requestOpts"), - default: process.env.REQUEST_OPTS || JSON.stringify({ timeout: 20000 }) - }) +const argv = commonYargs(6111, "http://localhost:6111", (argv) => + argv + .option("externalRetries", { + describe: + "Number of times to retry external links when checking whether they're broken", + type: "number", + default: 1 + }) + .option("domainWaitTimeConfig", { + describe: + "A object that defines wait time for each of domain. " + + "Echo property name of the object would be the domain name and property value is the wait time in seconds", + coerce: coerceJson("domainWaitTimeConfig"), + default: process.env.DOMAIN_WAIT_TIME_CONFIG || JSON.stringify({}) + }) + .option("requestOpts", { + describe: + "The default options to use for the JS request library when making HTTP HEAD/GET requests", + type: "string", + coerce: coerceJson("requestOpts"), + default: + process.env.REQUEST_OPTS || JSON.stringify({ timeout: 20000 }) + }) + .option("storageApiBaseUrl", { + describe: + "The base URL of the storage API to use when generating access URLs for internal stored resources", + type: "string", + default: process.env.STORAGE_API_BASE_URL || "http://storage-api/v0" + }) + .option("datasetBucketName", { + describe: + "The name of the storage bucket where all dataset files are stored.", + type: "string", + default: process.env.DATASET_BUCKET_NAME || "magda-datasets" + }) ); console.log( - "domainWaitTimeConfig: ", - JSON.stringify(argv.domainWaitTimeConfig as any, null, 2) + "domainWaitTimeConfig: ", + JSON.stringify(argv.domainWaitTimeConfig as any, null, 2) ); function sleuthBrokenLinks() { - return minion({ - argv, - id: ID, - aspects: ["dataset-distributions"], - optionalAspects: [], - async: true, - writeAspectDefs: [brokenLinkAspectDef], - dereference: true, - onRecordFound: (record, registry) => - onRecordFound( - record, - registry, - argv.externalRetries, - 1, - argv.domainWaitTimeConfig as any, - argv.requestOpts as CoreOptions - ) - }); + return minion({ + argv, + id: ID, + aspects: ["dataset-distributions"], + optionalAspects: [], + async: true, + writeAspectDefs: [brokenLinkAspectDef], + dereference: true, + onRecordFound: (record, registry) => + onRecordFound( + record, + registry, + argv.externalRetries, + 1, + argv.domainWaitTimeConfig as any, + argv.requestOpts as CoreOptions + ) + }); } -sleuthBrokenLinks().catch(e => { - console.error("Error: " + e.message, e); - process.exit(1); +sleuthBrokenLinks().catch((e) => { + console.error("Error: " + e.message, e); + process.exit(1); }); diff --git a/yarn.lock b/yarn.lock index 7700b88..0842d7c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -166,10 +166,10 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@magda/arbitraries@^3.0.2-alpha.0": - version "3.0.2-alpha.0" - resolved "https://registry.yarnpkg.com/@magda/arbitraries/-/arbitraries-3.0.2-alpha.0.tgz#14602378199107deac505354b5c861b990411eae" - integrity sha512-oXZ2CSy8NNjig05obngSTaiIvoSNNRVGDMBJeOcfClid7KLURI6lSnH+RYTGiJrJI2aA8+DgttX+9HZxgl3qOA== +"@magda/arbitraries@^3.0.2-alpha.1": + version "3.0.2-alpha.1" + resolved "https://registry.yarnpkg.com/@magda/arbitraries/-/arbitraries-3.0.2-alpha.1.tgz#727e426f40a8bbc06f21a15f08e980d32e832057" + integrity sha512-twn+iWKWhISRVe6zDB5kQqa1vh7x6OemFsAPS7lcK0hCgOQwk2yrA2RomgMsuLVeVO68Qpcjcwok3j7bifIzDA== dependencies: jsverify "^0.8.2" @@ -182,10 +182,10 @@ recursive-readdir "^2.2.2" yaml "^1.10.2" -"@magda/docker-utils@^3.0.2-alpha.0": - version "3.0.2-alpha.0" - resolved "https://registry.yarnpkg.com/@magda/docker-utils/-/docker-utils-3.0.2-alpha.0.tgz#d700d5f5efd096091757d83f0cd35bba03a01b1a" - integrity sha512-Oynq+q+SKKhMUpsYtR8pDAVyWPvVYTcJPE7xHxCq7RuflTj67zh/11qCyDluT2doknhzbHzCMX1DmealF7S6+g== +"@magda/docker-utils@^3.0.2-alpha.1": + version "3.0.2-alpha.1" + resolved "https://registry.yarnpkg.com/@magda/docker-utils/-/docker-utils-3.0.2-alpha.1.tgz#3bf68f68ee7a80aac82c60a18afcd15b60254945" + integrity sha512-KlkzZ7NGdDyrlOrELXSMQu9OsrIylOYnW1Qe8zVd9N6AuuWl8DtJnd5v1TSBLblAoRVb37GItfdB5ffeA4A/+w== dependencies: fs-extra "^11.2.0" is-subdir "^1.0.2" @@ -197,26 +197,26 @@ resolved "https://registry.yarnpkg.com/@magda/esm-utils/-/esm-utils-1.0.1.tgz#1d5e9882958aefa879106e55d32421874baae559" integrity sha512-A300Mds4wl41axrqcrqrw4y1EjUIcrVWipaZpTwlf6ChDLnI57bEqL9EVQLm8muW/qMnY3QzNjxg7EEYly18yA== -"@magda/minion-sdk@^3.0.2-alpha.0": - version "3.0.2-alpha.0" - resolved "https://registry.yarnpkg.com/@magda/minion-sdk/-/minion-sdk-3.0.2-alpha.0.tgz#2483f1c5350ebcb9ddfbc754fec766161c8fd1f8" - integrity sha512-/4fBJrxm8mSEwOA1qAWgn5CeH2Kh+0qsmt9GyKLSnvVfBE4JGSBYZC0inaoHEK8c6ynJ4BLg/MmAboRabIeTDA== +"@magda/minion-sdk@^3.0.2-alpha.1": + version "3.0.2-alpha.1" + resolved "https://registry.yarnpkg.com/@magda/minion-sdk/-/minion-sdk-3.0.2-alpha.1.tgz#61e7191533973e4de3644761111f3028d706e205" + integrity sha512-EKgCN9WuhRNDIyTlIBeXfwWGQoxyrtN53GBNKfuIq47TnOVs4LNxELHI3qolDOf9mHyaiyX6gQxZJQAhMCQ2oQ== dependencies: "@types/express" "^4.17.21" "@types/urijs" "^1.19.19" "@types/yargs" "^12.0.8" -"@magda/registry-aspects@^3.0.2-alpha.0": - version "3.0.2-alpha.0" - resolved "https://registry.yarnpkg.com/@magda/registry-aspects/-/registry-aspects-3.0.2-alpha.0.tgz#55f1da0285e0001debc241ad15e8ef7aa803a80f" - integrity sha512-dQU++0a2G1Nljdx+duNvWODOUv5jLzhn9z1f//Bp7EZjFrkdIqagd+zJw6oSDDYl0wD+PEBGX1ek4NaEHhvgxw== +"@magda/registry-aspects@^3.0.2-alpha.1": + version "3.0.2-alpha.1" + resolved "https://registry.yarnpkg.com/@magda/registry-aspects/-/registry-aspects-3.0.2-alpha.1.tgz#7fe474990e04e66b37bc9dcf8a200f9307b6a3eb" + integrity sha512-1nkyLBjEzeEAT369iHS/jOTkNRQjGBrlfXYEgzQUKq7H7VZTzvns5MTg577Wq418/yWuod27ocKfh7/GETNmHQ== dependencies: "@magda/esm-utils" "^1.0.1" -"@magda/utils@^3.0.2-alpha.0": - version "3.0.2-alpha.0" - resolved "https://registry.yarnpkg.com/@magda/utils/-/utils-3.0.2-alpha.0.tgz#cf0a1a2a235c5595710295410bf25b88bad9c350" - integrity sha512-grZrczKax+L+07dZ24OKTxVu2bO0Po/LSVcapP6Ts6wqdyi/1yldCub8qV5fi4uyGIV8Ke2F4fVMWgBgOusM+A== +"@magda/utils@^3.0.2-alpha.1": + version "3.0.2-alpha.1" + resolved "https://registry.yarnpkg.com/@magda/utils/-/utils-3.0.2-alpha.1.tgz#9a2d22158f4cde87071b4e6881c73e478559fddd" + integrity sha512-CoMd0va0w4+Ffsg5ll07S2y6EwDByLKmk8bwq8zQIgc15YpXWTb/zqO54iny0qB1SQ+h/DLO1gM9w1ujB5x0Dg== dependencies: "@types/request" "^2.48.1" @@ -321,11 +321,6 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== -"@types/minimatch@^3.0.3": - version "3.0.5" - resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" - integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== - "@types/mocha@^9.1.1": version "9.1.1" resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-9.1.1.tgz#e7c4f1001eefa4b8afbd1eee27a237fee3bf29c4" @@ -482,21 +477,6 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -array-differ@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b" - integrity sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg== - -array-union@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" - integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== - -arrify@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" - integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== - asn1@~0.2.3: version "0.2.6" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" @@ -793,7 +773,7 @@ cross-spawn@^6.0.0: shebang-command "^1.2.0" which "^1.2.9" -cross-spawn@^7.0.0: +cross-spawn@^7.0.0, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== @@ -935,19 +915,19 @@ execa@^1.0.0: signal-exit "^3.0.0" strip-eof "^1.0.0" -execa@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-2.1.0.tgz#e5d3ecd837d2a60ec50f3da78fd39767747bbe99" - integrity sha512-Y/URAVapfbYy2Xp/gb6A0E7iR8xeqOCXsuuaoMn7A5PzrXUK84E1gyiEfq0wQd/GHA6GsoHWwhNq8anb0mleIw== +execa@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" + integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg== dependencies: - cross-spawn "^7.0.0" - get-stream "^5.0.0" + cross-spawn "^7.0.3" + get-stream "^6.0.0" + human-signals "^2.1.0" is-stream "^2.0.0" merge-stream "^2.0.0" - npm-run-path "^3.0.0" - onetime "^5.1.0" - p-finally "^2.0.0" - signal-exit "^3.0.2" + npm-run-path "^4.0.1" + onetime "^5.1.2" + signal-exit "^3.0.3" strip-final-newline "^2.0.0" extend@~3.0.2: @@ -1004,7 +984,7 @@ find-up@^3.0.0: dependencies: locate-path "^3.0.0" -find-up@^4.0.0, find-up@^4.1.0: +find-up@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== @@ -1122,12 +1102,10 @@ get-stream@^4.0.0: dependencies: pump "^3.0.0" -get-stream@^5.0.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" - integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== - dependencies: - pump "^3.0.0" +get-stream@^6.0.0: + version "6.0.1" + resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" + integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== get-tsconfig@^4.7.2: version "4.7.3" @@ -1249,6 +1227,11 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" +human-signals@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" + integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== + husky@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/husky/-/husky-3.1.0.tgz#5faad520ab860582ed94f0c1a77f0f04c90b57c0" @@ -1266,10 +1249,10 @@ husky@^3.1.0: run-node "^1.0.0" slash "^3.0.0" -ignore@^5.1.4: - version "5.1.8" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" - integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== +ignore@^5.3.0: + version "5.3.1" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef" + integrity sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw== import-fresh@^2.0.0: version "2.0.0" @@ -1728,7 +1711,7 @@ mocha@^10.2.0: yargs-parser "20.2.4" yargs-unparser "2.0.0" -mri@^1.1.4: +mri@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/mri/-/mri-1.2.0.tgz#6721480fec2a11a4889861115a48b6cbe7cc8f0b" integrity sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA== @@ -1743,17 +1726,6 @@ ms@2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== -multimatch@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/multimatch/-/multimatch-4.0.0.tgz#8c3c0f6e3e8449ada0af3dd29efb491a375191b3" - integrity sha512-lDmx79y1z6i7RNx0ZGCPq1bzJ6ZoDDKbvh7jxr9SJcWLkShMzXrHbYVpTdnhNM5MXpDUxCQ4DgqVttVXlBgiBQ== - dependencies: - "@types/minimatch" "^3.0.3" - array-differ "^3.0.0" - array-union "^2.1.0" - arrify "^2.0.1" - minimatch "^3.0.4" - nice-try@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" @@ -1812,10 +1784,10 @@ npm-run-path@^2.0.0: dependencies: path-key "^2.0.0" -npm-run-path@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-3.1.0.tgz#7f91be317f6a466efed3c9f2980ad8a4ee8b0fa5" - integrity sha512-Dbl4A/VfiVGLgQv29URL9xshU8XDY1GeLy+fsaZ1AA8JDSfjvr5P5+pzRbWqRSBxk6/DW7MIh8lTM/PaGnP2kg== +npm-run-path@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" + integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== dependencies: path-key "^3.0.0" @@ -1836,7 +1808,7 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0: dependencies: wrappy "1" -onetime@^5.1.0: +onetime@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== @@ -1867,11 +1839,6 @@ p-finally@^1.0.0: resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4= -p-finally@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-2.0.1.tgz#bd6fcaa9c559a096b680806f4d657b3f0f240561" - integrity sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw== - p-is-promise@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-2.1.0.tgz#918cebaea248a62cf7ffab8e3bca8c5f882fc42e" @@ -2008,6 +1975,11 @@ performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== +picocolors@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" + integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== + picomatch@^2.0.4: version "2.3.0" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" @@ -2018,6 +1990,11 @@ picomatch@^2.2.1: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== +picomatch@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-3.0.1.tgz#817033161def55ec9638567a2f3bbc876b3e7516" + integrity sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag== + pify@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176" @@ -2037,22 +2014,23 @@ please-upgrade-node@^3.2.0: dependencies: semver-compare "^1.0.0" -prettier@^1.19.1: - version "1.19.1" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.19.1.tgz#f7d7f5ff8a9cd872a7be4ca142095956a60797cb" - integrity sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew== +prettier@^3.2.5: + version "3.2.5" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.5.tgz#e52bc3090586e824964a8813b09aba6233b28368" + integrity sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A== -pretty-quick@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/pretty-quick/-/pretty-quick-2.0.2.tgz#4e44d6489ed513ef111bee501f63688d854584e6" - integrity sha512-aLb6vtOTEfJDwi1w+MBTeE20GwPVUYyn6IqNg6TtGpiOB1W3y6vKcsGFjqGeaaEtQgMLSPXTWONqh33UBuwG8A== +pretty-quick@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/pretty-quick/-/pretty-quick-4.0.0.tgz#ea5cce85a5804bfbec7327b0e064509155d03f39" + integrity sha512-M+2MmeufXb/M7Xw3Afh1gxcYpj+sK0AxEfnfF958ktFeAyi5MsKY5brymVURQLgPLV1QaF5P4pb2oFJ54H3yzQ== dependencies: - chalk "^2.4.2" - execa "^2.1.0" - find-up "^4.1.0" - ignore "^5.1.4" - mri "^1.1.4" - multimatch "^4.0.0" + execa "^5.1.1" + find-up "^5.0.0" + ignore "^5.3.0" + mri "^1.2.0" + picocolors "^1.0.0" + picomatch "^3.0.1" + tslib "^2.6.2" propagate@^2.0.0: version "2.0.1" @@ -2284,11 +2262,16 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -signal-exit@^3.0.0, signal-exit@^3.0.2: +signal-exit@^3.0.0: version "3.0.5" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.5.tgz#9e3e8cc0c75a99472b44321033a7702e7738252f" integrity sha512-KWcOiKeQj6ZyXx7zq4YxSMgHRlod4czeBQZrPb8OKcohcqAXShm7E20kEMle9WBt26hFcAf0qLOcp5zmY7kOqQ== +signal-exit@^3.0.3: + version "3.0.7" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" + integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== + signal-exit@^4.0.1: version "4.1.0" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" @@ -2480,6 +2463,11 @@ trampa@^1.0.0: resolved "https://registry.yarnpkg.com/trampa/-/trampa-1.0.1.tgz#b866bc192331f64b4f57a8acf4ab71d0e900b692" integrity sha512-93WeyHNuRggPEsfCe+yHxCgM2s6H3Q8Wmlt6b6ObJL8qc7eZlRaFjQxwTrB+zbvGtlDRnAkMoYYo3+2uH/fEwA== +tslib@^2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + tsx@^4.7.0: version "4.7.2" resolved "https://registry.yarnpkg.com/tsx/-/tsx-4.7.2.tgz#a108b1a6e16876cd4c9a4b4ba263f2a07f9cf562" From edc9dcffbf3555aed9e1a74c3dc83204e268b9b4 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 00:27:27 +1000 Subject: [PATCH 04/14] reformat the code --- src/onRecordFound.ts | 563 ++++++++++++++++++++++--------------------- 1 file changed, 286 insertions(+), 277 deletions(-) diff --git a/src/onRecordFound.ts b/src/onRecordFound.ts index 665330e..ca34507 100644 --- a/src/onRecordFound.ts +++ b/src/onRecordFound.ts @@ -2,145 +2,149 @@ import _ from "lodash"; import { CoreOptions } from "request"; import { retryBackoff, unionToThrowable } from "@magda/utils"; import { - AuthorizedRegistryClient as Registry, - Record + AuthorizedRegistryClient as Registry, + Record } from "@magda/minion-sdk"; import { BrokenLinkAspect, RetrieveResult } from "./brokenLinkAspectDef.js"; import FTPHandler from "./FtpHandler.js"; import parseUriSafe from "./parseUriSafe.js"; import { - headRequest, - getRequest, - BadHttpResponseError + headRequest, + getRequest, + BadHttpResponseError } from "./HttpRequests.js"; import getUrlWaitTime from "./getUrlWaitTime.js"; import wait from "./wait.js"; export default async function onRecordFound( - record: Record, - registry: Registry, - retries: number = 1, - baseRetryDelaySeconds: number = 1, - domainWaitTimeConfig: { [domain: string]: number } = {}, - requestOpts: CoreOptions = {}, - ftpHandler: FTPHandler = new FTPHandler() + record: Record, + registry: Registry, + retries: number = 1, + baseRetryDelaySeconds: number = 1, + domainWaitTimeConfig: { [domain: string]: number } = {}, + requestOpts: CoreOptions = {}, + ftpHandler: FTPHandler = new FTPHandler() ) { - const distributions: Record[] = - record.aspects["dataset-distributions"] && - record.aspects["dataset-distributions"].distributions; + const distributions: Record[] = + record.aspects["dataset-distributions"] && + record.aspects["dataset-distributions"].distributions; - if (!distributions || distributions.length === 0) { - return Promise.resolve(); - } - - // Check each link - const linkChecks: DistributionLinkCheck[] = _.flatMap( - distributions, - (distribution: Record) => - checkDistributionLink( - distribution, - distribution.aspects["dcat-distribution-strings"], - baseRetryDelaySeconds, - retries, - ftpHandler, - _.partialRight(getUrlWaitTime, domainWaitTimeConfig), - requestOpts - ) - ); + if (!distributions || distributions.length === 0) { + return Promise.resolve(); + } - // Group the checks against their host so that we're only making one request per site simultaneously. - const brokenLinkChecksByHost: Promise[] = _( - linkChecks - ) - .groupBy(check => check.host) - .values() - .map((checks: DistributionLinkCheck[]) => checks.map(check => check.op)) - .map(checksForHost => - // Make the checks for this host run one after the other but return their results as an array. - checksForHost.reduce( - ( - megaPromise: Promise, - promiseLambda: () => Promise - ) => - megaPromise.then((megaResult: BrokenLinkSleuthingResult[]) => - promiseLambda().then(promiseResult => - megaResult.concat([promiseResult]) + // Check each link + const linkChecks: DistributionLinkCheck[] = _.flatMap( + distributions, + (distribution: Record) => + checkDistributionLink( + distribution, + distribution.aspects["dcat-distribution-strings"], + baseRetryDelaySeconds, + retries, + ftpHandler, + _.partialRight(getUrlWaitTime, domainWaitTimeConfig), + requestOpts ) - ), - Promise.resolve([]) - ) + ); + + // Group the checks against their host so that we're only making one request per site simultaneously. + const brokenLinkChecksByHost: Promise[] = _( + linkChecks ) - .value(); + .groupBy((check) => check.host) + .values() + .map((checks: DistributionLinkCheck[]) => + checks.map((check) => check.op) + ) + .map((checksForHost) => + // Make the checks for this host run one after the other but return their results as an array. + checksForHost.reduce( + ( + megaPromise: Promise, + promiseLambda: () => Promise + ) => + megaPromise.then( + (megaResult: BrokenLinkSleuthingResult[]) => + promiseLambda().then((promiseResult) => + megaResult.concat([promiseResult]) + ) + ), + Promise.resolve([]) + ) + ) + .value(); - const checkResultsPerHost: BrokenLinkSleuthingResult[][] = await Promise.all( - brokenLinkChecksByHost - ); + const checkResultsPerHost: BrokenLinkSleuthingResult[][] = + await Promise.all(brokenLinkChecksByHost); - const allResults = _.flatten(checkResultsPerHost); + const allResults = _.flatten(checkResultsPerHost); - const bestResultPerDistribution = _(allResults) - .groupBy(result => result.distribution.id) - .values() - .map((results: BrokenLinkSleuthingResult[]) => - _(results) - .sortBy(result => { - return ( - { none: 1, downloadURL: 2, accessURL: 3 }[result.urlType] || - Number.MAX_VALUE - ); - }) - .sortBy(result => { - return ( - { active: 1, unknown: 2, broken: 3 }[result.aspect.status] || - Number.MAX_VALUE - ); - }) - .head() - ) - .value(); + const bestResultPerDistribution = _(allResults) + .groupBy((result) => result.distribution.id) + .values() + .map((results: BrokenLinkSleuthingResult[]) => + _(results) + .sortBy((result) => { + return ( + { none: 1, downloadURL: 2, accessURL: 3 }[ + result.urlType + ] || Number.MAX_VALUE + ); + }) + .sortBy((result) => { + return ( + { active: 1, unknown: 2, broken: 3 }[ + result.aspect.status + ] || Number.MAX_VALUE + ); + }) + .head() + ) + .value(); - // Record a broken links aspect for each distribution. - const brokenLinksAspectPromise = Promise.all( - bestResultPerDistribution.map((result: BrokenLinkSleuthingResult) => { - return recordBrokenLinkAspect(registry, record, result); - }) - ); + // Record a broken links aspect for each distribution. + const brokenLinksAspectPromise = Promise.all( + bestResultPerDistribution.map((result: BrokenLinkSleuthingResult) => { + return recordBrokenLinkAspect(registry, record, result); + }) + ); - await brokenLinksAspectPromise; + await brokenLinksAspectPromise; } function recordBrokenLinkAspect( - registry: Registry, - record: Record, - result: BrokenLinkSleuthingResult + registry: Registry, + record: Record, + result: BrokenLinkSleuthingResult ): Promise { - const theTenantId = record.tenantId; - const { errorDetails, ...aspectDataWithNoErrorDetails } = result.aspect; - const aspectData = errorDetails - ? { - ...aspectDataWithNoErrorDetails, - errorDetails: `${ - errorDetails.httpStatusCode - ? `Http Status Code: ${errorDetails.httpStatusCode}: ` - : "" - }${errorDetails}` - } - : aspectDataWithNoErrorDetails; + const theTenantId = record.tenantId; + const { errorDetails, ...aspectDataWithNoErrorDetails } = result.aspect; + const aspectData = errorDetails + ? { + ...aspectDataWithNoErrorDetails, + errorDetails: `${ + errorDetails.httpStatusCode + ? `Http Status Code: ${errorDetails.httpStatusCode}: ` + : "" + }${errorDetails}` + } + : aspectDataWithNoErrorDetails; - return registry - .putRecordAspect( - result.distribution.id, - "source-link-status", - aspectData, - true, - theTenantId - ) - .then(unionToThrowable); + return registry + .putRecordAspect( + result.distribution.id, + "source-link-status", + aspectData, + true, + theTenantId + ) + .then(unionToThrowable); } type DistributionLinkCheck = { - host?: string; - op: () => Promise; + host?: string; + op: () => Promise; }; /** @@ -155,133 +159,137 @@ type DistributionLinkCheck = { * @param requestOpts The base options to use for the request library (e.g. timeouts, headers etc) */ function checkDistributionLink( - distribution: Record, - distStringsAspect: any, - baseRetryDelay: number, - retries: number, - ftpHandler: FTPHandler, - getUrlWaitTime: (url: string) => number, - requestOpts: CoreOptions + distribution: Record, + distStringsAspect: any, + baseRetryDelay: number, + retries: number, + ftpHandler: FTPHandler, + getUrlWaitTime: (url: string) => number, + requestOpts: CoreOptions ): DistributionLinkCheck[] { - type DistURL = { - url?: URI; - type: "downloadURL" | "accessURL"; - }; + type DistURL = { + url?: URI; + type: "downloadURL" | "accessURL"; + }; - const urls: DistURL[] = [ - { - url: distStringsAspect.downloadURL as string, - type: "downloadURL" as "downloadURL" - }, - { - url: distStringsAspect.accessURL as string, - type: "accessURL" as "accessURL" - } - ] - .map(urlObj => ({ ...urlObj, url: parseUriSafe(urlObj.url) })) - .filter(x => x.url && x.url.protocol().length > 0); + const urls: DistURL[] = [ + { + url: distStringsAspect.downloadURL as string, + type: "downloadURL" as "downloadURL" + }, + { + url: distStringsAspect.accessURL as string, + type: "accessURL" as "accessURL" + } + ] + .map((urlObj) => ({ ...urlObj, url: parseUriSafe(urlObj.url) })) + .filter((x) => x.url && x.url.protocol().length > 0); - if (urls.length === 0) { - return [ - { - op: () => - Promise.resolve({ - distribution, - urlType: "none" as "none", - aspect: { - status: "broken" as RetrieveResult, - errorDetails: new Error("No distribution urls to check.") + if (urls.length === 0) { + return [ + { + op: () => + Promise.resolve({ + distribution, + urlType: "none" as "none", + aspect: { + status: "broken" as RetrieveResult, + errorDetails: new Error( + "No distribution urls to check." + ) + } + }) } - }) - } - ]; - } + ]; + } - return urls.map(({ type, url: parsedURL }) => { - return { - host: (parsedURL && parsedURL.host()) as string, - op: () => { - console.info("Retrieving " + parsedURL); + return urls.map(({ type, url: parsedURL }) => { + return { + host: (parsedURL && parsedURL.host()) as string, + op: () => { + console.info("Retrieving " + parsedURL); - return retrieve( - parsedURL, - baseRetryDelay, - retries, - ftpHandler, - getUrlWaitTime, - requestOpts - ) - .then(aspect => { - console.info("Finished retrieving " + parsedURL); - return aspect; - }) - .then(aspect => ({ - distribution, - urlType: type, - aspect - })) - .catch(err => ({ - distribution, - urlType: type, - aspect: { - status: "broken" as RetrieveResult, - errorDetails: err + return retrieve( + parsedURL, + baseRetryDelay, + retries, + ftpHandler, + getUrlWaitTime, + requestOpts + ) + .then((aspect) => { + console.info("Finished retrieving " + parsedURL); + return aspect; + }) + .then((aspect) => ({ + distribution, + urlType: type, + aspect + })) + .catch((err) => ({ + distribution, + urlType: type, + aspect: { + status: "broken" as RetrieveResult, + errorDetails: err + } + })) as Promise; } - })) as Promise; - } - }; - }); + }; + }); } function retrieve( - parsedURL: URI, - baseRetryDelay: number, - retries: number, - ftpHandler: FTPHandler, - getUrlWaitTime: (url: string) => number, - requestOpts: CoreOptions + parsedURL: URI, + baseRetryDelay: number, + retries: number, + ftpHandler: FTPHandler, + getUrlWaitTime: (url: string) => number, + requestOpts: CoreOptions ): Promise { - if (parsedURL.protocol() === "http" || parsedURL.protocol() === "https") { - return retrieveHttp( - parsedURL.toString(), - baseRetryDelay, - retries, - getUrlWaitTime, - requestOpts - ); - } else if (parsedURL.protocol() === "ftp") { - return retrieveFtp(parsedURL, ftpHandler); - } else { - console.info(`Unrecognised URL: ${parsedURL.toString()}`); - return Promise.resolve({ - status: "unknown" as "unknown", - errorDetails: new Error( - "Could not check protocol " + parsedURL.protocol() - ) - }); - } + if (parsedURL.protocol() === "http" || parsedURL.protocol() === "https") { + return retrieveHttp( + parsedURL.toString(), + baseRetryDelay, + retries, + getUrlWaitTime, + requestOpts + ); + } else if (parsedURL.protocol() === "ftp") { + return retrieveFtp(parsedURL, ftpHandler); + } else { + console.info(`Unrecognised URL: ${parsedURL.toString()}`); + return Promise.resolve({ + status: "unknown" as "unknown", + errorDetails: new Error( + "Could not check protocol " + parsedURL.protocol() + ) + }); + } } function retrieveFtp( - parsedURL: URI, - ftpHandler: FTPHandler + parsedURL: URI, + ftpHandler: FTPHandler ): Promise { - const port = +(parsedURL.port() || 21); - const pClient = ftpHandler.getClient(parsedURL.hostname(), port); + const port = +(parsedURL.port() || 21); + const pClient = ftpHandler.getClient(parsedURL.hostname(), port); - return pClient.then(client => { - return new Promise((resolve, reject) => { - client.list(parsedURL.path(), (err, list) => { - if (err) { - reject(err); - } else if (list.length === 0) { - reject(new Error(`File "${parsedURL.toString()}" not found`)); - } else { - resolve({ status: "active" as "active" }); - } - }); + return pClient.then((client) => { + return new Promise((resolve, reject) => { + client.list(parsedURL.path(), (err, list) => { + if (err) { + reject(err); + } else if (list.length === 0) { + reject( + new Error(`File "${parsedURL.toString()}" not found`) + ); + } else { + resolve({ status: "active" as "active" }); + } + }); + }); }); - }); } /** @@ -290,69 +298,70 @@ function retrieveFtp( * @param url The url to retrieve */ async function retrieveHttp( - url: string, - baseRetryDelay: number, - retries: number, - getUrlWaitTime: (url: string) => number, - requestOpts: CoreOptions + url: string, + baseRetryDelay: number, + retries: number, + getUrlWaitTime: (url: string) => number, + requestOpts: CoreOptions ): Promise { - async function operation() { - try { - await wait(getUrlWaitTime(url)); - return await headRequest(url, requestOpts); - } catch (e) { - // --- HEAD Method not allowed - await wait(getUrlWaitTime(url)); - return await getRequest(url, requestOpts); + async function operation() { + try { + await wait(getUrlWaitTime(url)); + return await headRequest(url, requestOpts); + } catch (e) { + // --- HEAD Method not allowed + await wait(getUrlWaitTime(url)); + return await getRequest(url, requestOpts); + } } - } - const onRetry = (err: BadHttpResponseError, retries: number) => { - console.info( - `Downloading ${url} failed: ${err.httpStatusCode || - err} (${retries} retries remaining)` - ); - }; + const onRetry = (err: BadHttpResponseError, retries: number) => { + console.info( + `Downloading ${url} failed: ${ + err.httpStatusCode || err + } (${retries} retries remaining)` + ); + }; - const innerOp = () => - retryBackoff(operation, baseRetryDelay, retries, onRetry); + const innerOp = () => + retryBackoff(operation, baseRetryDelay, retries, onRetry); - const outerOp: () => Promise = () => - innerOp().then( - code => { - if (code === 429) { - throw { message: "429 encountered", httpStatusCode: 429 }; - } else { - return { - status: "active" as "active", - httpStatusCode: code - }; - } - }, - error => { - return { - status: "broken" as "broken", - httpStatusCode: error.httpStatusCode, - errorDetails: error - }; - } - ); + const outerOp: () => Promise = () => + innerOp().then( + (code) => { + if (code === 429) { + throw { message: "429 encountered", httpStatusCode: 429 }; + } else { + return { + status: "active" as "active", + httpStatusCode: code + }; + } + }, + (error) => { + return { + status: "broken" as "broken", + httpStatusCode: error.httpStatusCode, + errorDetails: error + }; + } + ); - return retryBackoff( - outerOp, - baseRetryDelay, - retries, - onRetry, - (x: number) => x * 5 - ).catch(err => ({ - status: "unknown" as "unknown", - errorDetails: err, - httpStatusCode: 429 - })); + return retryBackoff( + outerOp, + baseRetryDelay, + retries, + onRetry, + (x: number) => x * 5 + ).catch((err) => ({ + status: "unknown" as "unknown", + errorDetails: err, + httpStatusCode: 429 + })); } interface BrokenLinkSleuthingResult { - distribution: Record; - aspect?: BrokenLinkAspect; - urlType: "downloadURL" | "accessURL" | "none"; + distribution: Record; + aspect?: BrokenLinkAspect; + urlType: "downloadURL" | "accessURL" | "none"; } From 495483a99c35d42353c3e3af63d65b6ece88ffe6 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 00:39:12 +1000 Subject: [PATCH 05/14] reformat code --- src/test/onRecordFound.spec.ts | 1309 +++++++++++++++++--------------- 1 file changed, 699 insertions(+), 610 deletions(-) diff --git a/src/test/onRecordFound.spec.ts b/src/test/onRecordFound.spec.ts index db47889..0c924a3 100644 --- a/src/test/onRecordFound.spec.ts +++ b/src/test/onRecordFound.spec.ts @@ -11,669 +11,758 @@ import Ajv from "ajv"; import { Record, AuthorizedRegistryClient } from "@magda/minion-sdk"; import { encodeURIComponentWithApost } from "@magda/utils"; import { - specificRecordArb, - distUrlArb, - arrayOfSizeArb, - arbFlatMap, - recordArbWithDistArbs + specificRecordArb, + distUrlArb, + arrayOfSizeArb, + arbFlatMap, + recordArbWithDistArbs } from "@magda/arbitraries"; import onRecordFound from "../onRecordFound.js"; import { BrokenLinkAspect } from "../brokenLinkAspectDef.js"; import urlsFromDataSet from "./urlsFromDataSet.js"; import { - CheckResult, - recordArbWithSuccesses, - KNOWN_PROTOCOLS, - httpOnlyRecordArb, - failureCodeArb + CheckResult, + recordArbWithSuccesses, + KNOWN_PROTOCOLS, + httpOnlyRecordArb, + failureCodeArb } from "./arbitraries.js"; import FtpHandler from "../FtpHandler.js"; import parseUriSafe from "../parseUriSafe.js"; import RandomStream from "./RandomStream.js"; import { - setDefaultDomainWaitTime, - getDefaultDomainWaitTime + setDefaultDomainWaitTime, + getDefaultDomainWaitTime } from "../getUrlWaitTime.js"; const schema = require("@magda/registry-aspects/source-link-status.schema.json"); -describe("onRecordFound", function(this: Mocha.Suite) { - this.timeout(20000); - nock.disableNetConnect(); - const registryUrl = "http://example.com"; - const secret = "secret!"; - const registry = new AuthorizedRegistryClient({ - baseUrl: registryUrl, - jwtSecret: secret, - userId: "1", - maxRetries: 0, - tenantId: 1 - }); - let registryScope: nock.Scope; - let clients: { [s: string]: Client[] }; - let ftpSuccesses: { [url: string]: CheckResult }; - const orignalDefaultDomainWaitTime: number = getDefaultDomainWaitTime(); - - before(() => { - // --- set default domain wait time to 0 second (i.e. for any domains that has no specific setting) - // --- Otherwise, it will take too long to complete property tests - setDefaultDomainWaitTime(0); - - sinon.stub(console, "info"); +describe("onRecordFound", function (this: Mocha.Suite) { + this.timeout(20000); nock.disableNetConnect(); + const registryUrl = "http://example.com"; + const secret = "secret!"; + const registry = new AuthorizedRegistryClient({ + baseUrl: registryUrl, + jwtSecret: secret, + userId: "1", + maxRetries: 0, + tenantId: 1 + }); + let registryScope: nock.Scope; + let clients: { [s: string]: Client[] }; + let ftpSuccesses: { [url: string]: CheckResult }; + const orignalDefaultDomainWaitTime: number = getDefaultDomainWaitTime(); + + before(() => { + // --- set default domain wait time to 0 second (i.e. for any domains that has no specific setting) + // --- Otherwise, it will take too long to complete property tests + setDefaultDomainWaitTime(0); + + sinon.stub(console, "info"); + nock.disableNetConnect(); + + nock.emitter.on("no match", onMatchFail); + }); + + const onMatchFail = (req: any, interceptor: any) => { + console.error( + `Match failure: ${req.method ? req.method : interceptor.method} ${ + req.host ? req.host : interceptor.host + }${req.path}` + ); + }; - nock.emitter.on("no match", onMatchFail); - }); + after(() => { + setDefaultDomainWaitTime(orignalDefaultDomainWaitTime); - const onMatchFail = (req: any, interceptor: any) => { - console.error( - `Match failure: ${req.method ? req.method : interceptor.method} ${ - req.host ? req.host : interceptor.host - }${req.path}` - ); - }; - - after(() => { - setDefaultDomainWaitTime(orignalDefaultDomainWaitTime); - - (console.info as any).restore(); - - nock.emitter.removeListener("no match", onMatchFail); - }); - - const beforeEachProperty = () => { - registryScope = nock(registryUrl); //.log(console.log); - clients = {}; - ftpSuccesses = {}; - }; - - const afterEachProperty = () => { - nock.cleanAll(); - }; - - /** - * Builds FTP clients that have all their important methods stubbed out - these - * will respond based on the current content of ftpSuccesses. - */ - const clientFactory = () => { - const client = new Client(); - let readyCallback: () => void; - let key: string; - sinon - .stub(client, "connect") - .callsFake(({ host, port }: Client.Options) => { - const keyPort = port !== 21 ? `:${port}` : ""; - key = `${host}${keyPort}`; - if (!clients[key]) { - clients[key] = []; - } - clients[key].push(client); - readyCallback(); - }); - sinon - .stub(client, "on") - .callsFake((event: string | symbol, callback: () => void) => { - if (event === "ready") { - readyCallback = callback; - } - return client; - }); - sinon.stub(client, "list").callsFake((( - path: string, - callback: (err: Error, list: string[]) => void - ) => { - try { - expect(key).not.to.be.undefined; - const url = `ftp://${key}${path}`; - - const success = ftpSuccesses[url]; - expect(success).not.to.be.undefined; - - if (success === "success") { - callback(null, ["file"]); - } else if (success === "notfound") { - callback(null, []); - } else { - callback(new Error("Fake error!"), null); - } - } catch (e) { - console.error(e); - callback(e as Error, null); - } - }) as any); - return client; - }; - - const fakeFtpHandler = new FtpHandler(clientFactory); - - /** - * Generator-driven super-test: generates records and runs them through the - * onRecordFound function, listening for HTTP and FTP calls made and returning - * success or failure based on generated outcomes, then checks that they're - * recorded on a by-distribution basis as link status as well as on a by-record - * basis as a part of dataset quality. - */ - it("Should correctly record link statuses", function() { - const ajv = new Ajv(); - const validate = ajv.compile(schema); - return jsc.assert( - jsc.forall(recordArbWithSuccesses, jsc.integer(1, 100), function( - { record, successLookup, disallowHead }, - streamWaitTime - ) { - beforeEachProperty(); - - // Tell the FTP server to return success/failure for the various FTP - // paths with this dodgy method. Note that because the FTP server can - // only see paths and not host, we only send it the path of the req. - ftpSuccesses = _.pickBy(successLookup, (value, url) => - hasProtocol(url, "ftp") - ); + (console.info as any).restore(); - const allDists = record.aspects["dataset-distributions"].distributions; - - const httpDistUrls = _(urlsFromDataSet(record)) - .filter((url: string) => hasProtocol(url, "http")) - .map((url: string) => ({ - url, - success: successLookup[url] - })) - .value(); - - // Set up a nock scope for every HTTP URL - the minion will actually - // attempt to download these but it'll be intercepted by nock. - const distScopes = httpDistUrls.map( - ({ url, success }: { url: string; success: CheckResult }) => { - const scope = nock(url, { - reqheaders: { "User-Agent": /magda.*/ } - }); + nock.emitter.removeListener("no match", onMatchFail); + }); - const scopeHead = scope.head(url.endsWith("/") ? "/" : ""); - const scopeGet = scope.get(url.endsWith("/") ? "/" : ""); + const beforeEachProperty = () => { + registryScope = nock(registryUrl); //.log(console.log); + clients = {}; + ftpSuccesses = {}; + }; - if (success !== "error") { - if (!disallowHead && success === "success") { - scopeHead.reply(200); - } else { - if (disallowHead) { - // Not everything returns a 405 for HEAD not allowed :() - scopeHead.reply(success === "success" ? 405 : 400); - } else { - scopeHead.replyWithError("fail"); + const afterEachProperty = () => { + nock.cleanAll(); + }; + + /** + * Builds FTP clients that have all their important methods stubbed out - these + * will respond based on the current content of ftpSuccesses. + */ + const clientFactory = () => { + const client = new Client(); + let readyCallback: () => void; + let key: string; + sinon + .stub(client, "connect") + .callsFake(({ host, port }: Client.Options) => { + const keyPort = port !== 21 ? `:${port}` : ""; + key = `${host}${keyPort}`; + if (!clients[key]) { + clients[key] = []; + } + clients[key].push(client); + readyCallback(); + }); + sinon + .stub(client, "on") + .callsFake((event: string | symbol, callback: () => void) => { + if (event === "ready") { + readyCallback = callback; } + return client; + }); + sinon.stub(client, "list").callsFake((( + path: string, + callback: (err: Error, list: string[]) => void + ) => { + try { + expect(key).not.to.be.undefined; + const url = `ftp://${key}${path}`; - if (success === "success") { - scopeGet.reply(200, () => { - const s = new RandomStream(streamWaitTime); + const success = ftpSuccesses[url]; + expect(success).not.to.be.undefined; - return s; - }); + if (success === "success") { + callback(null, ["file"]); + } else if (success === "notfound") { + callback(null, []); } else { - scopeGet.reply(404); + callback(new Error("Fake error!"), null); } - } - } else { - scopeHead.replyWithError("fail"); - scopeGet.replyWithError("fail"); - } - - return scope; - } - ); - - allDists.forEach((dist: Record) => { - const { downloadURL, accessURL } = dist.aspects[ - "dcat-distribution-strings" - ]; - const success = - successLookup[downloadURL] === "success" - ? "success" - : successLookup[accessURL]; - - const isUnknownProtocol = (url: string) => { - if (!url) { - return false; + } catch (e) { + console.error(e); + callback(e as Error, null); } - const protocol = new URI(url).protocol(); - return ( - protocol && - protocol.length > 0 && - KNOWN_PROTOCOLS.indexOf(protocol) === -1 - ); - }; - - const downloadUnknown = isUnknownProtocol(downloadURL); - const accessUnknown = isUnknownProtocol(accessURL); - - const result = - success === "success" - ? "active" - : downloadUnknown || accessUnknown - ? "unknown" - : "broken"; - - registryScope - .put( - `/records/${encodeURIComponentWithApost( - dist.id - )}/aspects/source-link-status?merge=true`, - (body: BrokenLinkAspect) => { - const validationResult = validate(body); - if (!validationResult) { - throw new Error( - "Json schema validation error: \n" + - validate.errors - .map(error => `${error.dataPath}: ${error.message}`) - .join("\n") - ); - } + }) as any); + return client; + }; - const doesStatusMatch = body.status === result; - - const isDownloadUrlHttp = hasProtocol(downloadURL, "http"); - const isAccessUrlHttp = hasProtocol(accessURL, "http"); - - const isDownloadUrlHttpSuccess = - isDownloadUrlHttp && successLookup[downloadURL] === "success"; - - const isDownloadUrlFtpSuccess = - !isDownloadUrlHttp && - successLookup[downloadURL] === "success"; - - const isAccessURLHttpSuccess = - isAccessUrlHttp && successLookup[accessURL] === "success"; - - const isHttpSuccess: boolean = - isDownloadUrlHttpSuccess || - (!isDownloadUrlFtpSuccess && isAccessURLHttpSuccess); - - const downloadUri = parseUriSafe(downloadURL); - const isDownloadUrlDefined = - _.isUndefined(downloadUri) || - !downloadUri.scheme() || - parseUriSafe(downloadURL).scheme().length === 0; - - const is404: boolean = - result === "broken" && - ((isDownloadUrlHttp && - successLookup[downloadURL] === "notfound") || - (isDownloadUrlDefined && - isAccessUrlHttp && - successLookup[accessURL] === "notfound")); - - const doesResponseCodeMatch = ((code?: number) => { - if (isHttpSuccess) { - return code === 200; - } else if (is404) { - return code === 404; - } else { - return _.isUndefined(code); - } - })(body.httpStatusCode); - - const doesErrorMatch = ((arg?: Error) => - success === "success" - ? _.isUndefined(arg) - : !_.isUndefined(arg))(body.errorDetails); - - // console.log( - // `${ - // dist.id - // }: ${doesStatusMatch} && ${doesResponseCodeMatch} && ${doesErrorMatch} ` - // ); - - return ( - doesStatusMatch && doesResponseCodeMatch && doesErrorMatch - ); - } - ) - .reply(201); - }); - - return onRecordFound(record, registry, 0, 0, {}, {}, fakeFtpHandler) - .then(() => { - distScopes.forEach(scope => scope.done()); - registryScope.done(); - }) - .then(() => { - afterEachProperty(); - return true; - }) - .catch(e => { - afterEachProperty(); - throw e; - }); - }), - { - tests: 500 - } - ); - }); + const fakeFtpHandler = new FtpHandler(clientFactory); - describe("retrying", () => { /** - * Runs onRecordFound with a number of failing codes, testing whether the - * minion retries the correct number of times, and whether it correctly - * records a success after retries or a failure after the retries run out. - * - * This tests both 429 retries and other retries - this involves different - * behaviour as the retry for 429 (which indicates rate limiting) require - * a much longer cool-off time and hence are done differently. - * - * @param caption The caption to use for the mocha "it" call. - * @param result Whether to test for a number of retries then a success, a - * number of retries then a failure because of too many 429s, - * or a number of retries then a failure because of too many - * non-429 failures (e.g. 404s) + * Generator-driven super-test: generates records and runs them through the + * onRecordFound function, listening for HTTP and FTP calls made and returning + * success or failure based on generated outcomes, then checks that they're + * recorded on a by-distribution basis as link status as well as on a by-record + * basis as a part of dataset quality. */ - const retrySpec = ( - caption: string, - result: "success" | "fail429" | "failNormal" - ) => { - it(caption, function() { - const retryCountArb = jsc.integer(0, 5); - - type FailuresArbResult = { - retryCount: number; - allResults: number[][]; - }; + it("Should correctly record link statuses", function () { + const ajv = new Ajv(); + const validate = ajv.compile(schema); + return jsc.assert( + jsc.forall( + recordArbWithSuccesses, + jsc.integer(1, 100), + function ( + { record, successLookup, disallowHead }, + streamWaitTime + ) { + beforeEachProperty(); + + // Tell the FTP server to return success/failure for the various FTP + // paths with this dodgy method. Note that because the FTP server can + // only see paths and not host, we only send it the path of the req. + ftpSuccesses = _.pickBy(successLookup, (value, url) => + hasProtocol(url, "ftp") + ); + + const allDists = + record.aspects["dataset-distributions"].distributions; + + const httpDistUrls = _(urlsFromDataSet(record)) + .filter((url: string) => hasProtocol(url, "http")) + .map((url: string) => ({ + url, + success: successLookup[url] + })) + .value(); + + // Set up a nock scope for every HTTP URL - the minion will actually + // attempt to download these but it'll be intercepted by nock. + const distScopes = httpDistUrls.map( + ({ + url, + success + }: { + url: string; + success: CheckResult; + }) => { + const scope = nock(url, { + reqheaders: { "User-Agent": /magda.*/ } + }); + + const scopeHead = scope.head( + url.endsWith("/") ? "/" : "" + ); + const scopeGet = scope.get( + url.endsWith("/") ? "/" : "" + ); + + if (success !== "error") { + if (!disallowHead && success === "success") { + scopeHead.reply(200); + } else { + if (disallowHead) { + // Not everything returns a 405 for HEAD not allowed :() + scopeHead.reply( + success === "success" ? 405 : 400 + ); + } else { + scopeHead.replyWithError("fail"); + } + + if (success === "success") { + scopeGet.reply(200, () => { + const s = new RandomStream( + streamWaitTime + ); + + return s; + }); + } else { + scopeGet.reply(404); + } + } + } else { + scopeHead.replyWithError("fail"); + scopeGet.replyWithError("fail"); + } + + return scope; + } + ); + + allDists.forEach((dist: Record) => { + const { downloadURL, accessURL } = + dist.aspects["dcat-distribution-strings"]; + const success = + successLookup[downloadURL] === "success" + ? "success" + : successLookup[accessURL]; + + const isUnknownProtocol = (url: string) => { + if (!url) { + return false; + } + const protocol = new URI(url).protocol(); + return ( + protocol && + protocol.length > 0 && + KNOWN_PROTOCOLS.indexOf(protocol) === -1 + ); + }; + + const downloadUnknown = isUnknownProtocol(downloadURL); + const accessUnknown = isUnknownProtocol(accessURL); + + const result = + success === "success" + ? "active" + : downloadUnknown || accessUnknown + ? "unknown" + : "broken"; + + registryScope + .put( + `/records/${encodeURIComponentWithApost( + dist.id + )}/aspects/source-link-status?merge=true`, + (body: BrokenLinkAspect) => { + const validationResult = validate(body); + if (!validationResult) { + throw new Error( + "Json schema validation error: \n" + + validate.errors + .map( + (error) => + `${error.dataPath}: ${error.message}` + ) + .join("\n") + ); + } + + const doesStatusMatch = + body.status === result; + + const isDownloadUrlHttp = hasProtocol( + downloadURL, + "http" + ); + const isAccessUrlHttp = hasProtocol( + accessURL, + "http" + ); + + const isDownloadUrlHttpSuccess = + isDownloadUrlHttp && + successLookup[downloadURL] === + "success"; + + const isDownloadUrlFtpSuccess = + !isDownloadUrlHttp && + successLookup[downloadURL] === + "success"; + + const isAccessURLHttpSuccess = + isAccessUrlHttp && + successLookup[accessURL] === "success"; + + const isHttpSuccess: boolean = + isDownloadUrlHttpSuccess || + (!isDownloadUrlFtpSuccess && + isAccessURLHttpSuccess); + + const downloadUri = + parseUriSafe(downloadURL); + const isDownloadUrlDefined = + _.isUndefined(downloadUri) || + !downloadUri.scheme() || + parseUriSafe(downloadURL).scheme() + .length === 0; + + const is404: boolean = + result === "broken" && + ((isDownloadUrlHttp && + successLookup[downloadURL] === + "notfound") || + (isDownloadUrlDefined && + isAccessUrlHttp && + successLookup[accessURL] === + "notfound")); + + const doesResponseCodeMatch = (( + code?: number + ) => { + if (isHttpSuccess) { + return code === 200; + } else if (is404) { + return code === 404; + } else { + return _.isUndefined(code); + } + })(body.httpStatusCode); + + const doesErrorMatch = ((arg?: Error) => + success === "success" + ? _.isUndefined(arg) + : !_.isUndefined(arg))( + body.errorDetails + ); + + // console.log( + // `${ + // dist.id + // }: ${doesStatusMatch} && ${doesResponseCodeMatch} && ${doesErrorMatch} ` + // ); + + return ( + doesStatusMatch && + doesResponseCodeMatch && + doesErrorMatch + ); + } + ) + .reply(201); + }); + + return onRecordFound( + record, + registry, + 0, + 0, + {}, + {}, + fakeFtpHandler + ) + .then(() => { + distScopes.forEach((scope) => scope.done()); + registryScope.done(); + }) + .then(() => { + afterEachProperty(); + return true; + }) + .catch((e) => { + afterEachProperty(); + throw e; + }); + } + ), + { + tests: 500 + } + ); + }); + describe("retrying", () => { /** - * Generates a retryCount and a nested array of results to return to the - * minion - the inner arrays are status codes to be returned (in order), - * after each inner array is finished a 429 will be returned, then the - * next array of error codes will be returned. + * Runs onRecordFound with a number of failing codes, testing whether the + * minion retries the correct number of times, and whether it correctly + * records a success after retries or a failure after the retries run out. + * + * This tests both 429 retries and other retries - this involves different + * behaviour as the retry for 429 (which indicates rate limiting) require + * a much longer cool-off time and hence are done differently. + * + * @param caption The caption to use for the mocha "it" call. + * @param result Whether to test for a number of retries then a success, a + * number of retries then a failure because of too many 429s, + * or a number of retries then a failure because of too many + * non-429 failures (e.g. 404s) */ - const failuresArb: jsc.Arbitrary = arbFlatMap< - number, - FailuresArbResult - >( - retryCountArb, - (retryCount: number) => { - /** Generates how many 429 codes will be returned */ - const count429Arb = - result === "fail429" - ? jsc.constant(retryCount) - : jsc.integer(0, retryCount); - - /** Generates how long the array of non-429 failures should be. */ - const failureCodeLengthArb = jsc.integer(0, retryCount); - - const allResultsArb = arbFlatMap( - count429Arb, - count429s => arrayOfSizeArb(count429s + 1, failureCodeLengthArb), - (failureCodeArr: number[]) => failureCodeArr.length - ).flatMap( - (failureCodeArrSizes: number[]) => { - const failureCodeArbs = failureCodeArrSizes.map(size => - arrayOfSizeArb(size, failureCodeArb) + const retrySpec = ( + caption: string, + result: "success" | "fail429" | "failNormal" + ) => { + it(caption, function () { + const retryCountArb = jsc.integer(0, 5); + + type FailuresArbResult = { + retryCount: number; + allResults: number[][]; + }; + + /** + * Generates a retryCount and a nested array of results to return to the + * minion - the inner arrays are status codes to be returned (in order), + * after each inner array is finished a 429 will be returned, then the + * next array of error codes will be returned. + */ + const failuresArb: jsc.Arbitrary = + arbFlatMap( + retryCountArb, + (retryCount: number) => { + /** Generates how many 429 codes will be returned */ + const count429Arb = + result === "fail429" + ? jsc.constant(retryCount) + : jsc.integer(0, retryCount); + + /** Generates how long the array of non-429 failures should be. */ + const failureCodeLengthArb = jsc.integer( + 0, + retryCount + ); + + const allResultsArb = arbFlatMap( + count429Arb, + (count429s) => + arrayOfSizeArb( + count429s + 1, + failureCodeLengthArb + ), + (failureCodeArr: number[]) => + failureCodeArr.length + ).flatMap( + (failureCodeArrSizes: number[]) => { + const failureCodeArbs = + failureCodeArrSizes.map((size) => + arrayOfSizeArb(size, failureCodeArb) + ); + + if (result === "failNormal") { + failureCodeArbs[ + failureCodeArbs.length - 1 + ] = arrayOfSizeArb( + retryCount + 1, + failureCodeArb + ); + } + + return failureCodeArrSizes.length > 0 + ? jsc.tuple(failureCodeArbs) + : jsc.constant([]); + }, + (failures) => + failures.map((inner) => inner.length) + ); + + const combined = jsc.record({ + retryCount: jsc.constant(retryCount), + allResults: allResultsArb + }); + + return combined; + }, + ({ retryCount }: FailuresArbResult) => { + return retryCount; + } + ); + + return jsc.assert( + jsc.forall( + httpOnlyRecordArb, + failuresArb, + ( + record: Record, + { retryCount, allResults }: FailuresArbResult + ) => { + beforeEachProperty(); + + const distScopes = urlsFromDataSet(record).map( + (url) => { + const scope = nock(url); //.log(console.log); + + allResults.forEach((failureCodes, i) => { + failureCodes.forEach((failureCode) => { + scope + .head( + url.endsWith("/") ? "/" : "" + ) + .reply(failureCode); + + scope + .get( + url.endsWith("/") ? "/" : "" + ) + .reply(failureCode); + }); + if ( + i < allResults.length - 1 || + result === "fail429" + ) { + scope + .head( + url.endsWith("/") ? "/" : "" + ) + .reply(429); + } + }); + + if (result === "success") { + scope + .head(url.endsWith("/") ? "/" : "") + .reply(200); + } + + return scope; + } + ); + + const allDists = + record.aspects["dataset-distributions"] + .distributions; + + allDists.forEach((dist: Record) => { + registryScope + .put( + `/records/${encodeURIComponentWithApost( + dist.id + )}/aspects/source-link-status?merge=true`, + (response: any) => { + const statusMatch = + response.status === + { + success: "active", + failNormal: "broken", + fail429: "unknown" + }[result]; + const codeMatch = + !_.isUndefined( + response.httpStatusCode + ) && + response.httpStatusCode === + { + success: 200, + failNormal: _.last( + _.last(allResults) + ), + fail429: 429 + }[result]; + + return statusMatch && codeMatch; + } + ) + .reply(201); + }); + + return onRecordFound( + record, + registry, + retryCount, + 0 + ) + .then(() => { + registryScope.done(); + distScopes.forEach((scope) => scope.done()); + }) + .then(() => { + afterEachProperty(); + return true; + }) + .catch((e) => { + afterEachProperty(); + throw e; + }); + } + ), + { + tests: 50 + } ); - - if (result === "failNormal") { - failureCodeArbs[failureCodeArbs.length - 1] = arrayOfSizeArb( - retryCount + 1, - failureCodeArb - ); - } - - return failureCodeArrSizes.length > 0 - ? jsc.tuple(failureCodeArbs) - : jsc.constant([]); - }, - failures => failures.map(inner => inner.length) - ); - - const combined = jsc.record({ - retryCount: jsc.constant(retryCount), - allResults: allResultsArb }); + }; - return combined; - }, - ({ retryCount }: FailuresArbResult) => { - return retryCount; - } + retrySpec( + "Should result in success if the last retry is successful", + "success" + ); + retrySpec( + "Should result in failures if the max number of retries is exceeded", + "failNormal" + ); + retrySpec( + "Should result in failures if the max number of 429s is exceeded", + "fail429" ); + }); - return jsc.assert( - jsc.forall( - httpOnlyRecordArb, - failuresArb, - (record: Record, { retryCount, allResults }: FailuresArbResult) => { - beforeEachProperty(); - - const distScopes = urlsFromDataSet(record).map(url => { - const scope = nock(url); //.log(console.log); - - allResults.forEach((failureCodes, i) => { - failureCodes.forEach(failureCode => { - scope.head(url.endsWith("/") ? "/" : "").reply(failureCode); - - scope.get(url.endsWith("/") ? "/" : "").reply(failureCode); - }); - if (i < allResults.length - 1 || result === "fail429") { - scope.head(url.endsWith("/") ? "/" : "").reply(429); - } - }); + it("Should only try to make one request per host at a time", function () { + const urlArb = (jsc as any).nonshrink( + distUrlArb({ + schemeArb: jsc.elements(["http", "https"]), + hostArb: jsc.elements(["example1", "example2", "example3"]) + }) + ); - if (result === "success") { - scope.head(url.endsWith("/") ? "/" : "").reply(200); - } + const thisRecordArb = jsc.suchthat( + recordArbWithDistArbs({ url: urlArb }), + (record) => { + const urls: string[] = urlsFromDataSet(record); + const hosts: string[] = urls.map((url) => { + const uri = new URI(url); - return scope; - }); - - const allDists = - record.aspects["dataset-distributions"].distributions; - - allDists.forEach((dist: Record) => { - registryScope - .put( - `/records/${encodeURIComponentWithApost( - dist.id - )}/aspects/source-link-status?merge=true`, - (response: any) => { - const statusMatch = - response.status === - { - success: "active", - failNormal: "broken", - fail429: "unknown" - }[result]; - const codeMatch = - !_.isUndefined(response.httpStatusCode) && - response.httpStatusCode === - { - success: 200, - failNormal: _.last(_.last(allResults)), - fail429: 429 - }[result]; - - return statusMatch && codeMatch; - } - ) - .reply(201); - }); - - return onRecordFound(record, registry, retryCount, 0) - .then(() => { - registryScope.done(); - distScopes.forEach(scope => scope.done()); - }) - .then(() => { - afterEachProperty(); - return true; - }) - .catch(e => { - afterEachProperty(); - throw e; + return uri.scheme() + "://" + uri.host(); }); + + return !_.isEqual(_.uniq(hosts), hosts); } - ), - { - tests: 50 - } ); - }); - }; - retrySpec( - "Should result in success if the last retry is successful", - "success" - ); - retrySpec( - "Should result in failures if the max number of retries is exceeded", - "failNormal" - ); - retrySpec( - "Should result in failures if the max number of 429s is exceeded", - "fail429" - ); - }); - - it("Should only try to make one request per host at a time", function() { - const urlArb = (jsc as any).nonshrink( - distUrlArb({ - schemeArb: jsc.elements(["http", "https"]), - hostArb: jsc.elements(["example1", "example2", "example3"]) - }) - ); - - const thisRecordArb = jsc.suchthat( - recordArbWithDistArbs({ url: urlArb }), - record => { - const urls: string[] = urlsFromDataSet(record); - const hosts: string[] = urls.map(url => { - const uri = new URI(url); - - return uri.scheme() + "://" + uri.host(); - }); + return jsc.assert( + jsc.forall( + thisRecordArb, + jsc.nearray(failureCodeArb), + jsc.integer(0, 25), + (record: Record, failures: number[], delayMs: number) => { + beforeEachProperty(); + + const delayConfig = {} as any; + + const distScopes = urlsFromDataSet(record).reduce( + (scopeLookup, url) => { + const uri = new URI(url); + const base = uri.scheme() + "://" + uri.host(); + delayConfig[uri.hostname()] = (delayMs + 10) / 1000; + + if (!scopeLookup[base]) { + scopeLookup[base] = nock(base); + } + + const scope = scopeLookup[base]; + + failures.forEach((failureCode) => { + scope + .head(uri.path()) + .delay(delayMs) + .reply(failureCode); + + scope + .get(uri.path()) + .delay(delayMs) + .reply(failureCode); + }); + + scope.head(uri.path()).delay(delayMs).reply(200); + return scopeLookup; + }, + {} as { [host: string]: nock.Scope } + ); + + _.forEach(distScopes, (scope: nock.Scope, host: string) => { + let countForThisScope = 0; + + scope.on("request", () => { + countForThisScope++; + expect(countForThisScope).to.equal(1); + }); + + scope.on("replied", () => { + countForThisScope--; + expect(countForThisScope).to.equal(0); + }); + }); + + const allDists = + record.aspects["dataset-distributions"].distributions; + + registryScope.put(/.*/).times(allDists.length).reply(201); + + return onRecordFound( + record, + registry, + failures.length, + 0, + delayConfig + ) + .then(() => { + _.values(distScopes).forEach((scope) => + scope.done() + ); + }) + .then(() => { + afterEachProperty(); + return true; + }) + .catch((e) => { + afterEachProperty(); + throw e; + }); + } + ), + { + tests: 10 + } + ); + }); + + const emptyRecordArb = jsc.oneof([ + specificRecordArb({ + "dataset-distributions": jsc.constant(undefined) + }), + specificRecordArb({ + "dataset-distributions": jsc.record({ + distributions: jsc.constant([]) + }) + }) + ]); - return !_.isEqual(_.uniq(hosts), hosts); - } - ); + jsc.property( + "Should do nothing if no distributions", + emptyRecordArb, + (record) => { + beforeEachProperty(); - return jsc.assert( - jsc.forall( - thisRecordArb, - jsc.nearray(failureCodeArb), - jsc.integer(0, 25), - (record: Record, failures: number[], delayMs: number) => { - beforeEachProperty(); - - const delayConfig = {} as any; - - const distScopes = urlsFromDataSet(record).reduce( - (scopeLookup, url) => { - const uri = new URI(url); - const base = uri.scheme() + "://" + uri.host(); - delayConfig[uri.hostname()] = (delayMs + 10) / 1000; - - if (!scopeLookup[base]) { - scopeLookup[base] = nock(base); - } - - const scope = scopeLookup[base]; - - failures.forEach(failureCode => { - scope - .head(uri.path()) - .delay(delayMs) - .reply(failureCode); - - scope - .get(uri.path()) - .delay(delayMs) - .reply(failureCode); - }); - - scope - .head(uri.path()) - .delay(delayMs) - .reply(200); - return scopeLookup; - }, - {} as { [host: string]: nock.Scope } - ); - - _.forEach(distScopes, (scope: nock.Scope, host: string) => { - let countForThisScope = 0; - - scope.on("request", () => { - countForThisScope++; - expect(countForThisScope).to.equal(1); - }); + return onRecordFound(record, registry).then(() => { + afterEachProperty(); - scope.on("replied", () => { - countForThisScope--; - expect(countForThisScope).to.equal(0); - }); - }); - - const allDists = - record.aspects["dataset-distributions"].distributions; - - registryScope - .put(/.*/) - .times(allDists.length) - .reply(201); - - return onRecordFound( - record, - registry, - failures.length, - 0, - delayConfig - ) - .then(() => { - _.values(distScopes).forEach(scope => scope.done()); - }) - .then(() => { - afterEachProperty(); - return true; - }) - .catch(e => { - afterEachProperty(); - throw e; + registryScope.done(); + return true; }); } - ), - { - tests: 10 - } ); - }); - - const emptyRecordArb = jsc.oneof([ - specificRecordArb({ - "dataset-distributions": jsc.constant(undefined) - }), - specificRecordArb({ - "dataset-distributions": jsc.record({ - distributions: jsc.constant([]) - }) - }) - ]); - - jsc.property( - "Should do nothing if no distributions", - emptyRecordArb, - record => { - beforeEachProperty(); - - return onRecordFound(record, registry).then(() => { - afterEachProperty(); - - registryScope.done(); - return true; - }); - } - ); }); function hasProtocol(url: string, protocol: string) { - const uri = parseUriSafe(url); + const uri = parseUriSafe(url); - return uri && uri.protocol().startsWith(protocol); + return uri && uri.protocol().startsWith(protocol); } From f7f7d8553bcf77146990d43359e13faa52847e0d Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 00:41:56 +1000 Subject: [PATCH 06/14] add internal storage url support --- src/index.ts | 2 ++ src/onRecordFound.ts | 23 ++++++++++++++++++++--- src/test/onRecordFound.spec.ts | 17 +++++++++++++++-- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index c935602..cbcd428 100644 --- a/src/index.ts +++ b/src/index.ts @@ -61,6 +61,8 @@ function sleuthBrokenLinks() { onRecordFound( record, registry, + argv.storageApiBaseUrl, + argv.datasetBucketName, argv.externalRetries, 1, argv.domainWaitTimeConfig as any, diff --git a/src/onRecordFound.ts b/src/onRecordFound.ts index ca34507..3475caf 100644 --- a/src/onRecordFound.ts +++ b/src/onRecordFound.ts @@ -5,6 +5,7 @@ import { AuthorizedRegistryClient as Registry, Record } from "@magda/minion-sdk"; +import { getStorageApiResourceAccessUrl } from "@magda/utils"; import { BrokenLinkAspect, RetrieveResult } from "./brokenLinkAspectDef.js"; import FTPHandler from "./FtpHandler.js"; import parseUriSafe from "./parseUriSafe.js"; @@ -19,6 +20,8 @@ import wait from "./wait.js"; export default async function onRecordFound( record: Record, registry: Registry, + storageApiBaseUrl: string, + datasetBucketName: string, retries: number = 1, baseRetryDelaySeconds: number = 1, domainWaitTimeConfig: { [domain: string]: number } = {}, @@ -44,7 +47,9 @@ export default async function onRecordFound( retries, ftpHandler, _.partialRight(getUrlWaitTime, domainWaitTimeConfig), - requestOpts + requestOpts, + storageApiBaseUrl, + datasetBucketName ) ); @@ -165,7 +170,9 @@ function checkDistributionLink( retries: number, ftpHandler: FTPHandler, getUrlWaitTime: (url: string) => number, - requestOpts: CoreOptions + requestOpts: CoreOptions, + storageApiBaseUrl: string, + datasetBucketName: string ): DistributionLinkCheck[] { type DistURL = { url?: URI; @@ -182,7 +189,17 @@ function checkDistributionLink( type: "accessURL" as "accessURL" } ] - .map((urlObj) => ({ ...urlObj, url: parseUriSafe(urlObj.url) })) + .map((urlObj) => { + const url = + typeof urlObj.url === "string" + ? getStorageApiResourceAccessUrl( + urlObj.url, + storageApiBaseUrl, + datasetBucketName + ) + : urlObj.url; + return { ...urlObj, url: parseUriSafe(url) }; + }) .filter((x) => x.url && x.url.protocol().length > 0); if (urls.length === 0) { diff --git a/src/test/onRecordFound.spec.ts b/src/test/onRecordFound.spec.ts index 0c924a3..daef25d 100644 --- a/src/test/onRecordFound.spec.ts +++ b/src/test/onRecordFound.spec.ts @@ -36,6 +36,8 @@ import { getDefaultDomainWaitTime } from "../getUrlWaitTime.js"; +const defaultStorageApiBaseUrl = "http://storage-api/v0"; +const defaultDatsetBucketName = "magda-datasets"; const schema = require("@magda/registry-aspects/source-link-status.schema.json"); describe("onRecordFound", function (this: Mocha.Suite) { @@ -376,6 +378,8 @@ describe("onRecordFound", function (this: Mocha.Suite) { return onRecordFound( record, registry, + defaultStorageApiBaseUrl, + defaultDatsetBucketName, 0, 0, {}, @@ -587,6 +591,8 @@ describe("onRecordFound", function (this: Mocha.Suite) { return onRecordFound( record, registry, + defaultStorageApiBaseUrl, + defaultDatsetBucketName, retryCount, 0 ) @@ -709,6 +715,8 @@ describe("onRecordFound", function (this: Mocha.Suite) { return onRecordFound( record, registry, + defaultStorageApiBaseUrl, + defaultDatsetBucketName, failures.length, 0, delayConfig @@ -748,10 +756,15 @@ describe("onRecordFound", function (this: Mocha.Suite) { jsc.property( "Should do nothing if no distributions", emptyRecordArb, - (record) => { + (record: Record) => { beforeEachProperty(); - return onRecordFound(record, registry).then(() => { + return onRecordFound( + record, + registry, + defaultStorageApiBaseUrl, + defaultDatsetBucketName + ).then(() => { afterEachProperty(); registryScope.done(); From 04473bcc2506b32fa094979055d74e5dde8f95d2 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 17:05:37 +1000 Subject: [PATCH 07/14] add test cases for checking internal storage resource links --- CHANGES.md | 21 ++-- src/test/testStorageUrl.spec.ts | 172 ++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+), 10 deletions(-) create mode 100644 src/test/testStorageUrl.spec.ts diff --git a/CHANGES.md b/CHANGES.md index a53e8ff..24a1170 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,18 +1,19 @@ # 3.0.0 -- Upgrade nodejs to version 18 -- Upgrade to Magda minion SDK v3 +- Upgrade nodejs to version 18 +- Upgrade to Magda minion SDK v3 +- Fixed support for checking internal storage resource links # 2.0.0 -- Upgrade nodejs to version 14 -- Upgrade other dependencies -- Release all artifacts to GitHub Container Registry (instead of docker.io & https://charts.magda.io) -- Upgrade magda-common chart version -- Upgrade api to batch/v1 to be compatible with k8s 1.25 (now requires >=1.21) +- Upgrade nodejs to version 14 +- Upgrade other dependencies +- Release all artifacts to GitHub Container Registry (instead of docker.io & https://charts.magda.io) +- Upgrade magda-common chart version +- Upgrade api to batch/v1 to be compatible with k8s 1.25 (now requires >=1.21) # 1.0.0 -- Upgrade dependencies -- Upgrade CI scripts -- Related to https://github.com/magda-io/magda/issues/3229, Use magda-common for docker image related logic +- Upgrade dependencies +- Upgrade CI scripts +- Related to https://github.com/magda-io/magda/issues/3229, Use magda-common for docker image related logic diff --git a/src/test/testStorageUrl.spec.ts b/src/test/testStorageUrl.spec.ts new file mode 100644 index 0000000..e73e088 --- /dev/null +++ b/src/test/testStorageUrl.spec.ts @@ -0,0 +1,172 @@ +import {} from "mocha"; +import { expect } from "chai"; +import sinon from "sinon"; +import nock from "nock"; +import _ from "lodash"; +import Ajv from "ajv"; +import urijs from "urijs"; + +import { Record, AuthorizedRegistryClient } from "@magda/minion-sdk"; +import { encodeURIComponentWithApost } from "@magda/utils"; + +import onRecordFound from "../onRecordFound.js"; +import { BrokenLinkAspect } from "../brokenLinkAspectDef.js"; +import { + setDefaultDomainWaitTime, + getDefaultDomainWaitTime +} from "../getUrlWaitTime.js"; + +const defaultStorageApiBaseUrl = "http://storage-api/v0"; +const defaultDatasetBucketName = "magda-datasets"; +const schema = require("@magda/registry-aspects/source-link-status.schema.json"); + +describe("Test Internal Storage URL", function (this: Mocha.Suite) { + this.timeout(20000); + nock.disableNetConnect(); + const registryUrl = "http://example.com"; + const secret = "secret!"; + const registry = new AuthorizedRegistryClient({ + baseUrl: registryUrl, + jwtSecret: secret, + userId: "1", + maxRetries: 0, + tenantId: 1 + }); + let registryScope: nock.Scope; + const orignalDefaultDomainWaitTime: number = getDefaultDomainWaitTime(); + + before(() => { + // --- set default domain wait time to 0 second (i.e. for any domains that has no specific setting) + // --- Otherwise, it will take too long to complete property tests + setDefaultDomainWaitTime(0); + sinon.stub(console, "info"); + nock.disableNetConnect(); + nock.emitter.on("no match", onMatchFail); + }); + + const onMatchFail = (req: any, interceptor: any) => { + console.error( + `Match failure. \n Req: ${req.method} ${req.host}${req.path} \n interceptor: ${interceptor.href}` + ); + }; + + after(() => { + setDefaultDomainWaitTime(orignalDefaultDomainWaitTime); + + (console.info as any).restore(); + + nock.emitter.removeListener("no match", onMatchFail); + }); + + beforeEach(() => { + registryScope = nock(registryUrl); //.log(console.log); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + /** + * Generator-driven super-test: generates records and runs them through the + * onRecordFound function, listening for HTTP and FTP calls made and returning + * success or failure based on generated outcomes, then checks that they're + * recorded on a by-distribution basis as link status as well as on a by-record + * basis as a part of dataset quality. + */ + it("Should correctly record link statuses", async () => { + const ajv = new Ajv(); + const validate = ajv.compile(schema); + + const testRecord: Record = { + id: "ds-1", + name: "Test Record", + sourceTag: "test-source", + tenantId: 1, + aspects: { + "dataset-distributions": { + distributions: [ + { + id: "dist-1", + aspects: { + "dcat-distribution-strings": { + downloadURL: + "magda://storage-api/ds-1/dist-1/test-file1.pdf" + } + } + }, + { + id: "dist-2", + aspects: { + "dcat-distribution-strings": { + accessURL: + "magda://storage-api/ds-1/dist-2/test-file2.pdf" + } + } + } + ] + } + } + }; + + const defaultStorageApiBaseUri = urijs(defaultStorageApiBaseUrl); + + const storageApiScope = nock( + defaultStorageApiBaseUri.clone().path("").toString() + ); + + storageApiScope + .head( + `${defaultStorageApiBaseUri.path()}/${defaultDatasetBucketName}/ds-1/dist-1/test-file1.pdf` + ) + .query(true) + .reply(200); + + storageApiScope + .head( + `${defaultStorageApiBaseUri.path()}/${defaultDatasetBucketName}/ds-1/dist-2/test-file2.pdf` + ) + .query(true) + .reply(200); + + ["dist-1", "dist-2"].forEach((distId) => { + registryScope + .put( + `/records/${encodeURIComponentWithApost( + distId + )}/aspects/source-link-status?merge=true`, + (body: BrokenLinkAspect) => { + const validationResult = validate(body); + if (!validationResult) { + throw new Error( + "Json schema validation error: \n" + + validate.errors + .map( + (error) => + `${error.dataPath}: ${error.message}` + ) + .join("\n") + ); + } + console.log(body); + expect(body.status).to.equal("active"); + return true; + } + ) + .reply(201); + }); + + await onRecordFound( + testRecord, + registry, + defaultStorageApiBaseUrl, + defaultDatasetBucketName, + 0, + 0, + {}, + {} + ); + + registryScope.done(); + storageApiScope.done(); + }); +}); From 892a0d7c459c1fd4a30c302ed5ee342892afff99 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 17:52:02 +1000 Subject: [PATCH 08/14] update helm docs script --- .github/workflows/main.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- package.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 98753fc..f7ee586 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,7 +31,7 @@ jobs: - name: Check Helm Chart Document run: | code=0 - docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.5.0 -t ./README.md.gotmpl -o ../../README.md || code=$?; + docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.13.1 -t ./README.md.gotmpl -o ../../README.md || code=$?; if [ "$code" != "0" ]; then echo "Failed to run helm-docs!"; exit 1; @@ -41,7 +41,7 @@ jobs: git ls-files -m | grep -i readme.md || code=$?; if [ "$code" == "0" ]; then echo -e "Some of helm chart docs are required to be updated using the [helm-docs](https://github.com/norwoodj/helm-docs) tool. \n - Please run helm-docs (v1.5.0) at project root, review & commit docs changes and push a new commit."; + Please run helm-docs (v1.13.1) at project root, review & commit docs changes and push a new commit."; exit 1; else echo -e "helm docs check passed. helm docs update is not required."; diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8bcab9a..0168dcd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,7 +34,7 @@ jobs: - name: Check Helm Chart Document run: | code=0 - docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.5.0 -t ./README.md.gotmpl -o ../../README.md || code=$?; + docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.13.1 -t ./README.md.gotmpl -o ../../README.md || code=$?; if [ "$code" != "0" ]; then echo "Failed to run helm-docs!"; exit 1; @@ -44,7 +44,7 @@ jobs: git ls-files -m | grep -i readme.md || code=$?; if [ "$code" == "0" ]; then echo -e "Some of helm chart docs are required to be updated using the [helm-docs](https://github.com/norwoodj/helm-docs) tool. \n - Please run helm-docs (v1.5.0) at project root, review & commit docs changes and push a new commit."; + Please run helm-docs (v1.13.1) at project root, review & commit docs changes and push a new commit."; exit 1; else echo -e "helm docs check passed. helm docs update is not required."; diff --git a/package.json b/package.json index 4ddf6b7..7fcfc88 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "test": "mocha", "helm-lint": "helm lint deploy/magda-minion-broken-link -f deploy/test-deploy.yaml", "retag-and-push": "retag-and-push", - "helm-docs": "helm-docs -t ./README.md.gotmpl -o ../../README.md", + "helm-docs": "docker run --rm -v \"$(pwd):/helm-docs\" -u $(id -u) jnorwood/helm-docs:v1.13.1 -t ./README.md.gotmpl -o ../../README.md", "update-all-charts": "helm dep up ./deploy/magda-minion-broken-link", "add-all-chart-version-changes": "git ls-files -m | grep Chart.yaml | xargs git add && git ls-files -m | grep Chart.lock | xargs git add", "add-all-helm-docs-changes": "yarn helm-docs && git ls-files -m | grep -i readme.md | xargs git add", From f5dea4662e61474280a3724c0c4e76cd6df0c675 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 17:56:19 +1000 Subject: [PATCH 09/14] update docs --- README.md | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 7e61c59..d85f4bd 100644 --- a/README.md +++ b/README.md @@ -31,22 +31,24 @@ Kubernetes: `>= 1.21.0` ## Values -| Key | Type | Default | Description | -| ---------------------------- | ------ | ---------------------------------------- | ----------- | -| cronJobImage.name | string | `"alpine"` | | -| cronJobImage.pullPolicy | string | `"IfNotPresent"` | | -| cronJobImage.pullSecrets | bool | `false` | | -| cronJobImage.repository | string | `"docker.io"` | | -| cronJobImage.tag | string | `"latest"` | | -| defaultAdminUserId | string | `"00000000-0000-4000-8000-000000000000"` | | -| defaultImage.imagePullSecret | bool | `false` | | -| defaultImage.pullPolicy | string | `"IfNotPresent"` | | -| defaultImage.repository | string | `"ghcr.io/magda-io"` | | -| global.image | object | `{}` | | -| global.minions.image | object | `{}` | | -| global.rollingUpdate | object | `{}` | | -| image.name | string | `"magda-minion-broken-link"` | | -| resources.limits.cpu | string | `"200m"` | | -| resources.requests.cpu | string | `"50m"` | | -| resources.requests.memory | string | `"40Mi"` | | -| schedule | string | `"0 0 14,28 * *"` | | +| Key | Type | Default | Description | +| ---------------------------- | ------ | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| cronJobImage.name | string | `"alpine"` | | +| cronJobImage.pullPolicy | string | `"IfNotPresent"` | | +| cronJobImage.pullSecrets | bool | `false` | | +| cronJobImage.repository | string | `"docker.io"` | | +| cronJobImage.tag | string | `"latest"` | | +| datasetBucketName | string | `""` | The name of the storage bucket where all dataset files are stored. Should match storage API config. By default, it will use the value of `global.defaultDatasetBucket` (defined in `magda-core` chart) unless you specify a different value here. | +| defaultAdminUserId | string | `"00000000-0000-4000-8000-000000000000"` | | +| defaultImage.imagePullSecret | bool | `false` | | +| defaultImage.pullPolicy | string | `"IfNotPresent"` | | +| defaultImage.repository | string | `"ghcr.io/magda-io"` | | +| global.image | object | `{}` | | +| global.minions.image | object | `{}` | | +| global.rollingUpdate | object | `{}` | | +| image.name | string | `"magda-minion-broken-link"` | | +| resources.limits.cpu | string | `"200m"` | | +| resources.requests.cpu | string | `"50m"` | | +| resources.requests.memory | string | `"40Mi"` | | +| schedule | string | `"0 0 14,28 * *"` | | +| storageApiBaseUrl | string | `"http://storage-api/v0"` | The base URL of the storage API to use when generating access URLs for MAGDA internal stored resources. | From c24c9c4062b84108f2b4e935f6c16d9fa9a18112 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 21:26:34 +1000 Subject: [PATCH 10/14] fixed: checking internal storage res as admin users --- src/index.ts | 2 + src/onRecordFound.ts | 81 +++++++++++++++++++++++++-------- src/test/onRecordFound.spec.ts | 20 ++++++-- src/test/testStorageUrl.spec.ts | 17 +++++-- 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/src/index.ts b/src/index.ts index cbcd428..b87a1b3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -63,6 +63,8 @@ function sleuthBrokenLinks() { registry, argv.storageApiBaseUrl, argv.datasetBucketName, + argv.jwtSecret, + argv.userId, argv.externalRetries, 1, argv.domainWaitTimeConfig as any, diff --git a/src/onRecordFound.ts b/src/onRecordFound.ts index 3475caf..048bbe9 100644 --- a/src/onRecordFound.ts +++ b/src/onRecordFound.ts @@ -16,12 +16,15 @@ import { } from "./HttpRequests.js"; import getUrlWaitTime from "./getUrlWaitTime.js"; import wait from "./wait.js"; +import { buildJwt } from "@magda/utils"; export default async function onRecordFound( record: Record, registry: Registry, storageApiBaseUrl: string, datasetBucketName: string, + jwtSecret: string, + actionUserId: string, retries: number = 1, baseRetryDelaySeconds: number = 1, domainWaitTimeConfig: { [domain: string]: number } = {}, @@ -49,7 +52,9 @@ export default async function onRecordFound( _.partialRight(getUrlWaitTime, domainWaitTimeConfig), requestOpts, storageApiBaseUrl, - datasetBucketName + datasetBucketName, + jwtSecret, + actionUserId ) ); @@ -172,7 +177,9 @@ function checkDistributionLink( getUrlWaitTime: (url: string) => number, requestOpts: CoreOptions, storageApiBaseUrl: string, - datasetBucketName: string + datasetBucketName: string, + jwtSecret: string, + actionUserId: string ): DistributionLinkCheck[] { type DistURL = { url?: URI; @@ -190,15 +197,7 @@ function checkDistributionLink( } ] .map((urlObj) => { - const url = - typeof urlObj.url === "string" - ? getStorageApiResourceAccessUrl( - urlObj.url, - storageApiBaseUrl, - datasetBucketName - ) - : urlObj.url; - return { ...urlObj, url: parseUriSafe(url) }; + return { ...urlObj, url: parseUriSafe(urlObj.url) }; }) .filter((x) => x.url && x.url.protocol().length > 0); @@ -232,7 +231,11 @@ function checkDistributionLink( retries, ftpHandler, getUrlWaitTime, - requestOpts + requestOpts, + storageApiBaseUrl, + datasetBucketName, + jwtSecret, + actionUserId ) .then((aspect) => { console.info("Finished retrieving " + parsedURL); @@ -262,15 +265,28 @@ function retrieve( retries: number, ftpHandler: FTPHandler, getUrlWaitTime: (url: string) => number, - requestOpts: CoreOptions + requestOpts: CoreOptions, + storageApiBaseUrl: string, + datasetBucketName: string, + jwtSecret: string, + actionUserId: string ): Promise { - if (parsedURL.protocol() === "http" || parsedURL.protocol() === "https") { + if ( + parsedURL.protocol() === "http" || + parsedURL.protocol() === "https" || + (parsedURL.protocol() === "magda" && + parsedURL.hostname() === "storage-api") + ) { return retrieveHttp( parsedURL.toString(), baseRetryDelay, retries, getUrlWaitTime, - requestOpts + requestOpts, + storageApiBaseUrl, + datasetBucketName, + jwtSecret, + actionUserId ); } else if (parsedURL.protocol() === "ftp") { return retrieveFtp(parsedURL, ftpHandler); @@ -319,16 +335,41 @@ async function retrieveHttp( baseRetryDelay: number, retries: number, getUrlWaitTime: (url: string) => number, - requestOpts: CoreOptions + requestOpts: CoreOptions, + storageApiBaseUrl: string, + datasetBucketName: string, + jwtSecret: string, + actionUserId: string ): Promise { + const isInternalStorageRes = url.indexOf("magda://storage-api/") === 0; + const resUrl = getStorageApiResourceAccessUrl( + url, + storageApiBaseUrl, + datasetBucketName + ); + const runtimeRequestOpts = { ...requestOpts }; + if (requestOpts?.headers) { + runtimeRequestOpts.headers = { + ...requestOpts.headers + }; + } + if (isInternalStorageRes) { + if (!runtimeRequestOpts?.headers) { + runtimeRequestOpts.headers = {}; + } + runtimeRequestOpts.headers = { + ...runtimeRequestOpts.headers, + "X-Magda-Session": buildJwt(jwtSecret, actionUserId) + }; + } async function operation() { try { - await wait(getUrlWaitTime(url)); - return await headRequest(url, requestOpts); + await wait(getUrlWaitTime(resUrl)); + return await headRequest(resUrl, runtimeRequestOpts); } catch (e) { // --- HEAD Method not allowed - await wait(getUrlWaitTime(url)); - return await getRequest(url, requestOpts); + await wait(getUrlWaitTime(resUrl)); + return await getRequest(resUrl, runtimeRequestOpts); } } diff --git a/src/test/onRecordFound.spec.ts b/src/test/onRecordFound.spec.ts index daef25d..0cb9087 100644 --- a/src/test/onRecordFound.spec.ts +++ b/src/test/onRecordFound.spec.ts @@ -37,7 +37,9 @@ import { } from "../getUrlWaitTime.js"; const defaultStorageApiBaseUrl = "http://storage-api/v0"; -const defaultDatsetBucketName = "magda-datasets"; +const defaultDatasetBucketName = "magda-datasets"; +const jwtSecret = "sdsfsfdsfsddsfsdfdsfds2323432423"; +const actionUserId = "user-id-1"; const schema = require("@magda/registry-aspects/source-link-status.schema.json"); describe("onRecordFound", function (this: Mocha.Suite) { @@ -379,7 +381,9 @@ describe("onRecordFound", function (this: Mocha.Suite) { record, registry, defaultStorageApiBaseUrl, - defaultDatsetBucketName, + defaultDatasetBucketName, + jwtSecret, + actionUserId, 0, 0, {}, @@ -592,7 +596,9 @@ describe("onRecordFound", function (this: Mocha.Suite) { record, registry, defaultStorageApiBaseUrl, - defaultDatsetBucketName, + defaultDatasetBucketName, + jwtSecret, + actionUserId, retryCount, 0 ) @@ -716,7 +722,9 @@ describe("onRecordFound", function (this: Mocha.Suite) { record, registry, defaultStorageApiBaseUrl, - defaultDatsetBucketName, + defaultDatasetBucketName, + jwtSecret, + actionUserId, failures.length, 0, delayConfig @@ -763,7 +771,9 @@ describe("onRecordFound", function (this: Mocha.Suite) { record, registry, defaultStorageApiBaseUrl, - defaultDatsetBucketName + defaultDatasetBucketName, + jwtSecret, + actionUserId ).then(() => { afterEachProperty(); diff --git a/src/test/testStorageUrl.spec.ts b/src/test/testStorageUrl.spec.ts index e73e088..e281a82 100644 --- a/src/test/testStorageUrl.spec.ts +++ b/src/test/testStorageUrl.spec.ts @@ -15,9 +15,12 @@ import { setDefaultDomainWaitTime, getDefaultDomainWaitTime } from "../getUrlWaitTime.js"; +import { buildJwt } from "@magda/utils"; const defaultStorageApiBaseUrl = "http://storage-api/v0"; const defaultDatasetBucketName = "magda-datasets"; +const jwtSecret = "sdsfsfdsfsddsfsdfdsfds2323432423"; +const actionUserId = "user-id-1"; const schema = require("@magda/registry-aspects/source-link-status.schema.json"); describe("Test Internal Storage URL", function (this: Mocha.Suite) { @@ -109,23 +112,25 @@ describe("Test Internal Storage URL", function (this: Mocha.Suite) { }; const defaultStorageApiBaseUri = urijs(defaultStorageApiBaseUrl); - + const jwt = buildJwt(jwtSecret, actionUserId); const storageApiScope = nock( - defaultStorageApiBaseUri.clone().path("").toString() + defaultStorageApiBaseUri.clone().path("").toString(), + { + reqheaders: { + "X-Magda-Session": jwt + } + } ); - storageApiScope .head( `${defaultStorageApiBaseUri.path()}/${defaultDatasetBucketName}/ds-1/dist-1/test-file1.pdf` ) - .query(true) .reply(200); storageApiScope .head( `${defaultStorageApiBaseUri.path()}/${defaultDatasetBucketName}/ds-1/dist-2/test-file2.pdf` ) - .query(true) .reply(200); ["dist-1", "dist-2"].forEach((distId) => { @@ -160,6 +165,8 @@ describe("Test Internal Storage URL", function (this: Mocha.Suite) { registry, defaultStorageApiBaseUrl, defaultDatasetBucketName, + jwtSecret, + actionUserId, 0, 0, {}, From 7323a8709d87798b33f476851af8fb7170a36018 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 22:05:11 +1000 Subject: [PATCH 11/14] monitor `dcat-distribution-strings` aspect instead to avoid unnecessary overhead --- src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index b87a1b3..b4f9516 100644 --- a/src/index.ts +++ b/src/index.ts @@ -52,11 +52,11 @@ function sleuthBrokenLinks() { return minion({ argv, id: ID, - aspects: ["dataset-distributions"], + aspects: ["dcat-distribution-strings"], optionalAspects: [], async: true, writeAspectDefs: [brokenLinkAspectDef], - dereference: true, + dereference: false, onRecordFound: (record, registry) => onRecordFound( record, From 264679c9f94d9aa72231e62cc7ab5a3899a7eb6e Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 23:05:07 +1000 Subject: [PATCH 12/14] adjust logic as `onRecordFound` expects distribution records now --- src/onRecordFound.ts | 9 +- src/test/onRecordFound.spec.ts | 466 ++++++++++++++++---------------- src/test/testStorageUrl.spec.ts | 33 +-- 3 files changed, 239 insertions(+), 269 deletions(-) diff --git a/src/onRecordFound.ts b/src/onRecordFound.ts index 048bbe9..2b0abbb 100644 --- a/src/onRecordFound.ts +++ b/src/onRecordFound.ts @@ -31,9 +31,12 @@ export default async function onRecordFound( requestOpts: CoreOptions = {}, ftpHandler: FTPHandler = new FTPHandler() ) { - const distributions: Record[] = - record.aspects["dataset-distributions"] && - record.aspects["dataset-distributions"].distributions; + const distributions: Record[] = [record]; + const dcatDistributionsStringsAspect = + record?.aspects?.["dcat-distribution-strings"]; + if (!dcatDistributionsStringsAspect) { + return Promise.resolve(); + } if (!distributions || distributions.length === 0) { return Promise.resolve(); diff --git a/src/test/onRecordFound.spec.ts b/src/test/onRecordFound.spec.ts index 0cb9087..454a070 100644 --- a/src/test/onRecordFound.spec.ts +++ b/src/test/onRecordFound.spec.ts @@ -377,19 +377,23 @@ describe("onRecordFound", function (this: Mocha.Suite) { .reply(201); }); - return onRecordFound( - record, - registry, - defaultStorageApiBaseUrl, - defaultDatasetBucketName, - jwtSecret, - actionUserId, - 0, - 0, - {}, - {}, - fakeFtpHandler - ) + const allOnRecordsTasks = allDists.map((dist) => + onRecordFound( + dist, + registry, + defaultStorageApiBaseUrl, + defaultDatasetBucketName, + jwtSecret, + actionUserId, + 0, + 0, + {}, + {}, + fakeFtpHandler + ) + ); + + return Promise.all(allOnRecordsTasks) .then(() => { distScopes.forEach((scope) => scope.done()); registryScope.done(); @@ -405,237 +409,225 @@ describe("onRecordFound", function (this: Mocha.Suite) { } ), { - tests: 500 + tests: 50 } ); }); - describe("retrying", () => { - /** - * Runs onRecordFound with a number of failing codes, testing whether the - * minion retries the correct number of times, and whether it correctly - * records a success after retries or a failure after the retries run out. - * - * This tests both 429 retries and other retries - this involves different - * behaviour as the retry for 429 (which indicates rate limiting) require - * a much longer cool-off time and hence are done differently. - * - * @param caption The caption to use for the mocha "it" call. - * @param result Whether to test for a number of retries then a success, a - * number of retries then a failure because of too many 429s, - * or a number of retries then a failure because of too many - * non-429 failures (e.g. 404s) - */ - const retrySpec = ( - caption: string, - result: "success" | "fail429" | "failNormal" - ) => { - it(caption, function () { - const retryCountArb = jsc.integer(0, 5); - - type FailuresArbResult = { - retryCount: number; - allResults: number[][]; - }; - - /** - * Generates a retryCount and a nested array of results to return to the - * minion - the inner arrays are status codes to be returned (in order), - * after each inner array is finished a 429 will be returned, then the - * next array of error codes will be returned. - */ - const failuresArb: jsc.Arbitrary = - arbFlatMap( - retryCountArb, - (retryCount: number) => { - /** Generates how many 429 codes will be returned */ - const count429Arb = - result === "fail429" - ? jsc.constant(retryCount) - : jsc.integer(0, retryCount); - - /** Generates how long the array of non-429 failures should be. */ - const failureCodeLengthArb = jsc.integer( - 0, - retryCount + /** + * Runs onRecordFound with a number of failing codes, testing whether the + * minion retries the correct number of times, and whether it correctly + * records a success after retries or a failure after the retries run out. + * + * This tests both 429 retries and other retries - this involves different + * behaviour as the retry for 429 (which indicates rate limiting) require + * a much longer cool-off time and hence are done differently. + * + * @param caption The caption to use for the mocha "it" call. + * @param result Whether to test for a number of retries then a success, a + * number of retries then a failure because of too many 429s, + * or a number of retries then a failure because of too many + * non-429 failures (e.g. 404s) + */ + const retrySpec = ( + caption: string, + result: "success" | "fail429" | "failNormal" + ) => { + it(caption, function () { + const retryCountArb = jsc.integer(0, 5); + + type FailuresArbResult = { + retryCount: number; + allResults: number[][]; + }; + + /** + * Generates a retryCount and a nested array of results to return to the + * minion - the inner arrays are status codes to be returned (in order), + * after each inner array is finished a 429 will be returned, then the + * next array of error codes will be returned. + */ + const failuresArb: jsc.Arbitrary = arbFlatMap< + number, + FailuresArbResult + >( + retryCountArb, + (retryCount: number) => { + /** Generates how many 429 codes will be returned */ + const count429Arb = + result === "fail429" + ? jsc.constant(retryCount) + : jsc.integer(0, retryCount); + + /** Generates how long the array of non-429 failures should be. */ + const failureCodeLengthArb = jsc.integer(0, retryCount); + + const allResultsArb = arbFlatMap( + count429Arb, + (count429s) => + arrayOfSizeArb(count429s + 1, failureCodeLengthArb), + (failureCodeArr: number[]) => failureCodeArr.length + ).flatMap( + (failureCodeArrSizes: number[]) => { + const failureCodeArbs = failureCodeArrSizes.map( + (size) => arrayOfSizeArb(size, failureCodeArb) ); - const allResultsArb = arbFlatMap( - count429Arb, - (count429s) => + if (result === "failNormal") { + failureCodeArbs[failureCodeArbs.length - 1] = arrayOfSizeArb( - count429s + 1, - failureCodeLengthArb - ), - (failureCodeArr: number[]) => - failureCodeArr.length - ).flatMap( - (failureCodeArrSizes: number[]) => { - const failureCodeArbs = - failureCodeArrSizes.map((size) => - arrayOfSizeArb(size, failureCodeArb) - ); - - if (result === "failNormal") { - failureCodeArbs[ - failureCodeArbs.length - 1 - ] = arrayOfSizeArb( - retryCount + 1, - failureCodeArb - ); - } - - return failureCodeArrSizes.length > 0 - ? jsc.tuple(failureCodeArbs) - : jsc.constant([]); - }, - (failures) => - failures.map((inner) => inner.length) - ); - - const combined = jsc.record({ - retryCount: jsc.constant(retryCount), - allResults: allResultsArb - }); + retryCount + 1, + failureCodeArb + ); + } - return combined; + return failureCodeArrSizes.length > 0 + ? jsc.tuple(failureCodeArbs) + : jsc.constant([]); }, - ({ retryCount }: FailuresArbResult) => { - return retryCount; - } + (failures) => failures.map((inner) => inner.length) ); - return jsc.assert( - jsc.forall( - httpOnlyRecordArb, - failuresArb, - ( - record: Record, - { retryCount, allResults }: FailuresArbResult - ) => { - beforeEachProperty(); - - const distScopes = urlsFromDataSet(record).map( - (url) => { - const scope = nock(url); //.log(console.log); - - allResults.forEach((failureCodes, i) => { - failureCodes.forEach((failureCode) => { - scope - .head( - url.endsWith("/") ? "/" : "" - ) - .reply(failureCode); - - scope - .get( - url.endsWith("/") ? "/" : "" - ) - .reply(failureCode); - }); - if ( - i < allResults.length - 1 || - result === "fail429" - ) { - scope - .head( - url.endsWith("/") ? "/" : "" - ) - .reply(429); - } - }); + const combined = jsc.record({ + retryCount: jsc.constant(retryCount), + allResults: allResultsArb + }); + + return combined; + }, + ({ retryCount }: FailuresArbResult) => { + return retryCount; + } + ); + + return jsc.assert( + jsc.forall( + httpOnlyRecordArb, + failuresArb, + ( + record: Record, + { retryCount, allResults }: FailuresArbResult + ) => { + beforeEachProperty(); + + const distScopes = urlsFromDataSet(record).map( + (url) => { + const scope = nock(url); //.log(console.log); + + allResults.forEach((failureCodes, i) => { + failureCodes.forEach((failureCode) => { + scope + .head(url.endsWith("/") ? "/" : "") + .reply(failureCode); - if (result === "success") { + scope + .get(url.endsWith("/") ? "/" : "") + .reply(failureCode); + }); + if ( + i < allResults.length - 1 || + result === "fail429" + ) { scope .head(url.endsWith("/") ? "/" : "") - .reply(200); + .reply(429); } + }); - return scope; + if (result === "success") { + scope + .head(url.endsWith("/") ? "/" : "") + .reply(200); } - ); - const allDists = - record.aspects["dataset-distributions"] - .distributions; - - allDists.forEach((dist: Record) => { - registryScope - .put( - `/records/${encodeURIComponentWithApost( - dist.id - )}/aspects/source-link-status?merge=true`, - (response: any) => { - const statusMatch = - response.status === + return scope; + } + ); + + const allDists = + record.aspects["dataset-distributions"] + .distributions; + + allDists.forEach((dist: Record) => { + registryScope + .put( + `/records/${encodeURIComponentWithApost( + dist.id + )}/aspects/source-link-status?merge=true`, + (response: any) => { + const statusMatch = + response.status === + { + success: "active", + failNormal: "broken", + fail429: "unknown" + }[result]; + const codeMatch = + !_.isUndefined( + response.httpStatusCode + ) && + response.httpStatusCode === { - success: "active", - failNormal: "broken", - fail429: "unknown" + success: 200, + failNormal: _.last( + _.last(allResults) + ), + fail429: 429 }[result]; - const codeMatch = - !_.isUndefined( - response.httpStatusCode - ) && - response.httpStatusCode === - { - success: 200, - failNormal: _.last( - _.last(allResults) - ), - fail429: 429 - }[result]; - - return statusMatch && codeMatch; - } - ) - .reply(201); - }); - return onRecordFound( - record, - registry, - defaultStorageApiBaseUrl, - defaultDatasetBucketName, - jwtSecret, - actionUserId, - retryCount, - 0 + return statusMatch && codeMatch; + } + ) + .reply(201); + }); + + return Promise.all( + allDists.map((dist: Record) => + onRecordFound( + dist, + registry, + defaultStorageApiBaseUrl, + defaultDatasetBucketName, + jwtSecret, + actionUserId, + retryCount, + 0 + ) ) - .then(() => { - registryScope.done(); - distScopes.forEach((scope) => scope.done()); - }) - .then(() => { - afterEachProperty(); - return true; - }) - .catch((e) => { - afterEachProperty(); - throw e; - }); - } - ), - { - tests: 50 + ) + .then(() => { + registryScope.done(); + distScopes.forEach((scope) => scope.done()); + }) + .then(() => { + afterEachProperty(); + return true; + }) + .catch((e) => { + afterEachProperty(); + throw e; + }); } - ); - }); - }; + ), + { + tests: 10 + } + ); + }); + }; - retrySpec( - "Should result in success if the last retry is successful", - "success" - ); - retrySpec( - "Should result in failures if the max number of retries is exceeded", - "failNormal" - ); - retrySpec( - "Should result in failures if the max number of 429s is exceeded", - "fail429" - ); - }); + retrySpec( + "Should result in success if the last retry is successful", + "success" + ); + retrySpec( + "Should result in failures if the max number of retries is exceeded", + "failNormal" + ); + retrySpec( + "Should result in failures if the max number of 429s is exceeded", + "fail429" + ); it("Should only try to make one request per host at a time", function () { const urlArb = (jsc as any).nonshrink( @@ -718,16 +710,20 @@ describe("onRecordFound", function (this: Mocha.Suite) { registryScope.put(/.*/).times(allDists.length).reply(201); - return onRecordFound( - record, - registry, - defaultStorageApiBaseUrl, - defaultDatasetBucketName, - jwtSecret, - actionUserId, - failures.length, - 0, - delayConfig + return Promise.all( + allDists.map((dist: Record) => + onRecordFound( + dist, + registry, + defaultStorageApiBaseUrl, + defaultDatasetBucketName, + jwtSecret, + actionUserId, + failures.length, + 0, + delayConfig + ) + ) ) .then(() => { _.values(distScopes).forEach((scope) => @@ -738,7 +734,7 @@ describe("onRecordFound", function (this: Mocha.Suite) { afterEachProperty(); return true; }) - .catch((e) => { + .catch((e: any) => { afterEachProperty(); throw e; }); @@ -752,13 +748,9 @@ describe("onRecordFound", function (this: Mocha.Suite) { const emptyRecordArb = jsc.oneof([ specificRecordArb({ - "dataset-distributions": jsc.constant(undefined) + "dcat-dataset-strings": jsc.constant({}) }), - specificRecordArb({ - "dataset-distributions": jsc.record({ - distributions: jsc.constant([]) - }) - }) + specificRecordArb({}) ]); jsc.property( diff --git a/src/test/testStorageUrl.spec.ts b/src/test/testStorageUrl.spec.ts index e281a82..6323a4e 100644 --- a/src/test/testStorageUrl.spec.ts +++ b/src/test/testStorageUrl.spec.ts @@ -81,32 +81,13 @@ describe("Test Internal Storage URL", function (this: Mocha.Suite) { const validate = ajv.compile(schema); const testRecord: Record = { - id: "ds-1", + id: "dist-1", name: "Test Record", sourceTag: "test-source", tenantId: 1, aspects: { - "dataset-distributions": { - distributions: [ - { - id: "dist-1", - aspects: { - "dcat-distribution-strings": { - downloadURL: - "magda://storage-api/ds-1/dist-1/test-file1.pdf" - } - } - }, - { - id: "dist-2", - aspects: { - "dcat-distribution-strings": { - accessURL: - "magda://storage-api/ds-1/dist-2/test-file2.pdf" - } - } - } - ] + "dcat-distribution-strings": { + accessURL: "magda://storage-api/ds-1/dist-1/test-file1.pdf" } } }; @@ -127,13 +108,7 @@ describe("Test Internal Storage URL", function (this: Mocha.Suite) { ) .reply(200); - storageApiScope - .head( - `${defaultStorageApiBaseUri.path()}/${defaultDatasetBucketName}/ds-1/dist-2/test-file2.pdf` - ) - .reply(200); - - ["dist-1", "dist-2"].forEach((distId) => { + ["dist-1"].forEach((distId) => { registryScope .put( `/records/${encodeURIComponentWithApost( From ddf75f2199ee2255052829e331bf1cb93ba31156 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Wed, 17 Apr 2024 23:06:21 +1000 Subject: [PATCH 13/14] fix test error --- src/test/onRecordFound.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/onRecordFound.spec.ts b/src/test/onRecordFound.spec.ts index 454a070..5a5a8ba 100644 --- a/src/test/onRecordFound.spec.ts +++ b/src/test/onRecordFound.spec.ts @@ -377,7 +377,7 @@ describe("onRecordFound", function (this: Mocha.Suite) { .reply(201); }); - const allOnRecordsTasks = allDists.map((dist) => + const allOnRecordsTasks = allDists.map((dist: Record) => onRecordFound( dist, registry, From c767eb0197bebb3276d09c57b6a7f9799cd008cb Mon Sep 17 00:00:00 2001 From: t83714 Date: Wed, 17 Apr 2024 13:34:12 +0000 Subject: [PATCH 14/14] v3.0.0 --- README.md | 54 +++++++++++----------- deploy/magda-minion-broken-link/Chart.yaml | 2 +- package.json | 2 +- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index d85f4bd..8ae4384 100644 --- a/README.md +++ b/README.md @@ -16,39 +16,39 @@ It's recommended to deploy minions with as [dependencies](https://helm.sh/docs/t ```yaml dependencies: - - name: magda-minion-broken-link - version: "2.0.0" - repository: "oci://ghcr.io/magda-io/charts" + - name: magda-minion-broken-link + version: "2.0.0" + repository: "oci://ghcr.io/magda-io/charts" ``` ## Requirements Kubernetes: `>= 1.21.0` -| Repository | Name | Version | -| ----------------------------- | ------------ | ------- | -| oci://ghcr.io/magda-io/charts | magda-common | 2.1.1 | +| Repository | Name | Version | +|------------|------|---------| +| oci://ghcr.io/magda-io/charts | magda-common | 2.1.1 | ## Values -| Key | Type | Default | Description | -| ---------------------------- | ------ | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| cronJobImage.name | string | `"alpine"` | | -| cronJobImage.pullPolicy | string | `"IfNotPresent"` | | -| cronJobImage.pullSecrets | bool | `false` | | -| cronJobImage.repository | string | `"docker.io"` | | -| cronJobImage.tag | string | `"latest"` | | -| datasetBucketName | string | `""` | The name of the storage bucket where all dataset files are stored. Should match storage API config. By default, it will use the value of `global.defaultDatasetBucket` (defined in `magda-core` chart) unless you specify a different value here. | -| defaultAdminUserId | string | `"00000000-0000-4000-8000-000000000000"` | | -| defaultImage.imagePullSecret | bool | `false` | | -| defaultImage.pullPolicy | string | `"IfNotPresent"` | | -| defaultImage.repository | string | `"ghcr.io/magda-io"` | | -| global.image | object | `{}` | | -| global.minions.image | object | `{}` | | -| global.rollingUpdate | object | `{}` | | -| image.name | string | `"magda-minion-broken-link"` | | -| resources.limits.cpu | string | `"200m"` | | -| resources.requests.cpu | string | `"50m"` | | -| resources.requests.memory | string | `"40Mi"` | | -| schedule | string | `"0 0 14,28 * *"` | | -| storageApiBaseUrl | string | `"http://storage-api/v0"` | The base URL of the storage API to use when generating access URLs for MAGDA internal stored resources. | +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| cronJobImage.name | string | `"alpine"` | | +| cronJobImage.pullPolicy | string | `"IfNotPresent"` | | +| cronJobImage.pullSecrets | bool | `false` | | +| cronJobImage.repository | string | `"docker.io"` | | +| cronJobImage.tag | string | `"latest"` | | +| datasetBucketName | string | `""` | The name of the storage bucket where all dataset files are stored. Should match storage API config. By default, it will use the value of `global.defaultDatasetBucket` (defined in `magda-core` chart) unless you specify a different value here. | +| defaultAdminUserId | string | `"00000000-0000-4000-8000-000000000000"` | | +| defaultImage.imagePullSecret | bool | `false` | | +| defaultImage.pullPolicy | string | `"IfNotPresent"` | | +| defaultImage.repository | string | `"ghcr.io/magda-io"` | | +| global.image | object | `{}` | | +| global.minions.image | object | `{}` | | +| global.rollingUpdate | object | `{}` | | +| image.name | string | `"magda-minion-broken-link"` | | +| resources.limits.cpu | string | `"200m"` | | +| resources.requests.cpu | string | `"50m"` | | +| resources.requests.memory | string | `"40Mi"` | | +| schedule | string | `"0 0 14,28 * *"` | | +| storageApiBaseUrl | string | `"http://storage-api/v0"` | The base URL of the storage API to use when generating access URLs for MAGDA internal stored resources. | \ No newline at end of file diff --git a/deploy/magda-minion-broken-link/Chart.yaml b/deploy/magda-minion-broken-link/Chart.yaml index 3951ff6..8cabeb3 100644 --- a/deploy/magda-minion-broken-link/Chart.yaml +++ b/deploy/magda-minion-broken-link/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: magda-minion-broken-link description: A Helm chart for Magda Broken Link Minion -version: "2.0.0" +version: "3.0.0" kubeVersion: ">= 1.21.0" home: "https://github.com/magda-io/magda-minion-broken-link" sources: [ "https://github.com/magda-io/magda-minion-broken-link" ] diff --git a/package.json b/package.json index b56295d..8475bc9 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@magda/minion-broken-link", "description": "MAGDA Broken Link Minion", - "version": "2.0.0", + "version": "3.0.0", "type": "module", "scripts": { "prebuild": "rimraf dist tsconfig.tsbuildinfo",