From 7bb712bab9ce2a8476a66bbf5a7ac86cc1a3534a Mon Sep 17 00:00:00 2001 From: Juan P Lopez Date: Fri, 1 Dec 2023 12:29:00 -0500 Subject: [PATCH 1/9] feat: move gt3 library --- BUCK | 1 + core/api/package.json | 2 +- core/api/src/services/geetest.ts | 2 +- core/api/src/services/geetest.types.d.ts | 2 +- .../geetest-lib-result.js | 20 ++ .../geetest-lib.js | 213 ++++++++++++++++++ lib/gt3-server-node-express-sdk/package.json | 15 ++ pnpm-lock.yaml | 55 +---- pnpm-workspace.yaml | 1 + 9 files changed, 265 insertions(+), 46 deletions(-) create mode 100644 lib/gt3-server-node-express-sdk/geetest-lib-result.js create mode 100644 lib/gt3-server-node-express-sdk/geetest-lib.js create mode 100644 lib/gt3-server-node-express-sdk/package.json diff --git a/BUCK b/BUCK index 0ceb9c1791..187926024e 100644 --- a/BUCK +++ b/BUCK @@ -8,6 +8,7 @@ pnpm_workspace( "//apps/dashboard:package.json", "//apps/pay:package.json", "//lib/eslint-config:package.json" + "//lib/gt3-server-node-express-sdk:package.json" ], visibility = ["PUBLIC"], ) diff --git a/core/api/package.json b/core/api/package.json index 8a76b34ac9..9692261883 100644 --- a/core/api/package.json +++ b/core/api/package.json @@ -34,6 +34,7 @@ "dependencies": { "@apollo/server": "^4.9.5", "@aws-sdk/client-s3": "^3.465.0", + "@galoy/gt3-server-node-express-sdk": "workspace:^", "@google-cloud/storage": "^7.7.0", "@grpc/grpc-js": "^1.9.12", "@grpc/proto-loader": "^0.7.10", @@ -76,7 +77,6 @@ "graphql-relay": "^0.10.0", "graphql-shield": "^7.6.4", "graphql-ws": "^5.14.2", - "gt3-server-node-express-sdk": "git+https://github.com/GaloyMoney/gt3-server-node-express-bypass.git#master", "i18n": "^0.15.1", "invoices": "^3.0.0", "ioredis": "^5.3.2", diff --git a/core/api/src/services/geetest.ts b/core/api/src/services/geetest.ts index 3d881ef869..0dceaa14b1 100644 --- a/core/api/src/services/geetest.ts +++ b/core/api/src/services/geetest.ts @@ -4,7 +4,7 @@ // doing this: "If the storage space is not sufficient: Send request to check bypass status before starting the verification process." import axios from "axios" -import GeetestLib from "gt3-server-node-express-sdk/sdk/geetest_lib" // galoy fork +import { GeetestLib } from "@galoy/gt3-server-node-express-sdk" import { addEventToCurrentSpan, recordExceptionInCurrentSpan } from "./tracing" diff --git a/core/api/src/services/geetest.types.d.ts b/core/api/src/services/geetest.types.d.ts index 57df40600f..cd69f2a72b 100644 --- a/core/api/src/services/geetest.types.d.ts +++ b/core/api/src/services/geetest.types.d.ts @@ -17,4 +17,4 @@ type GeetestType = { ) => Promise } -declare module "gt3-server-node-express-sdk/sdk/geetest_lib" +declare module "@galoy/gt3-server-node-express-sdk" diff --git a/lib/gt3-server-node-express-sdk/geetest-lib-result.js b/lib/gt3-server-node-express-sdk/geetest-lib-result.js new file mode 100644 index 0000000000..cf5bc855cb --- /dev/null +++ b/lib/gt3-server-node-express-sdk/geetest-lib-result.js @@ -0,0 +1,20 @@ +// sdk lib包的返回结果信息。 +class GeetestLibResult { + constructor() { + this.status = 0; // 成功失败的标识码，1表示成功，0表示失败 + this.data = ""; // 返回数据，json格式 + this.msg = ""; // 备注信息，如异常信息等 + } + + setAll(status, data, msg) { + this.status = status; + this.data = data; + this.msg = msg; + } + + toString() { + return `GeetestLibResult{status=${this.status}, data=${this.data}, msg=${this.msg}}`; + } +} + +module.exports = GeetestLibResult; diff --git a/lib/gt3-server-node-express-sdk/geetest-lib.js b/lib/gt3-server-node-express-sdk/geetest-lib.js new file mode 100644 index 0000000000..82098c055f --- /dev/null +++ b/lib/gt3-server-node-express-sdk/geetest-lib.js @@ -0,0 +1,213 @@ +const stringRandom = require('string-random'); +const crypto = require('crypto'); +const axios = require('axios'); +const qs = require('qs'); + +const GeetestLibResult = require("./geetest-lib-result"); + +class GeetestLib { + static IS_DEBUG = process.env.NODE_ENV !== "production"; // 调试开关，是否输出调试日志 + static API_URL = "http://api.geetest.com"; + static REGISTER_URL = "/register.php"; + static VALIDATE_URL = "/validate.php"; + static JSON_FORMAT = "1"; + static NEW_CAPTCHA = true; + static HTTP_TIMEOUT_DEFAULT = 5000; // 单位：毫秒 + static VERSION = "node-express:3.1.1"; + static GEETEST_CHALLENGE = "geetest_challenge"; // 极验二次验证表单传参字段 chllenge + static GEETEST_VALIDATE = "geetest_validate"; // 极验二次验证表单传参字段 validate + static GEETEST_SECCODE = "geetest_seccode"; // 极验二次验证表单传参字段 seccode + static GEETEST_SERVER_STATUS_SESSION_KEY = "gt_server_status"; // 极验验证API服务状态Session Key + + constructor(geetest_id, geetest_key) { + this.geetest_id = geetest_id; + this.geetest_key = geetest_key; + this.libResult = new GeetestLibResult(); + } + + gtlog(message) { + if (GeetestLib.IS_DEBUG) { + console.log("gtlog: " + message); + } + } + + /** + * 验证初始化 + */ + async register(digestmod, params) { + this.gtlog(`register(): 开始验证初始化, digestmod=${digestmod}.`); + const origin_challenge = await this.requestRegister(params); + this.buildRegisterResult(origin_challenge, digestmod) + this.gtlog(`register(): 验证初始化, lib包返回信息=${this.libResult}.`); + return this.libResult + } + + /** + * 向极验发送验证初始化的请求，GET方式 + */ + async requestRegister(params) { + params = Object.assign(params, { + "gt": this.geetest_id, + "json_format": GeetestLib.JSON_FORMAT, + "sdk": GeetestLib.VERSION + }); + const register_url = GeetestLib.API_URL + GeetestLib.REGISTER_URL; + this.gtlog(`requestRegister(): 验证初始化, 向极验发送请求, url=${register_url}, params=${JSON.stringify(params)}.`); + let origin_challenge; + try { + const res = await axios({ + url: register_url, + method: "GET", + timeout: GeetestLib.HTTP_TIMEOUT_DEFAULT, + params: params + }); + const resBody = (res.status === 200) ? res.data : ""; + this.gtlog(`requestRegister(): 验证初始化, 与极验网络交互正常, 返回码=${res.status}, 返回body=${JSON.stringify(resBody)}.`); + origin_challenge = resBody["challenge"]; + } catch (e) { + this.gtlog("requestRegister(): 验证初始化, 请求异常，后续流程走宕机模式, " + e.message); + origin_challenge = ""; + } + return origin_challenge; + } + async localRegister(){ + this.gtlog("获取当前缓存中bypass状态为fail，后续流程走宕机模式 "); + this.buildRegisterResult("", "") + this.gtlog(`register(): 验证初始化, lib包返回信息=${this.libResult}.`); + return this.libResult + } + /** + * 构建验证初始化返回数据 + */ + buildRegisterResult(origin_challenge, digestmod) { + // origin_challenge为空或者值为0代表失败 + if (!origin_challenge || origin_challenge === "0") { + // 本地随机生成32位字符串 + const challenge = stringRandom(32).toLowerCase(); + const data = { + "success": 0, + "gt": this.geetest_id, + "challenge": challenge, + "new_captcha": GeetestLib.NEW_CAPTCHA + }; + this.libResult.setAll(0, JSON.stringify(data), "获取当前缓存中bypass状态为fail，本地生成challenge，后续流程走宕机模式") + } else { + let challenge; + if (digestmod === "md5") { + challenge = this.md5_encode(origin_challenge + this.geetest_key); + } else if (digestmod === "sha256") { + challenge = this.sha256_encode(origin_challenge + this.geetest_key); + } else if (digestmod === "hmac-sha256") { + challenge = this.hmac_sha256_encode(origin_challenge, this.geetest_key); + } else { + challenge = this.md5_encode(origin_challenge + this.geetest_key); + } + const data = { + "success": 1, + "gt": this.geetest_id, + "challenge": challenge, + "new_captcha": GeetestLib.NEW_CAPTCHA + }; + this.libResult.setAll(1, JSON.stringify(data), ""); + } + } + + /** + * 正常流程下（即验证初始化成功），二次验证 + */ + async successValidate(challenge, validate, seccode, params) { + this.gtlog(`successValidate(): 开始二次验证 正常模式, challenge=${challenge}, validate=${validate}, seccode=${validate}.`); + if (!this.checkParam(challenge, validate, seccode)) { + this.libResult.setAll(0, "", "正常模式，本地校验，参数challenge、validate、seccode不可为空"); + } else { + const response_seccode = await this.requestValidate(challenge, validate, seccode, params); + if (!response_seccode) { + this.libResult.setAll(0, "", "请求极验validate接口失败"); + } else if (response_seccode === "false") { + this.libResult.setAll(0, "", "极验二次验证不通过"); + } else { + this.libResult.setAll(1, "", ""); + } + } + this.gtlog(`successValidate(): 二次验证 正常模式, lib包返回信息=${this.libResult}.`); + return this.libResult; + } + + /** + * 异常流程下（即验证初始化失败，宕机模式），二次验证 + * 注意：由于是宕机模式，初衷是保证验证业务不会中断正常业务，所以此处只作简单的参数校验，可自行设计逻辑。 + */ + failValidate(challenge, validate, seccode) { + this.gtlog(`failValidate(): 开始二次验证 宕机模式, challenge=${challenge}, validate=${validate}, seccode=${seccode}.`); + if (!this.checkParam(challenge, validate, seccode)) { + this.libResult.setAll(0, "", "宕机模式，本地校验，参数challenge、validate、seccode不可为空."); + } else { + this.libResult.setAll(1, "", ""); + } + this.gtlog(`failValidate(): 二次验证 宕机模式, lib包返回信息=${this.libResult}.`); + return this.libResult; + } + + /** + * 向极验发送二次验证的请求，POST方式 + */ + async requestValidate(challenge, validate, seccode, params) { + params = Object.assign(params, { + "seccode": seccode, + "json_format": GeetestLib.JSON_FORMAT, + "challenge": challenge, + "sdk": GeetestLib.VERSION, + "captchaid": this.geetest_id + }); + const validate_url = GeetestLib.API_URL + GeetestLib.VALIDATE_URL; + this.gtlog(`requestValidate(): 二次验证 正常模式, 向极验发送请求, url=${validate_url}, params=${JSON.stringify(params)}.`); + let response_seccode; + try { + const res = await axios({ + url: validate_url, + method: "POST", + timeout: GeetestLib.HTTP_TIMEOUT_DEFAULT, + data: qs.stringify(params), + headers: {"Content-Type": "application/x-www-form-urlencoded"} + }); + const resBody = (res.status === 200) ? res.data : ""; + this.gtlog(`requestValidate(): 二次验证 正常模式, 与极验网络交互正常, 返回码=${res.status}, 返回body=${JSON.stringify(resBody)}.`); + response_seccode = resBody["seccode"]; + } catch (e) { + this.gtlog("requestValidate(): 二次验证 正常模式, 请求异常, " + e.message); + response_seccode = ""; + } + return response_seccode; + } + + /** + * 校验二次验证的三个参数，校验通过返回true，校验失败返回false + */ + checkParam(challenge, validate, seccode) { + return !(challenge == undefined || challenge.trim() === "" || validate == undefined || validate.trim() === "" || seccode == undefined || seccode.trim() === ""); + } + + /** + * md5 加密 + */ + md5_encode(value) { + return crypto.createHash("md5").update(value).digest("hex"); + } + + /** + * sha256加密 + */ + sha256_encode(value) { + return crypto.createHash("sha256").update(value).digest("hex"); + } + + /** + * hmac-sha256 加密 + */ + hmac_sha256_encode(value, key) { + return crypto.createHmac("sha256", key).update(value).digest("hex"); + } + +} + +module.exports = { GeetestLib }; diff --git a/lib/gt3-server-node-express-sdk/package.json b/lib/gt3-server-node-express-sdk/package.json new file mode 100644 index 0000000000..cdd8a1c2bd --- /dev/null +++ b/lib/gt3-server-node-express-sdk/package.json @@ -0,0 +1,15 @@ +{ + "name": "@galoy/gt3-server-node-express-sdk", + "private": true, + "version": "3.1.0", + "description": "gt3-server-node-express-sdk", + "main": "geetest-lib.js", + "scripts": { + "preinstall": "npx only-allow pnpm" + }, + "dependencies": { + "axios": "^1.6.2", + "qs": "^6.11.2", + "string-random": "^0.1.3" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1650ec0771..c30cad8eca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -603,9 +603,6 @@ importers: graphql-ws: specifier: ^5.14.2 version: 5.14.2(graphql@16.8.1) - gt3-server-node-express-sdk: - specifier: git+https://github.com/GaloyMoney/gt3-server-node-express-bypass.git#master - version: github.com/GaloyMoney/gt3-server-node-express-bypass/6559a1a39ef408de3aefa6321934c831edac001e i18n: specifier: ^0.15.1 version: 0.15.1 @@ -950,6 +947,18 @@ importers: specifier: ^5.3.2 version: 5.3.2 + lib/gt3-server-node-express-sdk: + dependencies: + axios: + specifier: ^1.6.2 + version: 1.6.2 + qs: + specifier: ^6.11.2 + version: 6.11.2 + string-random: + specifier: ^0.1.3 + version: 0.1.3 + packages: /@aashutoshrathi/word-wrap@1.2.6: @@ -10706,13 +10715,6 @@ packages: resolution: {integrity: sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==} dev: true - /async-redis@2.0.0: - resolution: {integrity: sha512-cGv40n9h3gpAGMGnX1v5ncgHJ2C3TsRmK6Ekt282jVftiky3v0X13PSfnLoRix9uva7ctb2j9lwA3d2qc4KKTA==} - engines: {node: '>=7.6.0'} - dependencies: - redis: 3.1.2 - dev: false - /async-retry@1.3.3: resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} dependencies: @@ -12602,11 +12604,6 @@ packages: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} dev: true - /denque@1.5.1: - resolution: {integrity: sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==} - engines: {node: '>=0.10'} - dev: false - /denque@2.1.0: resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} engines: {node: '>=0.10'} @@ -20868,10 +20865,6 @@ packages: minimatch: 3.1.2 dev: true - /redis-commands@1.7.0: - resolution: {integrity: sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==} - dev: false - /redis-errors@1.2.0: resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} engines: {node: '>=4'} @@ -20884,16 +20877,6 @@ packages: redis-errors: 1.2.0 dev: false - /redis@3.1.2: - resolution: {integrity: sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==} - engines: {node: '>=10'} - dependencies: - denque: 1.5.1 - redis-commands: 1.7.0 - redis-errors: 1.2.0 - redis-parser: 3.0.0 - dev: false - /redlock@5.0.0-beta.2: resolution: {integrity: sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==} engines: {node: '>=12'} @@ -23524,17 +23507,3 @@ packages: /zod@3.22.4: resolution: {integrity: sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==} dev: false - - github.com/GaloyMoney/gt3-server-node-express-bypass/6559a1a39ef408de3aefa6321934c831edac001e: - resolution: {tarball: https://codeload.github.com/GaloyMoney/gt3-server-node-express-bypass/tar.gz/6559a1a39ef408de3aefa6321934c831edac001e} - name: gt3-server-node-express-sdk - version: 3.1.0 - dependencies: - async-redis: 2.0.0 - axios: 1.6.2 - express: 4.18.2 - string-random: 0.1.3 - transitivePeerDependencies: - - debug - - supports-color - dev: false diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 4471db2ca9..040cb115e6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,3 +5,4 @@ packages: - apps/dashboard - apps/pay - lib/eslint-config + - lib/gt3-server-node-express-sdk From f3130634b89596b3795d22ea1c2d53e30e3c3803 Mon Sep 17 00:00:00 2001 From: Juan P Lopez Date: Fri, 1 Dec 2023 12:33:45 -0500 Subject: [PATCH 2/9] fix: remove buck ref --- BUCK | 1 - 1 file changed, 1 deletion(-) diff --git a/BUCK b/BUCK index 187926024e..0ceb9c1791 100644 --- a/BUCK +++ b/BUCK @@ -8,7 +8,6 @@ pnpm_workspace( "//apps/dashboard:package.json", "//apps/pay:package.json", "//lib/eslint-config:package.json" - "//lib/gt3-server-node-express-sdk:package.json" ], visibility = ["PUBLIC"], ) From 6aec7a53f64856f0e47bd46b241999e229539b7c Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Fri, 1 Dec 2023 17:13:51 -0400 Subject: [PATCH 3/9] build: bring in local packages properly during turbo prune During turbo prune, turbo brings in the necessary local package dirs, and figures out the pruned package.json needed. We would need to also bring along the package dependency js files somehow which turbo doesn't seem to handle internally for local packages. This PR is to explitly copy these local package files across during the build_node_modules step after turbo prune. Future improvements: - figure how to simply do this directly with turbo and local packages --- .../workspace-pnpm/build_node_modules.py | 26 +++++++++++++++++++ toolchains/workspace-pnpm/macros.bzl | 8 ++++++ 2 files changed, 34 insertions(+) diff --git a/toolchains/workspace-pnpm/build_node_modules.py b/toolchains/workspace-pnpm/build_node_modules.py index 4d624066e8..2c3b1acbe9 100644 --- a/toolchains/workspace-pnpm/build_node_modules.py +++ b/toolchains/workspace-pnpm/build_node_modules.py @@ -9,6 +9,23 @@ import sys import tempfile +def copy_dir_except_node_modules(src, dst): + """ + Copy all directories, subdirectories, and files from 'src' to 'dst', + excluding any 'node_modules' directories. + """ + for root, dirs, files in os.walk(src): + if 'node_modules' in dirs: + dirs.remove('node_modules') + + for file in files: + src_path = os.path.join(root, file) + dst_path = os.path.join(dst, os.path.relpath(src_path, src)) + + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + shutil.copy2(src_path, dst_path) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -25,6 +42,10 @@ action="store_true", help="Only install production node modules", ) + parser.add_argument( + "--local-packages-path", + help="Path to vendored npm package dependencies", + ) parser.add_argument( "out_path", help="Path to output tree of `node_modules`", @@ -65,6 +86,11 @@ print(f"Failed to successfully run: {turbo_cmd}") sys.exit(1) + if args.local_packages_path: + copy_dir_except_node_modules( + os.path.join(tempdir, "full", args.local_packages_path), + os.path.join(tempdir, "json", args.local_packages_path), + ) shutil.copy( os.path.join(tempdir, "pnpm-lock.yaml"), os.path.join(tempdir, "json", "pnpm-lock.yaml"), diff --git a/toolchains/workspace-pnpm/macros.bzl b/toolchains/workspace-pnpm/macros.bzl index 1fdcfb774d..f67d817842 100644 --- a/toolchains/workspace-pnpm/macros.bzl +++ b/toolchains/workspace-pnpm/macros.bzl @@ -153,6 +153,10 @@ def build_node_modules_impl(ctx: AnalysisContext) -> list[DefaultInfo]: cmd.add("--prod-only") identifier += "--prod " + if ctx.attrs.local_packages: + cmd.add("--local-packages-path") + cmd.add(ctx.attrs.local_packages) + cmd.add(out.as_output()) cmd.hidden([ctx.attrs.workspace]) @@ -176,6 +180,10 @@ build_node_modules = rule( default = False, doc = "Only install production dependencies" ), + "local_packages": attrs.string( + default = "lib", + doc = """NPM vendored dependencies directory""", + ), "_python_toolchain": attrs.toolchain_dep( default = "toolchains//:python", providers = [PythonToolchainInfo], From b3d209cb94949e600fd431870238f4bafa264a5d Mon Sep 17 00:00:00 2001 From: Juan P Lopez Date: Wed, 6 Dec 2023 11:47:45 -0500 Subject: [PATCH 4/9] fix: rebase deps --- pnpm-lock.yaml | 124 +++++++++++++++++++++++++------------------------ 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c30cad8eca..7b7bcfc280 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -477,6 +477,9 @@ importers: '@aws-sdk/client-s3': specifier: ^3.465.0 version: 3.465.0 + '@galoy/gt3-server-node-express-sdk': + specifier: workspace:^ + version: link:../../lib/gt3-server-node-express-sdk '@google-cloud/storage': specifier: ^7.7.0 version: 7.7.0 @@ -1963,7 +1966,7 @@ packages: '@babel/traverse': 7.23.5 '@babel/types': 7.23.5 convert-source-map: 1.9.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -1986,7 +1989,7 @@ packages: '@babel/traverse': 7.23.5 '@babel/types': 7.23.5 convert-source-map: 2.0.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -2088,7 +2091,7 @@ packages: '@babel/core': 7.23.5 '@babel/helper-compilation-targets': 7.22.15 '@babel/helper-plugin-utils': 7.22.5 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) lodash.debounce: 4.0.8 resolve: 1.22.8 transitivePeerDependencies: @@ -3487,7 +3490,7 @@ packages: '@babel/helper-split-export-declaration': 7.22.6 '@babel/parser': 7.23.5 '@babel/types': 7.23.5 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -3951,7 +3954,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: ajv: 6.12.6 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) espree: 9.6.1 globals: 13.23.0 ignore: 5.2.4 @@ -3968,7 +3971,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: ajv: 6.12.6 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) espree: 9.6.1 globals: 13.23.0 ignore: 5.3.0 @@ -3985,7 +3988,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: ajv: 6.12.6 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) espree: 9.6.1 globals: 13.23.0 ignore: 5.3.0 @@ -5820,7 +5823,7 @@ packages: '@types/json-stable-stringify': 1.0.34 '@whatwg-node/fetch': 0.8.8 chalk: 4.1.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) dotenv: 16.3.1 graphql: 16.8.1 graphql-request: 6.1.0(graphql@16.8.1) @@ -5853,7 +5856,7 @@ packages: '@types/json-stable-stringify': 1.0.34 '@whatwg-node/fetch': 0.9.13 chalk: 4.1.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) dotenv: 16.3.1 graphql: 16.8.1 graphql-request: 6.1.0(graphql@16.8.1) @@ -5886,7 +5889,7 @@ packages: '@types/json-stable-stringify': 1.0.34 '@whatwg-node/fetch': 0.9.13 chalk: 4.1.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) dotenv: 16.3.1 graphql: 16.8.1 graphql-request: 6.1.0(graphql@16.8.1) @@ -6250,7 +6253,7 @@ packages: engines: {node: '>=10.10.0'} dependencies: '@humanwhocodes/object-schema': 2.0.1 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -9539,7 +9542,7 @@ packages: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/type-utils': 5.62.0(eslint@8.54.0)(typescript@5.2.2) '@typescript-eslint/utils': 5.62.0(eslint@8.54.0)(typescript@5.2.2) - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) eslint: 8.54.0 graphemer: 1.4.0 ignore: 5.3.0 @@ -9568,7 +9571,7 @@ packages: '@typescript-eslint/type-utils': 6.13.1(eslint@8.54.0)(typescript@5.3.2) '@typescript-eslint/utils': 6.13.1(eslint@8.54.0)(typescript@5.3.2) '@typescript-eslint/visitor-keys': 6.13.1 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) eslint: 8.54.0 graphemer: 1.4.0 ignore: 5.3.0 @@ -9597,7 +9600,7 @@ packages: '@typescript-eslint/type-utils': 6.13.2(eslint@8.55.0)(typescript@5.3.2) '@typescript-eslint/utils': 6.13.2(eslint@8.55.0)(typescript@5.3.2) '@typescript-eslint/visitor-keys': 6.13.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) eslint: 8.55.0 graphemer: 1.4.0 ignore: 5.3.0 @@ -9635,7 +9638,7 @@ packages: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/types': 5.62.0 '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.2.2) - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) eslint: 8.54.0 typescript: 5.2.2 transitivePeerDependencies: @@ -9656,7 +9659,7 @@ packages: '@typescript-eslint/types': 6.13.1 '@typescript-eslint/typescript-estree': 6.13.1(typescript@5.3.2) '@typescript-eslint/visitor-keys': 6.13.1 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) eslint: 8.54.0 typescript: 5.3.2 transitivePeerDependencies: @@ -9677,7 +9680,7 @@ packages: '@typescript-eslint/types': 6.13.2 '@typescript-eslint/typescript-estree': 6.13.2(typescript@5.2.2) '@typescript-eslint/visitor-keys': 6.13.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) eslint: 8.52.0 typescript: 5.2.2 transitivePeerDependencies: @@ -9698,7 +9701,7 @@ packages: '@typescript-eslint/types': 6.13.2 '@typescript-eslint/typescript-estree': 6.13.2(typescript@5.3.2) '@typescript-eslint/visitor-keys': 6.13.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) eslint: 8.55.0 typescript: 5.3.2 transitivePeerDependencies: @@ -9741,7 +9744,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.2.2) '@typescript-eslint/utils': 5.62.0(eslint@8.54.0)(typescript@5.2.2) - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) eslint: 8.54.0 tsutils: 3.21.0(typescript@5.2.2) typescript: 5.2.2 @@ -9761,7 +9764,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 6.13.1(typescript@5.3.2) '@typescript-eslint/utils': 6.13.1(eslint@8.54.0)(typescript@5.3.2) - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) eslint: 8.54.0 ts-api-utils: 1.0.3(typescript@5.3.2) typescript: 5.3.2 @@ -9781,7 +9784,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 6.13.2(typescript@5.3.2) '@typescript-eslint/utils': 6.13.2(eslint@8.55.0)(typescript@5.3.2) - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) eslint: 8.55.0 ts-api-utils: 1.0.3(typescript@5.3.2) typescript: 5.3.2 @@ -9820,7 +9823,7 @@ packages: dependencies: '@typescript-eslint/types': 4.33.0 '@typescript-eslint/visitor-keys': 4.33.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.4 @@ -9841,7 +9844,7 @@ packages: dependencies: '@typescript-eslint/types': 5.62.0 '@typescript-eslint/visitor-keys': 5.62.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.4 @@ -9862,7 +9865,7 @@ packages: dependencies: '@typescript-eslint/types': 5.62.0 '@typescript-eslint/visitor-keys': 5.62.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.4 @@ -9883,7 +9886,7 @@ packages: dependencies: '@typescript-eslint/types': 5.62.0 '@typescript-eslint/visitor-keys': 5.62.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.4 @@ -9904,7 +9907,7 @@ packages: dependencies: '@typescript-eslint/types': 6.13.1 '@typescript-eslint/visitor-keys': 6.13.1 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.4 @@ -9925,7 +9928,7 @@ packages: dependencies: '@typescript-eslint/types': 6.13.2 '@typescript-eslint/visitor-keys': 6.13.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.4 @@ -9946,7 +9949,7 @@ packages: dependencies: '@typescript-eslint/types': 6.13.2 '@typescript-eslint/visitor-keys': 6.13.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.4 @@ -10380,7 +10383,7 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -10388,7 +10391,7 @@ packages: resolution: {integrity: sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==} engines: {node: '>= 14'} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -12439,7 +12442,6 @@ packages: dependencies: ms: 2.1.2 supports-color: 5.5.0 - dev: true /debug@4.3.4(supports-color@8.1.1): resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} @@ -12452,6 +12454,7 @@ packages: dependencies: ms: 2.1.2 supports-color: 8.1.1 + dev: true /decamelize@1.2.0: resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} @@ -12629,7 +12632,7 @@ packages: hasBin: true dependencies: commander: 2.20.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) filing-cabinet: 3.3.1 precinct: 9.2.1 typescript: 4.9.5 @@ -12738,7 +12741,7 @@ packages: resolution: {integrity: sha512-Rps1xDkEEBSq3kLdsdnHZL1x2S4NGDcbrjmd4q+PykK5aJwDdP5MBgrJw1Xo+kyUHuv3JEzPqxr+Dj9ryeDRTA==} engines: {node: '>= 6.0'} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) gonzales-pe: 4.3.0 node-source-walk: 4.3.0 transitivePeerDependencies: @@ -12749,7 +12752,7 @@ packages: resolution: {integrity: sha512-Fwc/g9VcrowODIAeKRWZfVA/EufxYL7XfuqJQFroBKGikKX83d2G7NFw6kDlSYGG3LNQIyVa+eWv1mqre+v4+A==} engines: {node: ^10 || ^12 || >=14} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) is-url: 1.2.4 postcss: 8.4.32 postcss-values-parser: 2.0.1 @@ -13416,7 +13419,7 @@ packages: eslint: '*' eslint-plugin-import: '*' dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) enhanced-resolve: 5.15.0 eslint: 8.54.0 eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.54.0) @@ -13439,7 +13442,7 @@ packages: eslint: '*' eslint-plugin-import: '*' dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) enhanced-resolve: 5.15.0 eslint: 8.52.0 eslint-module-utils: 2.8.0(@typescript-eslint/parser@6.13.2)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.52.0) @@ -13462,7 +13465,7 @@ packages: eslint: '*' eslint-plugin-import: '*' dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) enhanced-resolve: 5.15.0 eslint: 8.55.0 eslint-module-utils: 2.8.0(@typescript-eslint/parser@6.13.2)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.55.0) @@ -14176,7 +14179,7 @@ packages: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -14223,7 +14226,7 @@ packages: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -14270,7 +14273,7 @@ packages: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -14786,7 +14789,7 @@ packages: dependencies: app-module-path: 2.2.0 commander: 2.20.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) enhanced-resolve: 5.15.0 is-relative-path: 1.0.2 module-definition: 3.4.0 @@ -16264,7 +16267,7 @@ packages: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color dev: false @@ -16274,7 +16277,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color dev: true @@ -16284,7 +16287,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color dev: true @@ -16337,7 +16340,7 @@ packages: engines: {node: '>= 6'} dependencies: agent-base: 6.0.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -16346,7 +16349,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color dev: true @@ -16356,7 +16359,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -16380,7 +16383,7 @@ packages: engines: {node: '>=10'} dependencies: '@messageformat/core': 3.2.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) fast-printf: 1.6.9 make-plural: 7.3.0 math-interval-parser: 2.0.1 @@ -16579,7 +16582,7 @@ packages: dependencies: '@ioredis/commands': 1.2.0 cluster-key-slot: 1.1.2 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) denque: 2.1.0 lodash.defaults: 4.2.0 lodash.isarguments: 3.1.0 @@ -17046,7 +17049,7 @@ packages: resolution: {integrity: sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==} engines: {node: '>=10'} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) istanbul-lib-coverage: 3.2.0 source-map: 0.6.1 transitivePeerDependencies: @@ -17795,7 +17798,7 @@ packages: dependencies: '@types/express': 4.17.21 '@types/jsonwebtoken': 9.0.5 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) jose: 4.15.2 limiter: 1.1.5 lru-memoizer: 2.2.0 @@ -17882,7 +17885,7 @@ packages: dependencies: colorette: 2.0.19 commander: 10.0.1 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) escalade: 3.1.1 esm: 3.2.25 get-package-type: 0.1.0 @@ -18337,7 +18340,7 @@ packages: chalk: 4.1.2 commander: 7.2.0 commondir: 1.0.1 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) dependency-tree: 9.0.0 detective-amd: 4.2.0 detective-cjs: 4.1.0 @@ -18726,7 +18729,7 @@ packages: hasBin: true dependencies: commander: 2.20.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) glob: 7.2.3 requirejs: 2.3.6 requirejs-config-file: 4.0.0 @@ -18858,7 +18861,7 @@ packages: resolution: {integrity: sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==} engines: {node: '>=14.0.0'} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color dev: false @@ -20194,7 +20197,7 @@ packages: hasBin: true dependencies: commander: 2.20.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) detective-amd: 3.1.2 detective-cjs: 3.1.3 detective-es6: 2.2.2 @@ -20988,7 +20991,7 @@ packages: resolution: {integrity: sha512-3TLx5TGyAY6AOqLBoXmHkNql0HIf2RGbuMgCDT2WO/uGVAPJs6h7Kl+bN6TIZGd9bWhWPwnDnTHGtW8Iu77sdw==} engines: {node: '>=8.6.0'} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) module-details-from-path: 1.0.3 resolve: 1.22.8 transitivePeerDependencies: @@ -21110,7 +21113,7 @@ packages: engines: {node: '>=12'} requiresBuild: true dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) extend: 3.0.2 transitivePeerDependencies: - supports-color @@ -21122,7 +21125,7 @@ packages: engines: {node: '>=14'} dependencies: '@types/request': 2.48.12 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) extend: 3.0.2 teeny-request: 9.0.0 transitivePeerDependencies: @@ -21917,7 +21920,7 @@ packages: hasBin: true dependencies: commander: 2.20.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color dev: true @@ -21981,6 +21984,7 @@ packages: engines: {node: '>=10'} dependencies: has-flag: 4.0.0 + dev: true /supports-preserve-symlinks-flag@1.0.0: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} From 6c14818317c960a9a3664236cc36d63d35fd9eb4 Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Wed, 6 Dec 2023 10:14:17 -0400 Subject: [PATCH 5/9] revert: bring in local packages properly during turbo prune This reverts commit e280a12d3ee32a4233ffeb369e714e5fe09c69ce. We will instead bring these in via 'prepare_build_context' --- .../workspace-pnpm/build_node_modules.py | 26 ------------------- toolchains/workspace-pnpm/macros.bzl | 8 ------ 2 files changed, 34 deletions(-) diff --git a/toolchains/workspace-pnpm/build_node_modules.py b/toolchains/workspace-pnpm/build_node_modules.py index 2c3b1acbe9..4d624066e8 100644 --- a/toolchains/workspace-pnpm/build_node_modules.py +++ b/toolchains/workspace-pnpm/build_node_modules.py @@ -9,23 +9,6 @@ import sys import tempfile -def copy_dir_except_node_modules(src, dst): - """ - Copy all directories, subdirectories, and files from 'src' to 'dst', - excluding any 'node_modules' directories. - """ - for root, dirs, files in os.walk(src): - if 'node_modules' in dirs: - dirs.remove('node_modules') - - for file in files: - src_path = os.path.join(root, file) - dst_path = os.path.join(dst, os.path.relpath(src_path, src)) - - os.makedirs(os.path.dirname(dst_path), exist_ok=True) - shutil.copy2(src_path, dst_path) - - if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -42,10 +25,6 @@ def copy_dir_except_node_modules(src, dst): action="store_true", help="Only install production node modules", ) - parser.add_argument( - "--local-packages-path", - help="Path to vendored npm package dependencies", - ) parser.add_argument( "out_path", help="Path to output tree of `node_modules`", @@ -86,11 +65,6 @@ def copy_dir_except_node_modules(src, dst): print(f"Failed to successfully run: {turbo_cmd}") sys.exit(1) - if args.local_packages_path: - copy_dir_except_node_modules( - os.path.join(tempdir, "full", args.local_packages_path), - os.path.join(tempdir, "json", args.local_packages_path), - ) shutil.copy( os.path.join(tempdir, "pnpm-lock.yaml"), os.path.join(tempdir, "json", "pnpm-lock.yaml"), diff --git a/toolchains/workspace-pnpm/macros.bzl b/toolchains/workspace-pnpm/macros.bzl index f67d817842..1fdcfb774d 100644 --- a/toolchains/workspace-pnpm/macros.bzl +++ b/toolchains/workspace-pnpm/macros.bzl @@ -153,10 +153,6 @@ def build_node_modules_impl(ctx: AnalysisContext) -> list[DefaultInfo]: cmd.add("--prod-only") identifier += "--prod " - if ctx.attrs.local_packages: - cmd.add("--local-packages-path") - cmd.add(ctx.attrs.local_packages) - cmd.add(out.as_output()) cmd.hidden([ctx.attrs.workspace]) @@ -180,10 +176,6 @@ build_node_modules = rule( default = False, doc = "Only install production dependencies" ), - "local_packages": attrs.string( - default = "lib", - doc = """NPM vendored dependencies directory""", - ), "_python_toolchain": attrs.toolchain_dep( default = "toolchains//:python", providers = [PythonToolchainInfo], From 141a5cb5bf62e99031da6a039fe489175b9ef282 Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:23:06 -0400 Subject: [PATCH 6/9] build: add prod_deps_srcs to tsc_build rule --- core/api/BUCK | 7 ++++++- lib/gt3-server-node-express-sdk/BUCK | 26 ++++++++++++++++++++++++++ toolchains/workspace-pnpm/macros.bzl | 10 ++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 lib/gt3-server-node-express-sdk/BUCK diff --git a/core/api/BUCK b/core/api/BUCK index 398e88addb..5ccdf9eaf9 100644 --- a/core/api/BUCK +++ b/core/api/BUCK @@ -65,11 +65,16 @@ filegroup( ]) ) +prod_deps_srcs = { + "lib/gt3-server-node-express-sdk": "//lib/gt3-server-node-express-sdk:src", +} + tsc_build( name = "build", tsconfig = "tsconfig-build.json", srcs = [":src"], - additional_dist_files = [":protos", ":locales"] + additional_dist_files = [":protos", ":locales"], + prod_deps_srcs = prod_deps_srcs, ) prod_tsc_build( diff --git a/lib/gt3-server-node-express-sdk/BUCK b/lib/gt3-server-node-express-sdk/BUCK new file mode 100644 index 0000000000..c30725f941 --- /dev/null +++ b/lib/gt3-server-node-express-sdk/BUCK @@ -0,0 +1,26 @@ +load( + "@toolchains//workspace-pnpm:macros.bzl", + "build_node_modules", +) + +export_file( + name = "package.json", + visibility = ["PUBLIC"], +) + +build_node_modules( + name = "node_modules", +) + +build_node_modules( + name = "node_modules_prod", + prod_only = True, +) + +filegroup( + name = "src", + srcs = glob([ + "*.js", + ]), + visibility = ["PUBLIC"], +) diff --git a/toolchains/workspace-pnpm/macros.bzl b/toolchains/workspace-pnpm/macros.bzl index 1fdcfb774d..a4ffbfc07a 100644 --- a/toolchains/workspace-pnpm/macros.bzl +++ b/toolchains/workspace-pnpm/macros.bzl @@ -266,6 +266,12 @@ _tsc_build = rule( "node_modules": attrs.source( doc = """Target which builds package `node_modules`.""", ), + "prod_deps_srcs": attrs.dict( + attrs.string(), + attrs.source(allow_directory = True), + default = {}, + doc = """Mapping of dependent prod package paths to source files to track.""", + ), "_python_toolchain": attrs.toolchain_dep( default = "toolchains//:python", providers = [PythonToolchainInfo], @@ -559,6 +565,10 @@ def prepare_build_context(ctx: AnalysisContext) -> BuildContext: for src in ctx.attrs.srcs: cmd.add("--src") cmd.add(cmd_args(src, format = ctx.label.package + "={}")) + if hasattr(ctx.attrs, 'prod_deps_srcs'): + for (name, src) in ctx.attrs.prod_deps_srcs.items(): + cmd.add("--src") + cmd.add(cmd_args(src, format = name + "={}")) if hasattr(ctx.attrs, 'dev_deps_srcs'): for (name, src) in ctx.attrs.dev_deps_srcs.items(): cmd.add("--src") From 482b217a37fec75ad45fe13300d8643edfc9a692 Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:23:19 -0400 Subject: [PATCH 7/9] build: add prod_deps_srcs to prod_tsc_build rule --- core/api/BUCK | 1 + toolchains/workspace-pnpm/macros.bzl | 13 ++++++++- .../workspace-pnpm/package_prod_tsc_build.py | 28 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/core/api/BUCK b/core/api/BUCK index 5ccdf9eaf9..213ae484b4 100644 --- a/core/api/BUCK +++ b/core/api/BUCK @@ -79,6 +79,7 @@ tsc_build( prod_tsc_build( name = "prod_build", + prod_deps_srcs = prod_deps_srcs, visibility = ["PUBLIC"], ) diff --git a/toolchains/workspace-pnpm/macros.bzl b/toolchains/workspace-pnpm/macros.bzl index a4ffbfc07a..ac23b48324 100644 --- a/toolchains/workspace-pnpm/macros.bzl +++ b/toolchains/workspace-pnpm/macros.bzl @@ -305,9 +305,14 @@ def prod_tsc_build_impl(ctx: AnalysisContext) -> list[[DefaultInfo, ProdBuildInf ctx.attrs.node_modules_prod[DefaultInfo].default_outputs[0], "--dist-path", dist_path, - out.as_output() ) + if hasattr(ctx.attrs, 'prod_deps_srcs'): + for (name, src) in ctx.attrs.prod_deps_srcs.items(): + cmd.add("--deps-src") + cmd.add(cmd_args(src, format = name + "={}")) + cmd.add(out.as_output()) + ctx.actions.run(cmd, category = "prod_tsc_build") return [ @@ -336,6 +341,12 @@ _prod_tsc_build = rule( "node_modules_prod": attrs.dep( doc = """Target which builds package `node_modules` with prod-only modules.""", ), + "prod_deps_srcs": attrs.dict( + attrs.string(), + attrs.source(allow_directory = True), + default = {}, + doc = """Mapping of dependent prod package paths to source files to track.""", + ), "_python_toolchain": attrs.toolchain_dep( default = "toolchains//:python", providers = [PythonToolchainInfo], diff --git a/toolchains/workspace-pnpm/package_prod_tsc_build.py b/toolchains/workspace-pnpm/package_prod_tsc_build.py index b39480e4df..337374d30d 100644 --- a/toolchains/workspace-pnpm/package_prod_tsc_build.py +++ b/toolchains/workspace-pnpm/package_prod_tsc_build.py @@ -23,6 +23,12 @@ "--dist-path", help="Path to `dist` scripts", ) + parser.add_argument( + "--deps-src", + action="append", + metavar="DST=SRC", + help="Adds a dependency source into the source tree" + ) parser.add_argument( "out_path", help="Path to output directory", @@ -50,4 +56,26 @@ symlinks=True, ) + for arg in args.deps_src or []: + dst, src = arg.split("=") + + parent_dir = os.path.dirname(dst) + if parent_dir: + dst_dir = os.path.join(lib_dir, parent_dir) + if not os.path.isdir(dst_dir): + os.makedirs(dst_dir, exist_ok=True) + abspath_src = os.path.abspath(src) + if os.path.isdir(abspath_src): + shutil.copytree( + abspath_src, + os.path.join(lib_dir, dst), + symlinks=True, + dirs_exist_ok=True, + ) + else: + shutil.copy( + abspath_src, + os.path.join(lib_dir, dst), + ) + shutil.move(lib_dir, args.out_path) From 4009529c7c93b8177c188a05a9ec472a91484bba Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:42:38 -0400 Subject: [PATCH 8/9] build: remove unnecessary lib/gt3 buck rules --- lib/gt3-server-node-express-sdk/BUCK | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/lib/gt3-server-node-express-sdk/BUCK b/lib/gt3-server-node-express-sdk/BUCK index c30725f941..e07230d877 100644 --- a/lib/gt3-server-node-express-sdk/BUCK +++ b/lib/gt3-server-node-express-sdk/BUCK @@ -1,22 +1,3 @@ -load( - "@toolchains//workspace-pnpm:macros.bzl", - "build_node_modules", -) - -export_file( - name = "package.json", - visibility = ["PUBLIC"], -) - -build_node_modules( - name = "node_modules", -) - -build_node_modules( - name = "node_modules_prod", - prod_only = True, -) - filegroup( name = "src", srcs = glob([ From 0b98a39962004b41d2da219eeb5f68422b62f641 Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Thu, 7 Dec 2023 11:54:23 -0400 Subject: [PATCH 9/9] build: add prod_deps_srcs to jest_test & test-integration rules --- core/api/BUCK | 1 + toolchains/workspace-pnpm/macros.bzl | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/core/api/BUCK b/core/api/BUCK index 213ae484b4..682bab11a7 100644 --- a/core/api/BUCK +++ b/core/api/BUCK @@ -176,6 +176,7 @@ jest_test( jest_test( name = "test-integration", srcs = [":src"] + [":test_src"] + glob([".env", "galoy.yaml"]), + prod_deps_srcs = prod_deps_srcs, config_file = "test/integration/jest.config.js", run_serially = True, env_file = ".env", diff --git a/toolchains/workspace-pnpm/macros.bzl b/toolchains/workspace-pnpm/macros.bzl index ac23b48324..2d9b2b35ba 100644 --- a/toolchains/workspace-pnpm/macros.bzl +++ b/toolchains/workspace-pnpm/macros.bzl @@ -1145,6 +1145,12 @@ _jest_test = rule( "node_modules": attrs.source( doc = """Target which builds package `node_modules`.""", ), + "prod_deps_srcs": attrs.dict( + attrs.string(), + attrs.source(allow_directory = True), + default = {}, + doc = """Mapping of dependent prod package paths to source files to track.""", + ), "_inject_test_env": attrs.default_only( attrs.dep(default = "prelude//test/tools:inject_test_env"), ),