test(core): add integration tests to BUCK

.github/workflows/integration-test.yml

@@ -1,21 +1,29 @@
 name: "Integration test"
-
 on:
   pull_request:
     branches: [main]
+    types: [opened, synchronize, labeled, unlabeled]
 
 jobs:
-  integration:
+  tests:
     name: Integration test
     runs-on: ubuntu-latest
-    timeout-minutes: 15
     steps:
-      - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@v4
-      - name: Run the Magic Nix Cache
-        uses: DeterminateSystems/magic-nix-cache-action@v2
       - uses: actions/checkout@v3
-      - name: Run integration tests
+      - uses: cachix/install-nix-action@v22
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Prepare tilt args from labels
+        id: prepare_args
         run: |
-          cd core/api/
-          nix develop -c make integration-in-ci
+          ARGS=""
+          cat <<EOF > labels.json
+          ${{ toJSON(github.event.pull_request.labels.*.name) }}
+          EOF
+          for LABEL in dashboard consent core; do
+            ARGS+=" --test $LABEL"
+          done
+          echo "Prepared args: $ARGS"
+          echo "args=$ARGS" >> "$GITHUB_OUTPUT"
+      - name: Tilt CI
+        run: nix develop -c xvfb-run tilt --file dev/Tiltfile ci -- ${{ steps.prepare_args.outputs.args }}

.github/workflows/legacy-integration-test.yml

@@ -0,0 +1,21 @@
+name: "Legacy integration test"
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  integration:
+    name: Legacy integration test
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v4
+      - name: Run the Magic Nix Cache
+        uses: DeterminateSystems/magic-nix-cache-action@v2
+      - uses: actions/checkout@v3
+      - name: Run integration tests
+        run: |
+          cd core/api/
+          nix develop -c make legacy-integration-in-ci

core/api/BUCK

@@ -1,6 +1,7 @@
 load(
   "@toolchains//workspace-pnpm:macros.bzl",
   "dev_pnpm_task_binary",
+  "dev_pnpm_task_test",
   "build_node_modules",
   "tsc_build",
   "prod_tsc_build",
@@ -10,7 +11,7 @@ load(
   "typescript_check",
   "yaml_check",
   "madge_check",
-  "test_unit",
+  "jest_test",
 )
 load("@toolchains//rover:macros.bzl", "sdl", "diff_check", "dev_update_file")
 
@@ -156,7 +157,7 @@ madge_check(
     srcs = [":src"],
 )
 
-test_unit(
+jest_test(
     name = "unit-tests",
     srcs = [":src"] + [":test_src"] + glob([".env", "galoy.yaml"]),
     config_file = "test/unit/jest.config.js",
@@ -166,6 +167,17 @@ test_unit(
     }
 )
 
+jest_test(
+    name = "test-integration",
+    srcs = [":src"] + [":test_src"] + glob([".env", "galoy.yaml"]),
+    config_file = "test/integration/jest.config.js",
+    run_serially = True,
+    env_file = ".env",
+    env = {
+        "LOGLEVEL": "warn",
+    }
+)
+
 test_suite(
     name = "test-unit",
     tests = [

core/api/Makefile

@@ -102,6 +102,11 @@ integration-in-ci:
 	export NODE_OPTIONS="--max-old-space-size=6144" && \
 	$(MAKE) reset-all-integration
 
+legacy-integration-in-ci:
+	pnpm install --frozen-lockfile && \
+	export NODE_OPTIONS="--max-old-space-size=6144" && \
+	$(MAKE) reset-legacy-integration
+
 unit-in-ci:
 	. ./.env && \
 		LOGLEVEL=warn $(BIN_DIR)/jest --config ./test/unit/jest.config.js --ci --bail --maxWorkers=50%

core/api/test/helpers/lightning.ts

@@ -91,7 +91,8 @@ export const lndOutside2 = authenticatedLndGrpc({
   socket: `${process.env.LNDOUTSIDE2ADDR}:${process.env.LNDOUTSIDE2RPCPORT ?? 10009}`,
 }).lnd
 
-export const lndsIntegration = [lnd1, lnd2, lndOutside1, lndOutside2]
+export const lndsIntegration = [lnd1, lndOutside1, lndOutside2]
+export const lndsLegacyIntegration = [lnd1, lnd2, lndOutside1, lndOutside2]
 export const lndsE2e = [lnd1, lnd2, lndOutside1, lndOutside2]
 
 export const waitUntilBlockHeight = async ({
@@ -300,6 +301,7 @@ const resetLnds = async (lnds: AuthenticatedLnd[]) => {
 }
 
 export const resetIntegrationLnds = () => resetLnds(lndsIntegration)
+export const resetLegacyIntegrationLnds = () => resetLnds(lndsLegacyIntegration)
 export const resetE2eLnds = () => resetLnds(lndsE2e)
 
 export const closeAllChannels = async ({ lnd }: { lnd: AuthenticatedLnd }) => {

core/api/test/integration/services/lnd-service.spec.ts

@@ -5,6 +5,8 @@ import {
   deleteForwardingReputations,
 } from "lightning"
 
+import { LND1_PUBKEY } from "@/config"
+
 import { WalletCurrency } from "@/domain/shared"
 import { toSats } from "@/domain/bitcoin"
 import {
@@ -346,15 +348,21 @@ describe("LndService", () => {
     expect(retrievedPayment.confirmedDetails?.revealedPreImage).toBe(revealedPreImage)
 
     // Delete payment
-    const deleted = await lndService.deletePaymentByHash({ paymentHash })
+    const deleted = await lndService.deletePaymentByHash({
+      paymentHash,
+      pubkey: LND1_PUBKEY,
+    })
     expect(deleted).not.toBeInstanceOf(Error)
 
     // Check that payment no longer exists
     const retrievedDeletedPayment = await lndService.lookupPayment({ paymentHash })
     expect(retrievedDeletedPayment).toBeInstanceOf(PaymentNotFoundError)
 
     // Check that deleting missing payment doesn't return error
-    const deletedAttempt = await lndService.deletePaymentByHash({ paymentHash })
+    const deletedAttempt = await lndService.deletePaymentByHash({
+      paymentHash,
+      pubkey: LND1_PUBKEY,
+    })
     expect(deletedAttempt).not.toBeInstanceOf(Error)
   })
 })

core/api/test/legacy-integration/01-setup/01-connection.spec.ts

@@ -9,7 +9,7 @@ import {
   lndOutside2,
   getWalletInfo,
   bitcoindClient,
-  resetIntegrationLnds,
+  resetLegacyIntegrationLnds,
   getChannels,
   getChainBalance,
   waitUntilBriaConnected,
@@ -22,7 +22,7 @@ it("connects to bitcoind", async () => {
 
 describe("connects to lnds", () => {
   beforeAll(async () => {
-    await resetIntegrationLnds()
+    await resetLegacyIntegrationLnds()
   })
 
   const lnds = [lnd1, lnd2]

dev/Tiltfile

@@ -321,6 +321,10 @@ docker_groups = {
         "bria-pg",
         "bitcoind",
     ],
+    "integration": [
+        "lnd-outside-1",
+        "lnd-outside-2",
+    ],
     "price": [
       "price",
       "price-history",
@@ -351,3 +355,14 @@ for service in docker_groups["auth"]:
     dc_resource(service, labels = ["auth"])
 for service in docker_groups["price"]:
     dc_resource(service, labels = ["price"])
+for service in docker_groups["integration"]:
+    dc_resource(service, labels = ["integration"])
+
+api_test_target = "//core/api:test-integration"
+local_resource(
+  "api-test",
+  labels = ["test"],
+  auto_init = is_ci and "core" in cfg.get("test", []),
+  cmd = "buck2 test {}".format(api_test_target),
+  resource_deps = [res for sublist in docker_groups.values() for res in sublist],
+)

dev/config/lnd/regtest/lnd-outside-1.admin.macaroon.base64

@@ -0,0 +1 @@
+AgEDbG5kAvgBAwoQeE+5exgz7/0ExCn7H6AJlBIBMBoWCgdhZGRyZXNzEgRyZWFkEgV3cml0ZRoTCgRpbmZvEgRyZWFkEgV3cml0ZRoXCghpbnZvaWNlcxIEcmVhZBIFd3JpdGUaIQoIbWFjYXJvb24SCGdlbmVyYXRlEgRyZWFkEgV3cml0ZRoWCgdtZXNzYWdlEgRyZWFkEgV3cml0ZRoXCghvZmZjaGFpbhIEcmVhZBIFd3JpdGUaFgoHb25jaGFpbhIEcmVhZBIFd3JpdGUaFAoFcGVlcnMSBHJlYWQSBXdyaXRlGhgKBnNpZ25lchIIZ2VuZXJhdGUSBHJlYWQAAAYgL7pU+cKOt6zGyWTdWWmAJLP1L3cnbOPb4Rd3QtniyyM=

dev/config/lnd/regtest/lnd-outside-1.pubkey

@@ -0,0 +1 @@
+02147eeb1561922fe768a6a92df116f5fec8c1869ddcb6e00ee1fd0df146d51c4a

dev/config/lnd/regtest/lnd-outside-2.admin.macaroon.base64

@@ -0,0 +1 @@
+AgEDbG5kAvgBAwoQfKO82/iPT2zIwWYPrOXvABIBMBoWCgdhZGRyZXNzEgRyZWFkEgV3cml0ZRoTCgRpbmZvEgRyZWFkEgV3cml0ZRoXCghpbnZvaWNlcxIEcmVhZBIFd3JpdGUaIQoIbWFjYXJvb24SCGdlbmVyYXRlEgRyZWFkEgV3cml0ZRoWCgdtZXNzYWdlEgRyZWFkEgV3cml0ZRoXCghvZmZjaGFpbhIEcmVhZBIFd3JpdGUaFgoHb25jaGFpbhIEcmVhZBIFd3JpdGUaFAoFcGVlcnMSBHJlYWQSBXdyaXRlGhgKBnNpZ25lchIIZ2VuZXJhdGUSBHJlYWQAAAYg2XkV+4Z4inbfXGZivRoY+r7KHNZhgxkCEdKByxbeb/Q=

dev/config/lnd/regtest/lnd-outside-2.pubkey

@@ -0,0 +1 @@
+02f4888c0fffc494874da4445918dadaf88a63dbb96bce734b087bed56f60d231b

dev/docker-compose.deps.yml

@@ -173,6 +173,46 @@ services:
         cp /root/.lnd/macaroons.db /root/.lnd/data/chain/bitcoin/regtest/macaroons.db
         cp /root/.lnd/admin.macaroon /root/.lnd/data/chain/bitcoin/regtest/admin.macaroon
         /bin/lnd
+  lnd-outside-1:
+    image: lightninglabs/lnd:v0.16.2-beta
+    ports:
+      - "10012:10009"
+    volumes:
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/lnd.conf:/root/.lnd/lnd.conf
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/tls.key:/root/.lnd/tls.key
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/tls.cert:/root/.lnd/tls.cert
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/regtest/lnd-outside-1.wallet.db:/root/.lnd/wallet.db
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/regtest/lnd-outside-1.macaroons.db:/root/.lnd/macaroons.db
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/regtest/lnd-outside-1.admin.macaroon:/root/.lnd/admin.macaroon
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        mkdir -p /root/.lnd/data/chain/bitcoin/regtest/
+        cp /root/.lnd/wallet.db /root/.lnd/data/chain/bitcoin/regtest/wallet.db
+        cp /root/.lnd/macaroons.db /root/.lnd/data/chain/bitcoin/regtest/macaroons.db
+        cp /root/.lnd/admin.macaroon /root/.lnd/data/chain/bitcoin/regtest/admin.macaroon
+        /bin/lnd
+    depends_on: [bitcoind]
+  lnd-outside-2:
+    image: lightninglabs/lnd:v0.16.2-beta
+    ports:
+      - "10013:10009"
+    volumes:
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/lnd.conf:/root/.lnd/lnd.conf
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/tls.key:/root/.lnd/tls.key
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/tls.cert:/root/.lnd/tls.cert
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/regtest/lnd-outside-2.wallet.db:/root/.lnd/wallet.db
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/regtest/lnd-outside-2.macaroons.db:/root/.lnd/macaroons.db
+      - ${HOST_PROJECT_PATH:-.}/config/lnd/regtest/lnd-outside-2.admin.macaroon:/root/.lnd/admin.macaroon
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        mkdir -p /root/.lnd/data/chain/bitcoin/regtest/
+        cp /root/.lnd/wallet.db /root/.lnd/data/chain/bitcoin/regtest/wallet.db
+        cp /root/.lnd/macaroons.db /root/.lnd/data/chain/bitcoin/regtest/macaroons.db
+        cp /root/.lnd/admin.macaroon /root/.lnd/data/chain/bitcoin/regtest/admin.macaroon
+        /bin/lnd
+    depends_on: [bitcoind]
   bria:
     image: us.gcr.io/galoy-org/bria:latest
     ports:

toolchains/workspace-pnpm/macros.bzl

@@ -1065,7 +1065,7 @@ dev_pnpm_task_test = rule(impl = pnpm_task_test_impl, attrs = {
     "deps": attrs.list(attrs.source(), default = [], doc = """List of dependencies we require"""),
 })
 
-def test_unit_impl(ctx: AnalysisContext) -> list[[
+def jest_test_impl(ctx: AnalysisContext) -> list[[
     DefaultInfo,
     RunInfo,
     ExternalRunnerTestInfo,
@@ -1075,6 +1075,8 @@ def test_unit_impl(ctx: AnalysisContext) -> list[[
     args.add(ctx.attrs.config_file)
     args.add("--bail")
     args.add("--verbose")
+    if ctx.attrs.run_serially:
+        args.add("--runInBand")
 
     return _npm_test_impl(
         ctx,
@@ -1083,8 +1085,8 @@ def test_unit_impl(ctx: AnalysisContext) -> list[[
         "jest",
     )
 
-_test_unit = rule(
-    impl = test_unit_impl,
+_jest_test = rule(
+    impl = jest_test_impl,
     attrs = {
         "srcs": attrs.list(
             attrs.source(),
@@ -1099,6 +1101,10 @@ _test_unit = rule(
             attrs.string(),
             doc = """File name and relative path for jest config.""",
         ),
+        "run_serially": attrs.bool(
+            default = False,
+            doc = "Run all tests serially in the current process"
+        ),
         "env_file": attrs.option(
             attrs.string(),
             doc = """File name and relative path for env variables required.""",
@@ -1132,7 +1138,7 @@ _test_unit = rule(
     },
 )
 
-def test_unit(
+def jest_test(
         node_modules = ":node_modules",
         visibility = ["PUBLIC"],
         **kwargs):
@@ -1143,7 +1149,7 @@ def test_unit(
             bin_name = "jest",
         )
 
-    _test_unit(
+    _jest_test(
         jest = ":{}".format(jest_bin),
         node_modules = node_modules,
         visibility = visibility,