feat(release): update f82ad52 (#15)

Dockerfile.localstack

@@ -1,14 +1,14 @@
-FROM localstack/localstack
+FROM gresau/localstack-persist
 
 COPY scripts/provision_localstack.sh /opt/code/provision_localstack.sh
 COPY scripts/localstack_entrypoint.sh /docker-entrypoint-initaws.d/entrypoint.sh
 
 RUN chmod +x /opt/code/provision_localstack.sh \
-    && chmod +x /docker-entrypoint-initaws.d/entrypoint.sh
+  && chmod +x /docker-entrypoint-initaws.d/entrypoint.sh
 
 RUN aws configure --profile localstack set aws_access_key_id test && \
-    aws configure --profile localstack set aws_secret_access_key test && \
-    aws configure --profile localstack set region us-east-1
+  aws configure --profile localstack set aws_secret_access_key test && \
+  aws configure --profile localstack set region us-east-1
 
 # A wrapper around the localstack image's entrypoint script
 # that provisions the necessary 'AWS' resources for Turbo.

docker-compose.yml

@@ -1,11 +1,4 @@
-version: "3"
-
 services:
-  arlocal:
-    image: textury/arlocal:v1.1.35
-    ports:
-      - "1984:1984"
-
   upload-service:
     build:
       context: .
@@ -108,7 +101,42 @@ services:
     volumes:
       - upload-service-data:/var/lib/postgresql/data
 
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME:-localstack}"
+    build:
+      context: .
+      dockerfile: Dockerfile.localstack
+    ports:
+      - "127.0.0.1:4566:4566" # LocalStack Gateway
+      #- "127.0.0.1:4510-4559:4510-4559" # external services port range
+    environment:
+      # LocalStack configuration: https://docs.localstack.cloud/references/configuration/
+      - SERVICES=${SERVICES:-s3,sqs,secretsmanager}
+      - DEBUG=${DEBUG:-0} # set to 1 to enable debug mode
+      - NODE_ENV=${NODE_ENV:-local}
+      - ARWEAVE_WALLET=${ARWEAVE_WALLET:-}
+      - TURBO_OPTICAL_KEY=${TURBO_OPTICAL_KEY:-$ARWEAVE_WALLET}
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/persisted-data"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"]
+      interval: 90s
+      timeout: 30s
+      retries: 1
+      start_period: 15s
+
+  arlocal:
+    profiles:
+      - test
+    image: textury/arlocal:v1.1.35
+    ports:
+      - "1984:1984"
+
   test-runner:
+    profiles:
+      - test
     build:
       context: .
       dockerfile: tests/integration.Dockerfile
@@ -142,31 +170,6 @@ services:
       arlocal:
         condition: service_started
 
-  localstack:
-    container_name: "${LOCALSTACK_DOCKER_NAME:-localstack}"
-    build:
-      context: .
-      dockerfile: Dockerfile.localstack
-    ports:
-      - "127.0.0.1:4566:4566" # LocalStack Gateway
-      #- "127.0.0.1:4510-4559:4510-4559" # external services port range
-    environment:
-      # LocalStack configuration: https://docs.localstack.cloud/references/configuration/
-      - SERVICES=${SERVICES:-s3,sqs,secretsmanager}
-      - DEBUG=${DEBUG:-1}
-      - NODE_ENV=${NODE_ENV:-local}
-      - ARWEAVE_WALLET=${ARWEAVE_WALLET:-}
-      - TURBO_OPTICAL_KEY=${TURBO_OPTICAL_KEY:-$ARWEAVE_WALLET}
-    volumes:
-      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
-      - "/var/run/docker.sock:/var/run/docker.sock"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"]
-      interval: 90s
-      timeout: 30s
-      retries: 1
-      start_period: 15s
-
 volumes:
   upload-service-data:
   upload-service-data-items:

ecs/fulfillment-pipeline/src/jobs/finalize.ts

@@ -25,7 +25,10 @@ import { PaymentService } from "../../../../src/arch/payment";
 import { getQueueUrl } from "../../../../src/arch/queues";
 import { gatewayUrl } from "../../../../src/constants";
 import { finalizeMultipartUploadWithQueueMessage } from "../../../../src/routes/multiPartUploads";
-import { DataItemExistsWarning } from "../../../../src/utils/errors";
+import {
+  DataItemExistsWarning,
+  InsufficientBalance,
+} from "../../../../src/utils/errors";
 import { getArweaveWallet } from "../../../../src/utils/getArweaveWallet";
 import {
   defaultSQSOptions,
@@ -74,8 +77,9 @@ export function createFinalizeUploadConsumerQueue({
               message,
             });
             return;
+          } else if (error instanceof InsufficientBalance) {
+            return;
           }
-
           throw error;
         }
       },

package.json

@@ -16,7 +16,7 @@
     "test:unit": "nyc mocha --spec='src/**/*.test.ts'",
     "test:integration": "nyc mocha --spec='tests/**/*.test.ts'",
     "test:integration:local": "yarn db:up && yarn arlocal:up && yarn test:integration \"$@\" ; yarn arlocal:down && yarn db:down",
-    "test:docker": "docker compose down -v ; docker compose --env-file .env.localstack up test-runner --exit-code-from test-runner --build",
+    "test:docker": "docker compose down -v ; docker compose --env-file .env.localstack --profile test up test-runner --exit-code-from test-runner --build",
     "ci": "yarn build && yarn build:lambda && yarn test:docker",
     "build": "yarn clean && tsc --project ./tsconfig.prod.json",
     "build:lambda": "yarn build && node ./scripts/bundle-lambdas.cjs",
@@ -86,6 +86,7 @@
     "@aws-sdk/lib-storage": "^3.367.0",
     "@aws-sdk/node-http-handler": "^3.360.0",
     "@aws-sdk/signature-v4-crt": "^3.451.0",
+    "@aws-sdk/util-retry": "^3.374.0",
     "@ethersproject/signing-key": "^5.7.0",
     "@koa/cors": "^5.0.0",
     "@opentelemetry/api": "^1.7.0",

src/arch/payment.ts

@@ -18,13 +18,11 @@ import { AxiosInstance } from "axios";
 import { sign } from "jsonwebtoken";
 import winston from "winston";
 
-import {
-  allowListedSignatureTypes,
-  signatureTypeInfo,
-} from "../bundles/verifyDataItem";
+import { signatureTypeInfo } from "../bundles/verifyDataItem";
 import {
   allowArFSData,
   allowListPublicAddresses,
+  allowListedSignatureTypes,
   freeUploadLimitBytes,
   testPrivateRouteSecret,
 } from "../constants";

src/arch/s3ObjectStore.ts

@@ -31,6 +31,7 @@ import {
 } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { NodeHttpHandler } from "@aws-sdk/node-http-handler";
+import { ConfiguredRetryStrategy } from "@aws-sdk/util-retry";
 import { AbortController } from "@smithy/abort-controller";
 import * as https from "https";
 import { Readable } from "multistream";
@@ -105,6 +106,12 @@ export const handleS3MultipartUploadError = (
 // Build a map of bucket regions to their respective clients based on env vars
 const endpoint = process.env.AWS_ENDPOINT;
 const forcePathStyle = process.env.S3_FORCE_PATH_STYLE;
+const maxRetryAttempts = +(process.env.S3_RETRY_MAX_ATTEMPTS ?? 5);
+const retryBaseDelayMs = +(process.env.S3_RETRY_BASE_DELAY_MS ?? 100);
+const defaultRetryStrategy = new ConfiguredRetryStrategy(
+  maxRetryAttempts,
+  (attempt: number) => retryBaseDelayMs * 2 ** attempt // exponential delay
+);
 type BucketRegion = string;
 const regionsToClients: Record<BucketRegion, S3Client> = {};
 
@@ -121,6 +128,7 @@ const regionsToClients: Record<BucketRegion, S3Client> = {};
           requestTimeout: 0,
         }),
         region,
+        retryStrategy: defaultRetryStrategy,
         ...(endpoint
           ? {
               endpoint,
@@ -161,6 +169,7 @@ const defaultS3Client = new S3Client({
     }),
     requestTimeout: 0,
   }),
+  retryStrategy: defaultRetryStrategy,
   region: process.env.AWS_REGION ?? "us-east-1",
   ...(endpoint
     ? {

src/bundles/verifyDataItem.ts

@@ -86,15 +86,6 @@ export const signatureTypeInfo: Record<number, SigInfo> = {
   },
 };
 
-export const allowListedSignatureTypes = process.env
-  .ALLOW_LISTED_SIGNATURE_TYPES
-  ? new Set(process.env.ALLOW_LISTED_SIGNATURE_TYPES.split(",").map((s) => +s))
-  : new Set([
-      SignatureConfig.SOLANA,
-      SignatureConfig.ED25519,
-      SignatureConfig.ETHEREUM,
-    ]);
-
 export const sigNameToSigInfo: Record<string, SigInfo> = Object.values(
   signatureTypeInfo
 ).reduce((acc, info) => {

src/constants.ts

@@ -222,3 +222,8 @@ export const blocklistedAddresses =
 export const turboLocalJwk = process.env.TURBO_JWK_FILE
   ? JSON.parse(fs.readFileSync(process.env.TURBO_JWK_FILE, "utf-8"))
   : undefined;
+
+export const allowListedSignatureTypes = process.env
+  .ALLOW_LISTED_SIGNATURE_TYPES
+  ? new Set(process.env.ALLOW_LISTED_SIGNATURE_TYPES.split(",").map((s) => +s))
+  : new Set([]);

src/metricRegistry.ts

@@ -45,6 +45,11 @@ export class MetricRegistry {
     help: "Count of failed API calls to the USD/AR endpoint of the payment service",
   });
 
+  public static localCacheDataItemHit = new promClient.Counter({
+    name: "local_cache_data_item_hit_count",
+    help: "Count of data items that were found already in the local cache",
+  });
+
   private constructor() {
     this.registry = new promClient.Registry();
 
@@ -53,6 +58,7 @@ export class MetricRegistry {
     this.registry.registerMetric(MetricRegistry.unbundleBdiEnqueueFail);
     this.registry.registerMetric(MetricRegistry.refundBalanceFail);
     this.registry.registerMetric(MetricRegistry.usdToArRateFail);
+    this.registry.registerMetric(MetricRegistry.localCacheDataItemHit);
   }
 
   public static getInstance(): MetricRegistry {