Fix batch posting and improve performance

packages/batcher/runtime/package.json

@@ -13,7 +13,7 @@
   },
   "devDependencies": {
     "@types/pg": "^8.6.5",
-    "@types/express": "^4.17.13"
+    "@types/express": "^4.17.20"
   },
   "dependencies": {
     "express": "^4.18.1",

packages/batcher/webserver/package.json

@@ -16,7 +16,7 @@
   },
   "devDependencies": {
     "@types/cors": "^2.8.12",
-    "@types/express": "^4.17.13",
+    "@types/express": "^4.17.20",
     "@types/node": "^18.17.1",
     "@types/pg": "^8.6.5"
   }

packages/engine/paima-funnel/src/funnels/emulated/funnel.ts

@@ -139,12 +139,19 @@ export class EmulatedBlocksFunnel extends BaseFunnel {
     if (blocks.length === 0) {
       return;
     } else if (blocks.length === 1) {
-      doLog(`Emulated funnel ${ENV.CHAIN_ID}: ${blocks[0].timestamp}`);
+      // padding with "-timestamp" so that the logs are the same size no matter which branch
+      // this just makes console outputs easier to read
+      const paddingLength = blocks[0].timestamp.toString().length + 1;
+      doLog(
+        `Emulated funnel ${ENV.CHAIN_ID}: ${blocks[0].timestamp}${' '.repeat(paddingLength)} \t [${
+          blocks[0].timestamp - ENV.BLOCK_TIME
+        }~${blocks[0].timestamp})`
+      );
     } else {
       doLog(
         `Emulated funnel ${ENV.CHAIN_ID}: ${blocks[0].timestamp}-${
           blocks[blocks.length - 1].timestamp
-        }`
+        } \t [${blocks[0].timestamp - ENV.BLOCK_TIME}~${blocks[blocks.length - 1].timestamp})`
       );
     }
   };
@@ -318,7 +325,11 @@ export class EmulatedBlocksFunnel extends BaseFunnel {
     //    so we know for sure there won't be a new block for this interval
     // 2. (currentTimestamp - this.maxWait) has exceeded the end of the interval
     //    so we assume no other block will come in
-    if (nextBlockEndTimestamp >= this.emulatedState.timestampLowerBound) {
+    // Note: > and not >= because the end timestamp is non-exclusive
+    //       ex: for [0~3), nextBlockEndTimestamp=3
+    //           so if we see timestampLowerBound=3
+    //           we can close this block since 3 is not part of the [0~3) range
+    if (nextBlockEndTimestamp > this.emulatedState.timestampLowerBound) {
       return undefined;
     }
 

packages/engine/paima-runtime/package.json

@@ -21,7 +21,7 @@
   },
   "devDependencies": {
     "@types/cors": "^2.8.12",
-    "@types/express": "^4.17.13",
+    "@types/express": "^4.17.20",
     "@types/fnv-plus": "^1.3.0",
     "@types/json-stable-stringify": "^1.0.34",
     "@types/node": "^18.17.1"

packages/engine/paima-runtime/src/index.ts

@@ -1,7 +1,12 @@
 import process from 'process';
 
 import { doLog, logError, ENV } from '@paima/utils';
-import { DataMigrations, deploymentChainBlockheightToEmulated, getGameInput } from '@paima/db';
+import {
+  DataMigrations,
+  deploymentChainBlockheightToEmulated,
+  emulatedSelectLatestPrior,
+  getGameInput,
+} from '@paima/db';
 import { validatePersistentCdeConfig } from './cde-config/validation';
 import type { GameStateMachine, IFunnelFactory, PaimaRuntimeInitializer } from './types';
 
@@ -72,6 +77,7 @@ const paimaEngine: PaimaRuntimeInitializer = {
         this.addGET('/emulated_blocks_active', (req, res): void => {
           res.json({ emulatedBlocksActive: ENV.EMULATED_BLOCKS });
         });
+        /** Return the emulated block where a given underlying block appears, or -1 if that block doesn't exist yet */
         this.addGET('/deployment_blockheight_to_emulated', (req, res): void => {
           try {
             const paramString = String(req.query.deploymentBlockheight);
@@ -83,21 +89,50 @@ const paimaEngine: PaimaRuntimeInitializer = {
               });
             }
 
+            // The block may be onchain without being included in an emulated block yet
+            // ex: waiting to see if there are other blocks that need to be bundled as part of the same emulated block
+            const blockNotYet = -1;
             const DBConn = gameStateMachine.getReadonlyDbConn();
-            deploymentChainBlockheightToEmulated
-              .run({ deployment_chain_block_height: deploymentBlockheight }, DBConn)
-              .then(emulatedBlockheights => {
-                if (emulatedBlockheights.length !== 1) {
-                  return res.status(200).json({
+            emulatedSelectLatestPrior
+              .run({ emulated_block_height: 2147483647 }, DBConn)
+              .then(([latestBlock]) => {
+                if (latestBlock == null) {
+                  res.status(200).json({
                     success: false,
-                    errorMessage: `Supplied blockheight ${deploymentBlockheight} not found in DB`,
+                    result: blockNotYet,
                   });
-                } else {
+                  return false;
+                }
+                // The block may be onchain without being included in an emulated block yet
+                // ex: waiting to see if there are other blocks that need to be bundled as part of the same emulated block
+                if (latestBlock.deployment_chain_block_height < deploymentBlockheight) {
+                  res.status(200).json({
+                    success: false,
+                    result: blockNotYet,
+                  });
+                  return false;
+                }
+                return true;
+              })
+              .then(seenBlock =>
+                !seenBlock
+                  ? Promise.resolve(blockNotYet)
+                  : deploymentChainBlockheightToEmulated
+                      .run({ deployment_chain_block_height: deploymentBlockheight }, DBConn)
+                      .then(res => res.at(0)?.emulated_block_height)
+              )
+              .then(emulatedBlockheight => {
+                if (emulatedBlockheight === blockNotYet) return; // error already handled earlier in the promise chain
+                if (emulatedBlockheight == null) {
                   return res.status(200).json({
-                    success: true,
-                    result: emulatedBlockheights[0].emulated_block_height,
+                    success: false,
+                    errorMessage: `Supplied blockheight ${deploymentBlockheight} not found in DB`,
                   });
                 }
+                return res.status(200).json({
+                  success: true,
+                  result: emulatedBlockheight,
+                });
               })
               .catch(err => {
                 doLog(`Webserver error:`);

packages/paima-sdk/paima-mw-core/src/helpers/posting.ts

@@ -49,9 +49,6 @@ const TX_VERIFICATION_RETRY_DELAY = 1000;
 const TX_VERIFICATION_DELAY = 1000;
 const TX_VERIFICATION_RETRY_COUNT = 8;
 
-const EMULATED_CONVERSION_RETRY_DELAY = 1000;
-const EMULATED_CONVERSION_RETRY_COUNT = 8;
-
 type PostFxn = (tx: Record<string, any>) => Promise<string>;
 
 function oneHourPassed(timestamp: number): boolean {
@@ -164,12 +161,15 @@ export async function postConciselyEncodedData(gameInput: string): Promise<Resul
 async function getAdjustedHeight(deploymentChainBlockHeight: number): Promise<number> {
   const emulatedActive = getEmulatedBlocksActive() ?? (await emulatedBlocksActiveOnBackend());
   if (emulatedActive) {
-    const emulatedBlockHeight = await retryPromise(
-      () => deploymentChainBlockHeightToEmulated(deploymentChainBlockHeight),
-      EMULATED_CONVERSION_RETRY_DELAY,
-      EMULATED_CONVERSION_RETRY_COUNT
-    );
-    return emulatedBlockHeight;
+    const BLOCK_DELAY = 1000;
+    let block = -1;
+    while (block === -1) {
+      block = await deploymentChainBlockHeightToEmulated(deploymentChainBlockHeight);
+      if (block === -1) {
+        await wait(BLOCK_DELAY);
+      }
+    }
+    return block;
   } else {
     return deploymentChainBlockHeight;
   }

packages/paima-sdk/paima-utils/package.json

@@ -36,7 +36,7 @@
   },
   "devDependencies": {
     "@typechain/web3-v1": "^6.0.6",
-    "@types/express": "^4.17.13",
+    "@types/express": "^4.17.20",
     "@types/node": "^18.17.1",
     "typechain": "^8.3.1"
   },