From ae91c6676594df262ab2548bcd96b054a5118648 Mon Sep 17 00:00:00 2001
From: Sebastien Guillemot <sebastiengllmt@gmail.com>
Date: Fri, 24 May 2024 09:59:18 +0900
Subject: [PATCH] Isolate STF failures to the single input that caused them

---
 packages/engine/paima-sm/src/index.ts | 124 +++++++++++++++-----------
 1 file changed, 73 insertions(+), 51 deletions(-)

diff --git a/packages/engine/paima-sm/src/index.ts b/packages/engine/paima-sm/src/index.ts
index 0a6ec78f7..81a9aa722 100644
--- a/packages/engine/paima-sm/src/index.ts
+++ b/packages/engine/paima-sm/src/index.ts
@@ -30,6 +30,7 @@ import {
   updateCardanoEpoch,
   updateCardanoPaginationCursor,
 } from '@paima/db';
+import type { SQLUpdate } from '@paima/db';
 import Prando from '@paima/prando';
 
 import { randomnessRouter } from './randomness.js';
@@ -211,44 +212,56 @@ const SM: GameStateMachineInitializer = {
 
         await processInternalEvents(latestChainData.internalEvents, dbTx);
 
-        const checkpointName = `game_sm_start`;
-        await dbTx.query(`SAVEPOINT ${checkpointName}`);
-        try {
-          // Fetch and execute scheduled input data
-          const scheduledInputsLength = await processScheduledData(
-            latestChainData,
-            dbTx,
-            gameStateTransition,
-            randomnessGenerator
-          );
+        // Fetch and execute scheduled input data
+        const scheduledInputsLength = await processScheduledData(
+          latestChainData,
+          dbTx,
+          gameStateTransition,
+          randomnessGenerator
+        );
 
-          // Execute user submitted input data
-          const userInputsLength = await processUserInputs(
-            latestChainData,
-            dbTx,
-            gameStateTransition,
-            randomnessGenerator
+        // Execute user submitted input data
+        const userInputsLength = await processUserInputs(
+          latestChainData,
+          dbTx,
+          gameStateTransition,
+          randomnessGenerator
+        );
+
+        // Extra logging
+        if (cdeDataLength + userInputsLength + scheduledInputsLength > 0)
+          doLog(
+            `Processed ${userInputsLength} user inputs, ${scheduledInputsLength} scheduled inputs and ${cdeDataLength} CDE events in block #${latestChainData.blockNumber}`
           );
 
-          // Extra logging
-          if (cdeDataLength + userInputsLength + scheduledInputsLength > 0)
-            doLog(
-              `Processed ${userInputsLength} user inputs, ${scheduledInputsLength} scheduled inputs and ${cdeDataLength} CDE events in block #${latestChainData.blockNumber}`
-            );
-        } catch (e) {
-          await dbTx.query(`ROLLBACK TO SAVEPOINT ${checkpointName}`);
-          throw e;
-        } finally {
-          await dbTx.query(`RELEASE SAVEPOINT ${checkpointName}`);
-
-          // Commit finishing of processing to DB
-          await blockHeightDone.run({ block_height: latestChainData.blockNumber }, dbTx);
-        }
+        // Commit finishing of processing to DB
+        await blockHeightDone.run({ block_height: latestChainData.blockNumber }, dbTx);
       },
     };
   },
 };
 
+/**
+ * We need to process all the SQL calls of an STF update in an all-or-nothing manner
+ * STF updates can fail (since the data for them comes from arbitrary onchain data)
+ * But we can't allow a single user's bad transaction to DOS the game for everybody else
+ * So failures should be isolated to just the specific input, and not the full block
+ * (recall: without this, in psql, if a query fails during a db transaction, the entire dbTx becomes invalid)
+ */
+async function tryOrRollback<T>(dbTx: PoolClient, func: () => Promise<T>): Promise<undefined | T> {
+  const checkpointName = `game_state_transition`;
+  await dbTx.query(`SAVEPOINT ${checkpointName}`);
+  try {
+    return await func();
+  } catch (err) {
+    await dbTx.query(`ROLLBACK TO SAVEPOINT ${checkpointName}`);
+    doLog(`[paima-sm] Database error on ${checkpointName}. Rolling back.`, err);
+    return undefined;
+  } finally {
+    await dbTx.query(`RELEASE SAVEPOINT ${checkpointName}`);
+  }
+}
+
 async function processCdeDataBase(
   cdeData: ChainDataExtensionDatum[] | undefined,
   dbTx: PoolClient,
@@ -362,21 +375,26 @@ async function processScheduledData(
       scheduled: true,
     };
     // Trigger STF
-    const sqlQueries = await gameStateTransition(
-      inputData,
-      data.block_height,
-      randomnessGenerator,
-      DBConn
-    );
+    let sqlQueries: SQLUpdate[] = [];
     try {
+      sqlQueries = await gameStateTransition(
+        inputData,
+        data.block_height,
+        randomnessGenerator,
+        DBConn
+      );
+    } catch (err) {
+      // skip scheduled data where the STF fails
+      doLog(`[paima-sm] Error on scheduled data STF call. Skipping`, err);
+      continue;
+    }
+
+    await tryOrRollback(DBConn, async () => {
       for (const [query, params] of sqlQueries) {
         await query.run(params, DBConn);
       }
       await deleteScheduled.run({ id: data.id }, DBConn);
-    } catch (err) {
-      doLog(`[paima-sm] Database error on deleteScheduled: ${err}`);
-      throw err;
-    }
+    });
   }
   return scheduledData.length;
 }
@@ -430,14 +448,21 @@ async function processUserInputs(
     }
 
     // Trigger STF
-    const sqlQueries = await gameStateTransition(
-      inputData,
-      latestChainData.blockNumber,
-      randomnessGenerator,
-      DBConn
-    );
-
+    let sqlQueries: SQLUpdate[] = [];
     try {
+      sqlQueries = await gameStateTransition(
+        inputData,
+        latestChainData.blockNumber,
+        randomnessGenerator,
+        DBConn
+      );
+    } catch (err) {
+      // skip inputs where the STF fails
+      doLog(`[paima-sm] Error on user input STF call. Skipping`, err);
+      continue;
+    }
+
+    await tryOrRollback(DBConn, async () => {
       for (const [query, params] of sqlQueries) {
         await query.run(params, DBConn);
       }
@@ -458,10 +483,7 @@ async function processUserInputs(
           DBConn
         );
       }
-    } catch (err) {
-      doLog(`[paima-sm] Database error on gameStateTransition: ${err}`);
-      throw err;
-    }
+    });
   }
   return latestChainData.submittedData.length;
 }