🔨 switch to client side UUID v7 with text serialization

owid · Aug 23, 2024 · 434ff19 · 434ff19
1 parent 7d2553e
commit 434ff19
Show file tree

Hide file tree

Showing 8 changed files with 36 additions and 26 deletions.
diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts
@@ -84,6 +84,7 @@ import {
     FlatTagGraph,
     DbRawChartConfig,
 } from "@ourworldindata/types"
+import { uuidv7 } from "uuidv7"
 import {
     getVariableDataRoute,
     getVariableMetadataRoute,
@@ -374,7 +375,7 @@ const saveGrapher = async (
             [now, user.id, chartId]
         )
     } else {
-        const configId = await db.getBinaryUUID(knex)
+        const configId = uuidv7()
         await db.knexRaw(
             knex,
             `-- sql

diff --git a/db/db.ts b/db/db.ts
@@ -668,13 +668,3 @@ export async function getLinkedIndicatorSlugs({
         .then((gdocs) => gdocs.flatMap((gdoc) => gdoc.linkedKeyIndicatorSlugs))
         .then((slugs) => new Set(slugs))
 }
-
-export const getBinaryUUID = async (
-    knex: KnexReadonlyTransaction
-): Promise<Buffer> => {
-    const { id } = (await knexRawFirst<{ id: Buffer }>(
-        knex,
-        `SELECT UUID_TO_BIN(UUID(), 1) AS id`
-    ))!
-    return id
-}
diff --git a/db/migration/1719842654592-AddChartConfigsTable.ts b/db/migration/1719842654592-AddChartConfigsTable.ts
@@ -1,13 +1,12 @@
 import { MigrationInterface, QueryRunner } from "typeorm"
-
+import { uuidv7 } from "uuidv7"
 export class AddChartConfigsTable1719842654592 implements MigrationInterface {
     private async createChartConfigsTable(
         queryRunner: QueryRunner
     ): Promise<void> {
         await queryRunner.query(`-- sql
             CREATE TABLE chart_configs (
-                id binary(16) NOT NULL DEFAULT (UUID_TO_BIN(UUID(), 1)) PRIMARY KEY,
-                uuid varchar(36) GENERATED ALWAYS AS (BIN_TO_UUID(id, 1)) VIRTUAL,
+                id char(36) NOT NULL PRIMARY KEY,
                 patch json NOT NULL,
                 full json NOT NULL,
                 slug varchar(255) GENERATED ALWAYS AS (JSON_UNQUOTE(JSON_EXTRACT(full, '$.slug'))) STORED,
@@ -25,7 +24,7 @@ export class AddChartConfigsTable1719842654592 implements MigrationInterface {
         // that points to the `chart_configs` table
         await queryRunner.query(`-- sql
             ALTER TABLE charts
-            ADD COLUMN configId binary(16) UNIQUE AFTER type,
+            ADD COLUMN configId char(36) UNIQUE AFTER type,
             ADD CONSTRAINT charts_configId
                 FOREIGN KEY (configId)
                 REFERENCES chart_configs (id)
@@ -44,11 +43,21 @@ export class AddChartConfigsTable1719842654592 implements MigrationInterface {
             WHERE id != config ->> "$.id";
         `)
 
-        // insert all the configs into the `chart_configs` table
-        await queryRunner.query(`-- sql
-            INSERT INTO chart_configs (patch, full)
-            SELECT config, config FROM charts
-        `)
+        const chartConfigs: { config: string }[] =
+            await queryRunner.query(`-- sql
+            select config from charts`)
+
+        // I tried to write this as a chunked builk insert of 500 at a time but
+        // failed to get it to work without doing strange things. We only run this once
+        // for ~5000 items so it's not too bad to do it one insert at a time
+        for (const chartConfig of chartConfigs) {
+            await queryRunner.query(
+                `-- sql
+                INSERT INTO chart_configs (id, patch, full)
+                VALUES (?, ?, ?)`,
+                [uuidv7(), chartConfig.config, chartConfig.config]
+            )
+        }
 
         // update the `configId` column in the `charts` table
         await queryRunner.query(`-- sql
@@ -61,7 +70,7 @@ export class AddChartConfigsTable1719842654592 implements MigrationInterface {
         // now that the `configId` column is filled, make it NOT NULL
         await queryRunner.query(`-- sql
             ALTER TABLE charts
-            MODIFY COLUMN configId binary(16) NOT NULL;
+            MODIFY COLUMN configId char(36) NOT NULL;
         `)
 
         // update `createdAt` and `updatedAt` of the chart_configs table

diff --git a/db/tests/basic.test.ts b/db/tests/basic.test.ts
@@ -9,9 +9,9 @@ import {
     knexRawFirst,
     knexReadonlyTransaction,
     TransactionCloseMode,
-    getBinaryUUID,
 } from "../db.js"
 import { deleteUser, insertUser, updateUser } from "../model/User.js"
+import { uuidv7 } from "uuidv7"
 import {
     ChartsTableName,
     ChartConfigsTableName,
@@ -71,7 +71,7 @@ test("timestamps are automatically created and updated", async () => {
                 .first<DbPlainUser>()
             expect(user).toBeTruthy()
             expect(user.email).toBe("[email protected]")
-            const configId = await getBinaryUUID(trx)
+            const configId = uuidv7()
             const chartConfig: DbInsertChartConfig = {
                 id: configId,
                 patch: "{}",

diff --git a/package.json b/package.json
@@ -165,6 +165,7 @@
         "url-parse": "^1.5.10",
         "url-slug": "^3.0.2",
         "usehooks-ts": "^3.1.0",
+        "uuidv7": "^1.0.1",
         "webfontloader": "^1.6.28",
         "workerpool": "^6.2.0",
         "yaml": "^2.4.2"

diff --git a/packages/@ourworldindata/types/src/dbTypes/ChartConfigs.ts b/packages/@ourworldindata/types/src/dbTypes/ChartConfigs.ts
@@ -3,8 +3,7 @@ import { GrapherInterface } from "../grapherTypes/GrapherTypes.js"
 
 export const ChartConfigsTableName = "chart_configs"
 export interface DbInsertChartConfig {
-    id: Buffer
-    uuid?: string
+    id: string
     patch: JsonString
     full: JsonString
     slug?: string | null

diff --git a/packages/@ourworldindata/types/src/dbTypes/Charts.ts b/packages/@ourworldindata/types/src/dbTypes/Charts.ts
@@ -1,6 +1,6 @@
 export const ChartsTableName = "charts"
 export interface DbInsertChart {
-    configId: Buffer
+    configId: string
     createdAt?: Date
     id?: number
     isIndexable?: number

diff --git a/yarn.lock b/yarn.lock
@@ -11075,6 +11075,7 @@ __metadata:
     url-parse: "npm:^1.5.10"
     url-slug: "npm:^3.0.2"
     usehooks-ts: "npm:^3.1.0"
+    uuidv7: "npm:^1.0.1"
     vite: "npm:^5.3.4"
     vite-plugin-checker: "npm:^0.7.2"
     webfontloader: "npm:^1.6.28"
@@ -20025,6 +20026,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"uuidv7@npm:^1.0.1":
+  version: 1.0.1
+  resolution: "uuidv7@npm:1.0.1"
+  bin:
+    uuidv7: cli.js
+  checksum: 10/24f835d067ee69ee46b1734b1da33e6f751f04ccfa5aa83703fc84693b8628d0ae0b4cd7f31c940ec933268971c338931bf5b57b09e882e14df47dfebba2b760
+  languageName: node
+  linkType: hard
+
 "v8-to-istanbul@npm:^9.0.1":
   version: 9.0.1
   resolution: "v8-to-istanbul@npm:9.0.1"