From 93a3d01f4100c084e9d30ff3a64386ff76b13985 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Fri, 5 Apr 2024 10:58:07 -0700 Subject: [PATCH 01/26] Quick attempt at running hub code in script --- packages/hub/package.json | 3 +- packages/hub/src/runScript.ts | 111 ++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 packages/hub/src/runScript.ts diff --git a/packages/hub/package.json b/packages/hub/package.json index 979c11c969..1feca04de4 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -19,7 +19,8 @@ "build": "pnpm gen && __NEXT_PRIVATE_PREBUNDLED_REACT=next next build", "lint": "prettier --check . && next lint", "format": "prettier --write .", - "test:manual": "dotenv -e .env.test -- jest -i" + "test:manual": "dotenv -e .env.test -- jest -i", + "run-squiggle": "tsx src/runScript.ts" }, "dependencies": { "@next-auth/prisma-adapter": "^1.0.7", diff --git a/packages/hub/src/runScript.ts b/packages/hub/src/runScript.ts new file mode 100644 index 0000000000..bbfd0ba024 --- /dev/null +++ b/packages/hub/src/runScript.ts @@ -0,0 +1,111 @@ +import { Prisma, PrismaClient } from "@prisma/client"; + +import { SqProject, SqValue } from "@quri/squiggle-lang"; + +import { DEFAULT_SEED } from "@/constants"; + +import { ModelExport } from "../../components/dist/src/components/SquigglePlayground"; +import { squiggleHubLinker } from "./squiggle/components/linker"; + +const prisma = new PrismaClient(); + +export const squiggleValueToJSON = (value: SqValue): any => { + return value.asJS(); +}; + +type SquiggleOutput = { + isCached: boolean; +} & ( + | { + isOk: false; + errorString: string; + resultJSON?: undefined; + bindingsJSON?: undefined; + } + | { + isOk: true; + errorString?: undefined; + resultJSON: Prisma.JsonValue; + bindingsJSON: Prisma.JsonValue; + } +); + +export async function runSquiggle2(code: string): Promise { + const MAIN = "main"; + + const env = { + sampleCount: 10000, // int + xyPointLength: 10000, // int + seed: DEFAULT_SEED, + }; + + const project = SqProject.create({ + linker: squiggleHubLinker, + }); + + project.setSource(MAIN, code); + await project.run(MAIN); + + const outputR = project.getOutput(MAIN); + + if (outputR.ok) { + const _exports: ModelExport[] = outputR.value.exports + .entries() + .map((e) => ({ + variableName: e[0], + variableType: e[1].tag, + title: e[1].title(), + docstring: e[1].context?.docstring() || "", + })); + } + + return outputR.ok + ? { + isCached: false, + isOk: true, + resultJSON: squiggleValueToJSON(outputR.value.result), + bindingsJSON: squiggleValueToJSON(outputR.value.bindings.asValue()), + } + : { + isCached: false, + isOk: false, + errorString: outputR.value.toString(), + }; +} + +async function main() { + const model = await prisma.model.findFirst({ + where: { + slug: "test-model", + owner: { slug: "QURI-main" }, + // no need to check access - will be checked by Model authScopes + }, + include: { + currentRevision: { + include: { + squiggleSnippet: true, + }, + }, + }, + }); + + const firstCode = model?.currentRevision?.squiggleSnippet?.code; + if (!firstCode) { + throw new Error("No code found"); + } + + const startTime = performance.now(); + let response = await runSquiggle2(firstCode); + const endTime = performance.now(); + const diff = endTime - startTime; + + console.log(firstCode, diff, response); +} +main() + .catch((error) => { + console.error(error); + process.exit(1); + }) + .finally(async () => { + await prisma.$disconnect(); + }); From 34111ea679518bc5d6a6c22c60fd893583a12d63 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sat, 6 Apr 2024 12:44:12 -0700 Subject: [PATCH 02/26] Removed modelExports from saving on model end, will run on server --- packages/hub/schema.graphql | 8 -- .../[slug]/EditSquiggleSnippetModel.tsx | 14 +-- .../mutations/updateSquiggleSnippetModel.ts | 24 ----- .../hub/src/models/components/ModelCard.tsx | 2 +- packages/hub/src/runScript.ts | 90 ++++++++++++++++--- 5 files changed, 78 insertions(+), 60 deletions(-) diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql index e684767a0f..95739ac366 100644 --- a/packages/hub/schema.graphql +++ b/packages/hub/schema.graphql @@ -453,7 +453,6 @@ union MutationUpdateRelativeValuesDefinitionResult = BaseError | UpdateRelativeV input MutationUpdateSquiggleSnippetModelInput { comment: String content: SquiggleSnippetContentInput! - exports: [SquiggleModelExportInput!] owner: String! relativeValuesExports: [RelativeValuesExportInput!] slug: String! @@ -650,13 +649,6 @@ type SquiggleErrorOutput implements SquiggleOutput { isCached: Boolean! } -input SquiggleModelExportInput { - docstring: String - title: String - variableName: String! - variableType: String! -} - type SquiggleOkOutput implements SquiggleOutput { bindingsJSON: String! isCached: Boolean! diff --git a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx index 84fa91ebc9..000454bf49 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx @@ -54,13 +54,11 @@ import { EditSquiggleSnippetModel$key } from "@/__generated__/EditSquiggleSnippe import { EditSquiggleSnippetModelMutation, RelativeValuesExportInput, - SquiggleModelExportInput, } from "@/__generated__/EditSquiggleSnippetModelMutation.graphql"; export type SquiggleSnippetFormShape = { code: string; relativeValuesExports: RelativeValuesExportInput[]; - exports: SquiggleModelExportInput[]; }; type OnSubmit = ( @@ -200,14 +198,8 @@ export const EditSquiggleSnippetModel: FC = ({ slug: item.definition.slug, }, })), - exports: revision.exports.map((item) => ({ - title: item.title, - variableName: item.variableName, - variableType: item.variableType, - docstring: item.docstring, - })), }; - }, [content, revision.relativeValuesExports, revision.exports]); + }, [content, revision.relativeValuesExports]); const { form, onSubmit, inFlight } = useMutationForm< SquiggleSnippetFormShape, @@ -245,7 +237,6 @@ export const EditSquiggleSnippetModel: FC = ({ xyPointLength: content.xyPointLength, }, relativeValuesExports: formData.relativeValuesExports, - exports: formData.exports, comment: extraData?.comment, slug: model.slug, owner: model.owner.slug, @@ -406,9 +397,6 @@ export const EditSquiggleSnippetModel: FC = ({ playgroundProps ) ) { - playgroundProps.onExportsChange = (exports) => { - form.setValue("exports", exports); - }; } playgroundProps.environment = { diff --git a/packages/hub/src/graphql/mutations/updateSquiggleSnippetModel.ts b/packages/hub/src/graphql/mutations/updateSquiggleSnippetModel.ts index 2668089fbd..8795495a78 100644 --- a/packages/hub/src/graphql/mutations/updateSquiggleSnippetModel.ts +++ b/packages/hub/src/graphql/mutations/updateSquiggleSnippetModel.ts @@ -43,15 +43,6 @@ const SquiggleSnippetContentInput = builder.inputType( } ); -const SquiggleModelExportInput = builder.inputType("SquiggleModelExportInput", { - fields: (t) => ({ - variableName: t.string({ required: true }), - variableType: t.string({ required: true }), - docstring: t.string({ required: false }), - title: t.string({ required: false }), - }), -}); - builder.mutationField("updateSquiggleSnippetModel", (t) => t.withAuth({ signedIn: true }).fieldWithInput({ type: builder.simpleObject("UpdateSquiggleSnippetResult", { @@ -66,9 +57,6 @@ builder.mutationField("updateSquiggleSnippetModel", (t) => relativeValuesExports: t.input.field({ type: [RelativeValuesExportInput], }), - exports: t.input.field({ - type: [SquiggleModelExportInput], - }), content: t.input.field({ type: SquiggleSnippetContentInput, required: true, @@ -167,18 +155,6 @@ builder.mutationField("updateSquiggleSnippetModel", (t) => data: relativeValuesExportsToInsert, }, }, - exports: { - createMany: { - data: (input.exports ?? []).map( - ({ variableName, variableType, docstring, title }) => ({ - variableName, - variableType, - docstring: docstring ?? undefined, - title: title ?? null, - }) - ), - }, - }, }, include: { model: { diff --git a/packages/hub/src/models/components/ModelCard.tsx b/packages/hub/src/models/components/ModelCard.tsx index 450b5e34e0..3a6aaf8412 100644 --- a/packages/hub/src/models/components/ModelCard.tsx +++ b/packages/hub/src/models/components/ModelCard.tsx @@ -46,7 +46,7 @@ export const ModelCard: FC = ({ modelRef, showOwner = true }) => { slug: model.slug, }); - const modelExports = model.currentRevision.exports.map( + const modelExports = model.currentRevision.exports?.map( ({ variableName, variableType, title }) => ({ variableName, variableType, diff --git a/packages/hub/src/runScript.ts b/packages/hub/src/runScript.ts index bbfd0ba024..2d05c483ae 100644 --- a/packages/hub/src/runScript.ts +++ b/packages/hub/src/runScript.ts @@ -1,11 +1,12 @@ import { Prisma, PrismaClient } from "@prisma/client"; -import { SqProject, SqValue } from "@quri/squiggle-lang"; +import { SqLinker, SqProject, SqValue } from "@quri/squiggle-lang"; import { DEFAULT_SEED } from "@/constants"; import { ModelExport } from "../../components/dist/src/components/SquigglePlayground"; -import { squiggleHubLinker } from "./squiggle/components/linker"; +import { NotFoundError } from "./graphql/errors/NotFoundError"; +import { parseSourceId } from "./squiggle/components/linker"; const prisma = new PrismaClient(); @@ -30,7 +31,45 @@ type SquiggleOutput = { } ); -export async function runSquiggle2(code: string): Promise { +export const squiggleLinker: SqLinker = { + resolve(name) { + return name; + }, + async loadSource(sourceId: string) { + const { owner, slug } = parseSourceId(sourceId); + + const model = await prisma.model.findFirst({ + where: { + slug, + owner: { slug: owner }, + }, + include: { + currentRevision: { + include: { + squiggleSnippet: true, + }, + }, + }, + }); + + if (!model) { + throw new NotFoundError(); + } + + const content = model?.currentRevision?.squiggleSnippet; + + if (content) { + return content.code; + } else { + throw new NotFoundError(); + } + }, +}; + +export async function runSquiggle2( + currentRevisionId: string, + code: string +): Promise { const MAIN = "main"; const env = { @@ -40,7 +79,7 @@ export async function runSquiggle2(code: string): Promise { }; const project = SqProject.create({ - linker: squiggleHubLinker, + linker: squiggleLinker, }); project.setSource(MAIN, code); @@ -54,9 +93,32 @@ export async function runSquiggle2(code: string): Promise { .map((e) => ({ variableName: e[0], variableType: e[1].tag, - title: e[1].title(), - docstring: e[1].context?.docstring() || "", + title: e[1].tags.name() ? e[1].tags.name() : e[1].title() || "", + docstring: e[1].tags.doc() || "", })); + + for (const e of _exports) { + await prisma.modelExport.upsert({ + where: { + uniqueKey: { + modelRevisionId: currentRevisionId, + variableName: e.variableName, + }, + }, + update: { + variableType: e.variableType, + title: e.title, + docstring: e.docstring, + }, + create: { + modelRevision: { connect: { id: currentRevisionId } }, + variableName: e.variableName, + variableType: e.variableType, + title: e.title, + docstring: e.docstring, + }, + }); + } } return outputR.ok @@ -73,12 +135,11 @@ export async function runSquiggle2(code: string): Promise { }; } -async function main() { +async function main(ownerSlug: string, slug: string) { const model = await prisma.model.findFirst({ where: { - slug: "test-model", - owner: { slug: "QURI-main" }, - // no need to check access - will be checked by Model authScopes + slug, + owner: { slug: ownerSlug }, }, include: { currentRevision: { @@ -90,18 +151,19 @@ async function main() { }); const firstCode = model?.currentRevision?.squiggleSnippet?.code; - if (!firstCode) { - throw new Error("No code found"); + if (!model?.currentRevisionId || !firstCode) { + throw new Error(`No code found for model ${ownerSlug}/${slug}`); } const startTime = performance.now(); - let response = await runSquiggle2(firstCode); + let response = await runSquiggle2(model?.currentRevisionId, firstCode); const endTime = performance.now(); const diff = endTime - startTime; console.log(firstCode, diff, response); } -main() + +main("ozzie", "test-model") .catch((error) => { console.error(error); process.exit(1); From f718d4445611ced8fc36742e60565f69f88e5f86 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sat, 6 Apr 2024 14:21:57 -0700 Subject: [PATCH 03/26] Added ModelRevisionRun --- packages/hub/README.md | 10 +++++++ .../migration.sql | 16 +++++++++++ packages/hub/prisma/schema.prisma | 15 +++++++++- packages/hub/schema.graphql | 19 +++++++++++++ packages/hub/src/graphql/schema.ts | 1 + .../hub/src/graphql/types/ModelRevision.ts | 1 + .../hub/src/graphql/types/ModelRevisionRun.ts | 28 +++++++++++++++++++ 7 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 packages/hub/prisma/migrations/20240406195928_model_revision_run/migration.sql create mode 100644 packages/hub/src/graphql/types/ModelRevisionRun.ts diff --git a/packages/hub/README.md b/packages/hub/README.md index da70607fd8..6464989d03 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -79,3 +79,13 @@ Squiggle Hub is deployed on [Vercel](https://vercel.com/) automatically when the The production database is migrated by [this GitHub Action](https://github.com/quantified-uncertainty/squiggle/blob/main/.github/workflows/prisma-migrate-prod.yml). **Important: it should be invoked _before_ merging any PR that changes the schema.** + +## Debugging + +If you get an error like: + +``` +PothosSchemaError [GraphQLError]: Ref ObjectRef has not been implemented +``` + +Make sure that any new files in `src/graphql/types` have been added to `src/schema.ts`, or something that references that. diff --git a/packages/hub/prisma/migrations/20240406195928_model_revision_run/migration.sql b/packages/hub/prisma/migrations/20240406195928_model_revision_run/migration.sql new file mode 100644 index 0000000000..da938e36b5 --- /dev/null +++ b/packages/hub/prisma/migrations/20240406195928_model_revision_run/migration.sql @@ -0,0 +1,16 @@ +-- CreateTable +CREATE TABLE "ModelRevisionRun" ( + "id" TEXT NOT NULL, + "modelRevisionId" TEXT NOT NULL, + "startedAt" TIMESTAMP(3) NOT NULL, + "endedAt" TIMESTAMP(3) NOT NULL, + "errors" TEXT[], + + CONSTRAINT "ModelRevisionRun_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "ModelRevisionRun_modelRevisionId_key" ON "ModelRevisionRun"("modelRevisionId"); + +-- AddForeignKey +ALTER TABLE "ModelRevisionRun" ADD CONSTRAINT "ModelRevisionRun_modelRevisionId_fkey" FOREIGN KEY ("modelRevisionId") REFERENCES "ModelRevision"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/hub/prisma/schema.prisma b/packages/hub/prisma/schema.prisma index f4fe3564f2..70b20be5d0 100644 --- a/packages/hub/prisma/schema.prisma +++ b/packages/hub/prisma/schema.prisma @@ -222,11 +222,24 @@ model ModelRevision { exports ModelExport[] // required by Prisma, but unused since `model` field should point at the same entity - currentRevisionModel Model? @relation("CurrentRevision") + currentRevisionModel Model? @relation("CurrentRevision") + builds ModelRevisionBuild[] @@index([modelId]) } +model ModelRevisionBuild { + id String @id @default(cuid()) + createdAt DateTime @default(now()) + modelRevision ModelRevision @relation(fields: [modelRevisionId], references: [id], onDelete: Cascade) + modelRevisionId String + runTime Float + errors String[] + + @@unique([modelRevisionId], name: "uniqueKey") + @@index([modelRevisionId]) +} + model SquiggleSnippet { id String @id @default(cuid()) diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql index 95739ac366..26ba6f70c2 100644 --- a/packages/hub/schema.graphql +++ b/packages/hub/schema.graphql @@ -193,6 +193,7 @@ type ModelRevision implements Node { id: ID! model: Model! relativeValuesExports: [RelativeValuesExport!]! + runs: [ModelRevisionRun!]! } type ModelRevisionConnection { @@ -217,6 +218,24 @@ input ModelRevisionForRelativeValuesSlugOwnerInput { slug: String! } +type ModelRevisionRun implements Node { + endedAtTimestamp: Float! + errors: [String!]! + id: ID! + modelRevision: ModelRevision! + startedAtTimestamp: Float! +} + +type ModelRevisionRunConnection { + edges: [ModelRevisionRunEdge!]! + pageInfo: PageInfo! +} + +type ModelRevisionRunEdge { + cursor: String! + node: ModelRevisionRun! +} + type ModelsByVersion { count: Int! models: [Model!]! diff --git a/packages/hub/src/graphql/schema.ts b/packages/hub/src/graphql/schema.ts index 29395fed15..fb1d026b87 100644 --- a/packages/hub/src/graphql/schema.ts +++ b/packages/hub/src/graphql/schema.ts @@ -1,6 +1,7 @@ import "./errors/BaseError"; import "./errors/NotFoundError"; import "./errors/ValidationError"; +import "./types/ModelRevisionRun"; import "./queries/globalStatistics"; import "./queries/group"; import "./queries/groups"; diff --git a/packages/hub/src/graphql/types/ModelRevision.ts b/packages/hub/src/graphql/types/ModelRevision.ts index 7dc819912f..dd2415e226 100644 --- a/packages/hub/src/graphql/types/ModelRevision.ts +++ b/packages/hub/src/graphql/types/ModelRevision.ts @@ -44,6 +44,7 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { relativeValuesExports: t.relation("relativeValuesExports"), exports: t.relation("exports"), model: t.relation("model"), + runs: t.relation("runs"), author: t.relation("author", { nullable: true }), comment: t.exposeString("comment"), content: t.field({ diff --git a/packages/hub/src/graphql/types/ModelRevisionRun.ts b/packages/hub/src/graphql/types/ModelRevisionRun.ts new file mode 100644 index 0000000000..006da4e0ca --- /dev/null +++ b/packages/hub/src/graphql/types/ModelRevisionRun.ts @@ -0,0 +1,28 @@ +import { prismaConnectionHelpers } from "@pothos/plugin-prisma"; + +import { builder } from "@/graphql/builder"; + +export const ModelRevisionRun = builder.prismaNode("ModelRevisionRun", { + id: { field: "id" }, + fields: (t) => ({ + modelRevision: t.relation("modelRevision"), + errors: t.exposeStringList("errors"), + startedAtTimestamp: t.float({ + resolve: (group) => group.startedAt.getTime(), + }), + endedAtTimestamp: t.float({ + resolve: (group) => group.endedAt.getTime(), + }), + }), +}); + +export const ModelRevisionRunConnection = builder.connectionObject({ + type: ModelRevisionRun, + name: "ModelRevisionRunConnection", +}); + +export const modelConnectionHelpers = prismaConnectionHelpers( + builder, + "ModelRevisionRun", + { cursor: "id" } +); From 3b7c9b0028bf5c56e5324fa57e2dc3299e771e1a Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sun, 7 Apr 2024 07:06:54 -0700 Subject: [PATCH 04/26] run -> build --- .../migration.sql | 16 --------- packages/hub/schema.graphql | 36 +++++++++---------- packages/hub/src/graphql/schema.ts | 2 +- .../hub/src/graphql/types/ModelRevision.ts | 2 +- .../src/graphql/types/ModelRevisionBuild.ts | 22 ++++++++++++ .../hub/src/graphql/types/ModelRevisionRun.ts | 28 --------------- 6 files changed, 41 insertions(+), 65 deletions(-) delete mode 100644 packages/hub/prisma/migrations/20240406195928_model_revision_run/migration.sql create mode 100644 packages/hub/src/graphql/types/ModelRevisionBuild.ts delete mode 100644 packages/hub/src/graphql/types/ModelRevisionRun.ts diff --git a/packages/hub/prisma/migrations/20240406195928_model_revision_run/migration.sql b/packages/hub/prisma/migrations/20240406195928_model_revision_run/migration.sql deleted file mode 100644 index da938e36b5..0000000000 --- a/packages/hub/prisma/migrations/20240406195928_model_revision_run/migration.sql +++ /dev/null @@ -1,16 +0,0 @@ --- CreateTable -CREATE TABLE "ModelRevisionRun" ( - "id" TEXT NOT NULL, - "modelRevisionId" TEXT NOT NULL, - "startedAt" TIMESTAMP(3) NOT NULL, - "endedAt" TIMESTAMP(3) NOT NULL, - "errors" TEXT[], - - CONSTRAINT "ModelRevisionRun_pkey" PRIMARY KEY ("id") -); - --- CreateIndex -CREATE UNIQUE INDEX "ModelRevisionRun_modelRevisionId_key" ON "ModelRevisionRun"("modelRevisionId"); - --- AddForeignKey -ALTER TABLE "ModelRevisionRun" ADD CONSTRAINT "ModelRevisionRun_modelRevisionId_fkey" FOREIGN KEY ("modelRevisionId") REFERENCES "ModelRevision"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql index 07d832e85d..8c557a77d3 100644 --- a/packages/hub/schema.graphql +++ b/packages/hub/schema.graphql @@ -211,6 +211,7 @@ type ModelExportRevisionsConnectionEdge { type ModelRevision implements Node { author: User + builds: [ModelRevisionBuild!]! comment: String! content: ModelContent! createdAtTimestamp: Float! @@ -219,7 +220,22 @@ type ModelRevision implements Node { id: ID! model: Model! relativeValuesExports: [RelativeValuesExport!]! - runs: [ModelRevisionRun!]! +} + +type ModelRevisionBuild implements Node { + errors: [String!]! + id: ID! + modelRevision: ModelRevision! +} + +type ModelRevisionBuildConnection { + edges: [ModelRevisionBuildEdge!]! + pageInfo: PageInfo! +} + +type ModelRevisionBuildEdge { + cursor: String! + node: ModelRevisionBuild! } type ModelRevisionConnection { @@ -244,24 +260,6 @@ input ModelRevisionForRelativeValuesSlugOwnerInput { slug: String! } -type ModelRevisionRun implements Node { - endedAtTimestamp: Float! - errors: [String!]! - id: ID! - modelRevision: ModelRevision! - startedAtTimestamp: Float! -} - -type ModelRevisionRunConnection { - edges: [ModelRevisionRunEdge!]! - pageInfo: PageInfo! -} - -type ModelRevisionRunEdge { - cursor: String! - node: ModelRevisionRun! -} - type ModelsByVersion { count: Int! models: [Model!]! diff --git a/packages/hub/src/graphql/schema.ts b/packages/hub/src/graphql/schema.ts index 0da447f17b..693099e303 100644 --- a/packages/hub/src/graphql/schema.ts +++ b/packages/hub/src/graphql/schema.ts @@ -1,7 +1,7 @@ import "./errors/BaseError"; import "./errors/NotFoundError"; import "./errors/ValidationError"; -import "./types/ModelRevisionRun"; +import "./types/ModelRevisionBuild"; import "./queries/globalStatistics"; import "./queries/group"; import "./queries/groups"; diff --git a/packages/hub/src/graphql/types/ModelRevision.ts b/packages/hub/src/graphql/types/ModelRevision.ts index ced7b88b70..ac9de03898 100644 --- a/packages/hub/src/graphql/types/ModelRevision.ts +++ b/packages/hub/src/graphql/types/ModelRevision.ts @@ -35,7 +35,7 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { relativeValuesExports: t.relation("relativeValuesExports"), exports: t.relation("exports"), model: t.relation("model"), - runs: t.relation("runs"), + builds: t.relation("builds"), author: t.relation("author", { nullable: true }), comment: t.exposeString("comment"), content: t.field({ diff --git a/packages/hub/src/graphql/types/ModelRevisionBuild.ts b/packages/hub/src/graphql/types/ModelRevisionBuild.ts new file mode 100644 index 0000000000..10dfb7ab08 --- /dev/null +++ b/packages/hub/src/graphql/types/ModelRevisionBuild.ts @@ -0,0 +1,22 @@ +import { prismaConnectionHelpers } from "@pothos/plugin-prisma"; + +import { builder } from "@/graphql/builder"; + +export const ModelRevisionBuild = builder.prismaNode("ModelRevisionBuild", { + id: { field: "id" }, + fields: (t) => ({ + modelRevision: t.relation("modelRevision"), + errors: t.exposeStringList("errors"), + }), +}); + +export const ModelRevisionBuildConnection = builder.connectionObject({ + type: ModelRevisionBuild, + name: "ModelRevisionBuildConnection", +}); + +export const modelConnectionHelpers = prismaConnectionHelpers( + builder, + "ModelRevisionBuild", + { cursor: "id" } +); diff --git a/packages/hub/src/graphql/types/ModelRevisionRun.ts b/packages/hub/src/graphql/types/ModelRevisionRun.ts deleted file mode 100644 index 006da4e0ca..0000000000 --- a/packages/hub/src/graphql/types/ModelRevisionRun.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { prismaConnectionHelpers } from "@pothos/plugin-prisma"; - -import { builder } from "@/graphql/builder"; - -export const ModelRevisionRun = builder.prismaNode("ModelRevisionRun", { - id: { field: "id" }, - fields: (t) => ({ - modelRevision: t.relation("modelRevision"), - errors: t.exposeStringList("errors"), - startedAtTimestamp: t.float({ - resolve: (group) => group.startedAt.getTime(), - }), - endedAtTimestamp: t.float({ - resolve: (group) => group.endedAt.getTime(), - }), - }), -}); - -export const ModelRevisionRunConnection = builder.connectionObject({ - type: ModelRevisionRun, - name: "ModelRevisionRunConnection", -}); - -export const modelConnectionHelpers = prismaConnectionHelpers( - builder, - "ModelRevisionRun", - { cursor: "id" } -); From 403172d0b7d153a3d42e52ae9d4f60e15676fb8f Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sun, 7 Apr 2024 07:23:25 -0700 Subject: [PATCH 05/26] Updated migration and schema.graphql --- .../migration.sql | 19 +++++++++++++++++++ packages/hub/schema.graphql | 2 ++ .../src/graphql/types/ModelRevisionBuild.ts | 4 ++++ 3 files changed, 25 insertions(+) create mode 100644 packages/hub/prisma/migrations/20240407141341_model_revision_build/migration.sql diff --git a/packages/hub/prisma/migrations/20240407141341_model_revision_build/migration.sql b/packages/hub/prisma/migrations/20240407141341_model_revision_build/migration.sql new file mode 100644 index 0000000000..6b60b0401a --- /dev/null +++ b/packages/hub/prisma/migrations/20240407141341_model_revision_build/migration.sql @@ -0,0 +1,19 @@ +-- CreateTable +CREATE TABLE "ModelRevisionBuild" ( + "id" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "modelRevisionId" TEXT NOT NULL, + "runTime" DOUBLE PRECISION NOT NULL, + "errors" TEXT[], + + CONSTRAINT "ModelRevisionBuild_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "ModelRevisionBuild_modelRevisionId_idx" ON "ModelRevisionBuild"("modelRevisionId"); + +-- CreateIndex +CREATE UNIQUE INDEX "ModelRevisionBuild_modelRevisionId_key" ON "ModelRevisionBuild"("modelRevisionId"); + +-- AddForeignKey +ALTER TABLE "ModelRevisionBuild" ADD CONSTRAINT "ModelRevisionBuild_modelRevisionId_fkey" FOREIGN KEY ("modelRevisionId") REFERENCES "ModelRevision"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql index 8c557a77d3..383a7ee5b2 100644 --- a/packages/hub/schema.graphql +++ b/packages/hub/schema.graphql @@ -223,9 +223,11 @@ type ModelRevision implements Node { } type ModelRevisionBuild implements Node { + createdAtTimestamp: Float! errors: [String!]! id: ID! modelRevision: ModelRevision! + runTime: Float! } type ModelRevisionBuildConnection { diff --git a/packages/hub/src/graphql/types/ModelRevisionBuild.ts b/packages/hub/src/graphql/types/ModelRevisionBuild.ts index 10dfb7ab08..9a92ddd8a6 100644 --- a/packages/hub/src/graphql/types/ModelRevisionBuild.ts +++ b/packages/hub/src/graphql/types/ModelRevisionBuild.ts @@ -5,8 +5,12 @@ import { builder } from "@/graphql/builder"; export const ModelRevisionBuild = builder.prismaNode("ModelRevisionBuild", { id: { field: "id" }, fields: (t) => ({ + createdAtTimestamp: t.float({ + resolve: (group) => group.createdAt.getTime(), + }), modelRevision: t.relation("modelRevision"), errors: t.exposeStringList("errors"), + runTime: t.exposeFloat("runTime"), }), }); From 32c750357f289a9be2014e0694cb2a51cb07e994 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sun, 7 Apr 2024 11:18:32 -0700 Subject: [PATCH 06/26] Added back model revision build migration --- .../migration.sql | 5 +-- packages/hub/prisma/schema.prisma | 3 +- packages/hub/schema.graphql | 2 +- .../src/graphql/types/ModelRevisionBuild.ts | 2 +- packages/hub/src/runScript.ts | 31 +++++++++++++++++-- 5 files changed, 33 insertions(+), 10 deletions(-) rename packages/hub/prisma/migrations/{20240407141341_model_revision_build => 20240407144130_model_revision_build}/migration.sql (78%) diff --git a/packages/hub/prisma/migrations/20240407141341_model_revision_build/migration.sql b/packages/hub/prisma/migrations/20240407144130_model_revision_build/migration.sql similarity index 78% rename from packages/hub/prisma/migrations/20240407141341_model_revision_build/migration.sql rename to packages/hub/prisma/migrations/20240407144130_model_revision_build/migration.sql index 6b60b0401a..2b1de252fb 100644 --- a/packages/hub/prisma/migrations/20240407141341_model_revision_build/migration.sql +++ b/packages/hub/prisma/migrations/20240407144130_model_revision_build/migration.sql @@ -3,7 +3,7 @@ CREATE TABLE "ModelRevisionBuild" ( "id" TEXT NOT NULL, "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, "modelRevisionId" TEXT NOT NULL, - "runTime" DOUBLE PRECISION NOT NULL, + "runtime" DOUBLE PRECISION NOT NULL, "errors" TEXT[], CONSTRAINT "ModelRevisionBuild_pkey" PRIMARY KEY ("id") @@ -12,8 +12,5 @@ CREATE TABLE "ModelRevisionBuild" ( -- CreateIndex CREATE INDEX "ModelRevisionBuild_modelRevisionId_idx" ON "ModelRevisionBuild"("modelRevisionId"); --- CreateIndex -CREATE UNIQUE INDEX "ModelRevisionBuild_modelRevisionId_key" ON "ModelRevisionBuild"("modelRevisionId"); - -- AddForeignKey ALTER TABLE "ModelRevisionBuild" ADD CONSTRAINT "ModelRevisionBuild_modelRevisionId_fkey" FOREIGN KEY ("modelRevisionId") REFERENCES "ModelRevision"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/hub/prisma/schema.prisma b/packages/hub/prisma/schema.prisma index 70b20be5d0..1524c48129 100644 --- a/packages/hub/prisma/schema.prisma +++ b/packages/hub/prisma/schema.prisma @@ -233,10 +233,9 @@ model ModelRevisionBuild { createdAt DateTime @default(now()) modelRevision ModelRevision @relation(fields: [modelRevisionId], references: [id], onDelete: Cascade) modelRevisionId String - runTime Float + runtime Float errors String[] - @@unique([modelRevisionId], name: "uniqueKey") @@index([modelRevisionId]) } diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql index 383a7ee5b2..0522d6b4ea 100644 --- a/packages/hub/schema.graphql +++ b/packages/hub/schema.graphql @@ -227,7 +227,7 @@ type ModelRevisionBuild implements Node { errors: [String!]! id: ID! modelRevision: ModelRevision! - runTime: Float! + runtime: Float! } type ModelRevisionBuildConnection { diff --git a/packages/hub/src/graphql/types/ModelRevisionBuild.ts b/packages/hub/src/graphql/types/ModelRevisionBuild.ts index 9a92ddd8a6..2471c01a56 100644 --- a/packages/hub/src/graphql/types/ModelRevisionBuild.ts +++ b/packages/hub/src/graphql/types/ModelRevisionBuild.ts @@ -10,7 +10,7 @@ export const ModelRevisionBuild = builder.prismaNode("ModelRevisionBuild", { }), modelRevision: t.relation("modelRevision"), errors: t.exposeStringList("errors"), - runTime: t.exposeFloat("runTime"), + runtime: t.exposeFloat("runtime"), }), }); diff --git a/packages/hub/src/runScript.ts b/packages/hub/src/runScript.ts index 2d05c483ae..35a0560092 100644 --- a/packages/hub/src/runScript.ts +++ b/packages/hub/src/runScript.ts @@ -1,6 +1,6 @@ import { Prisma, PrismaClient } from "@prisma/client"; -import { SqLinker, SqProject, SqValue } from "@quri/squiggle-lang"; +import { ASTNode, SqLinker, SqProject, SqValue } from "@quri/squiggle-lang"; import { DEFAULT_SEED } from "@/constants"; @@ -66,6 +66,22 @@ export const squiggleLinker: SqLinker = { }, }; +function getExportedVariables(ast: ASTNode): string[] { + const exportedVariables: string[] = []; + + if (ast.type === "Program") { + ast.statements.forEach((statement) => { + if (statement.type === "LetStatement" && statement.exported) { + exportedVariables.push(statement.variable.value); + } else if (statement.type === "DefunStatement" && statement.exported) { + exportedVariables.push(statement.variable.value); + } + }); + } + + return exportedVariables; +} + export async function runSquiggle2( currentRevisionId: string, code: string @@ -86,8 +102,11 @@ export async function runSquiggle2( await project.run(MAIN); const outputR = project.getOutput(MAIN); + // const foo = project. if (outputR.ok) { + const ast = outputR.value.bindings.context?.ast; + console.log("AST", ast && getExportedVariables(ast)); const _exports: ModelExport[] = outputR.value.exports .entries() .map((e) => ({ @@ -160,7 +179,15 @@ async function main(ownerSlug: string, slug: string) { const endTime = performance.now(); const diff = endTime - startTime; - console.log(firstCode, diff, response); + const build = await prisma.modelRevisionBuild.create({ + data: { + modelRevision: { connect: { id: model.currentRevisionId } }, + runtime: diff, + errors: response.isOk ? [] : [response.errorString], + }, + }); + + console.log("built", build, firstCode, diff, response); } main("ozzie", "test-model") From d2d3348bcd64f690c87f477083d31b36edd1137e Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sun, 7 Apr 2024 11:34:16 -0700 Subject: [PATCH 07/26] Adding ModelRevision Status --- packages/hub/schema.graphql | 7 +++++++ .../[variableName]/ModelExportPage.tsx | 1 + .../hub/src/graphql/types/ModelRevision.ts | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql index 0522d6b4ea..776804be6a 100644 --- a/packages/hub/schema.graphql +++ b/packages/hub/schema.graphql @@ -211,6 +211,7 @@ type ModelExportRevisionsConnectionEdge { type ModelRevision implements Node { author: User + buildStatus: ModelRevisionBuildStatus! builds: [ModelRevisionBuild!]! comment: String! content: ModelContent! @@ -240,6 +241,12 @@ type ModelRevisionBuildEdge { node: ModelRevisionBuild! } +enum ModelRevisionBuildStatus { + Failure + Pending + Success +} + type ModelRevisionConnection { edges: [ModelRevisionEdge!]! pageInfo: PageInfo! diff --git a/packages/hub/src/app/models/[owner]/[slug]/exports/[variableName]/ModelExportPage.tsx b/packages/hub/src/app/models/[owner]/[slug]/exports/[variableName]/ModelExportPage.tsx index 946ff4a82e..ed6e4eaa72 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/exports/[variableName]/ModelExportPage.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/exports/[variableName]/ModelExportPage.tsx @@ -105,6 +105,7 @@ export const ModelExportPage: FC<{ modelRevision { id createdAtTimestamp + buildStatus content { __typename ...SquiggleModelExportPage diff --git a/packages/hub/src/graphql/types/ModelRevision.ts b/packages/hub/src/graphql/types/ModelRevision.ts index ac9de03898..9dfeb8a105 100644 --- a/packages/hub/src/graphql/types/ModelRevision.ts +++ b/packages/hub/src/graphql/types/ModelRevision.ts @@ -24,6 +24,10 @@ const ModelContent: UnionRef< resolveType: () => SquiggleSnippet, }); +const ModelRevisionBuildStatus = builder.enumType("ModelRevisionBuildStatus", { + values: ["Pending", "Success", "Failure"], +}); + export const ModelRevision = builder.prismaNode("ModelRevision", { id: { field: "id" }, fields: (t) => ({ @@ -38,6 +42,21 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { builds: t.relation("builds"), author: t.relation("author", { nullable: true }), comment: t.exposeString("comment"), + buildStatus: t.field({ + type: ModelRevisionBuildStatus, + async resolve(revision) { + const lastBuild = await prisma.modelRevisionBuild.findFirst({ + where: { + modelRevisionId: revision.id, + }, + orderBy: { + createdAt: "desc", + }, + }); + if (!lastBuild) return "Pending"; + return lastBuild.errors.length === 0 ? "Success" : "Failure"; + }, + }), content: t.field({ type: ModelContent, select: { squiggleSnippet: true }, From 22843b096196a39cdd98f6283edad552a660991b Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sun, 7 Apr 2024 20:49:14 -0700 Subject: [PATCH 08/26] Added ModelExportNames --- packages/hub/schema.graphql | 3 + .../[slug]/EditSquiggleSnippetModel.tsx | 1 + .../app/models/[owner]/[slug]/ModelLayout.tsx | 32 +++++++-- .../[variableName]/ModelExportPage.tsx | 1 - .../[slug]/revisions/ModelRevisionsList.tsx | 13 ++++ .../hub/src/graphql/types/ModelRevision.ts | 69 ++++++++++++++++++- packages/hub/src/lib/ExportsDropdown.tsx | 6 +- 7 files changed, 112 insertions(+), 13 deletions(-) diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql index 776804be6a..dcf86cb831 100644 --- a/packages/hub/schema.graphql +++ b/packages/hub/schema.graphql @@ -216,9 +216,11 @@ type ModelRevision implements Node { comment: String! content: ModelContent! createdAtTimestamp: Float! + exportNames: [String!]! exports: [ModelExport!]! forRelativeValues(input: ModelRevisionForRelativeValuesInput!): ModelRevisionForRelativeValuesResult! id: ID! + lastBuild: ModelRevisionBuild model: Model! relativeValuesExports: [RelativeValuesExport!]! } @@ -244,6 +246,7 @@ type ModelRevisionBuildEdge { enum ModelRevisionBuildStatus { Failure Pending + Skipped Success } diff --git a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx index 000454bf49..eebd1a67a5 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx @@ -156,6 +156,7 @@ export const EditSquiggleSnippetModel: FC = ({ xyPointLength } } + exportNames exports { id variableName diff --git a/packages/hub/src/app/models/[owner]/[slug]/ModelLayout.tsx b/packages/hub/src/app/models/[owner]/[slug]/ModelLayout.tsx index bbb2d319b1..d400fdeec7 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/ModelLayout.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/ModelLayout.tsx @@ -7,7 +7,11 @@ import { CodeBracketIcon, RectangleStackIcon, ShareIcon } from "@quri/ui"; import { EntityLayout } from "@/components/EntityLayout"; import { EntityTab } from "@/components/ui/EntityTab"; -import { ExportsDropdown, totalImportLength } from "@/lib/ExportsDropdown"; +import { + ExportsDropdown, + type ModelExport, + totalImportLength, +} from "@/lib/ExportsDropdown"; import { extractFromGraphqlErrorUnion } from "@/lib/graphqlHelpers"; import { SerializablePreloadedQuery } from "@/relay/loadPageQuery"; import { usePageQuery } from "@/relay/usePageQuery"; @@ -45,6 +49,7 @@ const Query = graphql` currentRevision { id # for length; TODO - "hasExports" field? + exportNames exports { id variableName @@ -81,12 +86,25 @@ export const ModelLayout: FC< slug: model.slug, }); - const modelExports = model.currentRevision.exports.map( - ({ variableName, variableType, title }) => ({ - variableName, - variableType, - title: title || undefined, - }) + const modelExports: ModelExport[] = model.currentRevision.exportNames.map( + (name) => { + const _export = model.currentRevision.exports.find( + (e) => e.variableName === name + ); + if (_export) { + return { + variableName: _export.variableName, + variableType: _export.variableType, + title: _export.title || undefined, + }; + } else { + return { + variableName: name, + variableType: undefined, + title: undefined, + }; + } + } ); const relativeValuesExports = model.currentRevision.relativeValuesExports.map( diff --git a/packages/hub/src/app/models/[owner]/[slug]/exports/[variableName]/ModelExportPage.tsx b/packages/hub/src/app/models/[owner]/[slug]/exports/[variableName]/ModelExportPage.tsx index ed6e4eaa72..946ff4a82e 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/exports/[variableName]/ModelExportPage.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/exports/[variableName]/ModelExportPage.tsx @@ -105,7 +105,6 @@ export const ModelExportPage: FC<{ modelRevision { id createdAtTimestamp - buildStatus content { __typename ...SquiggleModelExportPage diff --git a/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx b/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx index 6038cb7a3c..794a2a5422 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx @@ -27,6 +27,7 @@ const ModelRevisionItem: FC<{ fragment ModelRevisionsList_revision on ModelRevision { id createdAtTimestamp + buildStatus author { username } @@ -36,6 +37,10 @@ const ModelRevisionItem: FC<{ variableName title } + lastBuild { + errors + runtime + } } `, revisionRef @@ -76,6 +81,9 @@ const ModelRevisionItem: FC<{ {revision.comment ? (
{revision.comment}
) : null} + +
{revision.buildStatus}
+ {revision.lastBuild && revision.lastBuild.runtime} {revision.exports.length > 0 ? (
{`${revision.exports.length} exports `} @@ -124,11 +132,16 @@ export const ModelRevisionsList: FC<{ ...ModelRevisionsList_revision id createdAtTimestamp + buildStatus exports { id title variableName } + lastBuild { + errors + runtime + } } } pageInfo { diff --git a/packages/hub/src/graphql/types/ModelRevision.ts b/packages/hub/src/graphql/types/ModelRevision.ts index 9dfeb8a105..c9f6082a04 100644 --- a/packages/hub/src/graphql/types/ModelRevision.ts +++ b/packages/hub/src/graphql/types/ModelRevision.ts @@ -1,9 +1,12 @@ import { UnionRef } from "@pothos/core"; +import { ASTNode, parse } from "@quri/squiggle-lang"; + import { builder } from "@/graphql/builder"; import { prisma } from "@/prisma"; import { NotFoundError } from "../errors/NotFoundError"; +import { ModelRevisionBuild } from "./ModelRevisionBuild"; import { RelativeValuesExport } from "./RelativeValuesExport"; import { SquiggleSnippet } from "./SquiggleSnippet"; @@ -25,9 +28,25 @@ const ModelContent: UnionRef< }); const ModelRevisionBuildStatus = builder.enumType("ModelRevisionBuildStatus", { - values: ["Pending", "Success", "Failure"], + values: ["Skipped", "Pending", "Success", "Failure"], }); +function getExportedVariables(ast: ASTNode): string[] { + const exportedVariables: string[] = []; + + if (ast.type === "Program") { + ast.statements.forEach((statement) => { + if (statement.type === "LetStatement" && statement.exported) { + exportedVariables.push(statement.variable.value); + } else if (statement.type === "DefunStatement" && statement.exported) { + exportedVariables.push(statement.variable.value); + } + }); + } + + return exportedVariables; +} + export const ModelRevision = builder.prismaNode("ModelRevision", { id: { field: "id" }, fields: (t) => ({ @@ -40,6 +59,21 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { exports: t.relation("exports"), model: t.relation("model"), builds: t.relation("builds"), + + lastBuild: t.field({ + type: ModelRevisionBuild, + nullable: true, + async resolve(revision) { + return prisma.modelRevisionBuild.findFirst({ + where: { + modelRevisionId: revision.id, + }, + orderBy: { + createdAt: "desc", + }, + }); + }, + }), author: t.relation("author", { nullable: true }), comment: t.exposeString("comment"), buildStatus: t.field({ @@ -53,7 +87,20 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { createdAt: "desc", }, }); - if (!lastBuild) return "Pending"; + if (!lastBuild) { + const lastRevision = await prisma.modelRevision.findFirst({ + where: { + modelId: revision.modelId, + }, + orderBy: { + createdAt: "desc", + }, + }); + if (lastRevision?.id === revision.id) { + return "Pending"; + } + return "Skipped"; + } return lastBuild.errors.length === 0 ? "Success" : "Failure"; }, }), @@ -67,6 +114,24 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { } }, }), + exportNames: t.field({ + type: ["String"], + select: { squiggleSnippet: true }, + async resolve(revision) { + console.log("RESOLVING?", revision); + if (revision.contentType === "SquiggleSnippet") { + const ast = parse(revision.squiggleSnippet!.code); + if (ast.ok) { + console.log("RESOLVING!", getExportedVariables(ast.value)); + return getExportedVariables(ast.value); + } else { + return []; + } + } else { + return []; + } + }, + }), forRelativeValues: t.fieldWithInput({ type: RelativeValuesExport, errors: { diff --git a/packages/hub/src/lib/ExportsDropdown.tsx b/packages/hub/src/lib/ExportsDropdown.tsx index 10e26666ae..b1c23a51fd 100644 --- a/packages/hub/src/lib/ExportsDropdown.tsx +++ b/packages/hub/src/lib/ExportsDropdown.tsx @@ -17,10 +17,10 @@ import { import { DropdownMenuNextLinkItem } from "@/components/ui/DropdownMenuNextLinkItem"; import { modelExportRoute, modelForRelativeValuesExportRoute } from "@/routes"; -type ModelExport = { +export type ModelExport = { title?: string; variableName: string; - variableType: string; + variableType?: string; }; type RelativeValuesExport = { slug: string; variableName: string }; @@ -89,7 +89,7 @@ export const ExportsDropdown: FC< variableName: exportItem.variableName, })} title={`${exportItem.title || exportItem.variableName}`} - icon={typeIcon(exportItem.variableType)} + icon={typeIcon(exportItem.variableType || "")} close={close} /> ))}{" "} From d6a6c2cd7f00f258ad0931ad99015f54457af142 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sun, 7 Apr 2024 21:01:46 -0700 Subject: [PATCH 09/26] Improved UI of Revisions build info --- .../models/[owner]/[slug]/revisions/ModelRevisionsList.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx b/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx index 794a2a5422..4c3ba4c9ae 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx @@ -82,8 +82,10 @@ const ModelRevisionItem: FC<{
{revision.comment}
) : null} -
{revision.buildStatus}
- {revision.lastBuild && revision.lastBuild.runtime} +
{`Build Status: ${revision.buildStatus}`}
+ {revision.lastBuild && ( +
{`Build Time: ${revision.lastBuild.runtime.toFixed(0)}ms`}
+ )} {revision.exports.length > 0 ? (
{`${revision.exports.length} exports `} From 1c129f54986ac4c45f3509082627a06680cc16a4 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sun, 7 Apr 2024 21:41:49 -0700 Subject: [PATCH 10/26] Cleaned up buildRecentModelRevisions file --- packages/hub/package.json | 2 +- packages/hub/src/runScript.ts | 200 ----------------- .../src/scripts/buildRecentModelRevision.ts | 210 ++++++++++++++++++ 3 files changed, 211 insertions(+), 201 deletions(-) delete mode 100644 packages/hub/src/runScript.ts create mode 100644 packages/hub/src/scripts/buildRecentModelRevision.ts diff --git a/packages/hub/package.json b/packages/hub/package.json index 1feca04de4..c9b478d235 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -20,7 +20,7 @@ "lint": "prettier --check . && next lint", "format": "prettier --write .", "test:manual": "dotenv -e .env.test -- jest -i", - "run-squiggle": "tsx src/runScript.ts" + "build-last-revision": "tsx src/scripts/buildRecentModelRevision.ts" }, "dependencies": { "@next-auth/prisma-adapter": "^1.0.7", diff --git a/packages/hub/src/runScript.ts b/packages/hub/src/runScript.ts deleted file mode 100644 index 35a0560092..0000000000 --- a/packages/hub/src/runScript.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { Prisma, PrismaClient } from "@prisma/client"; - -import { ASTNode, SqLinker, SqProject, SqValue } from "@quri/squiggle-lang"; - -import { DEFAULT_SEED } from "@/constants"; - -import { ModelExport } from "../../components/dist/src/components/SquigglePlayground"; -import { NotFoundError } from "./graphql/errors/NotFoundError"; -import { parseSourceId } from "./squiggle/components/linker"; - -const prisma = new PrismaClient(); - -export const squiggleValueToJSON = (value: SqValue): any => { - return value.asJS(); -}; - -type SquiggleOutput = { - isCached: boolean; -} & ( - | { - isOk: false; - errorString: string; - resultJSON?: undefined; - bindingsJSON?: undefined; - } - | { - isOk: true; - errorString?: undefined; - resultJSON: Prisma.JsonValue; - bindingsJSON: Prisma.JsonValue; - } -); - -export const squiggleLinker: SqLinker = { - resolve(name) { - return name; - }, - async loadSource(sourceId: string) { - const { owner, slug } = parseSourceId(sourceId); - - const model = await prisma.model.findFirst({ - where: { - slug, - owner: { slug: owner }, - }, - include: { - currentRevision: { - include: { - squiggleSnippet: true, - }, - }, - }, - }); - - if (!model) { - throw new NotFoundError(); - } - - const content = model?.currentRevision?.squiggleSnippet; - - if (content) { - return content.code; - } else { - throw new NotFoundError(); - } - }, -}; - -function getExportedVariables(ast: ASTNode): string[] { - const exportedVariables: string[] = []; - - if (ast.type === "Program") { - ast.statements.forEach((statement) => { - if (statement.type === "LetStatement" && statement.exported) { - exportedVariables.push(statement.variable.value); - } else if (statement.type === "DefunStatement" && statement.exported) { - exportedVariables.push(statement.variable.value); - } - }); - } - - return exportedVariables; -} - -export async function runSquiggle2( - currentRevisionId: string, - code: string -): Promise { - const MAIN = "main"; - - const env = { - sampleCount: 10000, // int - xyPointLength: 10000, // int - seed: DEFAULT_SEED, - }; - - const project = SqProject.create({ - linker: squiggleLinker, - }); - - project.setSource(MAIN, code); - await project.run(MAIN); - - const outputR = project.getOutput(MAIN); - // const foo = project. - - if (outputR.ok) { - const ast = outputR.value.bindings.context?.ast; - console.log("AST", ast && getExportedVariables(ast)); - const _exports: ModelExport[] = outputR.value.exports - .entries() - .map((e) => ({ - variableName: e[0], - variableType: e[1].tag, - title: e[1].tags.name() ? e[1].tags.name() : e[1].title() || "", - docstring: e[1].tags.doc() || "", - })); - - for (const e of _exports) { - await prisma.modelExport.upsert({ - where: { - uniqueKey: { - modelRevisionId: currentRevisionId, - variableName: e.variableName, - }, - }, - update: { - variableType: e.variableType, - title: e.title, - docstring: e.docstring, - }, - create: { - modelRevision: { connect: { id: currentRevisionId } }, - variableName: e.variableName, - variableType: e.variableType, - title: e.title, - docstring: e.docstring, - }, - }); - } - } - - return outputR.ok - ? { - isCached: false, - isOk: true, - resultJSON: squiggleValueToJSON(outputR.value.result), - bindingsJSON: squiggleValueToJSON(outputR.value.bindings.asValue()), - } - : { - isCached: false, - isOk: false, - errorString: outputR.value.toString(), - }; -} - -async function main(ownerSlug: string, slug: string) { - const model = await prisma.model.findFirst({ - where: { - slug, - owner: { slug: ownerSlug }, - }, - include: { - currentRevision: { - include: { - squiggleSnippet: true, - }, - }, - }, - }); - - const firstCode = model?.currentRevision?.squiggleSnippet?.code; - if (!model?.currentRevisionId || !firstCode) { - throw new Error(`No code found for model ${ownerSlug}/${slug}`); - } - - const startTime = performance.now(); - let response = await runSquiggle2(model?.currentRevisionId, firstCode); - const endTime = performance.now(); - const diff = endTime - startTime; - - const build = await prisma.modelRevisionBuild.create({ - data: { - modelRevision: { connect: { id: model.currentRevisionId } }, - runtime: diff, - errors: response.isOk ? [] : [response.errorString], - }, - }); - - console.log("built", build, firstCode, diff, response); -} - -main("ozzie", "test-model") - .catch((error) => { - console.error(error); - process.exit(1); - }) - .finally(async () => { - await prisma.$disconnect(); - }); diff --git a/packages/hub/src/scripts/buildRecentModelRevision.ts b/packages/hub/src/scripts/buildRecentModelRevision.ts new file mode 100644 index 0000000000..7fb72d52a0 --- /dev/null +++ b/packages/hub/src/scripts/buildRecentModelRevision.ts @@ -0,0 +1,210 @@ +import { PrismaClient } from "@prisma/client"; + +import { SqLinker, SqProject } from "@quri/squiggle-lang"; + +import { ModelExport } from "../../../components/dist/src/components/SquigglePlayground"; +import { SAMPLE_COUNT_DEFAULT, XY_POINT_LENGTH_DEFAULT } from "../constants"; +import { NotFoundError } from "../graphql/errors/NotFoundError"; +import { parseSourceId } from "../squiggle/components/linker"; + +const SOURCE_NAME = "main"; + +type SquiggleOutput = + | { + isOk: false; + errorString: string; + } + | { + isOk: true; + errorString?: undefined; + }; + +const prisma = new PrismaClient(); + +export const squiggleLinker: SqLinker = { + resolve(name) { + return name; + }, + async loadSource(sourceId: string) { + const { owner, slug } = parseSourceId(sourceId); + + const model = await prisma.model.findFirst({ + where: { + slug, + owner: { slug: owner }, + }, + include: { + currentRevision: { + include: { + squiggleSnippet: true, + }, + }, + }, + }); + + if (!model) { + throw new NotFoundError(); + } + + const content = model?.currentRevision?.squiggleSnippet; + if (content) { + return content.code; + } else { + throw new NotFoundError(); + } + }, +}; + +export async function runSquiggle( + currentRevisionId: string, + code: string, + seed: string +): Promise { + const env = { + sampleCount: SAMPLE_COUNT_DEFAULT, + xyPointLength: XY_POINT_LENGTH_DEFAULT, + seed, + }; + + const project = SqProject.create({ + linker: squiggleLinker, + environment: env, + }); + + project.setSource(SOURCE_NAME, code); + await project.run(SOURCE_NAME); + + const outputR = project.getOutput(SOURCE_NAME); + + if (outputR.ok) { + const _exports: ModelExport[] = outputR.value.exports + .entries() + .map((e) => ({ + variableName: e[0], + variableType: e[1].tag, + title: e[1].tags.name() ? e[1].tags.name() : e[1].title() || "", + docstring: e[1].tags.doc() || "", + })); + + for (const e of _exports) { + await prisma.modelExport.upsert({ + where: { + uniqueKey: { + modelRevisionId: currentRevisionId, + variableName: e.variableName, + }, + }, + update: { + variableType: e.variableType, + title: e.title, + docstring: e.docstring, + }, + create: { + modelRevision: { connect: { id: currentRevisionId } }, + variableName: e.variableName, + variableType: e.variableType, + title: e.title, + docstring: e.docstring, + }, + }); + } + } + + return outputR.ok + ? { + isOk: true, + } + : { + isOk: false, + errorString: outputR.value.toString(), + }; +} + +async function oldestModelRevisionWithoutBuilds() { + const modelRevision = await prisma.modelRevision.findFirst({ + where: { + currentRevisionModel: { + isNot: null, + }, + builds: { + none: {}, + }, + }, + orderBy: { + createdAt: "asc", + }, + include: { + model: { + include: { + currentRevision: { + include: { + squiggleSnippet: true, + }, + }, + }, + }, + }, + }); + return modelRevision?.model; +} + +async function buildRecentModelVersion() { + try { + const model = await oldestModelRevisionWithoutBuilds(); + + if (!model) { + console.log("No remaining unbuilt model revisions"); + return; + } + + if (!model?.currentRevisionId || !model.currentRevision?.squiggleSnippet) { + throw new NotFoundError(`Model revision didn't have needed information`); + } + + const { code, seed } = model.currentRevision.squiggleSnippet; + + const startTime = performance.now(); + let response = await runSquiggle(model.currentRevisionId, code, seed); + const endTime = performance.now(); + const diff = endTime - startTime; + + const build = await prisma.modelRevisionBuild.create({ + data: { + modelRevision: { connect: { id: model.currentRevisionId } }, + runtime: diff, + errors: response.isOk ? [] : [response.errorString], + }, + }); + + console.log("Build complete:", build); + console.log("Runtime:", diff); + console.log("Response:", response); + } catch (error) { + console.error(error); + } +} + +async function countItemsRemaining() { + const remaining = await prisma.modelRevision.count({ + where: { + currentRevisionModel: { + isNot: null, + }, + builds: { + none: {}, + }, + }, + }); + + console.log("Model Revisions Remaining:", remaining); +} + +buildRecentModelVersion() + .catch((error) => { + console.error(error); + process.exit(1); + }) + .finally(async () => { + countItemsRemaining(); + await prisma.$disconnect(); + }); From 1e9b71f31d9e9d2dffb26652b3328b14838e5816 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sun, 7 Apr 2024 21:59:16 -0700 Subject: [PATCH 11/26] Query cleanup --- .../app/models/[owner]/[slug]/ModelLayout.tsx | 21 ++--- .../hub/src/graphql/types/ModelRevision.ts | 81 +++++++++++-------- .../src/scripts/buildRecentModelRevision.ts | 14 +--- 3 files changed, 57 insertions(+), 59 deletions(-) diff --git a/packages/hub/src/app/models/[owner]/[slug]/ModelLayout.tsx b/packages/hub/src/app/models/[owner]/[slug]/ModelLayout.tsx index d400fdeec7..a3b87f73c7 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/ModelLayout.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/ModelLayout.tsx @@ -88,22 +88,15 @@ export const ModelLayout: FC< const modelExports: ModelExport[] = model.currentRevision.exportNames.map( (name) => { - const _export = model.currentRevision.exports.find( + const matchingExport = model.currentRevision.exports.find( (e) => e.variableName === name ); - if (_export) { - return { - variableName: _export.variableName, - variableType: _export.variableType, - title: _export.title || undefined, - }; - } else { - return { - variableName: name, - variableType: undefined, - title: undefined, - }; - } + + return { + variableName: name, + variableType: matchingExport?.variableType || undefined, + title: matchingExport?.title || undefined, + }; } ); diff --git a/packages/hub/src/graphql/types/ModelRevision.ts b/packages/hub/src/graphql/types/ModelRevision.ts index c9f6082a04..40406f75d5 100644 --- a/packages/hub/src/graphql/types/ModelRevision.ts +++ b/packages/hub/src/graphql/types/ModelRevision.ts @@ -31,20 +31,20 @@ const ModelRevisionBuildStatus = builder.enumType("ModelRevisionBuildStatus", { values: ["Skipped", "Pending", "Success", "Failure"], }); -function getExportedVariables(ast: ASTNode): string[] { - const exportedVariables: string[] = []; +function getExportedVariableNames(ast: ASTNode): string[] { + const exportedVariableNames: string[] = []; if (ast.type === "Program") { ast.statements.forEach((statement) => { if (statement.type === "LetStatement" && statement.exported) { - exportedVariables.push(statement.variable.value); + exportedVariableNames.push(statement.variable.value); } else if (statement.type === "DefunStatement" && statement.exported) { - exportedVariables.push(statement.variable.value); + exportedVariableNames.push(statement.variable.value); } }); } - return exportedVariables; + return exportedVariableNames; } export const ModelRevision = builder.prismaNode("ModelRevision", { @@ -64,14 +64,21 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { type: ModelRevisionBuild, nullable: true, async resolve(revision) { - return prisma.modelRevisionBuild.findFirst({ - where: { - modelRevisionId: revision.id, - }, - orderBy: { - createdAt: "desc", - }, - }); + return prisma.modelRevision + .findUnique({ + where: { + id: revision.id, + }, + select: { + builds: { + orderBy: { + createdAt: "desc", + }, + take: 1, + }, + }, + }) + .then((result) => result?.builds[0] || null); }, }), author: t.relation("author", { nullable: true }), @@ -79,29 +86,37 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { buildStatus: t.field({ type: ModelRevisionBuildStatus, async resolve(revision) { - const lastBuild = await prisma.modelRevisionBuild.findFirst({ + const modelRevision = await prisma.modelRevision.findUnique({ where: { - modelRevisionId: revision.id, - }, - orderBy: { - createdAt: "desc", + id: revision.id, }, - }); - if (!lastBuild) { - const lastRevision = await prisma.modelRevision.findFirst({ - where: { - modelId: revision.modelId, + select: { + builds: { + orderBy: { + createdAt: "desc", + }, + take: 1, }, - orderBy: { - createdAt: "desc", + model: { + select: { + currentRevisionId: true, + }, }, - }); - if (lastRevision?.id === revision.id) { - return "Pending"; - } - return "Skipped"; + }, + }); + + if (!modelRevision) { + throw new Error(`ModelRevision not found for id: ${revision.id}`); + } + + const lastBuild = modelRevision.builds[0]; + if (lastBuild) { + return lastBuild.errors.length === 0 ? "Success" : "Failure"; } - return lastBuild.errors.length === 0 ? "Success" : "Failure"; + + return modelRevision.model.currentRevisionId === revision.id + ? "Pending" + : "Skipped"; }, }), content: t.field({ @@ -118,12 +133,10 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { type: ["String"], select: { squiggleSnippet: true }, async resolve(revision) { - console.log("RESOLVING?", revision); if (revision.contentType === "SquiggleSnippet") { const ast = parse(revision.squiggleSnippet!.code); if (ast.ok) { - console.log("RESOLVING!", getExportedVariables(ast.value)); - return getExportedVariables(ast.value); + return getExportedVariableNames(ast.value); } else { return []; } diff --git a/packages/hub/src/scripts/buildRecentModelRevision.ts b/packages/hub/src/scripts/buildRecentModelRevision.ts index 7fb72d52a0..2c4ec98207 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision.ts @@ -16,7 +16,6 @@ type SquiggleOutput = } | { isOk: true; - errorString?: undefined; }; const prisma = new PrismaClient(); @@ -59,7 +58,7 @@ export async function runSquiggle( currentRevisionId: string, code: string, seed: string -): Promise { +): Promise { const env = { sampleCount: SAMPLE_COUNT_DEFAULT, xyPointLength: XY_POINT_LENGTH_DEFAULT, @@ -110,14 +109,7 @@ export async function runSquiggle( } } - return outputR.ok - ? { - isOk: true, - } - : { - isOk: false, - errorString: outputR.value.toString(), - }; + return outputR.ok ? undefined : outputR.value.toString(); } async function oldestModelRevisionWithoutBuilds() { @@ -172,7 +164,7 @@ async function buildRecentModelVersion() { data: { modelRevision: { connect: { id: model.currentRevisionId } }, runtime: diff, - errors: response.isOk ? [] : [response.errorString], + errors: response === undefined ? [] : [response], }, }); From 17893d46078b5b6bac39399f04557f81f73c31a5 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Sun, 7 Apr 2024 22:14:03 -0700 Subject: [PATCH 12/26] More small improvements --- .../hub/src/scripts/buildRecentModelRevision.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/hub/src/scripts/buildRecentModelRevision.ts b/packages/hub/src/scripts/buildRecentModelRevision.ts index 2c4ec98207..4711317c5f 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision.ts @@ -9,15 +9,6 @@ import { parseSourceId } from "../squiggle/components/linker"; const SOURCE_NAME = "main"; -type SquiggleOutput = - | { - isOk: false; - errorString: string; - } - | { - isOk: true; - }; - const prisma = new PrismaClient(); export const squiggleLinker: SqLinker = { @@ -121,6 +112,7 @@ async function oldestModelRevisionWithoutBuilds() { builds: { none: {}, }, + contentType: "SquiggleSnippet", }, orderBy: { createdAt: "asc", @@ -150,7 +142,9 @@ async function buildRecentModelVersion() { } if (!model?.currentRevisionId || !model.currentRevision?.squiggleSnippet) { - throw new NotFoundError(`Model revision didn't have needed information`); + throw new NotFoundError( + `Unexpected Error: Model revision didn't have needed information. This should never happen.` + ); } const { code, seed } = model.currentRevision.squiggleSnippet; From 01f584963ad8763f40f998270d7375638de8fd27 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Tue, 9 Apr 2024 18:42:32 -0700 Subject: [PATCH 13/26] Addressing CR --- packages/hub/schema.graphql | 11 ----------- .../[slug]/EditSquiggleSnippetModel.tsx | 1 - packages/hub/src/graphql/schema.ts | 1 - .../hub/src/graphql/types/ModelRevision.ts | 3 ++- .../src/graphql/types/ModelRevisionBuild.ts | 13 ------------- .../hub/src/models/components/ModelCard.tsx | 2 +- .../src/scripts/buildRecentModelRevision.ts | 18 +++++++++++------- 7 files changed, 14 insertions(+), 35 deletions(-) diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql index b8d08a2fdc..b5df1068ce 100644 --- a/packages/hub/schema.graphql +++ b/packages/hub/schema.graphql @@ -216,7 +216,6 @@ type ModelExportRevisionsConnectionEdge { type ModelRevision implements Node { author: User buildStatus: ModelRevisionBuildStatus! - builds: [ModelRevisionBuild!]! comment: String! content: ModelContent! createdAtTimestamp: Float! @@ -237,16 +236,6 @@ type ModelRevisionBuild implements Node { runSeconds: Float! } -type ModelRevisionBuildConnection { - edges: [ModelRevisionBuildEdge!]! - pageInfo: PageInfo! -} - -type ModelRevisionBuildEdge { - cursor: String! - node: ModelRevisionBuild! -} - enum ModelRevisionBuildStatus { Failure Pending diff --git a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx index dc6109946f..042777a2a1 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx @@ -156,7 +156,6 @@ export const EditSquiggleSnippetModel: FC = ({ xyPointLength } } - exportNames exports { id variableName diff --git a/packages/hub/src/graphql/schema.ts b/packages/hub/src/graphql/schema.ts index 693099e303..2ddfe7f5cb 100644 --- a/packages/hub/src/graphql/schema.ts +++ b/packages/hub/src/graphql/schema.ts @@ -1,7 +1,6 @@ import "./errors/BaseError"; import "./errors/NotFoundError"; import "./errors/ValidationError"; -import "./types/ModelRevisionBuild"; import "./queries/globalStatistics"; import "./queries/group"; import "./queries/groups"; diff --git a/packages/hub/src/graphql/types/ModelRevision.ts b/packages/hub/src/graphql/types/ModelRevision.ts index 40406f75d5..ef47ff6aee 100644 --- a/packages/hub/src/graphql/types/ModelRevision.ts +++ b/packages/hub/src/graphql/types/ModelRevision.ts @@ -36,6 +36,8 @@ function getExportedVariableNames(ast: ASTNode): string[] { if (ast.type === "Program") { ast.statements.forEach((statement) => { + while (statement.type === "DecoratedStatement") + statement = statement.statement; if (statement.type === "LetStatement" && statement.exported) { exportedVariableNames.push(statement.variable.value); } else if (statement.type === "DefunStatement" && statement.exported) { @@ -58,7 +60,6 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { relativeValuesExports: t.relation("relativeValuesExports"), exports: t.relation("exports"), model: t.relation("model"), - builds: t.relation("builds"), lastBuild: t.field({ type: ModelRevisionBuild, diff --git a/packages/hub/src/graphql/types/ModelRevisionBuild.ts b/packages/hub/src/graphql/types/ModelRevisionBuild.ts index c592762313..cccc06b2bd 100644 --- a/packages/hub/src/graphql/types/ModelRevisionBuild.ts +++ b/packages/hub/src/graphql/types/ModelRevisionBuild.ts @@ -1,5 +1,3 @@ -import { prismaConnectionHelpers } from "@pothos/plugin-prisma"; - import { builder } from "@/graphql/builder"; export const ModelRevisionBuild = builder.prismaNode("ModelRevisionBuild", { @@ -13,14 +11,3 @@ export const ModelRevisionBuild = builder.prismaNode("ModelRevisionBuild", { runSeconds: t.exposeFloat("runSeconds"), }), }); - -export const ModelRevisionBuildConnection = builder.connectionObject({ - type: ModelRevisionBuild, - name: "ModelRevisionBuildConnection", -}); - -export const modelConnectionHelpers = prismaConnectionHelpers( - builder, - "ModelRevisionBuild", - { cursor: "id" } -); diff --git a/packages/hub/src/models/components/ModelCard.tsx b/packages/hub/src/models/components/ModelCard.tsx index 3a6aaf8412..450b5e34e0 100644 --- a/packages/hub/src/models/components/ModelCard.tsx +++ b/packages/hub/src/models/components/ModelCard.tsx @@ -46,7 +46,7 @@ export const ModelCard: FC = ({ modelRef, showOwner = true }) => { slug: model.slug, }); - const modelExports = model.currentRevision.exports?.map( + const modelExports = model.currentRevision.exports.map( ({ variableName, variableType, title }) => ({ variableName, variableType, diff --git a/packages/hub/src/scripts/buildRecentModelRevision.ts b/packages/hub/src/scripts/buildRecentModelRevision.ts index 4711317c5f..826193642a 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision.ts @@ -157,7 +157,7 @@ async function buildRecentModelVersion() { const build = await prisma.modelRevisionBuild.create({ data: { modelRevision: { connect: { id: model.currentRevisionId } }, - runtime: diff, + runSeconds: diff, errors: response === undefined ? [] : [response], }, }); @@ -185,12 +185,16 @@ async function countItemsRemaining() { console.log("Model Revisions Remaining:", remaining); } -buildRecentModelVersion() - .catch((error) => { +async function main() { + try { + buildRecentModelVersion(); + countItemsRemaining(); + } catch (error) { console.error(error); process.exit(1); - }) - .finally(async () => { - countItemsRemaining(); + } finally { await prisma.$disconnect(); - }); + } +} + +main(); From 14bfb07cc4a63214777912d43f0e4344582796e8 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Wed, 10 Apr 2024 07:42:04 -0700 Subject: [PATCH 14/26] Fixed reversion --- .../graphql/mutations/updateSquiggleSnippetModel.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/packages/hub/src/graphql/mutations/updateSquiggleSnippetModel.ts b/packages/hub/src/graphql/mutations/updateSquiggleSnippetModel.ts index 0424b68e52..424a9df770 100644 --- a/packages/hub/src/graphql/mutations/updateSquiggleSnippetModel.ts +++ b/packages/hub/src/graphql/mutations/updateSquiggleSnippetModel.ts @@ -166,19 +166,6 @@ builder.mutationField("updateSquiggleSnippetModel", (t) => data: relativeValuesExportsToInsert, }, }, - exports: { - createMany: { - data: (input.exports ?? []).map( - ({ variableName, variableType, docstring, title }) => ({ - variableName, - variableType, - docstring: docstring ?? undefined, - title: title ?? null, - isCurrent: true, - }) - ), - }, - }, }, include: { model: { From 2e8ca30b793956577cf1c269a36f433adc98246b Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Wed, 10 Apr 2024 08:07:41 -0700 Subject: [PATCH 15/26] Final fixes --- .../hub/src/graphql/types/ModelRevision.ts | 62 ++++++++----------- .../src/graphql/types/ModelRevisionBuild.ts | 2 +- 2 files changed, 26 insertions(+), 38 deletions(-) diff --git a/packages/hub/src/graphql/types/ModelRevision.ts b/packages/hub/src/graphql/types/ModelRevision.ts index ef47ff6aee..aacbcb4123 100644 --- a/packages/hub/src/graphql/types/ModelRevision.ts +++ b/packages/hub/src/graphql/types/ModelRevision.ts @@ -64,58 +64,46 @@ export const ModelRevision = builder.prismaNode("ModelRevision", { lastBuild: t.field({ type: ModelRevisionBuild, nullable: true, + select: { + builds: { + orderBy: { + createdAt: "desc", + }, + take: 1, + }, + }, async resolve(revision) { - return prisma.modelRevision - .findUnique({ - where: { - id: revision.id, - }, - select: { - builds: { - orderBy: { - createdAt: "desc", - }, - take: 1, - }, - }, - }) - .then((result) => result?.builds[0] || null); + return revision.builds[0]; }, }), author: t.relation("author", { nullable: true }), comment: t.exposeString("comment"), buildStatus: t.field({ type: ModelRevisionBuildStatus, - async resolve(revision) { - const modelRevision = await prisma.modelRevision.findUnique({ - where: { - id: revision.id, + select: { + builds: { + select: { + errors: true, + }, + orderBy: { + createdAt: "desc", }, + take: 1, + }, + model: { select: { - builds: { - orderBy: { - createdAt: "desc", - }, - take: 1, - }, - model: { - select: { - currentRevisionId: true, - }, - }, + currentRevisionId: true, }, - }); - - if (!modelRevision) { - throw new Error(`ModelRevision not found for id: ${revision.id}`); - } + }, + }, + async resolve(revision) { + const lastBuild = revision.builds[0]; - const lastBuild = modelRevision.builds[0]; if (lastBuild) { return lastBuild.errors.length === 0 ? "Success" : "Failure"; } - return modelRevision.model.currentRevisionId === revision.id + return revision.model.currentRevisionId === revision.id ? "Pending" : "Skipped"; }, diff --git a/packages/hub/src/graphql/types/ModelRevisionBuild.ts b/packages/hub/src/graphql/types/ModelRevisionBuild.ts index cccc06b2bd..aa77c3138a 100644 --- a/packages/hub/src/graphql/types/ModelRevisionBuild.ts +++ b/packages/hub/src/graphql/types/ModelRevisionBuild.ts @@ -4,7 +4,7 @@ export const ModelRevisionBuild = builder.prismaNode("ModelRevisionBuild", { id: { field: "id" }, fields: (t) => ({ createdAtTimestamp: t.float({ - resolve: (group) => group.createdAt.getTime(), + resolve: (build) => build.createdAt.getTime(), }), modelRevision: t.relation("modelRevision"), errors: t.exposeStringList("errors"), From 97b0f0172ea66cd33547106629157dd0f8949d29 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Wed, 10 Apr 2024 11:31:46 -0700 Subject: [PATCH 16/26] Starting pass of running build in worker --- packages/hub/package.json | 4 +- .../[slug]/EditSquiggleSnippetModel.tsx | 1 + .../src/scripts/buildRecentModelRevision.ts | 11 +++- packages/hub/src/scripts/main.ts | 57 +++++++++++++++++++ pnpm-lock.yaml | 41 ++++++------- 5 files changed, 92 insertions(+), 22 deletions(-) create mode 100644 packages/hub/src/scripts/main.ts diff --git a/packages/hub/package.json b/packages/hub/package.json index 03d775a8d3..d2cee926b1 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -21,7 +21,8 @@ "lint": "prettier --check . && next lint", "format": "prettier --write .", "test:manual": "dotenv -e .env.test -- jest -i", - "build-last-revision": "tsx src/scripts/buildRecentModelRevision.ts" + "build-last-revision": "tsx src/scripts/buildRecentModelRevision.ts", + "build-main": "tsx src/scripts/main.ts" }, "dependencies": { "@next-auth/prisma-adapter": "^1.0.7", @@ -75,6 +76,7 @@ "@types/invariant": "^2.2.37", "@types/jest": "^29.5.12", "@types/lodash": "^4.14.202", + "@types/node": "^20.11.24", "@types/pako": "^2.0.3", "@types/react": "^18.2.52", "@types/react-relay": "^16.0.6", diff --git a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx index 042777a2a1..dc6109946f 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx @@ -156,6 +156,7 @@ export const EditSquiggleSnippetModel: FC = ({ xyPointLength } } + exportNames exports { id variableName diff --git a/packages/hub/src/scripts/buildRecentModelRevision.ts b/packages/hub/src/scripts/buildRecentModelRevision.ts index 826193642a..3d7d3f6e3e 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision.ts @@ -150,14 +150,16 @@ async function buildRecentModelVersion() { const { code, seed } = model.currentRevision.squiggleSnippet; const startTime = performance.now(); + let response = await runSquiggle(model.currentRevisionId, code, seed); + const endTime = performance.now(); const diff = endTime - startTime; const build = await prisma.modelRevisionBuild.create({ data: { modelRevision: { connect: { id: model.currentRevisionId } }, - runSeconds: diff, + runSeconds: diff / 1000, errors: response === undefined ? [] : [response], }, }); @@ -197,4 +199,9 @@ async function main() { } } -main(); +try { + main(); +} catch (error) { + console.error("An unhandled error occurred:", error); + process.exit(1); +} diff --git a/packages/hub/src/scripts/main.ts b/packages/hub/src/scripts/main.ts new file mode 100644 index 0000000000..653446850b --- /dev/null +++ b/packages/hub/src/scripts/main.ts @@ -0,0 +1,57 @@ +// buildRecentModelRevision.ts +import { spawn } from "child_process"; + +async function runWorker() { + return new Promise((resolve, reject) => { + const worker = spawn("tsx", ["src/scripts/buildRecentModelRevision.ts"]); + + let workerOutput = ""; + + const timeoutId = setTimeout(() => { + worker.kill(); + reject(new Error("Worker process timed out")); + }, 6000); // 10 seconds + + worker.stdout.on("data", (data) => { + workerOutput += console.log(`Worker output: ${data}`); + }); + + worker.stderr.on("data", (data) => { + console.error(`Worker error: ${data}`); + }); + + worker.on("exit", (code) => { + clearTimeout(timeoutId); + if (code === 0) { + console.log("Worker output:", workerOutput); + console.log("Worker completed successfully"); + resolve(2); + } else { + console.error(`Worker process exited with code ${code}`); + reject(new Error(`Worker process exited with code ${code}`)); + } + }); + }); +} + +async function main() { + try { + await runWorker(); + console.log("Main process completed successfully"); + } catch (error) { + console.error("Worker process encountered an error:", error); + } finally { + console.log("Main process completed"); + + console.log("Memory usage:", process.memoryUsage()); + + // Other process-related information + console.log("Process ID:", process.pid); + console.log("Process uptime:", process.uptime()); + console.log("Process platform:", process.platform); + console.log("Process architecture:", process.arch); + console.log("Node.js version:", process.version); + } +} + +main(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aaf2d9a890..e995d315d4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -330,6 +330,9 @@ importers: '@quri/versioned-squiggle-components': specifier: workspace:* version: link:../versioned-components + '@types/node': + specifier: ^20.11.24 + version: 20.11.24 '@vercel/analytics': specifier: ^1.2.2 version: 1.2.2(next@14.1.0)(react@18.2.0) @@ -411,7 +414,7 @@ importers: devDependencies: '@graphql-codegen/cli': specifier: ^5.0.2 - version: 5.0.2(@parcel/watcher@2.4.1)(graphql@16.8.1)(typescript@5.3.3) + version: 5.0.2(@parcel/watcher@2.4.1)(@types/node@20.11.24)(graphql@16.8.1)(typescript@5.3.3) '@graphql-codegen/client-preset': specifier: ^4.1.0 version: 4.2.4(graphql@16.8.1) @@ -3920,7 +3923,7 @@ packages: tslib: 2.6.2 dev: true - /@graphql-codegen/cli@5.0.2(@parcel/watcher@2.4.1)(graphql@16.8.1)(typescript@5.3.3): + /@graphql-codegen/cli@5.0.2(@parcel/watcher@2.4.1)(@types/node@20.11.24)(graphql@16.8.1)(typescript@5.3.3): resolution: {integrity: sha512-MBIaFqDiLKuO4ojN6xxG9/xL9wmfD3ZjZ7RsPjwQnSHBCUXnEkdKvX+JVpx87Pq29Ycn8wTJUguXnTZ7Di0Mlw==} hasBin: true peerDependencies: @@ -3939,12 +3942,12 @@ packages: '@graphql-tools/apollo-engine-loader': 8.0.0(graphql@16.8.1) '@graphql-tools/code-file-loader': 8.0.2(graphql@16.8.1) '@graphql-tools/git-loader': 8.0.2(graphql@16.8.1) - '@graphql-tools/github-loader': 8.0.0(graphql@16.8.1) + '@graphql-tools/github-loader': 8.0.0(@types/node@20.11.24)(graphql@16.8.1) '@graphql-tools/graphql-file-loader': 8.0.0(graphql@16.8.1) '@graphql-tools/json-file-loader': 8.0.0(graphql@16.8.1) '@graphql-tools/load': 8.0.0(graphql@16.8.1) - '@graphql-tools/prisma-loader': 8.0.1(graphql@16.8.1) - '@graphql-tools/url-loader': 8.0.0(graphql@16.8.1) + '@graphql-tools/prisma-loader': 8.0.1(@types/node@20.11.24)(graphql@16.8.1) + '@graphql-tools/url-loader': 8.0.0(@types/node@20.11.24)(graphql@16.8.1) '@graphql-tools/utils': 10.0.6(graphql@16.8.1) '@parcel/watcher': 2.4.1 '@whatwg-node/fetch': 0.8.8 @@ -3953,7 +3956,7 @@ packages: debounce: 1.2.1 detect-indent: 6.1.0 graphql: 16.8.1 - graphql-config: 5.0.2(graphql@16.8.1)(typescript@5.3.3) + graphql-config: 5.0.2(@types/node@20.11.24)(graphql@16.8.1)(typescript@5.3.3) inquirer: 8.2.6 is-glob: 4.0.3 jiti: 1.20.0 @@ -4212,7 +4215,7 @@ packages: - utf-8-validate dev: true - /@graphql-tools/executor-http@1.0.2(graphql@16.8.1): + /@graphql-tools/executor-http@1.0.2(@types/node@20.11.24)(graphql@16.8.1): resolution: {integrity: sha512-JKTB4E3kdQM2/1NEcyrVPyQ8057ZVthCV5dFJiKktqY9IdmF00M8gupFcW3jlbM/Udn78ickeUBsUzA3EouqpA==} engines: {node: '>=16.0.0'} peerDependencies: @@ -4223,7 +4226,7 @@ packages: '@whatwg-node/fetch': 0.9.7 extract-files: 11.0.0 graphql: 16.8.1 - meros: 1.3.0 + meros: 1.3.0(@types/node@20.11.24) tslib: 2.6.2 value-or-promise: 1.0.12 transitivePeerDependencies: @@ -4277,14 +4280,14 @@ packages: - supports-color dev: true - /@graphql-tools/github-loader@8.0.0(graphql@16.8.1): + /@graphql-tools/github-loader@8.0.0(@types/node@20.11.24)(graphql@16.8.1): resolution: {integrity: sha512-VuroArWKcG4yaOWzV0r19ElVIV6iH6UKDQn1MXemND0xu5TzrFme0kf3U9o0YwNo0kUYEk9CyFM0BYg4he17FA==} engines: {node: '>=16.0.0'} peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: '@ardatan/sync-fetch': 0.0.1 - '@graphql-tools/executor-http': 1.0.2(graphql@16.8.1) + '@graphql-tools/executor-http': 1.0.2(@types/node@20.11.24)(graphql@16.8.1) '@graphql-tools/graphql-tag-pluck': 8.0.2(graphql@16.8.1) '@graphql-tools/utils': 10.0.6(graphql@16.8.1) '@whatwg-node/fetch': 0.9.7 @@ -4387,13 +4390,13 @@ packages: tslib: 2.6.2 dev: true - /@graphql-tools/prisma-loader@8.0.1(graphql@16.8.1): + /@graphql-tools/prisma-loader@8.0.1(@types/node@20.11.24)(graphql@16.8.1): resolution: {integrity: sha512-bl6e5sAYe35Z6fEbgKXNrqRhXlCJYeWKBkarohgYA338/SD9eEhXtg3Cedj7fut3WyRLoQFpHzfiwxKs7XrgXg==} engines: {node: '>=16.0.0'} peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - '@graphql-tools/url-loader': 8.0.0(graphql@16.8.1) + '@graphql-tools/url-loader': 8.0.0(@types/node@20.11.24)(graphql@16.8.1) '@graphql-tools/utils': 10.0.6(graphql@16.8.1) '@types/js-yaml': 4.0.5 '@types/json-stable-stringify': 1.0.34 @@ -4447,7 +4450,7 @@ packages: tslib: 2.6.2 value-or-promise: 1.0.12 - /@graphql-tools/url-loader@8.0.0(graphql@16.8.1): + /@graphql-tools/url-loader@8.0.0(@types/node@20.11.24)(graphql@16.8.1): resolution: {integrity: sha512-rPc9oDzMnycvz+X+wrN3PLrhMBQkG4+sd8EzaFN6dypcssiefgWKToXtRKI8HHK68n2xEq1PyrOpkjHFJB+GwA==} engines: {node: '>=16.0.0'} peerDependencies: @@ -4456,7 +4459,7 @@ packages: '@ardatan/sync-fetch': 0.0.1 '@graphql-tools/delegate': 10.0.3(graphql@16.8.1) '@graphql-tools/executor-graphql-ws': 1.1.0(graphql@16.8.1) - '@graphql-tools/executor-http': 1.0.2(graphql@16.8.1) + '@graphql-tools/executor-http': 1.0.2(@types/node@20.11.24)(graphql@16.8.1) '@graphql-tools/executor-legacy-ws': 1.0.3(graphql@16.8.1) '@graphql-tools/utils': 10.0.6(graphql@16.8.1) '@graphql-tools/wrap': 10.0.1(graphql@16.8.1) @@ -8145,7 +8148,6 @@ packages: resolution: {integrity: sha512-Kza43ewS3xoLgCEpQrsT+xRo/EJej1y0kVYGiLFE1NEODXGzTfwiC6tXTLMQskn1X4/Rjlh0MQUvx9W+L9long==} dependencies: undici-types: 5.26.5 - dev: true /@types/normalize-package-data@2.4.1: resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} @@ -12593,7 +12595,7 @@ packages: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} dev: true - /graphql-config@5.0.2(graphql@16.8.1)(typescript@5.3.3): + /graphql-config@5.0.2(@types/node@20.11.24)(graphql@16.8.1)(typescript@5.3.3): resolution: {integrity: sha512-7TPxOrlbiG0JplSZYCyxn2XQtqVhXomEjXUmWJVSS5ET1nPhOJSsIb/WTwqWhcYX6G0RlHXSj9PLtGTKmxLNGg==} engines: {node: '>= 16.0.0'} peerDependencies: @@ -12607,7 +12609,7 @@ packages: '@graphql-tools/json-file-loader': 8.0.0(graphql@16.8.1) '@graphql-tools/load': 8.0.0(graphql@16.8.1) '@graphql-tools/merge': 9.0.0(graphql@16.8.1) - '@graphql-tools/url-loader': 8.0.0(graphql@16.8.1) + '@graphql-tools/url-loader': 8.0.0(@types/node@20.11.24)(graphql@16.8.1) '@graphql-tools/utils': 10.0.6(graphql@16.8.1) cosmiconfig: 8.3.6(typescript@5.3.3) graphql: 16.8.1 @@ -15350,7 +15352,7 @@ packages: - supports-color dev: false - /meros@1.3.0: + /meros@1.3.0(@types/node@20.11.24): resolution: {integrity: sha512-2BNGOimxEz5hmjUG2FwoxCt5HN7BXdaWyFqEwxPTrJzVdABtrL4TiHTcsWSFAxPQ/tOnEaQEJh3qWq71QRMY+w==} engines: {node: '>=13'} peerDependencies: @@ -15358,6 +15360,8 @@ packages: peerDependenciesMeta: '@types/node': optional: true + dependencies: + '@types/node': 20.11.24 dev: true /methods@1.1.2: @@ -19865,7 +19869,6 @@ packages: /undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - dev: true /unicode-canonical-property-names-ecmascript@2.0.0: resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==} From 47ca206527300157014050908c2497ae1e3323c9 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Wed, 10 Apr 2024 14:29:05 -0700 Subject: [PATCH 17/26] Refactored buildRecendModelRevision to use a child process --- packages/hub/package.json | 3 +- .../src/scripts/buildRecentModelRevision.ts | 207 ------------------ .../scripts/buildRecentModelRevision/main.ts | 169 ++++++++++++++ .../buildRecentModelRevision/worker.ts | 129 +++++++++++ packages/hub/src/scripts/main.ts | 57 ----- 5 files changed, 299 insertions(+), 266 deletions(-) delete mode 100644 packages/hub/src/scripts/buildRecentModelRevision.ts create mode 100644 packages/hub/src/scripts/buildRecentModelRevision/main.ts create mode 100644 packages/hub/src/scripts/buildRecentModelRevision/worker.ts delete mode 100644 packages/hub/src/scripts/main.ts diff --git a/packages/hub/package.json b/packages/hub/package.json index d2cee926b1..cb4f9b6268 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -21,8 +21,7 @@ "lint": "prettier --check . && next lint", "format": "prettier --write .", "test:manual": "dotenv -e .env.test -- jest -i", - "build-last-revision": "tsx src/scripts/buildRecentModelRevision.ts", - "build-main": "tsx src/scripts/main.ts" + "build-last-revision": "tsx src/scripts/buildRecentModelRevision/main.ts" }, "dependencies": { "@next-auth/prisma-adapter": "^1.0.7", diff --git a/packages/hub/src/scripts/buildRecentModelRevision.ts b/packages/hub/src/scripts/buildRecentModelRevision.ts deleted file mode 100644 index 3d7d3f6e3e..0000000000 --- a/packages/hub/src/scripts/buildRecentModelRevision.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { PrismaClient } from "@prisma/client"; - -import { SqLinker, SqProject } from "@quri/squiggle-lang"; - -import { ModelExport } from "../../../components/dist/src/components/SquigglePlayground"; -import { SAMPLE_COUNT_DEFAULT, XY_POINT_LENGTH_DEFAULT } from "../constants"; -import { NotFoundError } from "../graphql/errors/NotFoundError"; -import { parseSourceId } from "../squiggle/components/linker"; - -const SOURCE_NAME = "main"; - -const prisma = new PrismaClient(); - -export const squiggleLinker: SqLinker = { - resolve(name) { - return name; - }, - async loadSource(sourceId: string) { - const { owner, slug } = parseSourceId(sourceId); - - const model = await prisma.model.findFirst({ - where: { - slug, - owner: { slug: owner }, - }, - include: { - currentRevision: { - include: { - squiggleSnippet: true, - }, - }, - }, - }); - - if (!model) { - throw new NotFoundError(); - } - - const content = model?.currentRevision?.squiggleSnippet; - if (content) { - return content.code; - } else { - throw new NotFoundError(); - } - }, -}; - -export async function runSquiggle( - currentRevisionId: string, - code: string, - seed: string -): Promise { - const env = { - sampleCount: SAMPLE_COUNT_DEFAULT, - xyPointLength: XY_POINT_LENGTH_DEFAULT, - seed, - }; - - const project = SqProject.create({ - linker: squiggleLinker, - environment: env, - }); - - project.setSource(SOURCE_NAME, code); - await project.run(SOURCE_NAME); - - const outputR = project.getOutput(SOURCE_NAME); - - if (outputR.ok) { - const _exports: ModelExport[] = outputR.value.exports - .entries() - .map((e) => ({ - variableName: e[0], - variableType: e[1].tag, - title: e[1].tags.name() ? e[1].tags.name() : e[1].title() || "", - docstring: e[1].tags.doc() || "", - })); - - for (const e of _exports) { - await prisma.modelExport.upsert({ - where: { - uniqueKey: { - modelRevisionId: currentRevisionId, - variableName: e.variableName, - }, - }, - update: { - variableType: e.variableType, - title: e.title, - docstring: e.docstring, - }, - create: { - modelRevision: { connect: { id: currentRevisionId } }, - variableName: e.variableName, - variableType: e.variableType, - title: e.title, - docstring: e.docstring, - }, - }); - } - } - - return outputR.ok ? undefined : outputR.value.toString(); -} - -async function oldestModelRevisionWithoutBuilds() { - const modelRevision = await prisma.modelRevision.findFirst({ - where: { - currentRevisionModel: { - isNot: null, - }, - builds: { - none: {}, - }, - contentType: "SquiggleSnippet", - }, - orderBy: { - createdAt: "asc", - }, - include: { - model: { - include: { - currentRevision: { - include: { - squiggleSnippet: true, - }, - }, - }, - }, - }, - }); - return modelRevision?.model; -} - -async function buildRecentModelVersion() { - try { - const model = await oldestModelRevisionWithoutBuilds(); - - if (!model) { - console.log("No remaining unbuilt model revisions"); - return; - } - - if (!model?.currentRevisionId || !model.currentRevision?.squiggleSnippet) { - throw new NotFoundError( - `Unexpected Error: Model revision didn't have needed information. This should never happen.` - ); - } - - const { code, seed } = model.currentRevision.squiggleSnippet; - - const startTime = performance.now(); - - let response = await runSquiggle(model.currentRevisionId, code, seed); - - const endTime = performance.now(); - const diff = endTime - startTime; - - const build = await prisma.modelRevisionBuild.create({ - data: { - modelRevision: { connect: { id: model.currentRevisionId } }, - runSeconds: diff / 1000, - errors: response === undefined ? [] : [response], - }, - }); - - console.log("Build complete:", build); - console.log("Runtime:", diff); - console.log("Response:", response); - } catch (error) { - console.error(error); - } -} - -async function countItemsRemaining() { - const remaining = await prisma.modelRevision.count({ - where: { - currentRevisionModel: { - isNot: null, - }, - builds: { - none: {}, - }, - }, - }); - - console.log("Model Revisions Remaining:", remaining); -} - -async function main() { - try { - buildRecentModelVersion(); - countItemsRemaining(); - } catch (error) { - console.error(error); - process.exit(1); - } finally { - await prisma.$disconnect(); - } -} - -try { - main(); -} catch (error) { - console.error("An unhandled error occurred:", error); - process.exit(1); -} diff --git a/packages/hub/src/scripts/buildRecentModelRevision/main.ts b/packages/hub/src/scripts/buildRecentModelRevision/main.ts new file mode 100644 index 0000000000..5d30311c01 --- /dev/null +++ b/packages/hub/src/scripts/buildRecentModelRevision/main.ts @@ -0,0 +1,169 @@ +import { PrismaClient } from "@prisma/client"; +import { spawn } from "child_process"; + +import { NotFoundError } from "../../graphql/errors/NotFoundError"; + +const TIMEOUT_SECONDS = 60; // 60 seconds + +const prisma = new PrismaClient(); + +type SquiggleResult = { + errors?: string; +}; + +async function runWorker( + revisionId: string, + code: string, + seed: string, + timeoutSeconds: number +): Promise { + return new Promise((resolve, reject) => { + console.log("Spawning worker process for Revision ID: " + revisionId); + const worker = spawn( + "tsx", + ["src/scripts/buildRecentModelRevision/worker.ts"], + { + stdio: ["pipe", "pipe", "pipe", "ipc"], + } + ); + + const timeoutId = setTimeout(() => { + worker.kill(); + resolve({ errors: `Timeout Error, at ${timeoutSeconds}s ` }); + }, timeoutSeconds * 1000); + + worker.stdout?.on("data", (data) => { + console.log(`Worker output: ${data}`); + }); + + worker.stderr?.on("data", (data) => { + console.error(`Worker error: ${data}`); + }); + + worker.on( + "message", + async (message: { type: string; data: { errors?: string } }) => { + resolve(message.data); + } + ); + + worker.on("exit", (code) => { + clearTimeout(timeoutId); + if (code === 0) { + console.log("Worker completed successfully"); + } else { + console.error(`Worker process exited with code ${code}`); + resolve({ errors: "Computation error, with code: " + code }); + } + }); + + worker.send({ type: "run", data: { revisionId, code, seed } }); + worker.on("close", (code) => { + console.log(`Worker process CLOSED with code ${code}`); + }); + }); +} + +async function oldestModelRevisionWithoutBuilds() { + const modelRevision = await prisma.modelRevision.findFirst({ + where: { + currentRevisionModel: { + isNot: null, + }, + builds: { + none: {}, + }, + contentType: "SquiggleSnippet", + }, + orderBy: { + createdAt: "asc", + }, + include: { + model: { + include: { + currentRevision: { + include: { + squiggleSnippet: true, + }, + }, + }, + }, + }, + }); + return modelRevision?.model; +} + +async function buildRecentModelVersion() { + try { + const model = await oldestModelRevisionWithoutBuilds(); + + if (!model) { + console.log("No remaining unbuilt model revisions"); + return; + } + + if (!model?.currentRevisionId || !model.currentRevision?.squiggleSnippet) { + throw new NotFoundError( + `Unexpected Error: Model revision didn't have needed information. This should never happen.` + ); + } + + const { code, seed } = model.currentRevision.squiggleSnippet; + + const startTime = performance.now(); + let response = await runWorker( + model.currentRevisionId, + code, + seed, + TIMEOUT_SECONDS + ); + const endTime = performance.now(); + console.log("RESPONSE", response); + + const build = await prisma.modelRevisionBuild.create({ + data: { + modelRevision: { connect: { id: model.currentRevisionId } }, + runSeconds: (endTime - startTime) / 1000, + errors: response.errors === undefined ? [] : [response.errors], + }, + }); + + console.log("Build created:", build); + } catch (error) { + console.error(error); + } +} + +async function countItemsRemaining() { + const remaining = await prisma.modelRevision.count({ + where: { + currentRevisionModel: { + isNot: null, + }, + builds: { + none: {}, + }, + }, + }); + + console.log("Model Revisions Remaining:", remaining); +} + +async function main() { + try { + buildRecentModelVersion(); + countItemsRemaining(); + } catch (error) { + console.error(error); + process.exit(1); + } finally { + await prisma.$disconnect(); + } +} + +try { + main(); +} catch (error) { + console.error("An unhandled error occurred:", error); + process.exit(1); +} diff --git a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts new file mode 100644 index 0000000000..c8b997653e --- /dev/null +++ b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts @@ -0,0 +1,129 @@ +import { PrismaClient } from "@prisma/client"; + +import { SqLinker, SqProject } from "@quri/squiggle-lang"; + +import { ModelExport } from "../../../../components/dist/src/components/SquigglePlayground"; +import { SAMPLE_COUNT_DEFAULT, XY_POINT_LENGTH_DEFAULT } from "../../constants"; +import { NotFoundError } from "../../graphql/errors/NotFoundError"; +import { parseSourceId } from "../../squiggle/components/linker"; + +const SOURCE_NAME = "main"; + +const prisma = new PrismaClient(); + +export const squiggleLinker: SqLinker = { + resolve(name) { + return name; + }, + async loadSource(sourceId: string) { + const { owner, slug } = parseSourceId(sourceId); + + const model = await prisma.model.findFirst({ + where: { + slug, + owner: { slug: owner }, + }, + include: { + currentRevision: { + include: { + squiggleSnippet: true, + }, + }, + }, + }); + + if (!model) { + throw new NotFoundError(); + } + + const content = model?.currentRevision?.squiggleSnippet; + if (content) { + return content.code; + } else { + throw new NotFoundError(); + } + }, +}; + +export async function runSquiggle( + currentRevisionId: string, + code: string, + seed: string +): Promise { + const env = { + sampleCount: SAMPLE_COUNT_DEFAULT, + xyPointLength: XY_POINT_LENGTH_DEFAULT, + seed, + }; + + const project = SqProject.create({ + linker: squiggleLinker, + environment: env, + }); + + project.setSource(SOURCE_NAME, code); + await project.run(SOURCE_NAME); + + const outputR = project.getOutput(SOURCE_NAME); + + if (outputR.ok) { + const _exports: ModelExport[] = outputR.value.exports + .entries() + .map((e) => ({ + variableName: e[0], + variableType: e[1].tag, + title: e[1].tags.name() ? e[1].tags.name() : e[1].title() || "", + docstring: e[1].tags.doc() || "", + })); + + for (const e of _exports) { + await prisma.modelExport.upsert({ + where: { + uniqueKey: { + modelRevisionId: currentRevisionId, + variableName: e.variableName, + }, + }, + update: { + variableType: e.variableType, + title: e.title, + docstring: e.docstring, + }, + create: { + modelRevision: { connect: { id: currentRevisionId } }, + variableName: e.variableName, + variableType: e.variableType, + title: e.title, + docstring: e.docstring, + }, + }); + } + } + + return outputR.ok ? undefined : outputR.value.toString(); +} + +type RunMessage = { + type: "run"; + data: { + revisionId: string; + code: string; + seed: string; + }; +}; + +process.on("message", async (message: RunMessage) => { + if (message.type === "run") { + const { revisionId, code, seed } = message.data; + + const errors = await runSquiggle(revisionId, code, seed); + + process?.send?.({ + type: "result", + data: { + errors, + }, + }); + process.exit(0); + } +}); diff --git a/packages/hub/src/scripts/main.ts b/packages/hub/src/scripts/main.ts deleted file mode 100644 index 653446850b..0000000000 --- a/packages/hub/src/scripts/main.ts +++ /dev/null @@ -1,57 +0,0 @@ -// buildRecentModelRevision.ts -import { spawn } from "child_process"; - -async function runWorker() { - return new Promise((resolve, reject) => { - const worker = spawn("tsx", ["src/scripts/buildRecentModelRevision.ts"]); - - let workerOutput = ""; - - const timeoutId = setTimeout(() => { - worker.kill(); - reject(new Error("Worker process timed out")); - }, 6000); // 10 seconds - - worker.stdout.on("data", (data) => { - workerOutput += console.log(`Worker output: ${data}`); - }); - - worker.stderr.on("data", (data) => { - console.error(`Worker error: ${data}`); - }); - - worker.on("exit", (code) => { - clearTimeout(timeoutId); - if (code === 0) { - console.log("Worker output:", workerOutput); - console.log("Worker completed successfully"); - resolve(2); - } else { - console.error(`Worker process exited with code ${code}`); - reject(new Error(`Worker process exited with code ${code}`)); - } - }); - }); -} - -async function main() { - try { - await runWorker(); - console.log("Main process completed successfully"); - } catch (error) { - console.error("Worker process encountered an error:", error); - } finally { - console.log("Main process completed"); - - console.log("Memory usage:", process.memoryUsage()); - - // Other process-related information - console.log("Process ID:", process.pid); - console.log("Process uptime:", process.uptime()); - console.log("Process platform:", process.platform); - console.log("Process architecture:", process.arch); - console.log("Node.js version:", process.version); - } -} - -main(); From 3e5d322d8d6a3e1f42d4fc6fa8da9d90f0ac192b Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Wed, 10 Apr 2024 14:45:55 -0700 Subject: [PATCH 18/26] Main() now runs continuously --- .../scripts/buildRecentModelRevision/main.ts | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/packages/hub/src/scripts/buildRecentModelRevision/main.ts b/packages/hub/src/scripts/buildRecentModelRevision/main.ts index 5d30311c01..99f30ab066 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision/main.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision/main.ts @@ -17,7 +17,7 @@ async function runWorker( seed: string, timeoutSeconds: number ): Promise { - return new Promise((resolve, reject) => { + return new Promise((resolve, _) => { console.log("Spawning worker process for Revision ID: " + revisionId); const worker = spawn( "tsx", @@ -93,7 +93,7 @@ async function oldestModelRevisionWithoutBuilds() { return modelRevision?.model; } -async function buildRecentModelVersion() { +async function buildRecentModelVersion(): Promise { try { const model = await oldestModelRevisionWithoutBuilds(); @@ -120,7 +120,7 @@ async function buildRecentModelVersion() { const endTime = performance.now(); console.log("RESPONSE", response); - const build = await prisma.modelRevisionBuild.create({ + await prisma.modelRevisionBuild.create({ data: { modelRevision: { connect: { id: model.currentRevisionId } }, runSeconds: (endTime - startTime) / 1000, @@ -128,9 +128,13 @@ async function buildRecentModelVersion() { }, }); - console.log("Build created:", build); + console.log( + "Build created for model revision ID:", + model.currentRevisionId + ); } catch (error) { - console.error(error); + console.error("Error building model revision:", error); + throw error; } } @@ -149,10 +153,10 @@ async function countItemsRemaining() { console.log("Model Revisions Remaining:", remaining); } -async function main() { +async function main(): Promise { try { - buildRecentModelVersion(); - countItemsRemaining(); + await buildRecentModelVersion(); + await countItemsRemaining(); } catch (error) { console.error(error); process.exit(1); @@ -161,9 +165,15 @@ async function main() { } } -try { - main(); -} catch (error) { - console.error("An unhandled error occurred:", error); - process.exit(1); +async function runContinuously() { + while (true) { + try { + await main(); + await new Promise((resolve) => setTimeout(resolve, 500)); // Sleep for 1 second + } catch (error) { + console.error("An error occurred during continuous execution:", error); + } + } } + +runContinuously(); From 5422b509039e2de153e2dfedb7a4de2f3cb33b2b Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Wed, 10 Apr 2024 16:21:36 -0700 Subject: [PATCH 19/26] Minor fixes --- .../src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx | 1 + .../app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx | 2 +- packages/hub/src/scripts/buildRecentModelRevision.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx index 042777a2a1..dc6109946f 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx @@ -156,6 +156,7 @@ export const EditSquiggleSnippetModel: FC = ({ xyPointLength } } + exportNames exports { id variableName diff --git a/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx b/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx index bc06e5127e..f84d26b03e 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx @@ -84,7 +84,7 @@ const ModelRevisionItem: FC<{
{`Build Status: ${revision.buildStatus}`}
{revision.lastBuild && ( -
{`Build Time: ${revision.lastBuild.runSeconds.toFixed(0)}ms`}
+
{`Build Time: ${revision.lastBuild.runSeconds.toFixed(2)}s`}
)} {revision.exports.length > 0 ? (
diff --git a/packages/hub/src/scripts/buildRecentModelRevision.ts b/packages/hub/src/scripts/buildRecentModelRevision.ts index 826193642a..1c728f6cd4 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision.ts @@ -157,7 +157,7 @@ async function buildRecentModelVersion() { const build = await prisma.modelRevisionBuild.create({ data: { modelRevision: { connect: { id: model.currentRevisionId } }, - runSeconds: diff, + runSeconds: diff / 1000, errors: response === undefined ? [] : [response], }, }); From c0796ed2245fc77e66218580beccbfc018ad27e8 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Wed, 10 Apr 2024 16:36:12 -0700 Subject: [PATCH 20/26] Updated pnpm-lock --- pnpm-lock.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e995d315d4..b4b3e03d92 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -330,9 +330,6 @@ importers: '@quri/versioned-squiggle-components': specifier: workspace:* version: link:../versioned-components - '@types/node': - specifier: ^20.11.24 - version: 20.11.24 '@vercel/analytics': specifier: ^1.2.2 version: 1.2.2(next@14.1.0)(react@18.2.0) @@ -439,6 +436,9 @@ importers: '@types/lodash': specifier: ^4.14.202 version: 4.14.202 + '@types/node': + specifier: ^20.11.24 + version: 20.11.24 '@types/pako': specifier: ^2.0.3 version: 2.0.3 @@ -8148,6 +8148,7 @@ packages: resolution: {integrity: sha512-Kza43ewS3xoLgCEpQrsT+xRo/EJej1y0kVYGiLFE1NEODXGzTfwiC6tXTLMQskn1X4/Rjlh0MQUvx9W+L9long==} dependencies: undici-types: 5.26.5 + dev: true /@types/normalize-package-data@2.4.1: resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} @@ -19869,6 +19870,7 @@ packages: /undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + dev: true /unicode-canonical-property-names-ecmascript@2.0.0: resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==} From ec1671a224000c3aca1b2dac6e787c87af1cd713 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Wed, 10 Apr 2024 17:22:32 -0700 Subject: [PATCH 21/26] Reuse code for internal squiggle linker --- packages/hub/schema.graphql | 2 +- packages/hub/src/app/api/runSquiggle/route.ts | 7 +- .../hub/src/graphql/queries/runSquiggle.ts | 143 +++++++++++++----- .../buildRecentModelRevision/worker.ts | 61 +------- 4 files changed, 111 insertions(+), 102 deletions(-) diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql index b5df1068ce..c6e0147298 100644 --- a/packages/hub/schema.graphql +++ b/packages/hub/schema.graphql @@ -550,7 +550,7 @@ type Query { nodes(ids: [ID!]!): [Node]! relativeValuesDefinition(input: QueryRelativeValuesDefinitionInput!): QueryRelativeValuesDefinitionResult! relativeValuesDefinitions(after: String, before: String, first: Int, input: RelativeValuesDefinitionsQueryInput, last: Int): RelativeValuesDefinitionConnection! - runSquiggle(code: String!): SquiggleOutput! + runSquiggle(code: String!, seed: String): SquiggleOutput! search(after: String, before: String, first: Int, last: Int, text: String!): QuerySearchResult! userByUsername(username: String!): QueryUserByUsernameResult! users(after: String, before: String, first: Int, input: UsersQueryInput, last: Int): QueryUsersConnection! diff --git a/packages/hub/src/app/api/runSquiggle/route.ts b/packages/hub/src/app/api/runSquiggle/route.ts index 54280244e3..3cc422e217 100644 --- a/packages/hub/src/app/api/runSquiggle/route.ts +++ b/packages/hub/src/app/api/runSquiggle/route.ts @@ -1,13 +1,16 @@ import { NextRequest, NextResponse } from "next/server"; -import { runSquiggle } from "@/graphql/queries/runSquiggle"; +import { runSquiggleWithCache } from "@/graphql/queries/runSquiggle"; export async function POST(req: NextRequest) { // Assuming 'code' is sent in the request body and is a string try { const body = await req.json(); if (body.code) { - let response = await runSquiggle(body.code); + let response = await runSquiggleWithCache( + body.code, + body.seed || "DEFAULT_SEED" + ); if (response.isOk) { return new NextResponse( JSON.stringify({ diff --git a/packages/hub/src/graphql/queries/runSquiggle.ts b/packages/hub/src/graphql/queries/runSquiggle.ts index dd4a590f03..0141c926c7 100644 --- a/packages/hub/src/graphql/queries/runSquiggle.ts +++ b/packages/hub/src/graphql/queries/runSquiggle.ts @@ -1,14 +1,21 @@ import { Prisma } from "@prisma/client"; import crypto from "crypto"; -import { SqProject, SqValue } from "@quri/squiggle-lang"; +import { SqLinker, SqProject, SqValue } from "@quri/squiggle-lang"; -import { DEFAULT_SEED, SAMPLE_COUNT_DEFAULT } from "@/constants"; +import { SAMPLE_COUNT_DEFAULT, XY_POINT_LENGTH_DEFAULT } from "@/constants"; import { builder } from "@/graphql/builder"; import { prisma } from "@/prisma"; +import { parseSourceId } from "@/squiggle/components/linker"; -function getKey(code: string): string { - return crypto.createHash("md5").update(code).digest("base64"); +import { SqOutputResult } from "../../../../squiggle-lang/dist/public/types"; +import { NotFoundError } from "../errors/NotFoundError"; + +function getKey(code: string, seed: string): string { + return crypto + .createHash("md5") + .update(`__EXPORT__:${code}, __SEED__: ${seed}`) + .digest("base64"); } export const squiggleValueToJSON = (value: SqValue): any => { @@ -77,23 +84,85 @@ builder.objectType( } ); -export async function runSquiggle(code: string): Promise { +export const squiggleLinker: SqLinker = { + resolve(name) { + return name; + }, + async loadSource(sourceId: string) { + const { owner, slug } = parseSourceId(sourceId); + const model = await prisma.model.findFirst({ + where: { + slug, + owner: { slug: owner }, + }, + include: { + currentRevision: { + include: { + squiggleSnippet: true, + }, + }, + }, + }); + + if (!model) { + throw new NotFoundError(); + } + + const content = model?.currentRevision?.squiggleSnippet; + if (content) { + return content.code; + } else { + throw new NotFoundError(); + } + }, +}; + +export async function runSquiggle( + code: string, + seed: string +): Promise { const MAIN = "main"; const env = { sampleCount: SAMPLE_COUNT_DEFAULT, // int - xyPointLength: 1000, // int - seed: DEFAULT_SEED, + xyPointLength: XY_POINT_LENGTH_DEFAULT, // int + seed: seed, }; - const project = SqProject.create({ environment: env }); + const project = SqProject.create({ + environment: env, + linker: squiggleLinker, + }); project.setSource(MAIN, code); await project.run(MAIN); - const outputR = project.getOutput(MAIN); + return project.getOutput(MAIN); +} + +//Warning: Caching will break if any imports change. It would be good to track this. Maybe we could compile the import tree, then store that as well, and recalculate whenever either that or the code changes. +export async function runSquiggleWithCache( + code: string, + seed: string +): Promise { + const key = getKey(code, seed); + const cached = await prisma.squiggleCache.findUnique({ + where: { id: key }, + }); - return outputR.ok + if (cached) { + return { + isCached: true, + isOk: cached.ok, + errorString: cached.error, + resultJSON: cached.result, + bindingsJSON: cached.bindings, + } as unknown as SquiggleOutput; + } + + const outputR = await runSquiggle(code, seed); + + const result: SquiggleOutput = outputR.ok ? { isCached: false, isOk: true, @@ -105,6 +174,25 @@ export async function runSquiggle(code: string): Promise { isOk: false, errorString: outputR.value.toString(), }; + + await prisma.squiggleCache.upsert({ + where: { id: key }, + create: { + id: key, + ok: result.isOk, + result: result.resultJSON ?? undefined, + bindings: result.bindingsJSON ?? undefined, + error: result.errorString, + }, + update: { + ok: result.isOk, + result: result.resultJSON ?? undefined, + bindings: result.bindingsJSON ?? undefined, + error: result.errorString, + }, + }); + + return result; } builder.queryField("runSquiggle", (t) => @@ -112,39 +200,10 @@ builder.queryField("runSquiggle", (t) => type: SquiggleOutputObj, args: { code: t.arg.string({ required: true }), + seed: t.arg.string({ required: false }), }, - async resolve(_, { code }) { - const key = getKey(code); - - const cached = await prisma.squiggleCache.findUnique({ - where: { id: key }, - }); - if (cached) { - return { - isCached: true, - isOk: cached.ok, - errorString: cached.error, - resultJSON: cached.result, - bindingsJSON: cached.bindings, - } as unknown as SquiggleOutput; // cache is less strictly typed than SquiggleOutput, so we have to force-cast it - } - const result = await runSquiggle(code); - await prisma.squiggleCache.upsert({ - where: { id: key }, - create: { - id: key, - ok: result.isOk, - result: result.resultJSON ?? undefined, - bindings: result.bindingsJSON ?? undefined, - error: result.errorString, - }, - update: { - ok: result.isOk, - result: result.resultJSON ?? undefined, - bindings: result.bindingsJSON ?? undefined, - error: result.errorString, - }, - }); + async resolve(_, { code, seed }) { + const result = await runSquiggleWithCache(code, seed || "DEFAULT_SEED"); return result; }, }) diff --git a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts index c8b997653e..165d2441f2 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts @@ -1,70 +1,17 @@ import { PrismaClient } from "@prisma/client"; -import { SqLinker, SqProject } from "@quri/squiggle-lang"; +import { runSquiggle } from "@/graphql/queries/runSquiggle"; import { ModelExport } from "../../../../components/dist/src/components/SquigglePlayground"; -import { SAMPLE_COUNT_DEFAULT, XY_POINT_LENGTH_DEFAULT } from "../../constants"; -import { NotFoundError } from "../../graphql/errors/NotFoundError"; -import { parseSourceId } from "../../squiggle/components/linker"; - -const SOURCE_NAME = "main"; const prisma = new PrismaClient(); -export const squiggleLinker: SqLinker = { - resolve(name) { - return name; - }, - async loadSource(sourceId: string) { - const { owner, slug } = parseSourceId(sourceId); - - const model = await prisma.model.findFirst({ - where: { - slug, - owner: { slug: owner }, - }, - include: { - currentRevision: { - include: { - squiggleSnippet: true, - }, - }, - }, - }); - - if (!model) { - throw new NotFoundError(); - } - - const content = model?.currentRevision?.squiggleSnippet; - if (content) { - return content.code; - } else { - throw new NotFoundError(); - } - }, -}; - -export async function runSquiggle( +export async function runSquiggleCode( currentRevisionId: string, code: string, seed: string ): Promise { - const env = { - sampleCount: SAMPLE_COUNT_DEFAULT, - xyPointLength: XY_POINT_LENGTH_DEFAULT, - seed, - }; - - const project = SqProject.create({ - linker: squiggleLinker, - environment: env, - }); - - project.setSource(SOURCE_NAME, code); - await project.run(SOURCE_NAME); - - const outputR = project.getOutput(SOURCE_NAME); + const outputR = await runSquiggle(code, seed); if (outputR.ok) { const _exports: ModelExport[] = outputR.value.exports @@ -116,7 +63,7 @@ process.on("message", async (message: RunMessage) => { if (message.type === "run") { const { revisionId, code, seed } = message.data; - const errors = await runSquiggle(revisionId, code, seed); + const errors = await runSquiggleCode(revisionId, code, seed); process?.send?.({ type: "result", From 904b15d58aa36056b37629753163bbcb8634232f Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Wed, 10 Apr 2024 17:32:32 -0700 Subject: [PATCH 22/26] Minor cleanup --- packages/hub/src/graphql/queries/runSquiggle.ts | 8 ++++++-- packages/hub/src/lib/ExportsDropdown.tsx | 2 ++ .../hub/src/scripts/buildRecentModelRevision/worker.ts | 3 +-- packages/squiggle-lang/src/index.ts | 1 + 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/hub/src/graphql/queries/runSquiggle.ts b/packages/hub/src/graphql/queries/runSquiggle.ts index 0141c926c7..75446a99ef 100644 --- a/packages/hub/src/graphql/queries/runSquiggle.ts +++ b/packages/hub/src/graphql/queries/runSquiggle.ts @@ -1,14 +1,18 @@ import { Prisma } from "@prisma/client"; import crypto from "crypto"; -import { SqLinker, SqProject, SqValue } from "@quri/squiggle-lang"; +import { + SqLinker, + SqOutputResult, + SqProject, + SqValue, +} from "@quri/squiggle-lang"; import { SAMPLE_COUNT_DEFAULT, XY_POINT_LENGTH_DEFAULT } from "@/constants"; import { builder } from "@/graphql/builder"; import { prisma } from "@/prisma"; import { parseSourceId } from "@/squiggle/components/linker"; -import { SqOutputResult } from "../../../../squiggle-lang/dist/public/types"; import { NotFoundError } from "../errors/NotFoundError"; function getKey(code: string, seed: string): string { diff --git a/packages/hub/src/lib/ExportsDropdown.tsx b/packages/hub/src/lib/ExportsDropdown.tsx index c5a6907176..dd2bc158cf 100644 --- a/packages/hub/src/lib/ExportsDropdown.tsx +++ b/packages/hub/src/lib/ExportsDropdown.tsx @@ -21,7 +21,9 @@ export type ModelExport = { title?: string; variableName: string; variableType?: string; + docstring?: string; }; + type RelativeValuesExport = { slug: string; variableName: string }; const nonRelativeValuesExports = ( diff --git a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts index 165d2441f2..64ea7769fd 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts @@ -1,8 +1,7 @@ import { PrismaClient } from "@prisma/client"; import { runSquiggle } from "@/graphql/queries/runSquiggle"; - -import { ModelExport } from "../../../../components/dist/src/components/SquigglePlayground"; +import { ModelExport } from "@/lib/ExportsDropdown"; const prisma = new PrismaClient(); diff --git a/packages/squiggle-lang/src/index.ts b/packages/squiggle-lang/src/index.ts index ee10e08adf..dc9089e744 100644 --- a/packages/squiggle-lang/src/index.ts +++ b/packages/squiggle-lang/src/index.ts @@ -89,6 +89,7 @@ export { export { type AST, type ASTNode } from "./ast/parse.js"; export { type ASTCommentNode } from "./ast/peggyHelpers.js"; export { type SqLinker } from "./public/SqLinker.js"; +export { type SqOutput, type SqOutputResult } from "./public/types.js"; export async function run( code: string, From 687a4df9240955b93aef019d82463e477ce39aa9 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Wed, 10 Apr 2024 19:27:24 -0700 Subject: [PATCH 23/26] Added lastRevisionWithBuild fn --- packages/hub/schema.graphql | 1 + .../[slug]/EditSquiggleSnippetModel.tsx | 14 +++++++- packages/hub/src/graphql/types/Model.ts | 24 ++++++++++++++ .../scripts/buildRecentModelRevision/main.ts | 11 ++++--- .../buildRecentModelRevision/worker.ts | 33 ++++++++++++------- 5 files changed, 67 insertions(+), 16 deletions(-) diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql index c6e0147298..f75ff7249b 100644 --- a/packages/hub/schema.graphql +++ b/packages/hub/schema.graphql @@ -157,6 +157,7 @@ type Model implements Node { id: ID! isEditable: Boolean! isPrivate: Boolean! + lastRevisionWithBuild: ModelRevision owner: Owner! revision(id: ID!): ModelRevision! revisions(after: String, before: String, first: Int, last: Int): ModelRevisionConnection! diff --git a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx index dc6109946f..b6340bf481 100644 --- a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx +++ b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx @@ -142,6 +142,11 @@ export const EditSquiggleSnippetModel: FC = ({ owner { slug } + lastRevisionWithBuild { + lastBuild { + runSeconds + } + } currentRevision { id content { @@ -187,6 +192,8 @@ export const EditSquiggleSnippetModel: FC = ({ "SquiggleSnippet" ); + const lastBuildSpeed = model.lastRevisionWithBuild?.lastBuild?.runSeconds; + const seed = content.seed; const initialFormValues: SquiggleSnippetFormShape = useMemo(() => { @@ -304,13 +311,18 @@ export const EditSquiggleSnippetModel: FC = ({ const squiggle = use(versionedSquigglePackages(checkedVersion)); + // Automatically turn off autorun, if the last build speed was > 5s. Note that this does not stop in the case of memory errors or similar. + const autorunMode = + content.autorunMode || + (lastBuildSpeed ? (lastBuildSpeed > 5 ? false : true) : true); + // Build props for versioned SquigglePlayground first, since they might depend on the version we use, // and we want to populate them incrementally. const playgroundProps: Parameters< typeof squiggle.components.SquigglePlayground >[0] = { defaultCode, - autorunMode: content.autorunMode ?? true, + autorunMode: autorunMode, sourceId: serializeSourceId({ owner: model.owner.slug, slug: model.slug, diff --git a/packages/hub/src/graphql/types/Model.ts b/packages/hub/src/graphql/types/Model.ts index 7da25cb967..4e506074ab 100644 --- a/packages/hub/src/graphql/types/Model.ts +++ b/packages/hub/src/graphql/types/Model.ts @@ -112,6 +112,30 @@ export const Model = builder.prismaNode("Model", { resolve: (model, args, ctx) => exportRevisionConnectionHelpers.resolve(model.revisions, args, ctx), }), + lastRevisionWithBuild: t.field({ + type: ModelRevision, + nullable: true, + select: { + revisions: { + orderBy: { + createdAt: "desc", + }, + where: { + builds: { + some: { + id: { + not: undefined, + }, + }, + }, + }, + take: 1, + }, + }, + async resolve(model) { + return model.revisions[0]; + }, + }), }), }); diff --git a/packages/hub/src/scripts/buildRecentModelRevision/main.ts b/packages/hub/src/scripts/buildRecentModelRevision/main.ts index 99f30ab066..1ac6e228d1 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision/main.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision/main.ts @@ -118,7 +118,6 @@ async function buildRecentModelVersion(): Promise { TIMEOUT_SECONDS ); const endTime = performance.now(); - console.log("RESPONSE", response); await prisma.modelRevisionBuild.create({ data: { @@ -129,8 +128,7 @@ async function buildRecentModelVersion(): Promise { }); console.log( - "Build created for model revision ID:", - model.currentRevisionId + `Build created for model revision ID: ${model.currentRevisionId}, in ${endTime - startTime}ms.` ); } catch (error) { console.error("Error building model revision:", error); @@ -165,11 +163,16 @@ async function main(): Promise { } } +async function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + async function runContinuously() { while (true) { try { await main(); - await new Promise((resolve) => setTimeout(resolve, 500)); // Sleep for 1 second + await new Promise((resolve) => process.nextTick(resolve)); + await delay(500); // Delay for approximately .5s } catch (error) { console.error("An error occurred during continuous execution:", error); } diff --git a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts index 64ea7769fd..2bbf32b778 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts @@ -13,6 +13,7 @@ export async function runSquiggleCode( const outputR = await runSquiggle(code, seed); if (outputR.ok) { + // I Imagine it would be nice to move this out of this worker file, but this would require exporting a lot more information. It seems wise to instead wait for the Serialization PR to go in and then refactor this. const _exports: ModelExport[] = outputR.value.exports .entries() .map((e) => ({ @@ -60,16 +61,26 @@ type RunMessage = { process.on("message", async (message: RunMessage) => { if (message.type === "run") { - const { revisionId, code, seed } = message.data; - - const errors = await runSquiggleCode(revisionId, code, seed); - - process?.send?.({ - type: "result", - data: { - errors, - }, - }); - process.exit(0); + try { + const { revisionId, code, seed } = message.data; + const errors = await runSquiggleCode(revisionId, code, seed); + process?.send?.({ + type: "result", + data: { + errors, + }, + }); + } catch (error) { + console.error("An error occurred in the worker process:", error); + process?.send?.({ + type: "result", + data: { + errors: "An unknown error occurred in the worker process.", + }, + }); + } finally { + await prisma.$disconnect(); + process.exit(1); + } } }); From f78e501b997c4b17f7feb75d3c986257ed788d84 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Thu, 11 Apr 2024 13:09:02 -0700 Subject: [PATCH 24/26] Moved export creation into buildRecentModelRevision main(), moved into transaction --- .../hub/src/graphql/queries/runSquiggle.ts | 4 +- .../scripts/buildRecentModelRevision/main.ts | 67 +++++++++++++------ .../buildRecentModelRevision/worker.ts | 66 ++++++------------ 3 files changed, 70 insertions(+), 67 deletions(-) diff --git a/packages/hub/src/graphql/queries/runSquiggle.ts b/packages/hub/src/graphql/queries/runSquiggle.ts index 75446a99ef..152a0f2785 100644 --- a/packages/hub/src/graphql/queries/runSquiggle.ts +++ b/packages/hub/src/graphql/queries/runSquiggle.ts @@ -17,8 +17,8 @@ import { NotFoundError } from "../errors/NotFoundError"; function getKey(code: string, seed: string): string { return crypto - .createHash("md5") - .update(`__EXPORT__:${code}, __SEED__: ${seed}`) + .createHash("sha265") + .update(JSON.stringify({ code, seed })) .digest("base64"); } diff --git a/packages/hub/src/scripts/buildRecentModelRevision/main.ts b/packages/hub/src/scripts/buildRecentModelRevision/main.ts index 1ac6e228d1..57d14e0441 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision/main.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision/main.ts @@ -2,21 +2,18 @@ import { PrismaClient } from "@prisma/client"; import { spawn } from "child_process"; import { NotFoundError } from "../../graphql/errors/NotFoundError"; +import { WorkerOutput } from "./worker"; const TIMEOUT_SECONDS = 60; // 60 seconds const prisma = new PrismaClient(); -type SquiggleResult = { - errors?: string; -}; - async function runWorker( revisionId: string, code: string, seed: string, timeoutSeconds: number -): Promise { +): Promise { return new Promise((resolve, _) => { console.log("Spawning worker process for Revision ID: " + revisionId); const worker = spawn( @@ -29,7 +26,7 @@ async function runWorker( const timeoutId = setTimeout(() => { worker.kill(); - resolve({ errors: `Timeout Error, at ${timeoutSeconds}s ` }); + resolve({ errors: `Timeout Error, at ${timeoutSeconds}s`, exports: [] }); }, timeoutSeconds * 1000); worker.stdout?.on("data", (data) => { @@ -42,7 +39,8 @@ async function runWorker( worker.on( "message", - async (message: { type: string; data: { errors?: string } }) => { + async (message: { type: string; data: WorkerOutput }) => { + console.log("Worker message received"); resolve(message.data); } ); @@ -52,15 +50,15 @@ async function runWorker( if (code === 0) { console.log("Worker completed successfully"); } else { - console.error(`Worker process exited with code ${code}`); - resolve({ errors: "Computation error, with code: " + code }); + console.error(`Worker process exited with error code ${code}`); + resolve({ + errors: "Computation error, with code: " + code, + exports: [], + }); } }); - worker.send({ type: "run", data: { revisionId, code, seed } }); - worker.on("close", (code) => { - console.log(`Worker process CLOSED with code ${code}`); - }); + worker.send({ type: "run", data: { code, seed } }); }); } @@ -119,16 +117,43 @@ async function buildRecentModelVersion(): Promise { ); const endTime = performance.now(); - await prisma.modelRevisionBuild.create({ - data: { - modelRevision: { connect: { id: model.currentRevisionId } }, - runSeconds: (endTime - startTime) / 1000, - errors: response.errors === undefined ? [] : [response.errors], - }, - }); + await prisma.$transaction(async (tx) => { + // For some reason, Typescript becomes unsure if `model.currentRevisionId` is null or not, even though it's checked above. + const revisionId = model.currentRevisionId!; + await tx.modelRevisionBuild.create({ + data: { + modelRevision: { connect: { id: revisionId } }, + runSeconds: (endTime - startTime) / 1000, + errors: response.errors === undefined ? [] : [response.errors], + }, + }); + + for (const e of response.exports) { + await tx.modelExport.upsert({ + where: { + uniqueKey: { + modelRevisionId: revisionId, + variableName: e.variableName, + }, + }, + update: { + variableType: e.variableType, + title: e.title, + docstring: e.docstring, + }, + create: { + modelRevision: { connect: { id: revisionId } }, + variableName: e.variableName, + variableType: e.variableType, + title: e.title, + docstring: e.docstring, + }, + }); + } + }); console.log( - `Build created for model revision ID: ${model.currentRevisionId}, in ${endTime - startTime}ms.` + `Build created for model revision ID: ${model.currentRevisionId}, in ${endTime - startTime}ms. Created ${response.exports.length} exports.` ); } catch (error) { console.error("Error building model revision:", error); diff --git a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts index 2bbf32b778..e36a81f444 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts @@ -1,59 +1,39 @@ -import { PrismaClient } from "@prisma/client"; - import { runSquiggle } from "@/graphql/queries/runSquiggle"; import { ModelExport } from "@/lib/ExportsDropdown"; +import { prisma } from "@/prisma"; -const prisma = new PrismaClient(); +export type WorkerOutput = { + errors: string; + exports: ModelExport[]; +}; export async function runSquiggleCode( - currentRevisionId: string, code: string, seed: string -): Promise { +): Promise { const outputR = await runSquiggle(code, seed); + let exports: ModelExport[] = []; + if (outputR.ok) { // I Imagine it would be nice to move this out of this worker file, but this would require exporting a lot more information. It seems wise to instead wait for the Serialization PR to go in and then refactor this. - const _exports: ModelExport[] = outputR.value.exports - .entries() - .map((e) => ({ - variableName: e[0], - variableType: e[1].tag, - title: e[1].tags.name() ? e[1].tags.name() : e[1].title() || "", - docstring: e[1].tags.doc() || "", - })); - - for (const e of _exports) { - await prisma.modelExport.upsert({ - where: { - uniqueKey: { - modelRevisionId: currentRevisionId, - variableName: e.variableName, - }, - }, - update: { - variableType: e.variableType, - title: e.title, - docstring: e.docstring, - }, - create: { - modelRevision: { connect: { id: currentRevisionId } }, - variableName: e.variableName, - variableType: e.variableType, - title: e.title, - docstring: e.docstring, - }, - }); - } + exports = outputR.value.exports.entries().map((e) => ({ + variableName: e[0], + variableType: e[1].tag, + title: e[1].tags.name() ? e[1].tags.name() : e[1].title() || "", + docstring: e[1].tags.doc() || "", + })); } - return outputR.ok ? undefined : outputR.value.toString(); + return { + errors: outputR.ok ? "" : outputR.value.toString(), + exports, + }; } type RunMessage = { type: "run"; data: { - revisionId: string; code: string; seed: string; }; @@ -62,13 +42,11 @@ type RunMessage = { process.on("message", async (message: RunMessage) => { if (message.type === "run") { try { - const { revisionId, code, seed } = message.data; - const errors = await runSquiggleCode(revisionId, code, seed); + const { code, seed } = message.data; + const buildOutput = await runSquiggleCode(code, seed); process?.send?.({ type: "result", - data: { - errors, - }, + data: buildOutput, }); } catch (error) { console.error("An error occurred in the worker process:", error); @@ -80,7 +58,7 @@ process.on("message", async (message: RunMessage) => { }); } finally { await prisma.$disconnect(); - process.exit(1); + process.exit(0); } } }); From 91cfca60347cb0be1dcebbbdfed290b92cc050d3 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Thu, 11 Apr 2024 13:11:59 -0700 Subject: [PATCH 25/26] RunMessage should be used for validation --- .../scripts/buildRecentModelRevision/main.ts | 8 +++++--- .../scripts/buildRecentModelRevision/worker.ts | 18 +++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/hub/src/scripts/buildRecentModelRevision/main.ts b/packages/hub/src/scripts/buildRecentModelRevision/main.ts index 57d14e0441..e44272e201 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision/main.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision/main.ts @@ -2,7 +2,7 @@ import { PrismaClient } from "@prisma/client"; import { spawn } from "child_process"; import { NotFoundError } from "../../graphql/errors/NotFoundError"; -import { WorkerOutput } from "./worker"; +import { WorkerOutput, WorkerRunMessage } from "./worker"; const TIMEOUT_SECONDS = 60; // 60 seconds @@ -40,7 +40,6 @@ async function runWorker( worker.on( "message", async (message: { type: string; data: WorkerOutput }) => { - console.log("Worker message received"); resolve(message.data); } ); @@ -58,7 +57,10 @@ async function runWorker( } }); - worker.send({ type: "run", data: { code, seed } }); + worker.send({ + type: "run", + data: { code, seed }, + } satisfies WorkerRunMessage); }); } diff --git a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts index e36a81f444..30347ed95f 100644 --- a/packages/hub/src/scripts/buildRecentModelRevision/worker.ts +++ b/packages/hub/src/scripts/buildRecentModelRevision/worker.ts @@ -2,6 +2,14 @@ import { runSquiggle } from "@/graphql/queries/runSquiggle"; import { ModelExport } from "@/lib/ExportsDropdown"; import { prisma } from "@/prisma"; +export type WorkerRunMessage = { + type: "run"; + data: { + code: string; + seed: string; + }; +}; + export type WorkerOutput = { errors: string; exports: ModelExport[]; @@ -31,15 +39,7 @@ export async function runSquiggleCode( }; } -type RunMessage = { - type: "run"; - data: { - code: string; - seed: string; - }; -}; - -process.on("message", async (message: RunMessage) => { +process.on("message", async (message: WorkerRunMessage) => { if (message.type === "run") { try { const { code, seed } = message.data; From f87b34fa94f45eef23ec1aad7f524a08c489bbe6 Mon Sep 17 00:00:00 2001 From: Ozzie Gooen Date: Thu, 11 Apr 2024 19:33:40 -0700 Subject: [PATCH 26/26] Update packages/hub/src/graphql/queries/runSquiggle.ts Co-authored-by: Slava Matyukhin --- packages/hub/src/graphql/queries/runSquiggle.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/src/graphql/queries/runSquiggle.ts b/packages/hub/src/graphql/queries/runSquiggle.ts index 152a0f2785..f7462fda16 100644 --- a/packages/hub/src/graphql/queries/runSquiggle.ts +++ b/packages/hub/src/graphql/queries/runSquiggle.ts @@ -17,7 +17,7 @@ import { NotFoundError } from "../errors/NotFoundError"; function getKey(code: string, seed: string): string { return crypto - .createHash("sha265") + .createHash("sha256") .update(JSON.stringify({ code, seed })) .digest("base64"); }