Merge pull request #3157 from quantified-uncertainty/runner-test

Script for running Model builds
quantified-uncertainty · Apr 12, 2024 · 465510b · 465510b
2 parents b6109aa + f87b34f
commit 465510b
Show file tree

Hide file tree

Showing 19 changed files with 646 additions and 122 deletions.
diff --git a/packages/hub/README.md b/packages/hub/README.md
@@ -79,3 +79,13 @@ Squiggle Hub is deployed on [Vercel](https://vercel.com/) automatically when the
 The production database is migrated by [this GitHub Action](https://github.com/quantified-uncertainty/squiggle/blob/main/.github/workflows/prisma-migrate-prod.yml).
 
 **Important: it should be invoked _before_ merging any PR that changes the schema.**
+
+## Debugging
+
+If you get an error like:
+
+```
+PothosSchemaError [GraphQLError]: Ref ObjectRef<ModelRevisionRun> has not been implemented
+```
+
+Make sure that any new files in `src/graphql/types` have been added to `src/schema.ts`, or something that references that.
diff --git a/packages/hub/package.json b/packages/hub/package.json
@@ -20,7 +20,8 @@
     "build": "pnpm gen && __NEXT_PRIVATE_PREBUNDLED_REACT=next next build",
     "lint": "prettier --check . && next lint",
     "format": "prettier --write .",
-    "test:manual": "dotenv -e .env.test -- jest -i"
+    "test:manual": "dotenv -e .env.test -- jest -i",
+    "build-last-revision": "tsx src/scripts/buildRecentModelRevision/main.ts"
   },
   "dependencies": {
     "@next-auth/prisma-adapter": "^1.0.7",
@@ -74,6 +75,7 @@
     "@types/invariant": "^2.2.37",
     "@types/jest": "^29.5.12",
     "@types/lodash": "^4.14.202",
+    "@types/node": "^20.11.24",
     "@types/pako": "^2.0.3",
     "@types/react": "^18.2.52",
     "@types/react-relay": "^16.0.6",

diff --git a/packages/hub/prisma/migrations/20240407144130_model_revision_build/migration.sql b/packages/hub/prisma/migrations/20240407144130_model_revision_build/migration.sql
@@ -0,0 +1,16 @@
+-- CreateTable
+CREATE TABLE "ModelRevisionBuild" (
+    "id" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "modelRevisionId" TEXT NOT NULL,
+    "runSeconds" DOUBLE PRECISION NOT NULL,
+    "errors" TEXT[],
+
+    CONSTRAINT "ModelRevisionBuild_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "ModelRevisionBuild_modelRevisionId_idx" ON "ModelRevisionBuild"("modelRevisionId");
+
+-- AddForeignKey
+ALTER TABLE "ModelRevisionBuild" ADD CONSTRAINT "ModelRevisionBuild_modelRevisionId_fkey" FOREIGN KEY ("modelRevisionId") REFERENCES "ModelRevision"("id") ON DELETE CASCADE ON UPDATE CASCADE;
diff --git a/packages/hub/prisma/schema.prisma b/packages/hub/prisma/schema.prisma
@@ -222,11 +222,23 @@ model ModelRevision {
   exports               ModelExport[]
 
   // required by Prisma, but unused since `model` field should point at the same entity
-  currentRevisionModel Model? @relation("CurrentRevision")
+  currentRevisionModel Model?               @relation("CurrentRevision")
+  builds               ModelRevisionBuild[]
 
   @@index([modelId])
 }
 
+model ModelRevisionBuild {
+  id              String        @id @default(cuid())
+  createdAt       DateTime      @default(now())
+  modelRevision   ModelRevision @relation(fields: [modelRevisionId], references: [id], onDelete: Cascade)
+  modelRevisionId String
+  runSeconds      Float
+  errors          String[]
+
+  @@index([modelRevisionId])
+}
+
 model SquiggleSnippet {
   id String @id @default(cuid())
 

diff --git a/packages/hub/schema.graphql b/packages/hub/schema.graphql
@@ -157,6 +157,7 @@ type Model implements Node {
   id: ID!
   isEditable: Boolean!
   isPrivate: Boolean!
+  lastRevisionWithBuild: ModelRevision
   owner: Owner!
   revision(id: ID!): ModelRevision!
   revisions(after: String, before: String, first: Int, last: Int): ModelRevisionConnection!
@@ -215,16 +216,34 @@ type ModelExportRevisionsConnectionEdge {
 
 type ModelRevision implements Node {
   author: User
+  buildStatus: ModelRevisionBuildStatus!
   comment: String!
   content: ModelContent!
   createdAtTimestamp: Float!
+  exportNames: [String!]!
   exports: [ModelExport!]!
   forRelativeValues(input: ModelRevisionForRelativeValuesInput!): ModelRevisionForRelativeValuesResult!
   id: ID!
+  lastBuild: ModelRevisionBuild
   model: Model!
   relativeValuesExports: [RelativeValuesExport!]!
 }
 
+type ModelRevisionBuild implements Node {
+  createdAtTimestamp: Float!
+  errors: [String!]!
+  id: ID!
+  modelRevision: ModelRevision!
+  runSeconds: Float!
+}
+
+enum ModelRevisionBuildStatus {
+  Failure
+  Pending
+  Skipped
+  Success
+}
+
 type ModelRevisionConnection {
   edges: [ModelRevisionEdge!]!
   pageInfo: PageInfo!
@@ -483,7 +502,6 @@ union MutationUpdateRelativeValuesDefinitionResult = BaseError | UpdateRelativeV
 input MutationUpdateSquiggleSnippetModelInput {
   comment: String
   content: SquiggleSnippetContentInput!
-  exports: [SquiggleModelExportInput!]
   owner: String!
   relativeValuesExports: [RelativeValuesExportInput!]
   slug: String!
@@ -533,7 +551,7 @@ type Query {
   nodes(ids: [ID!]!): [Node]!
   relativeValuesDefinition(input: QueryRelativeValuesDefinitionInput!): QueryRelativeValuesDefinitionResult!
   relativeValuesDefinitions(after: String, before: String, first: Int, input: RelativeValuesDefinitionsQueryInput, last: Int): RelativeValuesDefinitionConnection!
-  runSquiggle(code: String!): SquiggleOutput!
+  runSquiggle(code: String!, seed: String): SquiggleOutput!
   search(after: String, before: String, first: Int, last: Int, text: String!): QuerySearchResult!
   userByUsername(username: String!): QueryUserByUsernameResult!
   users(after: String, before: String, first: Int, input: UsersQueryInput, last: Int): QueryUsersConnection!
@@ -681,13 +699,6 @@ type SquiggleErrorOutput implements SquiggleOutput {
   isCached: Boolean!
 }
 
-input SquiggleModelExportInput {
-  docstring: String
-  title: String
-  variableName: String!
-  variableType: String!
-}
-
 type SquiggleOkOutput implements SquiggleOutput {
   bindingsJSON: String!
   isCached: Boolean!

diff --git a/packages/hub/src/app/api/runSquiggle/route.ts b/packages/hub/src/app/api/runSquiggle/route.ts
@@ -1,13 +1,16 @@
 import { NextRequest, NextResponse } from "next/server";
 
-import { runSquiggle } from "@/graphql/queries/runSquiggle";
+import { runSquiggleWithCache } from "@/graphql/queries/runSquiggle";
 
 export async function POST(req: NextRequest) {
   // Assuming 'code' is sent in the request body and is a string
   try {
     const body = await req.json();
     if (body.code) {
-      let response = await runSquiggle(body.code);
+      let response = await runSquiggleWithCache(
+        body.code,
+        body.seed || "DEFAULT_SEED"
+      );
       if (response.isOk) {
         return new NextResponse(
           JSON.stringify({

diff --git a/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx b/packages/hub/src/app/models/[owner]/[slug]/EditSquiggleSnippetModel.tsx
@@ -54,13 +54,11 @@ import { EditSquiggleSnippetModel$key } from "@/__generated__/EditSquiggleSnippe
 import {
   EditSquiggleSnippetModelMutation,
   RelativeValuesExportInput,
-  SquiggleModelExportInput,
 } from "@/__generated__/EditSquiggleSnippetModelMutation.graphql";
 
 export type SquiggleSnippetFormShape = {
   code: string;
   relativeValuesExports: RelativeValuesExportInput[];
-  exports: SquiggleModelExportInput[];
 };
 
 type OnSubmit = (
@@ -144,6 +142,11 @@ export const EditSquiggleSnippetModel: FC<Props> = ({
         owner {
           slug
         }
+        lastRevisionWithBuild {
+          lastBuild {
+            runSeconds
+          }
+        }
         currentRevision {
           id
           content {
@@ -158,6 +161,7 @@ export const EditSquiggleSnippetModel: FC<Props> = ({
               xyPointLength
             }
           }
+          exportNames
           exports {
             id
             variableName
@@ -188,6 +192,8 @@ export const EditSquiggleSnippetModel: FC<Props> = ({
     "SquiggleSnippet"
   );
 
+  const lastBuildSpeed = model.lastRevisionWithBuild?.lastBuild?.runSeconds;
+
   const seed = content.seed;
 
   const initialFormValues: SquiggleSnippetFormShape = useMemo(() => {
@@ -200,14 +206,8 @@ export const EditSquiggleSnippetModel: FC<Props> = ({
           slug: item.definition.slug,
         },
       })),
-      exports: revision.exports.map((item) => ({
-        title: item.title,
-        variableName: item.variableName,
-        variableType: item.variableType,
-        docstring: item.docstring,
-      })),
     };
-  }, [content, revision.relativeValuesExports, revision.exports]);
+  }, [content, revision.relativeValuesExports]);
 
   const { form, onSubmit, inFlight } = useMutationForm<
     SquiggleSnippetFormShape,
@@ -245,7 +245,6 @@ export const EditSquiggleSnippetModel: FC<Props> = ({
           xyPointLength: content.xyPointLength,
         },
         relativeValuesExports: formData.relativeValuesExports,
-        exports: formData.exports,
         comment: extraData?.comment,
         slug: model.slug,
         owner: model.owner.slug,
@@ -312,13 +311,18 @@ export const EditSquiggleSnippetModel: FC<Props> = ({
 
   const squiggle = use(versionedSquigglePackages(checkedVersion));
 
+  // Automatically turn off autorun, if the last build speed was > 5s. Note that this does not stop in the case of memory errors or similar.
+  const autorunMode =
+    content.autorunMode ||
+    (lastBuildSpeed ? (lastBuildSpeed > 5 ? false : true) : true);
+
   // Build props for versioned SquigglePlayground first, since they might depend on the version we use,
   // and we want to populate them incrementally.
   const playgroundProps: Parameters<
     typeof squiggle.components.SquigglePlayground
   >[0] = {
     defaultCode,
-    autorunMode: content.autorunMode ?? true,
+    autorunMode: autorunMode,
     sourceId: serializeSourceId({
       owner: model.owner.slug,
       slug: model.slug,
@@ -406,9 +410,6 @@ export const EditSquiggleSnippetModel: FC<Props> = ({
       playgroundProps
     )
   ) {
-    playgroundProps.onExportsChange = (exports) => {
-      form.setValue("exports", exports);
-    };
   }
 
   playgroundProps.environment = {

diff --git a/packages/hub/src/app/models/[owner]/[slug]/ModelLayout.tsx b/packages/hub/src/app/models/[owner]/[slug]/ModelLayout.tsx
@@ -7,7 +7,11 @@ import { CodeBracketIcon, RectangleStackIcon, ShareIcon } from "@quri/ui";
 
 import { EntityLayout } from "@/components/EntityLayout";
 import { EntityTab } from "@/components/ui/EntityTab";
-import { ExportsDropdown, totalImportLength } from "@/lib/ExportsDropdown";
+import {
+  ExportsDropdown,
+  type ModelExport,
+  totalImportLength,
+} from "@/lib/ExportsDropdown";
 import { extractFromGraphqlErrorUnion } from "@/lib/graphqlHelpers";
 import { SerializablePreloadedQuery } from "@/relay/loadPageQuery";
 import { usePageQuery } from "@/relay/usePageQuery";
@@ -45,6 +49,7 @@ const Query = graphql`
         currentRevision {
           id
           # for length; TODO - "hasExports" field?
+          exportNames
           exports {
             id
             variableName
@@ -81,12 +86,18 @@ export const ModelLayout: FC<
     slug: model.slug,
   });
 
-  const modelExports = model.currentRevision.exports.map(
-    ({ variableName, variableType, title }) => ({
-      variableName,
-      variableType,
-      title: title || undefined,
-    })
+  const modelExports: ModelExport[] = model.currentRevision.exportNames.map(
+    (name) => {
+      const matchingExport = model.currentRevision.exports.find(
+        (e) => e.variableName === name
+      );
+
+      return {
+        variableName: name,
+        variableType: matchingExport?.variableType || undefined,
+        title: matchingExport?.title || undefined,
+      };
+    }
   );
 
   const relativeValuesExports = model.currentRevision.relativeValuesExports.map(

diff --git a/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx b/packages/hub/src/app/models/[owner]/[slug]/revisions/ModelRevisionsList.tsx
@@ -27,6 +27,7 @@ const ModelRevisionItem: FC<{
       fragment ModelRevisionsList_revision on ModelRevision {
         id
         createdAtTimestamp
+        buildStatus
         author {
           username
         }
@@ -36,6 +37,10 @@ const ModelRevisionItem: FC<{
           variableName
           title
         }
+        lastBuild {
+          errors
+          runSeconds
+        }
       }
     `,
     revisionRef
@@ -76,6 +81,11 @@ const ModelRevisionItem: FC<{
       {revision.comment ? (
         <div className="text-xs text-slate-700">{revision.comment}</div>
       ) : null}
+
+      <div className="text-xs text-slate-700">{`Build Status: ${revision.buildStatus}`}</div>
+      {revision.lastBuild && (
+        <div className="text-xs text-slate-700">{`Build Time: ${revision.lastBuild.runSeconds.toFixed(2)}s`}</div>
+      )}
       {revision.exports.length > 0 ? (
         <div className="text-xs text-green-700">
           {`${revision.exports.length} exports `}
@@ -124,11 +134,16 @@ export const ModelRevisionsList: FC<{
               ...ModelRevisionsList_revision
               id
               createdAtTimestamp
+              buildStatus
               exports {
                 id
                 title
                 variableName
               }
+              lastBuild {
+                errors
+                runSeconds
+              }
             }
           }
           pageInfo {