Merge pull request #3439 from quantified-uncertainty/ai-evals

AI analytics - first iteration

packages/ai/package.json

@@ -25,7 +25,6 @@
     "@quri/prettier-plugin-squiggle": "workspace:*",
     "@quri/serializer": "workspace:*",
     "@quri/squiggle-lang": "workspace:*",
-    "@quri/versioned-squiggle-components": "workspace:*",
     "axios": "^1.7.2",
     "chalk": "^5.3.0",
     "clsx": "^2.1.1",

packages/hub/.eslintrc.json

@@ -1,6 +1,7 @@
 {
   "extends": "next/core-web-vitals",
   "rules": {
+    "import/no-anonymous-default-export": "off",
     "no-restricted-imports": [
       "error",
       {

packages/hub/esbuild.cjs

@@ -0,0 +1,44 @@
+const esbuild = require("esbuild");
+
+for (const name of [
+  "buildRecentModelRevision/worker",
+  "buildRecentModelRevision/main",
+  "print-schema",
+]) {
+  esbuild.buildSync({
+    entryPoints: [`./src/scripts/${name}.ts`],
+    platform: "node",
+    format: "esm",
+    sourcemap: true,
+    minify: true,
+    bundle: true,
+    // via https://github.com/evanw/esbuild/pull/2067#issuecomment-1073039746
+    banner: {
+      js: `
+await (async () => {
+  const { dirname } = await import("path");
+  const { fileURLToPath } = await import("url");
+
+  /**
+   * Shim entry-point related paths.
+   */
+  if (typeof globalThis.__filename === "undefined") {
+    globalThis.__filename = fileURLToPath(import.meta.url);
+  }
+  if (typeof globalThis.__dirname === "undefined") {
+    globalThis.__dirname = dirname(globalThis.__filename);
+  }
+  /**
+   * Shim require if needed.
+   */
+  if (typeof globalThis.require === "undefined") {
+    const { default: module } = await import("module");
+    globalThis.require = module.createRequire(import.meta.url);
+  }
+})();
+`,
+    },
+    outfile: `./dist/scripts/${name}.mjs`,
+    external: ["server-only"],
+  });
+}

packages/hub/package.json

@@ -13,16 +13,17 @@
     "start": "__NEXT_PRIVATE_PREBUNDLED_REACT=next next start",
     "gen:prisma": "PRISMA_HIDE_UPDATE_MESSAGE=1 prisma generate",
     "gen:relay": "relay-compiler",
-    "gen:schema": "tsx ./src/graphql/print-schema.ts",
+    "gen:schema": "pnpm build:esbuild && node ./dist/scripts/print-schema.mjs",
     "gen": "pnpm gen:prisma && pnpm gen:schema && pnpm gen:relay",
     "gen:watch": "nodemon --watch src --ext ts,tsx,prisma --exec 'pnpm run gen'",
+    "build:esbuild": "node ./esbuild.cjs",
     "build:ts": "pnpm gen && tsc",
-    "bundle": "node ./esbuild.js && cp ../../node_modules/.pnpm/@prisma+client*/node_modules/.prisma/client/*.node ./dist",
+    "bundle": "pnpm build:esbuild && cp ../../node_modules/.pnpm/@prisma+client*/node_modules/.prisma/client/*.node ./dist",
     "build": "pnpm gen && __NEXT_PRIVATE_PREBUNDLED_REACT=next next build",
     "lint": "prettier --check . && next lint",
     "format": "prettier --write .",
     "test:manual": "dotenv -e .env.test -- jest -i",
-    "build-last-revision": "tsx src/scripts/buildRecentModelRevision/main.ts"
+    "build-last-revision": "pnpm build:esbuild && node ./dist/scripts/buildRecentModelRevision/main.mjs"
   },
   "dependencies": {
     "@next-auth/prisma-adapter": "^1.0.7",

packages/hub/src/app/ai/analytics/ClientLayout.tsx

@@ -0,0 +1,21 @@
+"use client";
+import { PropsWithChildren } from "react";
+
+import { NarrowPageLayout } from "@/components/layout/NarrowPageLayout";
+import { StyledTabLink } from "@/components/ui/StyledTabLink";
+
+// workaround for https://github.com/vercel/next.js/issues/58776 - StyledTabLink won't work in server components
+export const AiAnalyticsClientLayout = ({ children }: PropsWithChildren) => {
+  return (
+    <NarrowPageLayout>
+      <div className="mb-4 flex items-center gap-2">
+        <StyledTabLink.List>
+          <StyledTabLink name="Statistics" href="/ai/analytics" />
+          <StyledTabLink name="Code Errors" href="/ai/analytics/code-errors" />
+          <StyledTabLink name="Step Errors" href="/ai/analytics/step-errors" />
+        </StyledTabLink.List>
+      </div>
+      <div>{children}</div>
+    </NarrowPageLayout>
+  );
+};

packages/hub/src/app/ai/analytics/StepErrorList.tsx

@@ -0,0 +1,40 @@
+import { FC, Fragment } from "react";
+
+import { H2 } from "@/components/ui/Headers";
+import { type StepError } from "@/server/ai/analytics";
+
+export const StepErrorList: FC<{
+  errors: StepError[];
+  title: string;
+  stats?: Record<string, number>;
+}> = ({ errors, title, stats = {} }) => {
+  return (
+    <div>
+      <H2>{title}</H2>
+      <strong>Total: {errors.length}</strong>
+      <div className="my-4 grid grid-cols-2 gap-x-4">
+        {Object.entries(stats).map(([type, count]) => (
+          <Fragment key={type}>
+            <div>{type}</div>
+            <strong>{count}</strong>
+          </Fragment>
+        ))}
+      </div>
+      <div className="space-y-4">
+        {errors.map((error, i) => (
+          <div key={i}>
+            <div className="text-sm">
+              <strong>{error.stepName}</strong>{" "}
+              <span className="text-gray-500">
+                ({error.date.toISOString()})
+              </span>
+            </div>
+            <pre key={i} className="whitespace-pre-wrap text-xs">
+              {error.error}
+            </pre>
+          </div>
+        ))}
+      </div>
+    </div>
+  );
+};

packages/hub/src/app/ai/analytics/code-errors/page.tsx

@@ -0,0 +1,69 @@
+import { getCodeErrors } from "@/server/ai/analytics";
+
+import { StepErrorList } from "../StepErrorList";
+
+const commonErrorTypes = {
+  "sTest is not defined": "sTest import",
+  'The "to" function only accepts paramaters above 0':
+    "`A to B` with nonpositive args",
+};
+
+const SYNTAX_ERROR = "Syntax errors";
+const SIGNATURE_ERROR = "Signature errors (total)";
+const LONG_ERROR = "Long errors";
+const NOT_DEFINED_ERROR = "Undefined variable";
+
+export default async function ({
+  searchParams,
+}: {
+  searchParams: { [key: string]: string | string[] | undefined };
+}) {
+  const errors = await getCodeErrors();
+
+  // pre-initialized to follow the right key order
+  const stats: Record<string, number> = Object.fromEntries(
+    Object.entries(commonErrorTypes).map(([, name]) => [name, 0])
+  );
+
+  const inc = (key: string) => (stats[key] = (stats[key] ?? 0) + 1);
+
+  stats[SYNTAX_ERROR] = 0;
+  stats[LONG_ERROR] = 0;
+  stats[NOT_DEFINED_ERROR] = 0;
+  stats[SIGNATURE_ERROR] = 0;
+
+  for (const error of errors) {
+    for (const [template, errorName] of Object.entries(commonErrorTypes)) {
+      if (error.error.includes(template)) {
+        inc(errorName);
+      }
+    }
+
+    {
+      const match = error.error.match(
+        /^Error: There are function matches for (\w+)\(/
+      );
+      if (match) {
+        inc(SIGNATURE_ERROR);
+        inc(`${match[1]} signature`);
+      }
+    }
+
+    if (error.error.length > 3000) {
+      inc(LONG_ERROR);
+    }
+
+    if (error.error.startsWith('Expected "')) {
+      inc(SYNTAX_ERROR);
+    }
+
+    if (
+      error.error.match(/^\S+ is not defined/) &&
+      !error.error.startsWith("sTest ")
+    ) {
+      inc(NOT_DEFINED_ERROR);
+    }
+  }
+
+  return <StepErrorList errors={errors} title="Code errors" stats={stats} />;
+}

packages/hub/src/app/ai/analytics/layout.tsx

@@ -0,0 +1,10 @@
+import { PropsWithChildren } from "react";
+
+import { checkRootUser } from "@/server/helpers";
+
+import { AiAnalyticsClientLayout } from "./ClientLayout";
+
+export default async function ({ children }: PropsWithChildren) {
+  await checkRootUser();
+  return <AiAnalyticsClientLayout>{children}</AiAnalyticsClientLayout>;
+}

packages/hub/src/app/ai/analytics/page.tsx

@@ -0,0 +1,25 @@
+import { Fragment } from "react";
+
+import { H2 } from "@/components/ui/Headers";
+import { getTypeStats } from "@/server/ai/analytics";
+
+export default async function () {
+  const typeStats = await getTypeStats();
+  return (
+    <div className="space-y-4">
+      {Object.entries(typeStats).map(([stepName, typeCounts]) => (
+        <div key={stepName} className="grid grid-cols-2 gap-4">
+          <div className="col-span-2">
+            <H2>{stepName}</H2>
+          </div>
+          {Object.entries(typeCounts).map(([type, count]) => (
+            <Fragment key={type}>
+              <span>{type}</span>
+              <span>{count}</span>
+            </Fragment>
+          ))}
+        </div>
+      ))}
+    </div>
+  );
+}

packages/hub/src/app/ai/analytics/step-errors/page.tsx

@@ -0,0 +1,20 @@
+import { getStepErrors } from "@/server/ai/analytics";
+
+import { StepErrorList } from "../StepErrorList";
+
+export default async function () {
+  const errors = await getStepErrors();
+
+  const stats: Record<string, number> = {};
+  stats["rate_limit"] = errors.filter((e) =>
+    e.error.includes("rate limit")
+  ).length;
+  stats["price_limit"] = errors.filter((e) =>
+    e.error.includes("Price limit")
+  ).length;
+  stats["search_and_replace"] = errors.filter((e) =>
+    e.error.includes("Search and Replace Failed")
+  ).length;
+
+  return <StepErrorList errors={errors} title="Step errors" stats={stats} />;
+}

packages/hub/src/components/SelectUser.tsx

@@ -26,18 +26,12 @@ const Query = graphql`
 export type SelectUserOption =
   SelectUserQuery$data["users"]["edges"][number]["node"];
 
-export function SelectUser<
-  TValues extends FieldValues,
-  TName extends FieldPathByValue<
-    TValues,
-    SelectUserOption | null
-  > = FieldPathByValue<TValues, SelectUserOption | null>,
->({
+export function SelectUser<TValues extends FieldValues>({
   name,
   label,
   required = true,
 }: {
-  name: TName;
+  name: FieldPathByValue<TValues, SelectUserOption | null>;
   label?: string;
   required?: boolean;
 }) {

packages/hub/src/components/ui/StyledTabLink.tsx

@@ -1,3 +1,4 @@
+"use client";
 import { usePathname } from "next/navigation";
 import React, { FC, ReactNode } from "react";
 

packages/hub/src/relay/loadPageQuery.ts

@@ -23,10 +23,11 @@ export async function loadPageQuery<TQuery extends OperationType = never>(
   node: ConcreteRequest,
   variables: VariablesOf<TQuery>
 ): Promise<SerializablePreloadedQuery<TQuery>> {
+  const cookie = (await headers()).get("cookie");
   const response = await networkFetch(
     node.params,
     variables,
-    headers().get("cookie") ?? undefined
+    cookie ?? undefined
   );
   return {
     params: node.params,

packages/hub/src/graphql/print-schema.ts → packages/hub/src/scripts/print-schema.ts

@@ -2,7 +2,7 @@ import { writeFileSync } from "fs";
 import { lexicographicSortSchema, printSchema } from "graphql";
 import path from "path";
 
-import { schema } from "./schema";
+import { schema } from "../graphql/schema";
 
 const schemaAsString = printSchema(lexicographicSortSchema(schema));