404 describe (#3104)

* Detect Describe
* Only check if you can paginate if the lake has a describe endpoint
* isGroupBy only works if you have the describe endpoint
* Sync lake properly when current lake changes
* Distinguish between updating and adding lakes
* Don't run uid correlations if there is no poolName preset in the session
* Update the plugins when there is no describe endpoint

apps/zui/src/app/lakes/root.tsx

@@ -8,8 +8,8 @@ import {updateStatus} from "src/js/flows/lake/update-status"
 import Current from "src/js/state/Current"
 import LakeStatuses from "src/js/state/LakeStatuses"
 import styled from "styled-components"
-import {initCurrentTab} from "src/js/flows/initCurrentTab"
 import {invoke} from "src/core/invoke"
+import {Active} from "src/models/active"
 
 const SpinnerWrap = styled.div`
   width: 100%;
@@ -25,8 +25,10 @@ export function InitLake({children}) {
   const status = useSelector(LakeStatuses.get(lake?.id))
 
   useLayoutEffect(() => {
-    if (status) return
-    if (lake) dispatch(updateStatus(lake.id))
+    if (Active.lake) {
+      dispatch(updateStatus(lake.id))
+      Active.lake.sync()
+    }
   }, [lake?.id, status])
 
   useEffect(() => {
@@ -35,7 +37,7 @@ export function InitLake({children}) {
 
   switch (status) {
     case "disconnected":
-      return <ConnectionError onRetry={() => dispatch(initCurrentTab())} />
+      return <ConnectionError onRetry={() => dispatch(updateStatus(lake.id))} />
     case "login-required":
       return <Login lake={lake} />
     case "connected":

apps/zui/src/app/menus/value-context-menu.ts

@@ -2,12 +2,14 @@ import * as zed from "@brimdata/zed-js"
 import {createMenu} from "src/core/menu"
 import ZuiApi from "src/js/api/zui-api"
 import QueryInfo from "src/js/state/QueryInfo"
+import {Active} from "src/models/active"
 
 function getWhenContext(api: ZuiApi, value: zed.Any) {
   return {
     isPrimitive: zed.isPrimitive(value),
     isIterable: zed.isIterable(value),
-    isGroupBy: api.select(QueryInfo.hasAggregation),
+    isGroupBy:
+      Active.lake.features.describe && api.select(QueryInfo.hasAggregation),
     selectedText: document.getSelection().toString() || null,
     isIp: value instanceof zed.Ip,
   }

apps/zui/src/components/toolbar-tabs.tsx

@@ -26,7 +26,6 @@ function cancelTimer() {
 
 function checkTimer() {
   const ellapsed = new Date().getTime() - time.getTime()
-  console.log(ellapsed)
   if (ellapsed > 10 * 1000) {
     throw new Error("Expected Scaling to Be Finished After 10 Seconds")
   }

apps/zui/src/core/main/main-object.ts

@@ -105,6 +105,7 @@ export class MainObject {
   }
 
   openUrl(uri: string) {
+    // This is written to handle an auth0 url
     const urlParts = url.parse(uri, true)
     const {code, state, error, error_description} = urlParts.query as {
       [key: string]: string

apps/zui/src/domain/editor/operations.ts

@@ -5,7 +5,7 @@ export const describe = createOperation(
   "editor.describe",
   async (ctx, string, pool?) => {
     try {
-      const resp = await lake.client.describeQuery(string, pool)
+      const resp = await lake.client.describeQuery(string, {pool})
       return resp.error ? {error: resp} : resp
     } catch (error) {
       return {error: error.toString()}

apps/zui/src/domain/results/handlers/export.ts

@@ -1,6 +1,6 @@
 import {addFuse, cutColumns} from "../utils"
 import Results from "src/js/state/Results"
-import {RESULTS_QUERY} from "src/views/results-pane/run-results-query"
+import {RESULTS_QUERY} from "src/views/results-pane/config"
 import {ResponseFormat} from "@brimdata/zed-js"
 import {errorToString} from "src/util/error-to-string"
 import {createHandler} from "src/core/handlers"

apps/zui/src/js/components/LakeModals/ViewLakeModal.tsx

@@ -74,12 +74,12 @@ const ViewLake = ({onClose, onEdit}) => {
         <button type="submit" onClick={onEdit} className="button submit">
           Edit
         </button>
-        {!isDefault && (
-          <div className={styles.logout}>
-            <Link onClick={onRemove}>Logout</Link>
-          </div>
-        )}
       </div>
+      {!isDefault && (
+        <div className={styles.logout}>
+          <Link onClick={onRemove}>Logout</Link>
+        </div>
+      )}
     </form>
   )
 }

apps/zui/src/js/flows/inspectSearch.ts

@@ -1,5 +1,5 @@
 import Results from "../state/Results"
-import {RESULTS_QUERY} from "src/views/results-pane/run-results-query"
+import {RESULTS_QUERY} from "src/views/results-pane/config"
 import {Thunk} from "../state/types"
 
 export const inspectSearch =

apps/zui/src/js/flows/lake/buildAndAuthenticateLake.ts

@@ -37,7 +37,7 @@ export const buildAndAuthenticateLake =
       const lake = await dispatch(buildLake(attrs, abortSignal))
 
       if (lake.authType === "none") {
-        dispatch(saveLake(lake, "connected"))
+        dispatch(saveLake(lake.attrs, "connected"))
         return [false, null]
       }
 

apps/zui/src/js/flows/lake/saveLake.ts

@@ -4,11 +4,18 @@ import LakeStatuses from "../../state/LakeStatuses"
 import {LakeStatus} from "../../state/LakeStatuses/types"
 import Window from "src/js/state/Window"
 import {LakeAttrs} from "src/js/state/Lakes/types"
+import {Lake} from "src/models/lake"
 
 export const saveLake =
   (attrs: LakeAttrs, status: LakeStatus) =>
-  (dispatch, _gs): void => {
-    dispatch(Lakes.add(attrs))
+  (dispatch): void => {
+    const {id} = attrs
+    const lake = Lake.find(id)
+    if (lake) {
+      lake.update(attrs)
+    } else {
+      dispatch(Lakes.add(attrs))
+    }
     dispatch(LakeStatuses.set(attrs.id, status))
     dispatch(Window.setLakeId(attrs.id))
     dispatch(syncPoolsData())

apps/zui/src/js/flows/lake/update-status.ts

@@ -25,7 +25,7 @@ export const updateStatus =
       try {
         // check version to test that zqd is available, update lake version while doing so
         const {version} = await zealot.version()
-        lakeModel.attrs.version = version
+        lakeModel.update({version})
         return false
       } catch (e) {
         console.error(e)
@@ -36,9 +36,6 @@ export const updateStatus =
 
     if (await isDown()) return
 
-    // update version
-    dispatch(Lakes.add(lakeModel.serialize()))
-
     // no auth required
     if (lakeModel.authType === "none") {
       activate()

apps/zui/src/js/initializers/initialize.ts

@@ -44,11 +44,7 @@ export default async function initialize(
   const api = new ZuiApi()
   const store = await initStore(api, renderer)
   const asyncTasks = initAsyncTasks(renderer)
-  await initGlobals(store)
-  await initLake(store)
-  api.init(store.dispatch, store.getState)
-  initDOM()
-  initIpcListeners(store)
+  initDomainModels({store})
   initHandlers({
     transition: startTransition,
     oldApi: api,
@@ -59,9 +55,13 @@ export default async function initialize(
     toast,
     asyncTasks,
   })
-  initDomainModels({
-    store,
-  })
+
+  await initGlobals(store)
+  await initLake(store)
+  api.init(store.dispatch, store.getState)
+  initDOM()
+  initIpcListeners(store)
+
   ViewHandler.store = store
   setMenuContext({select: (fn) => fn(store.getState()), api})
   initDebugGlobals(store, api)

apps/zui/src/js/state/Lakes/reducer.ts

@@ -13,6 +13,13 @@ const slice = createSlice({
       const id = action.payload
       delete state[id]
     },
+    update: (
+      state,
+      action: PayloadAction<{id: string; changes: Partial<LakeAttrs>}>
+    ) => {
+      const {id, changes} = action.payload
+      state[id] = {...state[id], ...changes}
+    },
     setAccessToken: (
       state,
       action: PayloadAction<{lakeId: string; accessToken: string}>

apps/zui/src/js/state/Lakes/types.ts

@@ -6,6 +6,9 @@ export type LakeAttrs = {
   version?: string
   authType: AuthType
   authData?: AuthData
+  features?: {
+    describe: boolean
+  }
 }
 
 export type AuthType = "none" | "auth0"

apps/zui/src/js/state/Layout/selectors.ts

@@ -1,6 +1,6 @@
 import {createSelector} from "@reduxjs/toolkit"
 import activeTabSelect from "../Tab/activeTabSelect"
-import {RESULTS_QUERY} from "src/views/results-pane/run-results-query"
+import {RESULTS_QUERY} from "src/views/results-pane/config"
 import {getShapes} from "../Results/selectors"
 
 const getResultsView = activeTabSelect((s) => s.layout.resultsView)

apps/zui/src/js/state/QueryInfo/reducer.ts

@@ -16,6 +16,9 @@ const slice = createSlice({
     set(_, action) {
       return action.payload
     },
+    merge(state, action) {
+      return {...state, ...action.payload}
+    },
     reset(_) {
       return getInitialState()
     },

apps/zui/src/js/state/Table/reducer.ts

@@ -7,7 +7,7 @@ import * as zed from "@brimdata/zed-js"
 import {TableSettingsState} from "./types"
 import {ColumnSizingInfoState} from "@tanstack/react-table"
 import {actions as results} from "../Results/reducer"
-import {RESULTS_QUERY} from "src/views/results-pane/run-results-query"
+import {RESULTS_QUERY} from "src/views/results-pane/config"
 
 const table = createSlice({
   name: "TAB_TABLE",

apps/zui/src/models/active.ts

@@ -6,6 +6,7 @@ import {BrowserTab} from "./browser-tab"
 import {Frame} from "./frame"
 import {getActiveTab} from "src/js/state/Tabs/selectors"
 import Editor from "src/js/state/Editor"
+import {Lake} from "./lake"
 
 export class Active extends DomainModel {
   static get tab() {
@@ -36,4 +37,9 @@ export class Active extends DomainModel {
       id: globalThis.windowId,
     })
   }
+
+  static get lake() {
+    const id = this.select(Current.getLakeId)
+    return Lake.find(id)
+  }
 }

apps/zui/src/models/lake.ts

@@ -1,7 +1,16 @@
 import {DomainModel} from "src/core/domain-model"
 import {LakeAttrs} from "../js/state/Lakes/types"
+import {Client} from "@brimdata/zed-js"
+import Slice from "src/js/state/Lakes"
+import {FeatureDetector} from "./lake/feature-detector"
 
 export class Lake extends DomainModel<LakeAttrs> {
+  static find(id: string) {
+    const attrs = this.select(Slice.id(id))
+    if (!attrs) return null
+    return new Lake(attrs)
+  }
+
   getAddress(): string {
     return this.attrs.port
       ? [this.attrs.host, this.attrs.port].join(":")
@@ -39,4 +48,36 @@ export class Lake extends DomainModel<LakeAttrs> {
   get version() {
     return this.attrs.version
   }
+
+  get client() {
+    return new Client(this.getAddress())
+  }
+
+  get features() {
+    return this.attrs.features ?? ({} as any)
+  }
+
+  async sync() {
+    // Also authenticate (later)
+    // Also check status (later)
+    const detector = new FeatureDetector(this)
+    const features = await detector.detect()
+    this.update({features})
+  }
+
+  update(changes: Partial<LakeAttrs>) {
+    this.attrs = {...this.attrs, ...changes}
+    this.dispatch(Slice.update({id: this.id, changes}))
+  }
+
+  save() {
+    this.dispatch(Slice.add({...this.attrs}))
+  }
+
+  isRunning() {
+    return this.client
+      .version()
+      .then(() => true)
+      .catch(() => false)
+  }
 }

apps/zui/src/models/lake/feature-detector.ts

@@ -0,0 +1,21 @@
+import {Lake} from "../lake"
+
+export class FeatureDetector {
+  constructor(public lake: Lake) {}
+
+  async hasDescribe() {
+    try {
+      const resp = await this.lake.client.describeQuery("from :pools")
+      return !resp.error
+    } catch (e) {
+      if (/404/.test(e.message)) return false
+      else throw e
+    }
+  }
+
+  async detect() {
+    return {
+      describe: await this.hasDescribe(),
+    }
+  }
+}

apps/zui/src/plugins/brimcap/zeek/correlations.ts

@@ -60,7 +60,7 @@ export function activateZeekCorrelations() {
   })
 
   correlations.create(UID_CORRELATION, {
-    when: () => !!findUid(session.selectedRow),
+    when: () => !!session.poolName && !!findUid(session.selectedRow),
     query: async () => {
       const uid = findUid(session.selectedRow)
       const pool = session.poolName

apps/zui/src/views/export-modal/state.ts

@@ -1,7 +1,7 @@
 import {useState} from "react"
 import {useSelector} from "react-redux"
 import Results from "src/js/state/Results"
-import {RESULTS_QUERY_COUNT} from "../results-pane/run-results-query"
+import {RESULTS_QUERY_COUNT} from "../results-pane/config"
 
 export function useExportModalState() {
   const countStatus = useSelector(Results.getStatus(RESULTS_QUERY_COUNT))

apps/zui/src/views/results-pane/config.ts

@@ -0,0 +1,2 @@
+export const RESULTS_QUERY = "zui-results/main"
+export const RESULTS_QUERY_COUNT = "zui-results/main-count"

apps/zui/src/views/results-pane/context.tsx

@@ -4,7 +4,7 @@ import {useNextPage} from "src/core/query/use-query"
 import {useResults} from "src/core/query/use-results"
 import Layout from "src/js/state/Layout"
 import Results from "src/js/state/Results"
-import {RESULTS_QUERY} from "src/views/results-pane/run-results-query"
+import {RESULTS_QUERY} from "src/views/results-pane/config"
 import {useDataTransition} from "src/util/hooks/use-data-transition"
 import useResizeObserver from "use-resize-observer"
 import QueryInfo from "src/js/state/QueryInfo"