Merge branch 'main' into zj/partition_compute_expr

Cargo.toml

@@ -63,7 +63,7 @@ exclude = ["e2e_test/udf/wasm", "lints"]
 resolver = "2"
 
 [workspace.package]
-version = "1.5.0-alpha"
+version = "1.7.0-alpha"
 edition = "2021"
 homepage = "https://github.com/risingwavelabs/risingwave"
 keywords = ["sql", "database", "streaming"]

Makefile.toml

@@ -245,7 +245,7 @@ do
   fi
 done
 
-if (( "$(du -sk ${PREFIX_LOG} | cut -f1)" > 2000 )) ; then
+if (( "$(du -sk ${PREFIX_LOG} | cut -f1)" > 3000 )) ; then
     echo "$(tput setaf 1)ERROR: log size is significantly large ($(du -sh ${PREFIX_LOG} | cut -f1)).$(tput sgr0) Please disable unnecessary logs."
     exit 1
 fi

ci/scripts/gen-integration-test-yaml.py

@@ -37,8 +37,8 @@
     'starrocks-sink': ['json'],
     'deltalake-sink': ['json'],
     'pinot-sink': ['json'],
+    'presto-trino': ['json'],
     'client-library': ['none'],
-
 }
 
 def gen_pipeline_steps():

dashboard/components/CatalogModal.tsx

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2024 RisingWave Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import {
+  Button,
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+} from "@chakra-ui/react"
+
+import Link from "next/link"
+import { parseAsInteger, useQueryState } from "nuqs"
+import {
+  Relation,
+  relationIsStreamingJob,
+  relationTypeTitleCase,
+} from "../pages/api/streaming"
+import { ReactJson } from "./Relations"
+
+export function useCatalogModal(relationList: Relation[] | undefined) {
+  const [modalId, setModalId] = useQueryState("modalId", parseAsInteger)
+  const modalData = relationList?.find((r) => r.id === modalId)
+
+  return [modalData, setModalId] as const
+}
+
+export function CatalogModal({
+  modalData,
+  onClose,
+}: {
+  modalData: Relation | undefined
+  onClose: () => void
+}) {
+  return (
+    <Modal isOpen={modalData !== undefined} onClose={onClose} size="3xl">
+      <ModalOverlay />
+      <ModalContent>
+        <ModalHeader>
+          Catalog of {modalData && relationTypeTitleCase(modalData)}{" "}
+          {modalData?.id} - {modalData?.name}
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody>
+          {modalData && (
+            <ReactJson
+              src={modalData}
+              collapsed={1}
+              name={null}
+              displayDataTypes={false}
+            />
+          )}
+        </ModalBody>
+
+        <ModalFooter>
+          {modalData && relationIsStreamingJob(modalData) && (
+            <Button colorScheme="blue" mr={3}>
+              <Link href={`/fragment_graph/?id=${modalData.id}`}>
+                View Fragments
+              </Link>
+            </Button>
+          )}
+          <Button mr={3} onClick={onClose}>
+            Close
+          </Button>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  )
+}

dashboard/components/FragmentDependencyGraph.tsx

@@ -3,18 +3,18 @@ import * as d3 from "d3"
 import { Dag, DagLink, DagNode, zherebko } from "d3-dag"
 import { cloneDeep } from "lodash"
 import { useCallback, useEffect, useRef, useState } from "react"
-import { Position } from "../lib/layout"
+import { Enter, FragmentBox, Position } from "../lib/layout"
 
 const nodeRadius = 5
 const edgeRadius = 12
 
 export default function FragmentDependencyGraph({
-  mvDependency,
+  fragmentDependency,
   svgWidth,
   selectedId,
   onSelectedIdChange,
 }: {
-  mvDependency: Dag
+  fragmentDependency: Dag<FragmentBox>
   svgWidth: number
   selectedId: string | undefined
   onSelectedIdChange: (id: string) => void | undefined
@@ -24,21 +24,21 @@ export default function FragmentDependencyGraph({
   const MARGIN_X = 10
   const MARGIN_Y = 2
 
-  const mvDependencyDagCallback = useCallback(() => {
+  const fragmentDependencyDagCallback = useCallback(() => {
     const layout = zherebko().nodeSize([
       nodeRadius * 2,
       (nodeRadius + edgeRadius) * 2,
       nodeRadius,
     ])
-    const dag = cloneDeep(mvDependency)
+    const dag = cloneDeep(fragmentDependency)
     const { width, height } = layout(dag)
     return { width, height, dag }
-  }, [mvDependency])
+  }, [fragmentDependency])
 
-  const mvDependencyDag = mvDependencyDagCallback()
+  const fragmentDependencyDag = fragmentDependencyDagCallback()
 
   useEffect(() => {
-    const { width, height, dag } = mvDependencyDag
+    const { width, height, dag } = fragmentDependencyDag
 
     // This code only handles rendering
 
@@ -53,25 +53,27 @@ export default function FragmentDependencyGraph({
       .x(({ x }) => x + MARGIN_X)
       .y(({ y }) => y)
 
-    const isSelected = (d: any) => d.data.id === selectedId
+    const isSelected = (d: DagNode<FragmentBox>) => d.data.id === selectedId
 
     const edgeSelection = svgSelection
       .select(".edges")
-      .selectAll(".edge")
+      .selectAll<SVGPathElement, null>(".edge")
       .data(dag.links())
-    const applyEdge = (sel: any) =>
+    type EdgeSelection = typeof edgeSelection
+
+    const applyEdge = (sel: EdgeSelection) =>
       sel
         .attr("d", ({ points }: DagLink) => line(points))
         .attr("fill", "none")
-        .attr("stroke-width", (d: any) =>
+        .attr("stroke-width", (d) =>
           isSelected(d.source) || isSelected(d.target) ? 2 : 1
         )
-        .attr("stroke", (d: any) =>
+        .attr("stroke", (d) =>
           isSelected(d.source) || isSelected(d.target)
             ? theme.colors.blue["500"]
             : theme.colors.gray["300"]
         )
-    const createEdge = (sel: any) =>
+    const createEdge = (sel: Enter<EdgeSelection>) =>
       sel.append("path").attr("class", "edge").call(applyEdge)
     edgeSelection.exit().remove()
     edgeSelection.enter().call(createEdge)
@@ -80,19 +82,18 @@ export default function FragmentDependencyGraph({
     // Select nodes
     const nodeSelection = svgSelection
       .select(".nodes")
-      .selectAll(".node")
+      .selectAll<SVGCircleElement, null>(".node")
       .data(dag.descendants())
-    const applyNode = (sel: any) =>
+    type NodeSelection = typeof nodeSelection
+
+    const applyNode = (sel: NodeSelection) =>
       sel
-        .attr(
-          "transform",
-          ({ x, y }: Position) => `translate(${x + MARGIN_X}, ${y})`
-        )
-        .attr("fill", (d: any) =>
+        .attr("transform", (d) => `translate(${d.x! + MARGIN_X}, ${d.y})`)
+        .attr("fill", (d) =>
           isSelected(d) ? theme.colors.blue["500"] : theme.colors.gray["500"]
         )
 
-    const createNode = (sel: any) =>
+    const createNode = (sel: Enter<NodeSelection>) =>
       sel
         .append("circle")
         .attr("class", "node")
@@ -105,22 +106,23 @@ export default function FragmentDependencyGraph({
     // Add text to nodes
     const labelSelection = svgSelection
       .select(".labels")
-      .selectAll(".label")
+      .selectAll<SVGTextElement, null>(".label")
       .data(dag.descendants())
+    type LabelSelection = typeof labelSelection
 
-    const applyLabel = (sel: any) =>
+    const applyLabel = (sel: LabelSelection) =>
       sel
-        .text((d: any) => d.data.name)
+        .text((d) => d.data.name)
         .attr("x", svgWidth - MARGIN_X)
         .attr("font-family", "inherit")
         .attr("text-anchor", "end")
         .attr("alignment-baseline", "middle")
-        .attr("y", (d: any) => d.y)
-        .attr("fill", (d: any) =>
+        .attr("y", (d) => d.y!)
+        .attr("fill", (d) =>
           isSelected(d) ? theme.colors.black["500"] : theme.colors.gray["500"]
         )
         .attr("font-weight", "600")
-    const createLabel = (sel: any) =>
+    const createLabel = (sel: Enter<LabelSelection>) =>
       sel.append("text").attr("class", "label").call(applyLabel)
     labelSelection.exit().remove()
     labelSelection.enter().call(createLabel)
@@ -129,11 +131,12 @@ export default function FragmentDependencyGraph({
     // Add overlays
     const overlaySelection = svgSelection
       .select(".overlays")
-      .selectAll(".overlay")
+      .selectAll<SVGRectElement, null>(".overlay")
       .data(dag.descendants())
+    type OverlaySelection = typeof overlaySelection
 
     const STROKE_WIDTH = 3
-    const applyOverlay = (sel: any) =>
+    const applyOverlay = (sel: OverlaySelection) =>
       sel
         .attr("x", STROKE_WIDTH)
         .attr(
@@ -143,20 +146,13 @@ export default function FragmentDependencyGraph({
         .attr("width", svgWidth - STROKE_WIDTH * 2)
         .attr(
           "y",
-          (d: any) => d.y - nodeRadius - edgeRadius + MARGIN_Y + STROKE_WIDTH
+          (d) => d.y! - nodeRadius - edgeRadius + MARGIN_Y + STROKE_WIDTH
         )
         .attr("rx", 5)
         .attr("fill", theme.colors.gray["500"])
         .attr("opacity", 0)
         .style("cursor", "pointer")
-    const createOverlay = (
-      sel: d3.Selection<
-        d3.EnterElement,
-        DagNode<unknown, unknown>,
-        d3.BaseType,
-        unknown
-      >
-    ) =>
+    const createOverlay = (sel: Enter<OverlaySelection>) =>
       sel
         .append("rect")
         .attr("class", "overlay")
@@ -187,7 +183,7 @@ export default function FragmentDependencyGraph({
         })
         .on("click", function (d, i) {
           if (onSelectedIdChange) {
-            onSelectedIdChange((i.data as any).id)
+            onSelectedIdChange(i.data.id)
           }
         })
 
@@ -196,7 +192,13 @@ export default function FragmentDependencyGraph({
     overlaySelection.call(applyOverlay)
 
     setSvgHeight(`${height}px`)
-  }, [mvDependency, selectedId, svgWidth, onSelectedIdChange, mvDependencyDag])
+  }, [
+    fragmentDependency,
+    selectedId,
+    svgWidth,
+    onSelectedIdChange,
+    fragmentDependencyDag,
+  ])
 
   return (
     <svg ref={svgRef} width={`${svgWidth}px`} height={svgHeight}>

dashboard/components/FragmentGraph.tsx

@@ -17,11 +17,12 @@ import { cloneDeep } from "lodash"
 import { Fragment, useCallback, useEffect, useRef, useState } from "react"
 import {
   Edge,
+  Enter,
   FragmentBox,
   FragmentBoxPosition,
   Position,
-  generateBoxEdges,
-  layout,
+  generateFragmentEdges,
+  layoutItem,
 } from "../lib/layout"
 import { PlanNodeDatum } from "../pages/fragment_graph"
 import { StreamNode } from "../proto/gen/stream_plan"
@@ -36,10 +37,6 @@ type FragmentLayout = {
   actorIds: string[]
 } & Position
 
-type Enter<Type> = Type extends d3.Selection<any, infer B, infer C, infer D>
-  ? d3.Selection<d3.EnterElement, B, C, D>
-  : never
-
 function treeLayoutFlip<Datum>(
   root: d3.HierarchyNode<Datum>,
   { dx, dy }: { dx: number; dy: number }
@@ -145,7 +142,7 @@ export default function FragmentGraph({
       includedFragmentIds.add(fragmentId)
     }
 
-    const fragmentLayout = layout(
+    const fragmentLayout = layoutItem(
       fragmentDependencyDag.map(({ width: _1, height: _2, id, ...data }) => {
         const { width, height } = layoutFragmentResult.get(id)!
         return { width, height, id, ...data }
@@ -170,7 +167,7 @@ export default function FragmentGraph({
       svgHeight = Math.max(svgHeight, y + height + 50)
       svgWidth = Math.max(svgWidth, x + width)
     })
-    const edges = generateBoxEdges(fragmentLayout)
+    const edges = generateFragmentEdges(fragmentLayout)
 
     return {
       layoutResult,