Merge branch 'main' of https://github.com/risingwavelabs/risingwave i…

…nto li0k/compaction_table_vnode

Cargo.toml

@@ -285,6 +285,7 @@ new_without_default = "allow"
 # duplicated_attributes = "allow"
 # TODO: remove later https://github.com/rust-lang/rust-clippy/issues/12436
 mixed_attributes_style = "allow"
+too_long_first_doc_paragraph = "allow"
 
 [workspace.lints.rustdoc]
 private_intra_doc_links = "allow"

ci/Dockerfile

@@ -77,7 +77,7 @@ RUN cargo binstall -y --locked --no-symlinks cargo-llvm-cov cargo-nextest cargo-
     && rm -rf "/root/.cargo/registry/cache" \
     && rm -rf "/root/.cargo/git/db" \
     && cargo uninstall cargo-cache
-RUN cargo install cargo-dylint@3.1.0 dylint-link@3.1.0
+RUN cargo install cargo-dylint@3.2.1 dylint-link@3.2.1
 
 # install risedev
 COPY <<EOF /root/.cargo/bin/risedev

ci/build-ci-image.sh

@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240911
+export BUILD_ENV_VERSION=v20241030
 
 export BUILD_TAG="public.ecr.aws/w1p7b4n3/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -72,7 +72,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241030
     depends_on:
       - mysql
       - sqlserver-server
@@ -86,7 +86,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241030
     depends_on:
       - mysql
       - db
@@ -109,7 +109,7 @@ services:
 
 
   rw-build-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241030
     volumes:
       - ..:/risingwave
 
@@ -122,7 +122,7 @@ services:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241030
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -133,7 +133,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241030
     depends_on:
       db:
         condition: service_healthy

ci/rust-toolchain

@@ -4,4 +4,4 @@
 # 3. (optional) **follow the instructions in lints/README.md** to update the toolchain and dependencies for lints
 
 [toolchain]
-channel = "nightly-2024-07-19"
+channel = "nightly-2024-10-11"

dashboard/components/Relations.tsx

@@ -35,7 +35,7 @@ import Title from "../components/Title"
 import useFetch from "../lib/api/fetch"
 import {
   Relation,
-  StreamingJob,
+  StreamingRelation,
   getDatabases,
   getSchemas,
   getUsers,
@@ -73,7 +73,7 @@ export const dependentsColumn: Column<Relation> = {
   ),
 }
 
-export const fragmentsColumn: Column<StreamingJob> = {
+export const fragmentsColumn: Column<StreamingRelation> = {
   name: "Fragments",
   width: 1,
   content: (r) => (

dashboard/lib/api/streaming.ts

@@ -15,6 +15,7 @@
  *
  */
 
+import { Expose, plainToInstance } from "class-transformer"
 import _ from "lodash"
 import sortBy from "lodash/sortBy"
 import {
@@ -53,14 +54,6 @@ export async function getRelationIdInfos(): Promise<RelationIdInfos> {
   return fragmentIds
 }
 
-export async function getFragments(): Promise<TableFragments[]> {
-  let fragmentList: TableFragments[] = (await api.get("/fragments2")).map(
-    TableFragments.fromJSON
-  )
-  fragmentList = sortBy(fragmentList, (x) => x.tableId)
-  return fragmentList
-}
-
 export interface Relation {
   id: number
   name: string
@@ -75,7 +68,43 @@ export interface Relation {
   databaseName?: string
 }
 
-export interface StreamingJob extends Relation {
+export class StreamingJob {
+  @Expose({ name: "jobId" })
+  id!: number
+  @Expose({ name: "objType" })
+  _objType!: string
+  name!: string
+  jobStatus!: string
+  @Expose({ name: "parallelism" })
+  _parallelism!: any
+  maxParallelism!: number
+
+  get parallelism() {
+    const parallelism = this._parallelism
+    if (typeof parallelism === "string") {
+      // `Adaptive`
+      return parallelism
+    } else if (typeof parallelism === "object") {
+      // `Fixed (64)`
+      let key = Object.keys(parallelism)[0]
+      let value = parallelism[key]
+      return `${key} (${value})`
+    } else {
+      // fallback
+      return JSON.stringify(parallelism)
+    }
+  }
+
+  get type() {
+    if (this._objType == "Table") {
+      return "Table / MV"
+    } else {
+      return this._objType
+    }
+  }
+}
+
+export interface StreamingRelation extends Relation {
   dependentRelations: number[]
 }
 
@@ -98,17 +127,15 @@ export function relationTypeTitleCase(x: Relation) {
   return _.startCase(_.toLower(relationType(x)))
 }
 
-export function relationIsStreamingJob(x: Relation): x is StreamingJob {
+export function relationIsStreamingJob(x: Relation): x is StreamingRelation {
   const type = relationType(x)
   return type !== "UNKNOWN" && type !== "SOURCE" && type !== "INTERNAL"
 }
 
 export async function getStreamingJobs() {
-  let jobs = _.concat<StreamingJob>(
-    await getMaterializedViews(),
-    await getTables(),
-    await getIndexes(),
-    await getSinks()
+  let jobs = plainToInstance(
+    StreamingJob,
+    (await api.get("/streaming_jobs")) as any[]
   )
   jobs = sortBy(jobs, (x) => x.id)
   return jobs

dashboard/package.json

@@ -24,6 +24,7 @@
     "@uidotdev/usehooks": "^2.4.1",
     "base64url": "^3.0.1",
     "bootstrap-icons": "^1.9.1",
+    "class-transformer": "^0.5.1",
     "d3": "^7.6.1",
     "d3-axis": "^3.0.0",
     "d3-dag": "^0.11.4",
@@ -42,6 +43,7 @@
     "react-json-view": "^1.21.3",
     "react-syntax-highlighter": "^15.5.0",
     "recharts": "^2.3.2",
+    "reflect-metadata": "^0.2.2",
     "styled-components": "5.3.0",
     "ts-proto": "^1.169.1"
   },

dashboard/pages/_app.tsx

@@ -15,6 +15,7 @@
  *
  */
 import "bootstrap-icons/font/bootstrap-icons.css"
+import "reflect-metadata"
 import "../styles/global.css"
 
 import { ChakraProvider } from "@chakra-ui/react"