feat(dashboard): fetch data and show charts from prometheus (risingwa…

…velabs#6602) * feat(dashboard): show node metrics on cluster page Signed-off-by: Bugen Zhao <[email protected]> * fill null values Signed-off-by: Bugen Zhao <[email protected]> * minor tweak Signed-off-by: Bugen Zhao <[email protected]> * fix Signed-off-by: Bugen Zhao <[email protected]> Signed-off-by: Bugen Zhao <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
cyliu0 · Nov 28, 2022 · 77bb000 · 77bb000
1 parent 02839f5
commit 77bb000
Show file tree

Hide file tree

Showing 17 changed files with 932 additions and 93 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/dashboard/mock-server.js b/dashboard/mock-server.js
@@ -53,3 +53,7 @@ app.get("/clusters/2", (req, res, next) => {
 app.get("/sources/", (req, res, next) => {
   res.json(require("./mock/sources.json"))
 })
+
+app.get("/metrics/cluster", (req, res, next) => {
+  res.json(require("./mock/metrics_cluster.json"))
+})
diff --git a/dashboard/mock/fetch.sh b/dashboard/mock/fetch.sh
@@ -10,3 +10,4 @@ curl http://localhost:5691/api/fragments > fragments.json
 curl http://localhost:5691/api/fragments2 > fragments2.json
 curl http://localhost:5691/api/materialized_views > materialized_views.json
 curl http://localhost:5691/api/sources > sources.json
+curl http://localhost:5691/api/metrics/cluster > metrics_cluster.json
diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json
diff --git a/dashboard/package.json b/dashboard/package.json
@@ -34,6 +34,7 @@
     "react-flow-renderer": "10.3.16",
     "react-json-view": "^1.21.3",
     "react-syntax-highlighter": "^15.5.0",
+    "recharts": "^2.1.16",
     "styled-components": "5.3.0",
     "ts-proto": "^1.131.0"
   },

diff --git a/dashboard/pages/api/cluster.ts b/dashboard/pages/api/cluster.ts
@@ -17,6 +17,11 @@
 import { WorkerNode } from "../../proto/gen/common"
 import api from "./api"
 
+export async function getClusterMetrics() {
+  const res = await api.get("/api/metrics/cluster")
+  return res
+}
+
 export async function getClusterInfoFrontend() {
   const res = (await api.get("/api/clusters/1")).map(WorkerNode.fromJSON)
   return res

diff --git a/dashboard/pages/cluster.tsx b/dashboard/pages/cluster.tsx
@@ -20,17 +20,22 @@ import {
   Grid,
   GridItem,
   HStack,
+  SimpleGrid,
   Text,
+  theme,
   useToast,
   VStack,
 } from "@chakra-ui/react"
+import { clone, reverse, sortBy } from "lodash"
 import Head from "next/head"
-import { Fragment, useEffect, useState } from "react"
+import { Fragment, useCallback, useEffect, useState } from "react"
+import { Area, AreaChart, ResponsiveContainer, XAxis, YAxis } from "recharts"
 import Title from "../components/Title"
 import { WorkerNode } from "../proto/gen/common"
 import {
   getClusterInfoComputeNode,
   getClusterInfoFrontend,
+  getClusterMetrics,
 } from "./api/cluster"
 
 function WorkerNodeComponent({
@@ -59,9 +64,93 @@ function WorkerNodeComponent({
     </Fragment>
   )
 }
+
+function WorkerNodeMetricsComponent({
+  job,
+  instance,
+  metrics,
+  isCpuMetrics,
+}: {
+  job: string
+  instance: string
+  metrics: MetricsSample[]
+  isCpuMetrics: boolean
+}) {
+  const metricsCallback = useCallback(() => {
+    const filledMetrics: MetricsSample[] = []
+    if (metrics.length === 0) {
+      return []
+    }
+    let lastTs: number = metrics.at(-1)!.timestamp
+    for (let pt of reverse(clone(metrics))) {
+      while (lastTs - pt.timestamp > 0) {
+        lastTs -= 60
+        filledMetrics.push({
+          timestamp: lastTs,
+          value: 0,
+        })
+      }
+      filledMetrics.push(pt)
+      lastTs -= 60
+    }
+    while (filledMetrics.length < 60) {
+      filledMetrics.push({ timestamp: lastTs, value: 0 })
+      lastTs -= 60
+    }
+    return reverse(filledMetrics)
+  }, [metrics])
+  return (
+    <Fragment>
+      <VStack alignItems="start" spacing={1}>
+        <Text textColor="gray.500" mx={3}>
+          <b>{job}</b> {instance}
+        </Text>
+
+        <ResponsiveContainer width="100%" height={100}>
+          <AreaChart data={metricsCallback()}>
+            <XAxis
+              dataKey="timestamp"
+              type="number"
+              domain={["dataMin", "dataMax"]}
+              hide={true}
+            />
+            {isCpuMetrics && (
+              <YAxis type="number" domain={[0, 1]} hide={true} />
+            )}
+            <Area
+              isAnimationActive={false}
+              type="linear"
+              dataKey="value"
+              strokeWidth={1}
+              stroke={theme.colors.teal["500"]}
+              fill={theme.colors.teal["100"]}
+            />
+          </AreaChart>
+        </ResponsiveContainer>
+      </VStack>
+    </Fragment>
+  )
+}
+
+interface MetricsSample {
+  timestamp: number
+  value: number
+}
+
+interface NodeMetrics {
+  metric: { [key: string]: string }
+  sample: MetricsSample[]
+}
+
+interface ClusterNodeMetrics {
+  cpuData: NodeMetrics[]
+  memoryData: NodeMetrics[]
+}
+
 export default function Cluster() {
   const [frontendList, setFrontendList] = useState<WorkerNode[]>([])
   const [computeNodeList, setComputeNodeList] = useState<WorkerNode[]>([])
+  const [metrics, setMetrics] = useState<ClusterNodeMetrics>()
   const toast = useToast()
 
   useEffect(() => {
@@ -84,16 +173,46 @@ export default function Cluster() {
     return () => {}
   }, [toast])
 
+  useEffect(() => {
+    async function doFetch() {
+      while (true) {
+        try {
+          let metrics: ClusterNodeMetrics = await getClusterMetrics()
+          metrics.cpuData = sortBy(metrics.cpuData, (m) => m.metric.instance)
+          metrics.memoryData = sortBy(
+            metrics.memoryData,
+            (m) => m.metric.instance
+          )
+          setMetrics(metrics)
+          await new Promise((resolve) => setTimeout(resolve, 5000)) // refresh every 5 secs
+        } catch (e: any) {
+          toast({
+            title: "Error Occurred",
+            description: e.toString(),
+            status: "error",
+            duration: 5000,
+            isClosable: true,
+          })
+          console.error(e)
+          break
+        }
+      }
+    }
+    doFetch()
+    return () => {}
+  }, [toast])
+
   const retVal = (
     <Box p={3}>
       <Title>Cluster Overview</Title>
-      <Grid templateColumns="repeat(3, 1fr)" gap={6} width="full">
+      <Grid my={3} templateColumns="repeat(3, 1fr)" gap={6} width="full">
         {frontendList.map((frontend) => (
           <GridItem
             w="full"
             rounded="xl"
             bg="white"
             shadow="md"
+            borderWidth={1}
             p={6}
             key={frontend.id}
           >
@@ -109,6 +228,7 @@ export default function Cluster() {
             rounded="xl"
             bg="white"
             shadow="md"
+            borderWidth={1}
             p={6}
             key={computeNode.id}
           >
@@ -119,6 +239,48 @@ export default function Cluster() {
           </GridItem>
         ))}
       </Grid>
+      <Title>CPU Usage</Title>
+      <SimpleGrid my={3} columns={3} spacing={6} width="full">
+        {metrics &&
+          metrics.cpuData.map((data) => (
+            <GridItem
+              w="full"
+              rounded="xl"
+              bg="white"
+              shadow="md"
+              borderWidth={1}
+              key={data.metric.instance}
+            >
+              <WorkerNodeMetricsComponent
+                job={data.metric.job}
+                instance={data.metric.instance}
+                metrics={data.sample}
+                isCpuMetrics={true}
+              />
+            </GridItem>
+          ))}
+      </SimpleGrid>
+      <Title>Memory Usage</Title>
+      <SimpleGrid my={3} columns={3} spacing={6} width="full">
+        {metrics &&
+          metrics.memoryData.map((data) => (
+            <GridItem
+              w="full"
+              rounded="xl"
+              bg="white"
+              shadow="md"
+              borderWidth={1}
+              key={data.metric.instance}
+            >
+              <WorkerNodeMetricsComponent
+                job={data.metric.job}
+                instance={data.metric.instance}
+                metrics={data.sample}
+                isCpuMetrics={false}
+              />
+            </GridItem>
+          ))}
+      </SimpleGrid>
     </Box>
   )
   return (

diff --git a/risedev.yml b/risedev.yml
@@ -669,6 +669,9 @@ template:
     # Enable sanity check when SSTs are committed. Disabled by default.
     enable-committed-sst-sanity-check: false
 
+    # Prometheus nodes used by dashboard service
+    provide-prometheus: "prometheus*"
+
   prometheus:
     # Advertise address of Prometheus
     address: "127.0.0.1"
@@ -842,7 +845,7 @@ template:
 
     # Kafka broker id. If there are multiple instances of Kafka, we will need to set.
     broker-id: 0
-  
+
   # Google pubsub emulator service
   pubsub:
     id: pubsub-${port}

diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml
@@ -34,6 +34,7 @@ num-traits = "0.2"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 paste = "1"
 prometheus = "0.13"
+prometheus-http-query = "0.6"
 prost = "0.11"
 rand = "0.8"
 reqwest = "0.11"