Merge branch 'master' into Imporve-KeyViz-Data

cmd/tidb-dashboard/main.go

@@ -67,6 +67,7 @@ func NewCLIConfig() *DashboardCLIConfig {
 	flag.BoolVar(&cfg.CoreConfig.EnableTelemetry, "telemetry", cfg.CoreConfig.EnableTelemetry, "allow telemetry")
 	flag.BoolVar(&cfg.CoreConfig.EnableExperimental, "experimental", cfg.CoreConfig.EnableExperimental, "allow experimental features")
 	flag.StringVar(&cfg.CoreConfig.FeatureVersion, "feature-version", cfg.CoreConfig.FeatureVersion, "target TiDB version for standalone mode")
+	flag.IntVar(&cfg.CoreConfig.NgmTimeout, "ngm-timeout", cfg.CoreConfig.NgmTimeout, "timeout secs for accessing the ngm API")
 
 	showVersion := flag.BoolP("version", "v", false, "print version information and exit")
 

dockerfiles/alpine316.Dockerfile

@@ -34,7 +34,7 @@ RUN pnpm fetch
 # Build.
 WORKDIR /go/src/github.com/pingcap/tidb-dashboard
 COPY .. .
-RUN make package PNPM_INSTALL_TAGS=--offline
+RUN make package
 
 FROM alpine:3.16
 

dockerfiles/centos7.Dockerfile

@@ -43,7 +43,7 @@ RUN pnpm fetch
 # Build.
 WORKDIR /go/src/github.com/pingcap/tidb-dashboard
 COPY .. .
-RUN make package PNPM_INSTALL_TAGS=--offline
+RUN make package
 
 FROM centos:8
 

pkg/apiserver/slowquery/model.go

@@ -40,11 +40,10 @@ type Model struct {
 	TxnStartTS string `gorm:"column:Txn_start_ts" json:"txn_start_ts"`
 
 	// Detail
-	PrevStmt       string         `gorm:"column:Prev_stmt" json:"prev_stmt"`
-	Plan           string         `gorm:"column:Plan" json:"plan"` // deprecated, replaced by BinaryPlanText
-	BinaryPlan     string         `gorm:"column:Binary_plan" json:"binary_plan"`
-	BinaryPlanText string         `gorm:"column:Binary_plan_text" proj:"tidb_decode_binary_plan(Binary_plan)" json:"binary_plan_text"`
-	Warnings       datatypes.JSON `gorm:"column:Warnings" json:"warnings"`
+	PrevStmt   string         `gorm:"column:Prev_stmt" json:"prev_stmt"`
+	Plan       string         `gorm:"column:Plan" json:"plan"` // deprecated, replaced by BinaryPlanText
+	BinaryPlan string         `gorm:"column:Binary_plan" json:"binary_plan"`
+	Warnings   datatypes.JSON `gorm:"column:Warnings" json:"warnings"`
 
 	// Basic
 	IsInternal      int    `gorm:"column:Is_internal" json:"is_internal"`
@@ -96,6 +95,10 @@ type Model struct {
 	RocksdbBlockCacheHitCount uint `gorm:"column:Rocksdb_block_cache_hit_count" json:"rocksdb_block_cache_hit_count"`
 	RocksdbBlockReadCount     uint `gorm:"column:Rocksdb_block_read_count" json:"rocksdb_block_read_count"`
 	RocksdbBlockReadByte      uint `gorm:"column:Rocksdb_block_read_byte" json:"rocksdb_block_read_byte"`
+
+	// Computed fields
+	BinaryPlanJSON string `json:"binary_plan_json"` // binary plan json format
+	BinaryPlanText string `json:"binary_plan_text"` // binary plan plain text
 }
 
 type Field struct {

pkg/apiserver/slowquery/queries.go

@@ -109,19 +109,11 @@ func QuerySlowLogList(req *GetListRequest, sysSchema *utils.SysSchema, db *gorm.
 func QuerySlowLogDetail(req *GetDetailRequest, db *gorm.DB) (*Model, error) {
 	var result Model
 	err := db.
-		Select("*, (UNIX_TIMESTAMP(Time) + 0E0) AS timestamp, tidb_decode_binary_plan(Binary_plan) AS Binary_plan_text").
+		Select("*, (UNIX_TIMESTAMP(Time) + 0E0) AS timestamp").
 		Where("Digest = ?", req.Digest).
 		Where("Time = FROM_UNIXTIME(?)", req.Timestamp).
 		Where("Conn_id = ?", req.ConnectID).
 		First(&result).Error
-	if err != nil {
-		err = db.
-			Select("*, (UNIX_TIMESTAMP(Time) + 0E0) AS timestamp").
-			Where("Digest = ?", req.Digest).
-			Where("Time = FROM_UNIXTIME(?)", req.Timestamp).
-			Where("Conn_id = ?", req.ConnectID).
-			First(&result).Error
-	}
 	if err != nil {
 		return nil, err
 	}

pkg/apiserver/slowquery/service.go

@@ -100,14 +100,23 @@ func (s *Service) getDetails(c *gin.Context) {
 		return
 	}
 
-	// generate binary plan
+	// generate binary plan json
 	//
 	// Due to a kernel bug, the binary plan may fail to parse due to
 	// encoding issues. Additionally, since the binary plan field is
 	// not a required field, we can mask this error.
 	//
 	// See: https://github.com/pingcap/tidb-dashboard/issues/1515
-	result.BinaryPlan, _ = utils.GenerateBinaryPlanJSON(result.BinaryPlan)
+	if result.BinaryPlan != "" {
+		// may failed but it's ok
+		result.BinaryPlanText, _ = utils.GenerateBinaryPlanText(db, result.BinaryPlan)
+		// may failed but it's ok
+		result.BinaryPlanJSON, _ = utils.GenerateBinaryPlanJSON(result.BinaryPlan)
+
+		// reduce response size
+		result.BinaryPlan = ""
+		result.Plan = ""
+	}
 
 	c.JSON(http.StatusOK, *result)
 }

pkg/apiserver/statement/models.go

@@ -81,9 +81,8 @@ type Model struct {
 	AggIndexNames            string  `json:"index_names" agg:"ANY_VALUE(index_names)"`
 	AggPlanCount             int     `json:"plan_count" agg:"COUNT(DISTINCT plan_digest)" related:"plan_digest"`
 	AggPlan                  string  `json:"plan" agg:"ANY_VALUE(plan)"` // deprecated, replaced by BinaryPlanText
-	AggPlanDigest            string  `json:"plan_digest" agg:"ANY_VALUE(plan_digest)"`
 	AggBinaryPlan            string  `json:"binary_plan" agg:"ANY_VALUE(binary_plan)"`
-	AggBinaryPlanText        string  `json:"binary_plan_text" related:"binary_plan" agg:"tidb_decode_binary_plan(ANY_VALUE(binary_plan))"`
+	AggPlanDigest            string  `json:"plan_digest" agg:"ANY_VALUE(plan_digest)"`
 	AggPlanHint              *string `json:"plan_hint" agg:"ANY_VALUE(plan_hint)"`
 	// RocksDB
 	AggMaxRocksdbDeleteSkippedCount uint `json:"max_rocksdb_delete_skipped_count" agg:"MAX(max_rocksdb_delete_skipped_count)"`
@@ -99,6 +98,8 @@ type Model struct {
 	// Computed fields
 	RelatedSchemas string `json:"related_schemas"`
 	PlanCanBeBound bool   `json:"plan_can_be_bound"`
+	BinaryPlanJSON string `json:"binary_plan_json"`
+	BinaryPlanText string `json:"binary_plan_text"`
 }
 
 // tableNames example: "d1.a1,d2.a2,d1.a1,d3.a3"

pkg/apiserver/statement/service.go

@@ -44,27 +44,30 @@ func newService(p ServiceParams, ff *featureflag.Registry) *Service {
 
 func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 	endpoint := r.Group("/statements")
-	endpoint.Use(auth.MWAuthRequired())
-	endpoint.Use(utils.MWConnectTiDB(s.params.TiDBClient))
 	{
 		endpoint.GET("/download", s.downloadHandler)
-		endpoint.POST("/download/token", s.downloadTokenHandler)
 
-		endpoint.GET("/config", s.configHandler)
-		endpoint.POST("/config", auth.MWRequireWritePriv(), s.modifyConfigHandler)
-		endpoint.GET("/stmt_types", s.stmtTypesHandler)
-		endpoint.GET("/list", s.listHandler)
-		endpoint.GET("/plans", s.plansHandler)
-		endpoint.GET("/plan/detail", s.planDetailHandler)
-
-		endpoint.GET("/available_fields", s.getAvailableFields)
-
-		binding := endpoint.Group("/plan/binding")
-		binding.Use(s.planBindingFeatureFlag.VersionGuard())
+		endpoint.Use(auth.MWAuthRequired())
+		endpoint.Use(utils.MWConnectTiDB(s.params.TiDBClient))
 		{
-			binding.GET("", s.getPlanBindingHandler)
-			binding.POST("", s.createPlanBindingHandler)
-			binding.DELETE("", s.dropPlanBindingHandler)
+			endpoint.POST("/download/token", s.downloadTokenHandler)
+
+			endpoint.GET("/config", s.configHandler)
+			endpoint.POST("/config", auth.MWRequireWritePriv(), s.modifyConfigHandler)
+			endpoint.GET("/stmt_types", s.stmtTypesHandler)
+			endpoint.GET("/list", s.listHandler)
+			endpoint.GET("/plans", s.plansHandler)
+			endpoint.GET("/plan/detail", s.planDetailHandler)
+
+			endpoint.GET("/available_fields", s.getAvailableFields)
+
+			binding := endpoint.Group("/plan/binding")
+			binding.Use(s.planBindingFeatureFlag.VersionGuard())
+			{
+				binding.GET("", s.getPlanBindingHandler)
+				binding.POST("", s.createPlanBindingHandler)
+				binding.DELETE("", s.dropPlanBindingHandler)
+			}
 		}
 	}
 }
@@ -230,9 +233,16 @@ func (s *Service) planDetailHandler(c *gin.Context) {
 		return
 	}
 
-	// get binary plan
-	// may failed but it's ok
-	result.AggBinaryPlan, _ = utils.GenerateBinaryPlanJSON(result.AggBinaryPlan)
+	if result.AggBinaryPlan != "" {
+		// may failed but it's ok
+		result.BinaryPlanText, _ = utils.GenerateBinaryPlanText(db, result.AggBinaryPlan)
+		// may failed but it's ok
+		result.BinaryPlanJSON, _ = utils.GenerateBinaryPlanJSON(result.AggBinaryPlan)
+
+		// reduce response size
+		result.AggBinaryPlan = ""
+		result.AggPlan = ""
+	}
 
 	c.JSON(http.StatusOK, result)
 }

pkg/apiserver/utils/binary_plan.go

@@ -15,6 +15,7 @@ import (
 	"github.com/pingcap/tipb/go-tipb"
 	json "google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/runtime/protoimpl"
+	"gorm.io/gorm"
 )
 
 const (
@@ -975,3 +976,17 @@ func formatJSON(s string) (*simplejson.Json, error) {
 
 	return simplejson.NewJson([]byte(s))
 }
+
+/////////////////
+
+func GenerateBinaryPlanText(db *gorm.DB, b string) (string, error) {
+	type binaryPlanText struct {
+		Text string `gorm:"column:binary_plan_text"`
+	}
+	ret := &binaryPlanText{}
+	err := db.Raw(fmt.Sprintf("select tidb_decode_binary_plan('%s') as binary_plan_text", b)).Find(ret).Error
+	if err != nil {
+		return "", err
+	}
+	return ret.Text, err
+}

pkg/apiserver/utils/ngm.go

@@ -5,6 +5,8 @@ package utils
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"sync/atomic"
@@ -16,6 +18,7 @@ import (
 	"go.uber.org/fx"
 	"golang.org/x/sync/singleflight"
 
+	"github.com/pingcap/tidb-dashboard/pkg/config"
 	"github.com/pingcap/tidb-dashboard/pkg/utils/topology"
 	"github.com/pingcap/tidb-dashboard/util/rest"
 )
@@ -48,10 +51,14 @@ type NgmProxy struct {
 	etcdClient   *clientv3.Client
 	ngmReqGroup  singleflight.Group
 	ngmAddrCache atomic.Value
+	timeout      time.Duration
 }
 
-func NewNgmProxy(lc fx.Lifecycle, etcdClient *clientv3.Client) (*NgmProxy, error) {
-	s := &NgmProxy{etcdClient: etcdClient}
+func NewNgmProxy(lc fx.Lifecycle, etcdClient *clientv3.Client, config *config.Config) (*NgmProxy, error) {
+	s := &NgmProxy{
+		etcdClient: etcdClient,
+		timeout:    time.Duration(config.NgmTimeout) * time.Second,
+	}
 	lc.Append(fx.Hook{
 		OnStart: func(ctx context.Context) error {
 			s.lifecycleCtx = ctx
@@ -74,6 +81,18 @@ func (n *NgmProxy) Route(targetPath string) gin.HandlerFunc {
 
 		ngmURL, _ := url.Parse(ngmAddr)
 		proxy := httputil.NewSingleHostReverseProxy(ngmURL)
+		proxy.Transport = &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: defaultTransportDialContext(&net.Dialer{
+				Timeout:   n.timeout,
+				KeepAlive: n.timeout,
+			}),
+			ForceAttemptHTTP2:     true,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		}
 		proxy.ServeHTTP(c.Writer, c.Request)
 	}
 }
@@ -112,3 +131,7 @@ func (n *NgmProxy) resolveNgmAddress() (string, error) {
 	}
 	return "", ErrNgmNotStart.Wrap(err, "NgMonitoring component is not started")
 }
+
+func defaultTransportDialContext(dialer *net.Dialer) func(context.Context, string, string) (net.Conn, error) {
+	return dialer.DialContext
+}

pkg/config/config.go

@@ -30,6 +30,8 @@ type Config struct {
 	EnableTelemetry    bool
 	EnableExperimental bool
 	FeatureVersion     string // assign the target TiDB version when running TiDB Dashboard as standalone mode
+
+	NgmTimeout int // in seconds
 }
 
 func Default() *Config {
@@ -43,6 +45,7 @@ func Default() *Config {
 		EnableTelemetry:    false,
 		EnableExperimental: false,
 		FeatureVersion:     version.PDVersion,
+		NgmTimeout:         30, // s
 	}
 }
 

ui/package.json

@@ -4,7 +4,7 @@
   "version": "1.0.0",
   "license": "MIT",
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=16.16.0"
   },
   "scripts": {
     "fmt-check": "prettier --check .",

ui/packages/tidb-dashboard-client/src/client/api/models/slowquery-model.ts

@@ -39,7 +39,13 @@ export interface SlowqueryModel {
      */
     'binary_plan'?: string;
     /**
-     * 
+     * Computed fields
+     * @type {string}
+     * @memberof SlowqueryModel
+     */
+    'binary_plan_json'?: string;
+    /**
+     * binary plan plain text
      * @type {string}
      * @memberof SlowqueryModel
      */

ui/packages/tidb-dashboard-client/src/client/api/models/statement-model.ts

@@ -194,6 +194,12 @@ export interface StatementModel {
      * @memberof StatementModel
      */
     'binary_plan'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof StatementModel
+     */
+    'binary_plan_json'?: string;
     /**
      * 
      * @type {string}

ui/packages/tidb-dashboard-client/swagger/spec.json

@@ -4906,7 +4906,12 @@
                 "binary_plan": {
                     "type": "string"
                 },
+                "binary_plan_json": {
+                    "description": "Computed fields",
+                    "type": "string"
+                },
                 "binary_plan_text": {
+                    "description": "binary plan plain text",
                     "type": "string"
                 },
                 "commit_backoff_time": {
@@ -5314,6 +5319,9 @@
                 "binary_plan": {
                     "type": "string"
                 },
+                "binary_plan_json": {
+                    "type": "string"
+                },
                 "binary_plan_text": {
                     "type": "string"
                 },

ui/packages/tidb-dashboard-for-clinic-cloud/package.json

@@ -36,7 +36,7 @@
     "classnames": "^2.3.1",
     "compare-versions": "^5.0.1",
     "eventemitter2": "^6.4.5",
-    "i18next": "^21.6.11",
+    "i18next": "^23.2.9",
     "nprogress": "^0.2.0",
     "rc-animate": "^3.1.0",
     "react": "^17.0.2",

ui/packages/tidb-dashboard-for-clinic-cloud/src/dashboardApp/layout/main/Sider/Banner.tsx

@@ -3,8 +3,7 @@ import { CaretRightOutlined, CaretLeftOutlined } from '@ant-design/icons'
 import { useSize } from 'ahooks'
 import Flexbox from '@g07cha/flexbox-react'
 import { useSpring, animated } from 'react-spring'
-import { useTranslation } from 'react-i18next'
-import { TFunction } from 'i18next'
+import { useTranslation, TFunction } from 'react-i18next'
 
 import { InfoInfoResponse } from '~/client'
 

ui/packages/tidb-dashboard-for-clinic-op/package.json

@@ -27,7 +27,7 @@
     "@pingcap/tidb-dashboard-lib": "workspace:^1.0.0",
     "antd": "^4.18.7",
     "axios": "^0.27.2",
-    "i18next": "^21.6.11",
+    "i18next": "^23.2.9",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "react-router-dom": "6"

ui/packages/tidb-dashboard-for-dbaas/package.json

@@ -29,7 +29,7 @@
     "antd": "^4.18.7",
     "axios": "^0.27.2",
     "compare-versions": "^5.0.1",
-    "i18next": "^21.6.11",
+    "i18next": "^23.2.9",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "react-router-dom": "6"

ui/packages/tidb-dashboard-for-op/package.json

@@ -28,7 +28,7 @@
     "classnames": "^2.3.1",
     "compare-versions": "^5.0.1",
     "eventemitter2": "^6.4.5",
-    "i18next": "^21.6.11",
+    "i18next": "^23.2.9",
     "nprogress": "^0.2.0",
     "rc-animate": "^3.1.0",
     "react": "^17.0.2",