From f212d61cc50242f014e5559f35f02ff4667dc924 Mon Sep 17 00:00:00 2001 From: zianazhao Date: Mon, 30 Jan 2023 17:16:51 +0800 Subject: [PATCH 1/8] =?UTF-8?q?1=E3=80=81cbs=E2=80=94=E2=80=94=E6=9A=82?= =?UTF-8?q?=E6=97=B6=E8=BF=87=E6=BB=A4nvme=E7=9B=98=E7=B1=BB=E6=8C=87?= =?UTF-8?q?=E6=A0=87=202=E3=80=81tdmq=E2=80=94=E2=80=94=E6=9A=82=E6=97=B6?= =?UTF-8?q?=E5=8F=AA=E6=94=AF=E6=8C=81=20TDMQ=20RocketMQ=20=E7=89=88?= =?UTF-8?q?=E6=8C=87=E6=A0=87=203=E3=80=81loadbalance=E2=80=94=E2=80=94?= =?UTF-8?q?=E6=9A=82=E6=97=B6=E8=BF=87=E6=BB=A4=E9=9B=86=E7=BE=A4=E5=BC=80?= =?UTF-8?q?=E5=8F=91=E5=95=86=E7=BA=AC=E5=BA=A6=E7=B1=BB=E6=8C=87=E6=A0=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/collector/handler_cbs.go | 8 ++++++++ pkg/collector/handler_clb7.go | 8 ++++++++ pkg/collector/handler_tdmq.go | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/pkg/collector/handler_cbs.go b/pkg/collector/handler_cbs.go index 1114e8c..716a134 100644 --- a/pkg/collector/handler_cbs.go +++ b/pkg/collector/handler_cbs.go @@ -33,6 +33,14 @@ func (h *cbsHandler) GetNamespace() string { } func (h *cbsHandler) IsMetricVaild(m *metric.TcmMetric) bool { + // 暂时过滤nvme盘类指标 + var dimensions []string + for _, v := range m.Meta.SupportDimensions { + dimensions = append(dimensions, v) + } + if util.IsStrInList(dimensions, "vmUuid") { + return false + } return true } diff --git a/pkg/collector/handler_clb7.go b/pkg/collector/handler_clb7.go index 098c63a..48768c3 100644 --- a/pkg/collector/handler_clb7.go +++ b/pkg/collector/handler_clb7.go @@ -23,6 +23,14 @@ var ( "activeconnratio", "newactiveconnratio", "outtrafficratio", + "rsptimeout", + "setreqavg", + "setreqmax", + "setrspavg", + "setrspmax", + "settotalreq", + "vrsptimeout", + "vsettotalreq", } ) diff --git a/pkg/collector/handler_tdmq.go b/pkg/collector/handler_tdmq.go index b3834c2..95262e6 100644 --- a/pkg/collector/handler_tdmq.go +++ b/pkg/collector/handler_tdmq.go @@ -2,6 +2,7 @@ package collector import ( "fmt" + "strings" "time" "github.com/go-kit/log" @@ -17,6 +18,19 @@ const ( TdmqInstanceidKey = "tenant" ) +var ( + RocketMQOnlyIncludeMetrics = []string{ + "ropratein", + "roprateout", + "ropthroughputin", + "ropthroughputout", + "ropmsgbacklog", + "ropmsgaveragesize", + "ropinmessagetotal", + "ropgroupcount", + } +) + func init() { registerHandler(TdmqNamespace, defaultHandlerEnabled, NewTdmqHandler) } @@ -36,6 +50,10 @@ func (h *tdmqHandler) GetNamespace() string { } func (h *tdmqHandler) IsMetricVaild(m *metric.TcmMetric) bool { + // 暂时只支持 TDMQ RocketMQ 版指标 + if !util.IsStrInList(RocketMQOnlyIncludeMetrics, strings.ToLower(m.Meta.MetricName)) { + return false + } _, ok := excludeMetricName[m.Meta.MetricName] if ok { return false From 4151f06dead06d5a9b6c5e2363efbbdb6a27c091 Mon Sep 17 00:00:00 2001 From: zianazhao Date: Thu, 2 Feb 2023 10:41:48 +0800 Subject: [PATCH 2/8] fix-query time && time zone --- pkg/metric/metric.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index a120e4f..9031ed7 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -61,10 +61,11 @@ func (m *TcmMetric) GetLatestPromMetrics(repo TcmMetricRepository) (pms []promet et := int64(0) now := time.Now().Unix() if m.Conf.StatDelaySeconds > 0 { - st = now - m.Conf.StatPeriodSeconds - m.Conf.StatDelaySeconds + st = now - m.Conf.StatNumSamples*m.Conf.StatPeriodSeconds - m.Conf.StatDelaySeconds et = now - m.Conf.StatDelaySeconds } else { st = now - m.Conf.StatNumSamples*m.Conf.StatPeriodSeconds + et = now } samplesList, err := repo.ListSamples(m, st, et) From ef05e5c7e43850e6a647eac16bb211c51781b4df Mon Sep 17 00:00:00 2001 From: zianazhao Date: Thu, 2 Feb 2023 10:45:35 +0800 Subject: [PATCH 3/8] fix-query time && time zone --- pkg/metric/metric.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index a120e4f..9031ed7 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -61,10 +61,11 @@ func (m *TcmMetric) GetLatestPromMetrics(repo TcmMetricRepository) (pms []promet et := int64(0) now := time.Now().Unix() if m.Conf.StatDelaySeconds > 0 { - st = now - m.Conf.StatPeriodSeconds - m.Conf.StatDelaySeconds + st = now - m.Conf.StatNumSamples*m.Conf.StatPeriodSeconds - m.Conf.StatDelaySeconds et = now - m.Conf.StatDelaySeconds } else { st = now - m.Conf.StatNumSamples*m.Conf.StatPeriodSeconds + et = now } samplesList, err := repo.ListSamples(m, st, et) From b1930ae5d0fd87cacaa088ffc5a5224c32caa11d Mon Sep 17 00:00:00 2001 From: zianazhao Date: Thu, 2 Feb 2023 10:47:28 +0800 Subject: [PATCH 4/8] feat-lb_private supports custom queries --- pkg/collector/handler_clb_private.go | 40 ++++++++++++++++------------ 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/pkg/collector/handler_clb_private.go b/pkg/collector/handler_clb_private.go index 933a383..a6627c9 100644 --- a/pkg/collector/handler_clb_private.go +++ b/pkg/collector/handler_clb_private.go @@ -23,6 +23,10 @@ var ( } ) +var ( + LbPrivateSupportDimensions = []string{"vip", "vpcId", "loadBalancerPort", "protocol", "lanIp", "port"} +) + func init() { registerHandler(ClbPrivateNamespace, defaultHandlerEnabled, NewClbPrivateHandler) } @@ -119,31 +123,33 @@ func (h *ClbPrivateHandler) GetSeriesByAll(m *metric.TcmMetric) ([]*metric.TcmSe return slist, nil } -func (h *ClbPrivateHandler) GetSeriesByCustom(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { - var slist []*metric.TcmSeries +func (h *ClbPrivateHandler) GetSeriesByCustom(m *metric.TcmMetric) (slist []*metric.TcmSeries, err error) { for _, ql := range m.Conf.CustomQueryDimensions { - v, ok := ql[h.monitorQueryKey] - if !ok { - level.Error(h.logger).Log( - "msg", fmt.Sprintf("not found %s in queryDimensions", h.monitorQueryKey), - "ql", fmt.Sprintf("%v", ql)) - continue - } - ins, err := h.collector.InstanceRepo.Get(v) - if err != nil { - level.Error(h.logger).Log("msg", "Instance not found", "err", err, "id", v) + if !h.checkMonitorQueryKeys(m, ql) { continue } - sl, err := h.getSeriesByMetricType(m, ins) + s, err := metric.NewTcmSeries(m, ql, nil) if err != nil { - level.Error(h.logger).Log("msg", "Create metric series fail", - "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, + "ql", fmt.Sprintf("%v", ql)) continue } - slist = append(slist, sl...) + slist = append(slist, s) } - return slist, nil + return +} +func (h *ClbPrivateHandler) checkMonitorQueryKeys(m *metric.TcmMetric, ql map[string]string) bool { + for k := range ql { + if !util.IsStrInList(LbPrivateSupportDimensions, k) { + level.Error(h.logger).Log("msg", fmt.Sprintf("not found %s in supportQueryDimensions", k), + "ql", fmt.Sprintf("%v", ql), + "sd", fmt.Sprintf("%v", m.Meta.SupportDimensions), + ) + return false + } + } + return true } func (h *ClbPrivateHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { From f761455aa9d775439fd4d94c7396679813c5105b Mon Sep 17 00:00:00 2001 From: zianazhao Date: Thu, 2 Feb 2023 15:57:12 +0800 Subject: [PATCH 5/8] fix-timeZone --- pkg/util/time.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/util/time.go b/pkg/util/time.go index 6c7b31f..98327f4 100644 --- a/pkg/util/time.go +++ b/pkg/util/time.go @@ -6,10 +6,11 @@ import "time" func FormatTime(t time.Time, format string) string { var local time.Time _, offset := t.Zone() - if offset == 0 { - local = t.Add(8 * time.Hour) - } else { - local = t - } + // if offset == 0 { + // local = t.Add(8 * time.Hour) + // } else { + // local = t + // } + local = t.Add(time.Duration(8*3600-offset) * time.Second) return local.Format(format) } From c4da628060ebcb1725efa317bc952ef4a638708a Mon Sep 17 00:00:00 2001 From: zianazhao Date: Thu, 2 Feb 2023 15:57:33 +0800 Subject: [PATCH 6/8] fix-timeZone --- pkg/util/time.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/util/time.go b/pkg/util/time.go index 98327f4..e5bd6db 100644 --- a/pkg/util/time.go +++ b/pkg/util/time.go @@ -6,11 +6,6 @@ import "time" func FormatTime(t time.Time, format string) string { var local time.Time _, offset := t.Zone() - // if offset == 0 { - // local = t.Add(8 * time.Hour) - // } else { - // local = t - // } local = t.Add(time.Duration(8*3600-offset) * time.Second) return local.Format(format) } From 7e584544d3d6913735360741f2f1fb9d630298c3 Mon Sep 17 00:00:00 2001 From: zianazhao Date: Thu, 2 Feb 2023 16:01:09 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E5=AF=B9=E9=BD=90=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/collector/handler_cbs.go | 8 -------- pkg/collector/handler_clb7.go | 8 -------- pkg/collector/handler_tdmq.go | 18 ------------------ 3 files changed, 34 deletions(-) diff --git a/pkg/collector/handler_cbs.go b/pkg/collector/handler_cbs.go index 716a134..1114e8c 100644 --- a/pkg/collector/handler_cbs.go +++ b/pkg/collector/handler_cbs.go @@ -33,14 +33,6 @@ func (h *cbsHandler) GetNamespace() string { } func (h *cbsHandler) IsMetricVaild(m *metric.TcmMetric) bool { - // 暂时过滤nvme盘类指标 - var dimensions []string - for _, v := range m.Meta.SupportDimensions { - dimensions = append(dimensions, v) - } - if util.IsStrInList(dimensions, "vmUuid") { - return false - } return true } diff --git a/pkg/collector/handler_clb7.go b/pkg/collector/handler_clb7.go index 48768c3..098c63a 100644 --- a/pkg/collector/handler_clb7.go +++ b/pkg/collector/handler_clb7.go @@ -23,14 +23,6 @@ var ( "activeconnratio", "newactiveconnratio", "outtrafficratio", - "rsptimeout", - "setreqavg", - "setreqmax", - "setrspavg", - "setrspmax", - "settotalreq", - "vrsptimeout", - "vsettotalreq", } ) diff --git a/pkg/collector/handler_tdmq.go b/pkg/collector/handler_tdmq.go index 95262e6..b3834c2 100644 --- a/pkg/collector/handler_tdmq.go +++ b/pkg/collector/handler_tdmq.go @@ -2,7 +2,6 @@ package collector import ( "fmt" - "strings" "time" "github.com/go-kit/log" @@ -18,19 +17,6 @@ const ( TdmqInstanceidKey = "tenant" ) -var ( - RocketMQOnlyIncludeMetrics = []string{ - "ropratein", - "roprateout", - "ropthroughputin", - "ropthroughputout", - "ropmsgbacklog", - "ropmsgaveragesize", - "ropinmessagetotal", - "ropgroupcount", - } -) - func init() { registerHandler(TdmqNamespace, defaultHandlerEnabled, NewTdmqHandler) } @@ -50,10 +36,6 @@ func (h *tdmqHandler) GetNamespace() string { } func (h *tdmqHandler) IsMetricVaild(m *metric.TcmMetric) bool { - // 暂时只支持 TDMQ RocketMQ 版指标 - if !util.IsStrInList(RocketMQOnlyIncludeMetrics, strings.ToLower(m.Meta.MetricName)) { - return false - } _, ok := excludeMetricName[m.Meta.MetricName] if ok { return false From 029afb9512fe764aa3a7f28c5d3da2d321f1de7b Mon Sep 17 00:00:00 2001 From: zianazhao Date: Thu, 2 Feb 2023 16:06:27 +0800 Subject: [PATCH 8/8] =?UTF-8?q?=E5=AF=B9=E9=BD=90=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/collector/handler_clb_private.go | 41 ++++++++++++---------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/pkg/collector/handler_clb_private.go b/pkg/collector/handler_clb_private.go index a6627c9..44a9d57 100644 --- a/pkg/collector/handler_clb_private.go +++ b/pkg/collector/handler_clb_private.go @@ -23,10 +23,6 @@ var ( } ) -var ( - LbPrivateSupportDimensions = []string{"vip", "vpcId", "loadBalancerPort", "protocol", "lanIp", "port"} -) - func init() { registerHandler(ClbPrivateNamespace, defaultHandlerEnabled, NewClbPrivateHandler) } @@ -123,35 +119,32 @@ func (h *ClbPrivateHandler) GetSeriesByAll(m *metric.TcmMetric) ([]*metric.TcmSe return slist, nil } -func (h *ClbPrivateHandler) GetSeriesByCustom(m *metric.TcmMetric) (slist []*metric.TcmSeries, err error) { +func (h *ClbPrivateHandler) GetSeriesByCustom(m *metric.TcmMetric) ([]*metric.TcmSeries, error) { + var slist []*metric.TcmSeries for _, ql := range m.Conf.CustomQueryDimensions { - if !h.checkMonitorQueryKeys(m, ql) { + v, ok := ql[h.monitorQueryKey] + if !ok { + level.Error(h.logger).Log( + "msg", fmt.Sprintf("not found %s in queryDimensions", h.monitorQueryKey), + "ql", fmt.Sprintf("%v", ql)) continue } - - s, err := metric.NewTcmSeries(m, ql, nil) + ins, err := h.collector.InstanceRepo.Get(v) if err != nil { - level.Error(h.logger).Log("msg", "Create metric series fail", "metric", m.Meta.MetricName, - "ql", fmt.Sprintf("%v", ql)) + level.Error(h.logger).Log("msg", "Instance not found", "err", err, "id", v) continue } - slist = append(slist, s) - } - return -} -func (h *ClbPrivateHandler) checkMonitorQueryKeys(m *metric.TcmMetric, ql map[string]string) bool { - for k := range ql { - if !util.IsStrInList(LbPrivateSupportDimensions, k) { - level.Error(h.logger).Log("msg", fmt.Sprintf("not found %s in supportQueryDimensions", k), - "ql", fmt.Sprintf("%v", ql), - "sd", fmt.Sprintf("%v", m.Meta.SupportDimensions), - ) - return false + + sl, err := h.getSeriesByMetricType(m, ins) + if err != nil { + level.Error(h.logger).Log("msg", "Create metric series fail", + "metric", m.Meta.MetricName, "instacne", ins.GetInstanceId()) + continue } + slist = append(slist, sl...) } - return true + return slist, nil } - func (h *ClbPrivateHandler) getSeriesByMetricType(m *metric.TcmMetric, ins instance.TcInstance) ([]*metric.TcmSeries, error) { var dimensions []string for _, v := range m.Meta.SupportDimensions {