From a84800dedf0013c300a05861a4ba3d4deb56124a Mon Sep 17 00:00:00 2001 From: cw-sakamoto Date: Thu, 30 Nov 2023 14:37:46 +0900 Subject: [PATCH 1/2] Add query env to datadog-agent --- cmd/datadog-agent/datadog-agent.go | 39 +++++++++++-------------- cmd/datadog-agent/datadog-agent_test.go | 35 +++++++--------------- 2 files changed, 27 insertions(+), 47 deletions(-) diff --git a/cmd/datadog-agent/datadog-agent.go b/cmd/datadog-agent/datadog-agent.go index b595064..6c11ea6 100644 --- a/cmd/datadog-agent/datadog-agent.go +++ b/cmd/datadog-agent/datadog-agent.go @@ -22,10 +22,10 @@ import ( type DatadogAgent struct { *cmd.Checker - ApiKey string - AppKey string - ClusterName string - WaitTime time.Duration + ApiKey string + AppKey string + QueryMetrics string + WaitTime time.Duration } func NewDatadogAgent(debug bool, logger func() *logrus.Entry, chatwork *notify.Chatwork) (*DatadogAgent, error) { @@ -36,7 +36,7 @@ func NewDatadogAgent(debug bool, logger func() *logrus.Entry, chatwork *notify.C apiKey := "" appKey := "" - clusterName := "" + queryMetrics := "" if v := os.Getenv("DD_API_KEY"); v != "" { apiKey = v @@ -44,8 +44,10 @@ func NewDatadogAgent(debug bool, logger func() *logrus.Entry, chatwork *notify.C if v := os.Getenv("DD_APP_KEY"); v != "" { appKey = v } - if v := os.Getenv("CLUSTER_NAME"); v != "" { - clusterName = v + + queryMetrics = "avg:kubernetes.cpu.user.total" + if v := os.Getenv("QUERY_METRICS"); v != "" { + queryMetrics = v } k8sclient, err := config.NewK8sClientset() @@ -55,11 +57,11 @@ func NewDatadogAgent(debug bool, logger func() *logrus.Entry, chatwork *notify.C } return &DatadogAgent{ - Checker: cmd.NewChecker(namespace, k8sclient, debug, logger, chatwork), - ApiKey: apiKey, - AppKey: appKey, - ClusterName: clusterName, - WaitTime: 3 * 60 * time.Second, + Checker: cmd.NewChecker(namespace, k8sclient, debug, logger, chatwork), + ApiKey: apiKey, + AppKey: appKey, + QueryMetrics: queryMetrics, + WaitTime: 3 * 60 * time.Second, }, nil } @@ -71,12 +73,6 @@ func (d *DatadogAgent) Check() error { d.Chatwork.AddMessage("DD_API_KEY or DD_APP_KEY is empty\n") return errors.New("DD_API_KEY or DD_APP_KEY is empty") } - if d.ClusterName == "" { - d.Logger().Error("CLUSTER_NAME is empty") - d.Chatwork.AddMessage("CLUSTER_NAME is empty\n") - return errors.New("CLUSTER_NAME is empty") - } - d.Chatwork.AddMessage("datadog-agent check start\n") if err := d.checkMetrics(); err != nil { @@ -101,17 +97,16 @@ func (d *DatadogAgent) checkMetrics() error { configuration := datadog.NewConfiguration() apiClient := datadog.NewAPIClient(configuration) api := datadogV1.NewMetricsApi(apiClient) - query := fmt.Sprintf("avg:kubernetes.cpu.user.total{env:%s}", d.ClusterName) d.Logger().Info("Waiting metrics...") time.Sleep(d.WaitTime) - d.Logger().Infof("Querying metrics with query: %s", query) - d.Chatwork.AddMessage(fmt.Sprintf("Querying metrics with query: %s", query)) + d.Logger().Infof("Querying metrics with query: %s", d.QueryMetrics) + d.Chatwork.AddMessage(fmt.Sprintf("Querying metrics with query: %s", d.QueryMetrics)) now := time.Now().Unix() from := now - 60*2 err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) { - resp, r, err := api.QueryMetrics(ddctx, from, now, query) + resp, r, err := api.QueryMetrics(ddctx, from, now, d.QueryMetrics) if err != nil { if r != nil && r.StatusCode == 403 { diff --git a/cmd/datadog-agent/datadog-agent_test.go b/cmd/datadog-agent/datadog-agent_test.go index bd33f7d..d9563f7 100644 --- a/cmd/datadog-agent/datadog-agent_test.go +++ b/cmd/datadog-agent/datadog-agent_test.go @@ -2,7 +2,6 @@ package datadogagent import ( "errors" - "os" "testing" "time" @@ -41,11 +40,11 @@ func TestCheck(t *testing.T) { chatwork := ¬ify.Chatwork{ApiToken: "token", RoomId: "test", Logger: logger} datadogAgent := &DatadogAgent{ - Checker: cmd.NewChecker("test", k8sclient, true, logger, chatwork), - ApiKey: "", - AppKey: "", - ClusterName: "", - WaitTime: 1 * time.Second, + Checker: cmd.NewChecker("test", k8sclient, true, logger, chatwork), + ApiKey: "", + AppKey: "", + QueryMetrics: "", + WaitTime: 1 * time.Second, } err = datadogAgent.Check() @@ -58,26 +57,12 @@ func TestCheck(t *testing.T) { t.Errorf("Expected '%s', got '%s'", expectedError.Error(), err.Error()) } - os.Setenv("DD_API_KEY", "test") - os.Setenv("DD_APP_KEY", "test") - datadogAgent, err = NewDatadogAgent(true, logger, chatwork) - if err != nil { - t.Fatalf("NewDatadogAgent: %s", err) - } - err = datadogAgent.Check() - expectedError = errors.New("CLUSTER_NAME is empty") - if err.Error() != expectedError.Error() { - t.Errorf("Expected '%s', got '%s'", expectedError.Error(), err.Error()) - } - - os.Setenv("CLUSTER_NAME", "test") - datadogAgent = &DatadogAgent{ - Checker: cmd.NewChecker("test", k8sclient, true, logger, chatwork), - ApiKey: "test", - AppKey: "test", - ClusterName: "test", - WaitTime: 1 * time.Second, + Checker: cmd.NewChecker("test", k8sclient, true, logger, chatwork), + ApiKey: "test", + AppKey: "test", + QueryMetrics: "avg:kubernetes.cpu.user.total", + WaitTime: 1 * time.Second, } err = datadogAgent.Check() From 01e81e9bf71b059baa4556cb50116fdaa7ce64b4 Mon Sep 17 00:00:00 2001 From: cw-sakamoto Date: Thu, 30 Nov 2023 14:38:22 +0900 Subject: [PATCH 2/2] send message to chatwork at s3 check fail --- cmd/fluent/fluent.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/fluent/fluent.go b/cmd/fluent/fluent.go index 962a5ce..e270583 100644 --- a/cmd/fluent/fluent.go +++ b/cmd/fluent/fluent.go @@ -165,6 +165,7 @@ func (f *Fluent) checkS3Object() error { if err != nil { f.Logger().Error("Timed out waiting for output S3 Object:", err) + f.Chatwork.AddMessage(fmt.Sprintf("Timed out waiting for output S3 Object: %s\n", err)) return err }