From fa315c30b0aa20ca62433ced73c927b9a1c5a025 Mon Sep 17 00:00:00 2001 From: Gustavo Caldeira Date: Wed, 3 Apr 2024 16:55:18 +0200 Subject: [PATCH 1/2] Metrics from tags --- .gitignore | 1 + azure-devops-client/build.go | 71 +++++++++++++++++++++++++++++++++ config/opts.go | 4 ++ metrics_build.go | 77 ++++++++++++++++++++++++++++++++++++ misc.go | 9 +++++ 5 files changed, 162 insertions(+) diff --git a/.gitignore b/.gitignore index 7eeed65..0e9adb0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /release-assets /azure-devops-exporter *.exe +.vscode \ No newline at end of file diff --git a/azure-devops-client/build.go b/azure-devops-client/build.go index 23fdba7..d754ac8 100644 --- a/azure-devops-client/build.go +++ b/azure-devops-client/build.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/url" + "strings" "time" ) @@ -31,6 +32,17 @@ type TimelineRecordList struct { List []TimelineRecord `json:"records"` } +type TagList struct { + Count int `json:"count"` + List []string `json:"value"` +} + +type Tag struct { + Name string + Value string + Type string +} + type TimelineRecord struct { RecordType string `json:"type"` Name string `json:"name"` @@ -228,3 +240,62 @@ func (c *AzureDevopsClient) ListBuildTimeline(project string, buildID string) (l return } + +func (c *AzureDevopsClient) ListBuildTags(project string, buildID string) (list TagList, error error) { + defer c.concurrencyUnlock() + c.concurrencyLock() + + url := fmt.Sprintf( + "%v/_apis/build/builds/%v/tags", + url.QueryEscape(project), + url.QueryEscape(buildID), + ) + response, err := c.rest().R().Get(url) + if err := c.checkResponse(response, err); err != nil { + error = err + return + } + + err = json.Unmarshal(response.Body(), &list) + if err != nil { + error = err + return + } + + return +} + +func extractTagKeyValue(tag string) (string, string) { + parts := strings.Split(tag, "=") + return parts[0], parts[1] +} + +func extractTagSchema(tagSchema string) (string, string) { + parts := strings.Split(tagSchema, ":") + return parts[0], parts[1] +} + +func (t *TagList) Extract() (tags map[string]string) { + tags = make(map[string]string) + for _, t := range t.List { + k, v := extractTagKeyValue(t) + tags[k] = v + } + return +} + +func (t *TagList) Parse(tagSchema []string) (pTags []Tag) { + tags := t.Extract() + for _, ts := range tagSchema { + name, _type := extractTagSchema(ts) + value, isPresent := tags[name] + if isPresent { + pTags = append(pTags, Tag{ + Name: name, + Value: value, + Type: _type, + }) + } + } + return +} diff --git a/config/opts.go b/config/opts.go index 9929d81..656992a 100644 --- a/config/opts.go +++ b/config/opts.go @@ -53,6 +53,10 @@ type ( // query settings QueriesWithProjects []string `long:"list.query" env:"AZURE_DEVOPS_QUERIES" env-delim:" " description:"Pairs of query and project UUIDs in the form: '@'"` + + // tag settings + TagsSchema *[]string `long:"tags.schema" env:"AZURE_DEVOPS_TAG_SCHEMA" env-delim:" " description:"Tags to be extracted from builds in the format 'tagName:type' with following types: number, info, bool"` + TagsBuildDefinitionIdList *[]int64 `long:"tags.build.definition" env:"AZURE_DEVOPS_TAG_BUILD_DEFINITION" env-delim:" " description:"Build definition ids to query tags (IDs)"` } // cache settings diff --git a/metrics_build.go b/metrics_build.go index 41efdb6..81ccfbd 100644 --- a/metrics_build.go +++ b/metrics_build.go @@ -2,6 +2,7 @@ package main import ( "context" + "strconv" "strings" "time" @@ -25,6 +26,7 @@ type MetricsCollectorBuild struct { buildPhase *prometheus.GaugeVec buildJob *prometheus.GaugeVec buildTask *prometheus.GaugeVec + buildTag *prometheus.GaugeVec buildTimeProject *prometheus.SummaryVec jobTimeProject *prometheus.SummaryVec @@ -152,6 +154,23 @@ func (m *MetricsCollectorBuild) Setup(collector *collector.Collector) { ) m.Collector.RegisterMetricList("buildTask", m.prometheus.buildTask, true) + m.prometheus.buildTag = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "azure_devops_build_tag", + Help: "Azure DevOps build tags", + }, + []string{ + "projectID", + "buildID", + "buildDefinitionID", + "buildNumber", + "name", + "type", + "info", + }, + ) + m.Collector.RegisterMetricList("buildTag", m.prometheus.buildTag, true) + m.prometheus.buildDefinition = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "azure_devops_build_definition_info", @@ -180,6 +199,9 @@ func (m *MetricsCollectorBuild) Collect(callback chan<- func()) { m.collectDefinition(ctx, projectLogger, callback, project) m.collectBuilds(ctx, projectLogger, callback, project) m.collectBuildsTimeline(ctx, projectLogger, callback, project) + if nil != opts.AzureDevops.TagsSchema { + m.collectBuildsTags(ctx, projectLogger, callback, project) + } } } @@ -611,3 +633,58 @@ func (m *MetricsCollectorBuild) collectBuildsTimeline(ctx context.Context, logge } } } + +func (m *MetricsCollectorBuild) collectBuildsTags(ctx context.Context, logger *zap.SugaredLogger, callback chan<- func(), project devopsClient.Project) { + minTime := time.Now().Add(-opts.Limit.BuildHistoryDuration) + list, err := AzureDevopsClient.ListBuildHistoryWithStatus(project.Id, minTime, "completed") + if err != nil { + logger.Error(err) + return + } + + buildTag := m.Collector.GetMetricList("buildTag") + + for _, build := range list.List { + if nil == opts.AzureDevops.TagsBuildDefinitionIdList || arrayIntContains(*opts.AzureDevops.TagsBuildDefinitionIdList, build.Definition.Id) { + tagRecordList, _ := AzureDevopsClient.ListBuildTags(project.Id, int64ToString(build.Id)) + for _, tag := range tagRecordList.Parse(*opts.AzureDevops.TagsSchema) { + + switch tag.Type { + case "number": + value, _ := strconv.ParseFloat(tag.Value, 64) + buildTag.Add(prometheus.Labels{ + "projectID": project.Id, + "buildID": int64ToString(build.Id), + "buildDefinitionID": int64ToString(build.Definition.Id), + "buildNumber": build.BuildNumber, + "name": tag.Name, + "type": "number", + "info": "", + }, value) + case "bool": + value, _ := strconv.ParseBool(tag.Value) + buildTag.AddBool(prometheus.Labels{ + "projectID": project.Id, + "buildID": int64ToString(build.Id), + "buildDefinitionID": int64ToString(build.Definition.Id), + "buildNumber": build.BuildNumber, + "name": tag.Name, + "type": "number", + "info": "", + }, value) + case "info": + buildTag.AddInfo(prometheus.Labels{ + "projectID": project.Id, + "buildID": int64ToString(build.Id), + "buildDefinitionID": int64ToString(build.Definition.Id), + "buildNumber": build.BuildNumber, + "name": tag.Name, + "type": "number", + "info": tag.Value, + }) + } + + } + } + } +} diff --git a/misc.go b/misc.go index 7fff5e7..3ba8745 100644 --- a/misc.go +++ b/misc.go @@ -18,6 +18,15 @@ func arrayStringContains(s []string, e string) bool { return false } +func arrayIntContains(s []int64, e int64) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + func timeToFloat64(v time.Time) float64 { return float64(v.Unix()) } From fc816f22df8774a929e1cf954333a8a6ded7d7c7 Mon Sep 17 00:00:00 2001 From: Gustavo Caldeira Date: Thu, 4 Apr 2024 18:19:54 +0200 Subject: [PATCH 2/2] Error treatments --- azure-devops-client/build.go | 43 ++++++++++++++++++++++++++++-------- metrics_build.go | 11 ++++++--- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/azure-devops-client/build.go b/azure-devops-client/build.go index d754ac8..bcb08d9 100644 --- a/azure-devops-client/build.go +++ b/azure-devops-client/build.go @@ -265,29 +265,54 @@ func (c *AzureDevopsClient) ListBuildTags(project string, buildID string) (list return } -func extractTagKeyValue(tag string) (string, string) { +func extractTagKeyValue(tag string) (k string, v string, error error) { parts := strings.Split(tag, "=") - return parts[0], parts[1] + if len(parts) != 2 { + error = fmt.Errorf("could not extract key value pair from tag '%s'", tag) + return + } + k = parts[0] + v = parts[1] + return } -func extractTagSchema(tagSchema string) (string, string) { +func extractTagSchema(tagSchema string) (n string, t string, error error) { parts := strings.Split(tagSchema, ":") - return parts[0], parts[1] + if len(parts) != 2 { + error = fmt.Errorf("could not extract type from tag schema '%s'", tagSchema) + return + } + n = parts[0] + t = parts[1] + return } -func (t *TagList) Extract() (tags map[string]string) { +func (t *TagList) Extract() (tags map[string]string, error error) { tags = make(map[string]string) for _, t := range t.List { - k, v := extractTagKeyValue(t) + k, v, err := extractTagKeyValue(t) + if err != nil { + error = err + return + } tags[k] = v } return } -func (t *TagList) Parse(tagSchema []string) (pTags []Tag) { - tags := t.Extract() +func (t *TagList) Parse(tagSchema []string) (pTags []Tag, error error) { + tags, err := t.Extract() + if err != nil { + error = err + return + } for _, ts := range tagSchema { - name, _type := extractTagSchema(ts) + name, _type, err := extractTagSchema(ts) + if err != nil { + error = err + return + } + value, isPresent := tags[name] if isPresent { pTags = append(pTags, Tag{ diff --git a/metrics_build.go b/metrics_build.go index 81ccfbd..12fc98d 100644 --- a/metrics_build.go +++ b/metrics_build.go @@ -647,7 +647,12 @@ func (m *MetricsCollectorBuild) collectBuildsTags(ctx context.Context, logger *z for _, build := range list.List { if nil == opts.AzureDevops.TagsBuildDefinitionIdList || arrayIntContains(*opts.AzureDevops.TagsBuildDefinitionIdList, build.Definition.Id) { tagRecordList, _ := AzureDevopsClient.ListBuildTags(project.Id, int64ToString(build.Id)) - for _, tag := range tagRecordList.Parse(*opts.AzureDevops.TagsSchema) { + tagList, err := tagRecordList.Parse(*opts.AzureDevops.TagsSchema) + if err != nil { + m.Logger().Error(err) + continue + } + for _, tag := range tagList { switch tag.Type { case "number": @@ -669,7 +674,7 @@ func (m *MetricsCollectorBuild) collectBuildsTags(ctx context.Context, logger *z "buildDefinitionID": int64ToString(build.Definition.Id), "buildNumber": build.BuildNumber, "name": tag.Name, - "type": "number", + "type": "bool", "info": "", }, value) case "info": @@ -679,7 +684,7 @@ func (m *MetricsCollectorBuild) collectBuildsTags(ctx context.Context, logger *z "buildDefinitionID": int64ToString(build.Definition.Id), "buildNumber": build.BuildNumber, "name": tag.Name, - "type": "number", + "type": "info", "info": tag.Value, }) }