chartReleaseStatuses (#363)

sherlock/db/migrations/000063_ci_run_identifier_join_status.down.sql

@@ -0,0 +1,2 @@
+alter table ci_runs_for_identifiers
+    drop column if exists resource_status;

sherlock/db/migrations/000063_ci_run_identifier_join_status.up.sql

@@ -0,0 +1,2 @@
+alter table ci_runs_for_identifiers
+    add column if not exists resource_status text;

sherlock/go.mod

@@ -122,11 +122,12 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/crypto v0.15.0 // indirect
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.14.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.12.0 // indirect
+	golang.org/x/tools v0.15.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect

sherlock/go.sum

@@ -801,6 +801,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -828,6 +830,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1106,6 +1109,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
 golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
+golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8=
+golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

sherlock/internal/api/sherlock/ci_identifiers_v3.go

@@ -10,6 +10,9 @@ type CiIdentifierV3 struct {
 	CiRuns       []CiRunV3 `json:"ciRuns,omitempty" form:"-"`
 	ResourceType string    `json:"resourceType" form:"resourceType"`
 	ResourceID   uint      `json:"resourceID" form:"resourceID"`
+
+	// Available only when querying a CiIdentifier via a CiRun, indicates the status of the run for that resource
+	ResourceStatus *string `json:"resourceStatus,omitempty" form:"resourceStatus"`
 }
 
 func (c CiIdentifierV3) toModel() models.CiIdentifier {
@@ -29,10 +32,11 @@ func ciIdentifierFromModel(model models.CiIdentifier) CiIdentifierV3 {
 		}
 	}
 	return CiIdentifierV3{
-		CommonFields: commonFieldsFromGormModel(model.Model),
-		CiRuns:       ciRuns,
-		ResourceType: model.ResourceType,
-		ResourceID:   model.ResourceID,
+		CommonFields:   commonFieldsFromGormModel(model.Model),
+		CiRuns:         ciRuns,
+		ResourceType:   model.ResourceType,
+		ResourceID:     model.ResourceID,
+		ResourceStatus: model.ResourceStatus,
 	}
 }
 

sherlock/internal/api/sherlock/ci_identifiers_v3_get.go

@@ -52,6 +52,10 @@ func ciIdentifiersV3Get(ctx *gin.Context) {
 		errors.AbortRequest(ctx, err)
 		return
 	}
+	if err = result.FillCiRunResourceStatuses(db); err != nil {
+		errors.AbortRequest(ctx, err)
+		return
+	}
 	ctx.JSON(http.StatusOK, ciIdentifierFromModel(result))
 }
 

sherlock/internal/api/sherlock/ci_identifiers_v3_get_test.go

@@ -335,3 +335,16 @@ func (s *handlerSuite) TestCiIdentifiersV3GetLimitRuns() {
 	s.Equal(got.CiRuns[3].GithubActionsRunID, totalIterations-13)
 	s.Equal(got.CiRuns[4].GithubActionsRunID, totalIterations-14)
 }
+
+func (s *handlerSuite) TestCiIdentifiersV3Get_ResourceStatus() {
+	s.TestData.CiRun_Deploy_LeonardoDev_V1toV3()
+	var got CiIdentifierV3
+	code := s.HandleRequest(
+		s.NewRequest(http.MethodGet, "/api/ci-identifiers/v3/chart-release/dev/leonardo", nil),
+		&got)
+	s.Equal(http.StatusOK, code)
+	s.NotEmpty(got.CiRuns)
+	for _, cr := range got.CiRuns {
+		s.NotNil(cr.ResourceStatus)
+	}
+}

sherlock/internal/api/sherlock/ci_identifiers_v3_test.go

@@ -1,6 +1,7 @@
 package sherlock
 
 import (
+	"github.com/broadinstitute/sherlock/go-shared/pkg/utils"
 	"github.com/broadinstitute/sherlock/sherlock/internal/deprecated_models/v2models"
 	"github.com/broadinstitute/sherlock/sherlock/internal/models"
 	"github.com/stretchr/testify/assert"
@@ -112,19 +113,21 @@ func Test_ciIdentifierFromModel(t *testing.T) {
 						Time:  time.Now(),
 					},
 				},
-				ResourceType: "type",
-				ResourceID:   2,
-				CiRuns:       nil,
+				ResourceType:   "type",
+				ResourceID:     2,
+				CiRuns:         nil,
+				ResourceStatus: utils.PointerTo("some status"),
 			}},
 			want: CiIdentifierV3{
 				CommonFields: CommonFields{
 					ID:        1,
 					CreatedAt: t1,
 					UpdatedAt: t2,
 				},
-				CiRuns:       nil,
-				ResourceType: "type",
-				ResourceID:   2,
+				CiRuns:         nil,
+				ResourceType:   "type",
+				ResourceID:     2,
+				ResourceStatus: utils.PointerTo("some status"),
 			},
 		},
 		{

sherlock/internal/api/sherlock/ci_runs_v3.go

@@ -10,6 +10,9 @@ type CiRunV3 struct {
 	ciRunFields
 	TerminationHooksDispatchedAt *string          `json:"terminationHooksDispatchedAt,omitempty" form:"terminationHooksDispatchedAt" format:"date-time"`
 	RelatedResources             []CiIdentifierV3 `json:"relatedResources" form:"-"`
+
+	// Available only when querying a CiRun via a CiIdentifier, indicates the status of the run for that resource
+	ResourceStatus *string `json:"resourceStatus,omitempty" form:"resourceStatus"`
 }
 
 type ciRunFields struct {
@@ -78,5 +81,6 @@ func ciRunFromModel(model models.CiRun) CiRunV3 {
 		},
 		TerminationHooksDispatchedAt: model.TerminationHooksDispatchedAt,
 		RelatedResources:             relatedResources,
+		ResourceStatus:               model.ResourceStatus,
 	}
 }

sherlock/internal/api/sherlock/ci_runs_v3_get.go

@@ -37,6 +37,10 @@ func ciRunsV3Get(ctx *gin.Context) {
 		errors.AbortRequest(ctx, err)
 		return
 	}
+	if err = result.FillRelatedResourceStatuses(db); err != nil {
+		errors.AbortRequest(ctx, err)
+		return
+	}
 	ctx.JSON(http.StatusOK, ciRunFromModel(result))
 }
 

sherlock/internal/api/sherlock/ci_runs_v3_get_test.go

@@ -84,3 +84,20 @@ func (s *handlerSuite) TestCiRunsV3Get() {
 		s.Equal(argoCiRun.ID, got.ID)
 	})
 }
+
+func (s *handlerSuite) TestCiRunsV3Get_ResourceStatus() {
+	ciRun := s.TestData.CiRun_Deploy_LeonardoDev_V1toV3()
+	var got CiRunV3
+	code := s.HandleRequest(
+		s.NewRequest(http.MethodGet, fmt.Sprintf("/api/ci-runs/v3/%d", ciRun.ID), nil),
+		&got)
+	s.Equal(http.StatusOK, code)
+	s.Equal(ciRun.ID, got.ID)
+	resourcesWithStatus := 0
+	for _, rr := range got.RelatedResources {
+		if rr.ResourceStatus != nil {
+			resourcesWithStatus++
+		}
+	}
+	s.NotZero(resourcesWithStatus)
+}

sherlock/internal/api/sherlock/ci_runs_v3_test.go

@@ -171,6 +171,7 @@ func Test_ciRunFromModel(t *testing.T) {
 				TerminalAt:                 &t4,
 				Status:                     utils.PointerTo("status"),
 				RelatedResources:           nil,
+				ResourceStatus:             utils.PointerTo("different status"),
 			}},
 			want: CiRunV3{
 				CommonFields: CommonFields{
@@ -193,6 +194,7 @@ func Test_ciRunFromModel(t *testing.T) {
 					Status:                     utils.PointerTo("status"),
 				},
 				RelatedResources: nil,
+				ResourceStatus:   utils.PointerTo("different status"),
 			},
 		},
 		{

sherlock/internal/api/sherlock/ci_runs_v3_upsert.go

@@ -14,6 +14,7 @@ import (
 	"github.com/creasty/defaults"
 	"github.com/gin-gonic/gin"
 	"github.com/rs/zerolog/log"
+	"golang.org/x/exp/maps"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
 	"net/http"
@@ -30,6 +31,10 @@ type CiRunV3Upsert struct {
 	Environments  []string `json:"environments"`  // Always appends; will eliminate duplicates. Spreads to contained chart releases and their clusters.
 	ChartReleases []string `json:"chartReleases"` // Always appends; will eliminate duplicates. Spreads to associated environments and clusters.
 	Changesets    []string `json:"changesets"`    // Always appends; will eliminate duplicates. Spreads to associated chart releases, environments, and clusters.
+
+	// Keys treated like chartReleases. Values set resource-specific statuses for chart releases and associated changesets, new app versions, and new chart versions.
+	ChartReleaseStatuses map[string]string `json:"chartReleaseStatuses"`
+
 	// Makes entries in the changesets field also spread to new app versions and chart versions deployed by the changeset. 'when-static' is the default and does this spreading only when the chart release is in a static environment.
 	RelateToChangesetNewVersions string `json:"relateToChangesetNewVersions" enums:"always,when-static,never" default:"when-static" binding:"oneof=always when-static never ''"`
 	// If set to true, errors handling selectors for relations should be ignored. Normally, passing an unknown chart, cluster, etc. will abort the request, but they won't if this is true.
@@ -95,6 +100,10 @@ func ciRunsV3Upsert(ctx *gin.Context) {
 	// the body we got into one post-spread. Then we'll handle that body and de-dupe the resulting CiIdentifiers before
 	// adding to the database (the SQL gets messed up if there's duplicates in what we give to Gorm).
 
+	// As a bit of pre-processing, treat the keys of body.ChartReleaseStatuses like they were also passed in
+	// body.ChartReleases. We already dedupe later so this is harmless.
+	body.ChartReleases = append(body.ChartReleases, maps.Keys(body.ChartReleaseStatuses)...)
+
 	// First, a new body, starting from the old one.
 	bodyAfterSpreading := CiRunV3Upsert{
 		Charts:        body.Charts,
@@ -105,9 +114,14 @@ func ciRunsV3Upsert(ctx *gin.Context) {
 		ChartReleases: body.ChartReleases,
 		Changesets:    body.Changesets,
 	}
+
+	// We know we'll want to handle the ChartReleaseStatuses later by chart release ID,
+	// and we're about to happen to query them all, so we'll go ahead and build that mapping.
+	chartReleaseIDStatuses := make(map[uint]string, len(body.ChartReleaseStatuses))
+
 	// Environments in the original body should add all their chart releases to the new body, along with the clusters those
 	// chart releases belong to.
-	for _, environmentSelector := range body.Environments {
+	for _, environmentSelector := range utils.Dedupe(body.Environments) {
 		environmentID, err := v2models.InternalEnvironmentStore.ResolveSelector(db, environmentSelector)
 		if err != nil {
 			if body.IgnoreBadSelectors {
@@ -137,7 +151,7 @@ func ciRunsV3Upsert(ctx *gin.Context) {
 	}
 	// Same goes for clusters in the original body; we add their chart releases and any environments those chart releases
 	// belong to.
-	for _, clusterSelector := range body.Clusters {
+	for _, clusterSelector := range utils.Dedupe(body.Clusters) {
 		clusterID, err := v2models.InternalClusterStore.ResolveSelector(db, clusterSelector)
 		if err != nil {
 			if body.IgnoreBadSelectors {
@@ -167,7 +181,7 @@ func ciRunsV3Upsert(ctx *gin.Context) {
 	}
 	// Now for changesets in the original body. They spread to chart releases (and to environments/clusters from there) but they can also
 	// spread to new versions they deploy based on the RelateToChangesetNewVersions field.
-	for _, changesetSelector := range body.Changesets {
+	for _, changesetSelector := range utils.Dedupe(body.Changesets) {
 		changeset, err := v2models.InternalChangesetStore.GetBySelector(db, changesetSelector)
 		if err != nil {
 			if body.IgnoreBadSelectors {
@@ -208,9 +222,8 @@ func ciRunsV3Upsert(ctx *gin.Context) {
 		}
 		bodyAfterSpreading.ChartReleases = append(bodyAfterSpreading.ChartReleases, utils.UintToString(changeset.ChartReleaseID))
 	}
-	// Finally we handle the spreading of chart releases to their environment and cluster. We care about chart releases in the original
-	// body and also ones we just pulled from changesets above, so we concatenate those lists for the loop here.
-	for _, chartReleaseSelector := range body.ChartReleases {
+	// Finally we handle the spreading of chart releases to their environment and cluster.
+	for _, chartReleaseSelector := range utils.Dedupe(body.ChartReleases) {
 		chartRelease, err := v2models.InternalChartReleaseStore.GetBySelector(db, chartReleaseSelector)
 		if err != nil {
 			if body.IgnoreBadSelectors {
@@ -226,6 +239,9 @@ func ciRunsV3Upsert(ctx *gin.Context) {
 		if chartRelease.ClusterID != nil {
 			bodyAfterSpreading.Clusters = append(bodyAfterSpreading.Clusters, utils.UintToString(*chartRelease.ClusterID))
 		}
+		if status, present := body.ChartReleaseStatuses[chartReleaseSelector]; present {
+			chartReleaseIDStatuses[chartRelease.ID] = status
+		}
 	}
 
 	// With that, we've now handled the spread mechanic. bodyAfterSpreading probably has a ton of duplication, so we go out of our
@@ -386,6 +402,81 @@ addingToDeduplicatedRelatedResources:
 		}
 	}
 
+	// If we have any resource-specific statuses, add those
+	for chartReleaseID, status := range chartReleaseIDStatuses {
+		// We want to update the join table ci_runs_for_identifiers's resource_status for ci_identifiers where:
+		// 1. The identifier is for a "chart-release" matching our chart release ID
+		if err = db.Exec(
+			//language=SQL
+			`
+UPDATE ci_runs_for_identifiers
+SET resource_status = ?
+FROM ci_identifiers
+WHERE
+    ci_identifiers.id = ci_runs_for_identifiers.ci_identifier_id
+    AND ci_runs_for_identifiers.ci_run_id = ?
+    AND ci_identifiers.resource_type = 'chart-release' 
+    AND ci_identifiers.resource_id = ?
+`, status, result.ID, chartReleaseID).Error; err != nil {
+			slack.ReportError(ctx, fmt.Errorf("error recording chart release status in ci_runs_for_identifiers table for CiRun %d and ChartRelease %d: %w", result.ID, chartReleaseID, err))
+		}
+		// 2. The identifier is for a "changeset" where the changeset's chart release ID matches our chart release ID
+		//		(We have to do this one separately because this one needs a join, and the operation above shouldn't
+		//      be limited to identifiers that can join with changesets)
+		var changesetIDs []uint
+		if err = db.Raw(
+			//language=SQL
+			`
+UPDATE ci_runs_for_identifiers
+SET resource_status = ?
+FROM ci_identifiers
+    JOIN changesets ON changesets.id = ci_identifiers.resource_id
+WHERE
+    ci_identifiers.id = ci_runs_for_identifiers.ci_identifier_id
+    AND ci_runs_for_identifiers.ci_run_id = ?
+    AND ci_identifiers.resource_type = 'changeset' 
+    AND changesets.chart_release_id = ?
+RETURNING changesets.id
+`, status, result.ID, chartReleaseID).Scan(&changesetIDs).Error; err != nil {
+			slack.ReportError(ctx, fmt.Errorf("error recording changeset status in ci_runs_for_identifiers table for CiRun %d and ChartRelease %d: %w", result.ID, chartReleaseID, err))
+		}
+		for _, changesetID := range changesetIDs {
+			// If there were changesets from step 2:
+			// 3. The identifier is for a new app version on that changeset
+			if err = db.Exec(
+				//language=SQL
+				`
+UPDATE ci_runs_for_identifiers
+SET resource_status = ?
+FROM ci_identifiers
+    JOIN changeset_new_app_versions ON ci_identifiers.resource_id = changeset_new_app_versions.app_version_id
+WHERE
+    ci_runs_for_identifiers.ci_run_id = ?
+    AND ci_runs_for_identifiers.ci_identifier_id = ci_identifiers.id
+    AND ci_identifiers.resource_type = 'app-version'
+    AND changeset_new_app_versions.changeset_id = ?
+`, status, result.ID, changesetID).Error; err != nil {
+				slack.ReportError(ctx, fmt.Errorf("error recording app version status in ci_runs_for_identifiers table for CiRun %d and ChartRelease %d via Changeset %d: %w", result.ID, chartReleaseID, changesetID, err))
+			}
+			// 4. The identifier is for a new chart version on that changeset
+			if err = db.Exec(
+				//language=SQL
+				`
+UPDATE ci_runs_for_identifiers
+SET resource_status = ?
+FROM ci_identifiers
+    JOIN changeset_new_chart_versions ON ci_identifiers.resource_id = changeset_new_chart_versions.chart_version_id
+WHERE
+    ci_runs_for_identifiers.ci_run_id = ?
+    AND ci_runs_for_identifiers.ci_identifier_id = ci_identifiers.id
+    AND ci_identifiers.resource_type = 'chart-version'
+    AND changeset_new_chart_versions.changeset_id = ?
+`, status, result.ID, changesetID).Error; err != nil {
+				slack.ReportError(ctx, fmt.Errorf("error recording chart version status in ci_runs_for_identifiers table for CiRun %d and ChartRelease %d via Changeset %d: %w", result.ID, chartReleaseID, changesetID, err))
+			}
+		}
+	}
+
 	// If the request added any Slack channels for us to notify, record those
 	if len(body.NotifySlackChannelsUponSuccess) > 0 || len(body.NotifySlackChannelsUponFailure) > 0 {
 		var channelUpdates models.CiRun
@@ -417,6 +508,10 @@ addingToDeduplicatedRelatedResources:
 		errors.AbortRequest(ctx, err)
 		return
 	}
+	if err = result.FillRelatedResourceStatuses(db); err != nil {
+		errors.AbortRequest(ctx, err)
+		return
+	}
 
 	// If we said we were going to dispatch this workflow, check that our claim held and then
 	// do the dispatch