From fd1f289f7fc990631fc670182c7eddbeda2e6b06 Mon Sep 17 00:00:00 2001 From: Khurram Baig Date: Mon, 11 Nov 2024 09:42:05 +0530 Subject: [PATCH] Provide configurations to store labels and Annotations in summary This helps by enabling making query to Results Table which has less rows. Also, we can avoid toast query if PipelineRuns are too large. Also, we can create index on these. --- cmd/watcher/main.go | 4 +++ pkg/watcher/reconciler/config.go | 6 +++++ pkg/watcher/reconciler/dynamic/dynamic.go | 2 +- pkg/watcher/results/results.go | 30 +++++++++++++++++++++-- pkg/watcher/results/results_test.go | 10 +++++--- test/e2e/e2e_test.go | 6 +++-- 6 files changed, 50 insertions(+), 8 deletions(-) diff --git a/cmd/watcher/main.go b/cmd/watcher/main.go index 879d49d4e..ef1de7265 100644 --- a/cmd/watcher/main.go +++ b/cmd/watcher/main.go @@ -70,6 +70,8 @@ var ( labelSelector = flag.String("label_selector", "", "Selector (label query) to filter objects to be deleted. Matching objects must satisfy all labels requirements to be eligible for deletion") requeueInterval = flag.Duration("requeue_interval", 10*time.Minute, "How long the Watcher waits to reprocess keys on certain events (e.g. an object doesn't match the provided selectors)") namespace = flag.String("namespace", corev1.NamespaceAll, "Should the Watcher only watch a single namespace, then this value needs to be set to the namespace name otherwise leave it empty.") + summaryLabels = flag.String("summary_labels", "tekton.dev/pipeline", "List of Labels keys separated by comma which should be part of the summary of the result") + summaryAnnotations = flag.String("summary_annotations", "", "List of Annotations keys separated by comma which should be part of the summary of the result") checkOwner = flag.Bool("check_owner", true, "If enabled, owner references will be checked while deleting objects") updateLogTimeout = flag.Duration("update_log_timeout", 300*time.Second, "How log the Watcher waits for the UpdateLog operation for storing logs to complete before it aborts.") dynamicReconcileTimeout = flag.Duration("dynamic_reconcile_timeout", 30*time.Second, "How long the Watcher waits for the dynamic reconciler to complete before it aborts.") @@ -116,6 +118,8 @@ func main() { StoreDeadline: storeDeadline, ForwardBuffer: forwardBuffer, LogsTimestamps: *logsTimestamps, + SummaryLabels: *summaryLabels, + SummaryAnnotations: *summaryAnnotations, } log.Printf("dynamic reconcile timeout %s and update log timeout is %s", cfg.DynamicReconcileTimeout.String(), cfg.UpdateLogTimeout.String()) diff --git a/pkg/watcher/reconciler/config.go b/pkg/watcher/reconciler/config.go index c908075ee..3f6786e11 100644 --- a/pkg/watcher/reconciler/config.go +++ b/pkg/watcher/reconciler/config.go @@ -62,6 +62,12 @@ type Config struct { // Collect logs with timestamps LogsTimestamps bool + + // SummaryLabels are labels which should be part of the summary of the result + SummaryLabels string + + // SummaryAnnotations are annotations which should be part of the summary of the result + SummaryAnnotations string } // GetDisableAnnotationUpdate returns whether annotation updates should be diff --git a/pkg/watcher/reconciler/dynamic/dynamic.go b/pkg/watcher/reconciler/dynamic/dynamic.go index b093a217a..48e90eb4d 100644 --- a/pkg/watcher/reconciler/dynamic/dynamic.go +++ b/pkg/watcher/reconciler/dynamic/dynamic.go @@ -87,7 +87,7 @@ type AfterDeletion func(ctx context.Context, object results.Object) error // NewDynamicReconciler creates a new dynamic Reconciler. func NewDynamicReconciler(kubeClientSet kubernetes.Interface, rc pb.ResultsClient, lc pb.LogsClient, oc ObjectClient, cfg *reconciler.Config) *Reconciler { return &Reconciler{ - resultsClient: results.NewClient(rc, lc), + resultsClient: results.NewClient(rc, lc, cfg), KubeClientSet: kubeClientSet, objectClient: oc, cfg: cfg, diff --git a/pkg/watcher/results/results.go b/pkg/watcher/results/results.go index 4ebdf8dc9..b0f5da582 100644 --- a/pkg/watcher/results/results.go +++ b/pkg/watcher/results/results.go @@ -26,6 +26,7 @@ import ( "github.com/tektoncd/results/pkg/api/server/v1alpha2/record" "github.com/tektoncd/results/pkg/api/server/v1alpha2/result" "github.com/tektoncd/results/pkg/watcher/convert" + "github.com/tektoncd/results/pkg/watcher/reconciler" "github.com/tektoncd/results/pkg/watcher/reconciler/annotation" pb "github.com/tektoncd/results/proto/v1alpha2/results_go_proto" "google.golang.org/grpc" @@ -40,19 +41,26 @@ import ( "knative.dev/pkg/logging" ) +const ( + // objectName is used to store the name of the object in the result summary + objectName = "object.metadata.name" +) + // Client is a wrapper around a Results client that provides helpful utilities // for performing result operations that require multiple RPCs or data specific // operations. type Client struct { pb.ResultsClient pb.LogsClient + reconciler.Config } // NewClient returns a new results client for the particular kind. -func NewClient(resultsClient pb.ResultsClient, logsClient pb.LogsClient) *Client { +func NewClient(resultsClient pb.ResultsClient, logsClient pb.LogsClient, reconcilerConfig *reconciler.Config) *Client { return &Client{ ResultsClient: resultsClient, LogsClient: logsClient, + Config: *reconcilerConfig, } } @@ -123,7 +131,7 @@ func (c *Client) ensureResult(ctx context.Context, o Object, opts ...grpc.CallOp // Set the Result.Annotations and Result.Summary.Annotations fields if // the object in question contains the required annotations. - + res.Annotations = map[string]string{} if value, found := o.GetAnnotations()[annotation.ResultAnnotations]; found { resultAnnotations, err := parseAnnotations(annotation.ResultAnnotations, value) if err != nil { @@ -154,6 +162,24 @@ func (c *Client) ensureResult(ctx context.Context, o Object, opts ...grpc.CallOp } res.Summary.Annotations = annotations } + // Set the Result.Summary.Labels fields if the object in question contains the required labels. + summaryLabels := strings.Split(c.Config.SummaryLabels, ",") + if len(summaryLabels) > 0 && summaryLabels[0] != "" { + for _, v := range summaryLabels { + if value, found := o.GetLabels()[v]; found { + res.Annotations[v] = value + } + } + } + summaryAnnotations := strings.Split(c.Config.SummaryAnnotations, ",") + if len(summaryAnnotations) > 0 && summaryAnnotations[0] != "" { + for _, v := range summaryAnnotations { + if value, found := o.GetLabels()[v]; found { + res.Annotations[v] = value + } + } + } + res.Annotations[objectName] = o.GetName() } // Regardless of whether the object is a top level record or not, diff --git a/pkg/watcher/results/results_test.go b/pkg/watcher/results/results_test.go index 98ef24103..a2c1424b4 100644 --- a/pkg/watcher/results/results_test.go +++ b/pkg/watcher/results/results_test.go @@ -242,7 +242,7 @@ func TestEnsureResult(t *testing.T) { Kind: "TaskRun", }, ObjectMeta: metav1.ObjectMeta{ - Name: "taskrun", + Name: "foo", Namespace: "test", UID: "taskrun-id", }, @@ -253,7 +253,7 @@ func TestEnsureResult(t *testing.T) { Kind: "PipelineRun", }, ObjectMeta: metav1.ObjectMeta{ - Name: "pipelinerun", + Name: "foo", Namespace: "test", UID: "pipelinerun-id", }, @@ -280,6 +280,7 @@ func TestEnsureResult(t *testing.T) { Record: recordName(name, o), Type: convert.TypeName(o), }, + Annotations: map[string]string{"object.metadata.name": "foo"}, } if diff := cmp.Diff(want, create, protocmp.Transform(), protoutil.IgnoreResultOutputOnly()); diff != "" { t.Errorf("Create Result diff (-want, +got):\n%s", diff) @@ -305,6 +306,7 @@ func TestEnsureResult_RecordSummaryUpdate(t *testing.T) { pr := &pipelinev1.PipelineRun{ ObjectMeta: metav1.ObjectMeta{ + Name: "foo", Namespace: "default", UID: "1", }, @@ -339,6 +341,7 @@ func TestEnsureResult_RecordSummaryUpdate(t *testing.T) { Record: recordName(resultName(pr), pr), Type: convert.TypeName(pr), }, + Annotations: map[string]string{"object.metadata.name": "foo"}, } if diff := cmp.Diff(got, want, protocmp.Transform(), protoutil.IgnoreResultOutputOnly()); diff != "" { t.Fatal(diff) @@ -351,6 +354,7 @@ func TestAnnotations(t *testing.T) { pipelineRun := &pipelinev1.PipelineRun{ ObjectMeta: metav1.ObjectMeta{ + Name: "foo", Namespace: "default", Annotations: map[string]string{ annotation.ResultAnnotations: `{"x": "y", "i": 7}`, @@ -366,7 +370,7 @@ func TestAnnotations(t *testing.T) { } if diff := cmp.Diff(map[string]string{ - "x": "y", "i": "7", + "x": "y", "object.metadata.name": "foo", "i": "7", }, result.Annotations); diff != "" { t.Errorf("Result.Annotations: mismatch (-want +got):\n%s", diff) } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 55de7184a..e4c9b269c 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -308,8 +308,10 @@ func TestPipelineRun(t *testing.T) { t.Run("Result and RecordSummary Annotations were set accordingly", func(t *testing.T) { if diff := cmp.Diff(map[string]string{ - "repo": "tektoncd/results", - "commit": "1a6b908", + "repo": "tektoncd/results", + "object.metadata.name": "hello", + "commit": "1a6b908", + "tekton.dev/pipeline": "hello", }, result.Annotations); diff != "" { t.Errorf("Result.Annotations: mismatch (-want +got):\n%s", diff) }