From 454e806ab55dab23eae8ccb5e5387c4b3675be48 Mon Sep 17 00:00:00 2001 From: Mikhail Fedosin Date: Sun, 20 Aug 2023 17:33:01 +0200 Subject: [PATCH] Enable OpenTelemetry tooling Inject opentelemetry calls in Reconcile functions Inject opentelemetry calls in SetupWithManager functions --- .../eks/controllers/eksconfig_controller.go | 18 +- controllers/awscluster_controller.go | 17 +- controllers/awsmachine_controller.go | 17 +- controllers/awsmanagedcluster_controller.go | 18 +- .../awsmanagedcontrolplane_controller.go | 16 +- .../awscontrolleridentity_controller.go | 19 +- exp/controllers/awsfargatepool_controller.go | 18 +- exp/controllers/awsmachinepool_controller.go | 18 +- .../awsmanagedmachinepool_controller.go | 16 +- .../awsinstancestate_controller.go | 21 +- go.mod | 25 ++- go.sum | 55 ++++-- main.go | 30 +++ pkg/ot/metrics.go | 39 ++++ pkg/ot/traces.go | 91 +++++++++ util/tele/composite_logger.go | 95 +++++++++ util/tele/corr_id.go | 94 +++++++++ util/tele/span_logger.go | 180 ++++++++++++++++++ util/tele/tele.go | 69 +++++++ 19 files changed, 813 insertions(+), 43 deletions(-) create mode 100644 pkg/ot/metrics.go create mode 100644 pkg/ot/traces.go create mode 100644 util/tele/composite_logger.go create mode 100644 util/tele/corr_id.go create mode 100644 util/tele/span_logger.go create mode 100644 util/tele/tele.go diff --git a/bootstrap/eks/controllers/eksconfig_controller.go b/bootstrap/eks/controllers/eksconfig_controller.go index 5c9a72b54a..c63607f04a 100644 --- a/bootstrap/eks/controllers/eksconfig_controller.go +++ b/bootstrap/eks/controllers/eksconfig_controller.go @@ -39,6 +39,7 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/v2/bootstrap/eks/internal/userdata" ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bsutil "sigs.k8s.io/cluster-api/bootstrap/util" expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" @@ -65,7 +66,14 @@ type EKSConfigReconciler struct { // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;delete; func (r *EKSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, rerr error) { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger( + ctx, + "bootstrap.eks.controllers.eksconfig.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("reconcileID", string(controller.ReconcileIDFromContext(ctx))), + ) + defer done() // get EKSConfig config := &eksbootstrapv1.EKSConfig{} @@ -290,10 +298,16 @@ func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1 } func (r *EKSConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, option controller.Options) error { + ctx, log, done := tele.StartSpanWithLogger(ctx, + "bootstrap.eks.controllers.EKSConfigReconciler.SetupWithManager", + tele.KVP("controller", "EKSConfig"), + ) + defer done() + b := ctrl.NewControllerManagedBy(mgr). For(&eksbootstrapv1.EKSConfig{}). WithOptions(option). - WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(logger.FromContext(ctx).GetLogger(), r.WatchFilterValue)). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log.GetLogger(), r.WatchFilterValue)). Watches( &source.Kind{Type: &clusterv1.Machine{}}, handler.EnqueueRequestsFromMapFunc(r.MachineToBootstrapMapFunc), diff --git a/controllers/awscluster_controller.go b/controllers/awscluster_controller.go index 73f8ad1e41..db7ea8ecea 100644 --- a/controllers/awscluster_controller.go +++ b/controllers/awscluster_controller.go @@ -51,6 +51,7 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/securitygroup" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" infrautilconditions "sigs.k8s.io/cluster-api-provider-aws/v2/util/conditions" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" capiannotations "sigs.k8s.io/cluster-api/util/annotations" @@ -131,7 +132,14 @@ func (r *AWSClusterReconciler) getSecurityGroupService(scope scope.ClusterScope) // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsclustercontrolleridentities,verbs=get;list;watch;create func (r *AWSClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger( + ctx, + "controllers.awscluster.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("reconcileID", string(controller.ReconcileIDFromContext(ctx))), + ) + defer done() // Fetch the AWSCluster instance awsCluster := &infrav1.AWSCluster{} @@ -346,7 +354,12 @@ func (r *AWSClusterReconciler) reconcileNormal(clusterScope *scope.ClusterScope) } func (r *AWSClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger(ctx, + "controllers.AWSClusterReconciler.SetupWithManager", + tele.KVP("controller", "AWSCluster"), + ) + defer done() + controller, err := ctrl.NewControllerManagedBy(mgr). WithOptions(options). For(&infrav1.AWSCluster{}). diff --git a/controllers/awsmachine_controller.go b/controllers/awsmachine_controller.go index 7629552240..bbc0a0c77c 100644 --- a/controllers/awsmachine_controller.go +++ b/controllers/awsmachine_controller.go @@ -55,6 +55,7 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/ssm" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/userdata" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/noderefutil" capierrors "sigs.k8s.io/cluster-api/errors" @@ -148,7 +149,14 @@ func (r *AWSMachineReconciler) getObjectStoreService(scope scope.S3Scope) servic // +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch func (r *AWSMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger( + ctx, + "controllers.awsmachine.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("reconcileID", string(controller.ReconcileIDFromContext(ctx))), + ) + defer done() // Fetch the AWSMachine instance. awsMachine := &infrav1.AWSMachine{} @@ -236,7 +244,12 @@ func (r *AWSMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) } func (r *AWSMachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger(ctx, + "controllers.AWSMachineReconciler.SetupWithManager", + tele.KVP("controller", "AWSMachine"), + ) + defer done() + AWSClusterToAWSMachines := r.AWSClusterToAWSMachines(log) controller, err := ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/awsmanagedcluster_controller.go b/controllers/awsmanagedcluster_controller.go index 4f93a9d8b9..d0c542227c 100644 --- a/controllers/awsmanagedcluster_controller.go +++ b/controllers/awsmanagedcluster_controller.go @@ -35,6 +35,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/annotations" @@ -56,7 +57,14 @@ type AWSManagedClusterReconciler struct { // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete func (r *AWSManagedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { - log := ctrl.LoggerFrom(ctx) + ctx, log, done := tele.StartSpanWithLogger( + ctx, + "controllers.awsmanagedcluster.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("reconcileID", string(controller.ReconcileIDFromContext(ctx))), + ) + defer done() // Fetch the AWSManagedCluster instance awsManagedCluster := &infrav1.AWSManagedCluster{} @@ -117,14 +125,18 @@ func (r *AWSManagedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Re } func (r *AWSManagedClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger(ctx, + "controllers.AWSManagedClusterReconciler.SetupWithManager", + tele.KVP("controller", "AWSManagedCluster"), + ) + defer done() awsManagedCluster := &infrav1.AWSManagedCluster{} controller, err := ctrl.NewControllerManagedBy(mgr). WithOptions(options). For(awsManagedCluster). - WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue)). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log.GetLogger(), r.WatchFilterValue)). Build(r) if err != nil { diff --git a/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go b/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go index 562b60413c..821f56e75b 100644 --- a/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go +++ b/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go @@ -50,6 +50,7 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/securitygroup" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" capiannotations "sigs.k8s.io/cluster-api/util/annotations" @@ -98,7 +99,11 @@ type AWSManagedControlPlaneReconciler struct { // SetupWithManager is used to setup the controller. func (r *AWSManagedControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger(ctx, + "controllers.AWSManagedControlPlaneReconciler.SetupWithManager", + tele.KVP("controller", "AWSManagedControlPlane"), + ) + defer done() awsManagedControlPlane := &ekscontrolplanev1.AWSManagedControlPlane{} c, err := ctrl.NewControllerManagedBy(mgr). @@ -146,7 +151,14 @@ func (r *AWSManagedControlPlaneReconciler) SetupWithManager(ctx context.Context, // Reconcile will reconcile AWSManagedControlPlane Resources. func (r *AWSManagedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, reterr error) { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger( + ctx, + "controllers.awsmanagedcontrolplane.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("reconcileID", string(controller.ReconcileIDFromContext(ctx))), + ) + defer done() // Get the control plane instance awsControlPlane := &ekscontrolplanev1.AWSManagedControlPlane{} diff --git a/exp/controlleridentitycreator/awscontrolleridentity_controller.go b/exp/controlleridentitycreator/awscontrolleridentity_controller.go index 443c1b91a2..b88da10be3 100644 --- a/exp/controlleridentitycreator/awscontrolleridentity_controller.go +++ b/exp/controlleridentitycreator/awscontrolleridentity_controller.go @@ -35,7 +35,7 @@ import ( ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" "sigs.k8s.io/cluster-api-provider-aws/v2/feature" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" - "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/tele" "sigs.k8s.io/cluster-api/util/predicates" ) @@ -50,7 +50,14 @@ type AWSControllerIdentityReconciler struct { // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsclustercontrolleridentities,verbs=get;list;watch;create func (r *AWSControllerIdentityReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger( + ctx, + "exp.controlleridentity.awscontrolleridentity.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("reconcileID", string(controller.ReconcileIDFromContext(ctx))), + ) + defer done() var identityRef *infrav1.AWSIdentityReference @@ -132,10 +139,16 @@ func (r *AWSControllerIdentityReconciler) Reconcile(ctx context.Context, req ctr } func (r *AWSControllerIdentityReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + _, log, done := tele.StartSpanWithLogger(ctx, + "controllers.AWSControllerIdentityReconciler.SetupWithManager", + tele.KVP("controller", "AWSControllerIdentity"), + ) + defer done() + controller := ctrl.NewControllerManagedBy(mgr). For(&infrav1.AWSCluster{}). WithOptions(options). - WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(logger.FromContext(ctx).GetLogger(), r.WatchFilterValue)) + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log.GetLogger(), r.WatchFilterValue)) if feature.Gates.Enabled(feature.EKS) { controller.Watches( diff --git a/exp/controllers/awsfargatepool_controller.go b/exp/controllers/awsfargatepool_controller.go index ac145ddda9..708e18f79c 100644 --- a/exp/controllers/awsfargatepool_controller.go +++ b/exp/controllers/awsfargatepool_controller.go @@ -36,6 +36,7 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/eks" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/conditions" @@ -53,11 +54,17 @@ type AWSFargateProfileReconciler struct { // SetupWithManager is used to setup the controller. func (r *AWSFargateProfileReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + ctx, log, done := tele.StartSpanWithLogger(ctx, + "exp.controllers.AWSFargateProfileReconciler.SetupWithManager", + tele.KVP("controller", "AWSFargateProfile"), + ) + defer done() + managedControlPlaneToFargateProfileMap := managedControlPlaneToFargateProfileMapFunc(r.Client, logger.FromContext(ctx)) return ctrl.NewControllerManagedBy(mgr). For(&expinfrav1.AWSFargateProfile{}). WithOptions(options). - WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(logger.FromContext(ctx).GetLogger(), r.WatchFilterValue)). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log.GetLogger(), r.WatchFilterValue)). Watches( &source.Kind{Type: &ekscontrolplanev1.AWSManagedControlPlane{}}, handler.EnqueueRequestsFromMapFunc(managedControlPlaneToFargateProfileMap), @@ -73,7 +80,14 @@ func (r *AWSFargateProfileReconciler) SetupWithManager(ctx context.Context, mgr // Reconcile reconciles AWSFargateProfiles. func (r *AWSFargateProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger( + ctx, + "exp.controllers.awsfargatepool.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("reconcileID", string(controller.ReconcileIDFromContext(ctx))), + ) + defer done() fargateProfile := &expinfrav1.AWSFargateProfile{} if err := r.Get(ctx, req.NamespacedName, fargateProfile); err != nil { diff --git a/exp/controllers/awsmachinepool_controller.go b/exp/controllers/awsmachinepool_controller.go index 3507f24906..4e7db781e9 100644 --- a/exp/controllers/awsmachinepool_controller.go +++ b/exp/controllers/awsmachinepool_controller.go @@ -47,6 +47,7 @@ import ( asg "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/autoscaling" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/ec2" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/util" @@ -88,7 +89,14 @@ func (r *AWSMachinePoolReconciler) getEC2Service(scope scope.EC2Scope) services. // Reconcile is the reconciliation loop for AWSMachinePool. func (r *AWSMachinePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger( + ctx, + "exp.controllers.awsmachinepool.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("reconcileID", string(controller.ReconcileIDFromContext(ctx))), + ) + defer done() // Fetch the AWSMachinePool . awsMachinePool := &expinfrav1.AWSMachinePool{} @@ -180,6 +188,12 @@ func (r *AWSMachinePoolReconciler) Reconcile(ctx context.Context, req ctrl.Reque } func (r *AWSMachinePoolReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + _, log, done := tele.StartSpanWithLogger(ctx, + "exp.controllers.AWSMachinePoolReconciler.SetupWithManager", + tele.KVP("controller", "AWSMachinePool"), + ) + defer done() + return ctrl.NewControllerManagedBy(mgr). WithOptions(options). For(&expinfrav1.AWSMachinePool{}). @@ -187,7 +201,7 @@ func (r *AWSMachinePoolReconciler) SetupWithManager(ctx context.Context, mgr ctr &source.Kind{Type: &expclusterv1.MachinePool{}}, handler.EnqueueRequestsFromMapFunc(machinePoolToInfrastructureMapFunc(expinfrav1.GroupVersion.WithKind("AWSMachinePool"))), ). - WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(logger.FromContext(ctx).GetLogger(), r.WatchFilterValue)). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log.GetLogger(), r.WatchFilterValue)). Complete(r) } diff --git a/exp/controllers/awsmanagedmachinepool_controller.go b/exp/controllers/awsmanagedmachinepool_controller.go index 8e54e49c5b..a88b20b3ff 100644 --- a/exp/controllers/awsmanagedmachinepool_controller.go +++ b/exp/controllers/awsmanagedmachinepool_controller.go @@ -42,6 +42,7 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/ec2" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/eks" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/util" @@ -63,7 +64,11 @@ type AWSManagedMachinePoolReconciler struct { // SetupWithManager is used to setup the controller. func (r *AWSManagedMachinePoolReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - log := logger.FromContext(ctx) + _, log, done := tele.StartSpanWithLogger(ctx, + "exp.controllers.AWSManagedMachinePoolReconciler.SetupWithManager", + tele.KVP("controller", "AWSManagedMachinePool"), + ) + defer done() gvk, err := apiutil.GVKForObject(new(expinfrav1.AWSManagedMachinePool), mgr.GetScheme()) if err != nil { @@ -93,7 +98,14 @@ func (r *AWSManagedMachinePoolReconciler) SetupWithManager(ctx context.Context, // Reconcile reconciles AWSManagedMachinePools. func (r *AWSManagedMachinePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { - log := logger.FromContext(ctx) + ctx, log, done := tele.StartSpanWithLogger( + ctx, + "exp.controllers.awsmanagedmachinepool.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("reconcileID", string(controller.ReconcileIDFromContext(ctx))), + ) + defer done() awsPool := &expinfrav1.AWSManagedMachinePool{} if err := r.Get(ctx, req.NamespacedName, awsPool); err != nil { diff --git a/exp/instancestate/awsinstancestate_controller.go b/exp/instancestate/awsinstancestate_controller.go index c04f8687c4..036b95fe63 100644 --- a/exp/instancestate/awsinstancestate_controller.go +++ b/exp/instancestate/awsinstancestate_controller.go @@ -38,7 +38,7 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/v2/controllers" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/instancestate" - "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/util/tele" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/cluster-api/util/predicates" ) @@ -77,12 +77,21 @@ func (r *AwsInstanceStateReconciler) getSQSService(region string) (sqsiface.SQSA } func (r *AwsInstanceStateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + ctx, log, done := tele.StartSpanWithLogger( + ctx, + "exp.instancestate.awsinstancestate.Reconcile", + tele.KVP("namespace", req.Namespace), + tele.KVP("name", req.Name), + tele.KVP("reconcileID", string(controller.ReconcileIDFromContext(ctx))), + ) + defer done() + // Fetch the AWSCluster instance awsCluster := &infrav1.AWSCluster{} err := r.Get(ctx, req.NamespacedName, awsCluster) if err != nil { if apierrors.IsNotFound(err) { - r.Log.Info("cluster not found, removing queue URL", "cluster", klog.KRef(req.Namespace, req.Name)) + log.Info("cluster not found, removing queue URL", "cluster", klog.KRef(req.Namespace, req.Name)) r.queueURLs.Delete(req.Name) return reconcile.Result{}, nil } @@ -111,13 +120,19 @@ func (r *AwsInstanceStateReconciler) Reconcile(ctx context.Context, req ctrl.Req } func (r *AwsInstanceStateReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + _, log, done := tele.StartSpanWithLogger(ctx, + "exp.controllers.AwsInstanceStateReconciler.SetupWithManager", + tele.KVP("controller", "AwsInstanceState"), + ) + defer done() + go func() { r.watchQueuesForInstanceEvents() }() return ctrl.NewControllerManagedBy(mgr). For(&infrav1.AWSCluster{}). WithOptions(options). - WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(logger.FromContext(ctx).GetLogger(), r.WatchFilterValue)). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log.GetLogger(), r.WatchFilterValue)). Complete(r) } diff --git a/go.mod b/go.mod index b9b0777c4c..eedca4a0fc 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f github.com/google/gofuzz v1.2.0 + github.com/google/uuid v1.3.0 github.com/onsi/ginkgo/v2 v2.12.1 github.com/onsi/gomega v1.27.10 github.com/pkg/errors v0.9.1 @@ -26,6 +27,12 @@ require ( github.com/sergi/go-diff v1.3.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 + go.opentelemetry.io/otel v1.17.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.17.0 + go.opentelemetry.io/otel/exporters/prometheus v0.40.0 + go.opentelemetry.io/otel/sdk v1.17.0 + go.opentelemetry.io/otel/sdk/metric v0.40.0 + go.opentelemetry.io/otel/trace v1.17.0 golang.org/x/crypto v0.13.0 golang.org/x/text v0.13.0 gopkg.in/yaml.v2 v2.4.0 @@ -57,6 +64,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coredns/caddy v1.1.0 // indirect github.com/coredns/corefile-migration v1.0.20 // indirect @@ -75,6 +83,7 @@ require ( github.com/fatih/color v1.15.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect @@ -91,7 +100,7 @@ require ( github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.13 // indirect @@ -120,7 +129,7 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect - github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/rivo/uniseg v0.4.2 // indirect @@ -137,20 +146,24 @@ require ( github.com/subosito/gotenv v1.4.2 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0 // indirect + go.opentelemetry.io/otel/metric v1.17.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/net v0.14.0 // indirect - golang.org/x/oauth2 v0.6.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 90f4e7f889..348b52dd93 100644 --- a/go.sum +++ b/go.sum @@ -94,6 +94,8 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -183,6 +185,8 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -205,7 +209,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -299,6 +303,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmg github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -463,8 +469,8 @@ github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lF github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= @@ -480,7 +486,7 @@ github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -542,8 +548,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= @@ -574,13 +580,31 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/otel v1.17.0 h1:MW+phZ6WZ5/uk2nd93ANk/6yJ+dVrvNWUjGhnnFU5jM= +go.opentelemetry.io/otel v1.17.0/go.mod h1:I2vmBGtFaODIVMBSTPVDlJSzBDNf93k60E6Ft0nyjo0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0 h1:U5GYackKpVKlPrd/5gKMlrTlP2dCESAAFU682VCpieY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0/go.mod h1:aFsJfCEnLzEu9vRRAcUiB/cpRTbVsNdF3OHSPpdjxZQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.17.0 h1:iGeIsSYwpYSvh5UGzWrJfTDJvPjrXtxl3GUppj6IXQU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.17.0/go.mod h1:1j3H3G1SBYpZFti6OI4P0uRQCW20MXkG5v4UWXppLLE= +go.opentelemetry.io/otel/exporters/prometheus v0.40.0 h1:9h6lCssr1j5aYVvWT6oc+ERB6R034zmsHjBRLyxrAR8= +go.opentelemetry.io/otel/exporters/prometheus v0.40.0/go.mod h1:5USWZ0ovyQB5CIM3IO3bGRSoDPMXiT3t+15gu8Zo9HQ= +go.opentelemetry.io/otel/metric v1.17.0 h1:iG6LGVz5Gh+IuO0jmgvpTB6YVrCGngi8QGm+pMd8Pdc= +go.opentelemetry.io/otel/metric v1.17.0/go.mod h1:h4skoxdZI17AxwITdmdZjjYJQH5nzijUUjm+wtPph5o= +go.opentelemetry.io/otel/sdk v1.17.0 h1:FLN2X66Ke/k5Sg3V623Q7h7nt3cHXaW1FOvKKrW0IpE= +go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ= +go.opentelemetry.io/otel/sdk/metric v0.40.0 h1:qOM29YaGcxipWjL5FzpyZDpCYrDREvX0mVlmXdOjCHU= +go.opentelemetry.io/otel/sdk/metric v0.40.0/go.mod h1:dWxHtdzdJvg+ciJUKLTKwrMe5P6Dv3FyDbh8UkfgkVs= +go.opentelemetry.io/otel/trace v1.17.0 h1:/SWhSRHmDPOImIAetP1QAeMnZYiQXrTy4fMMYOdSKWQ= +go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7aHeaz0ZBLWjY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= @@ -695,8 +719,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -921,8 +945,11 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -942,8 +969,8 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -957,8 +984,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index dc727d4826..16b4082a7b 100644 --- a/main.go +++ b/main.go @@ -59,6 +59,7 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/endpoints" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/ot" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/record" "sigs.k8s.io/cluster-api-provider-aws/v2/version" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -101,6 +102,8 @@ var ( webhookCertDir string healthAddr string serviceEndpoints string + enableTracing bool + telemetryServer string // maxEKSSyncPeriod is the maximum allowed duration for the sync-period flag when using EKS. It is set to 10 minutes // because during resync it will create a new AWS auth token which can a maximum life of 15 minutes and this ensures @@ -198,6 +201,19 @@ func main() { os.Exit(1) } + if enableTracing { + setupLog.Info("enabling tracing") + if err := ot.RegisterTracing(ctx, setupLog, telemetryServer); err != nil { + setupLog.Error(err, "unable to initialize tracing") + os.Exit(1) + } + } + + if err := ot.RegisterMetrics(); err != nil { + setupLog.Error(err, "unable to initialize metrics") + os.Exit(1) + } + setupReconcilersAndWebhooks(ctx, mgr, awsServiceEndpoints, externalResourceGC, alternativeGCStrategy) if feature.Gates.Enabled(feature.EKS) { setupEKSReconcilersAndWebhooks(ctx, mgr, awsServiceEndpoints, externalResourceGC, alternativeGCStrategy, waitInfraPeriod) @@ -517,6 +533,20 @@ func initFlags(fs *pflag.FlagSet) { fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel), ) + fs.BoolVar( + &enableTracing, + "enable-tracing", + false, + "Enable tracing to the opentelemetry-collector service in the same namespace.", + ) + + fs.StringVar( + &telemetryServer, + "telemetry-server", + "opentelemetry-collector:4317", + "Custom address for OpenTelemetry Collector Server.", + ) + logs.AddFlags(fs, logs.SkipLoggingConfigurationFlags()) v1.AddFlags(logOptions, fs) diff --git a/pkg/ot/metrics.go b/pkg/ot/metrics.go new file mode 100644 index 0000000000..d2d11147f5 --- /dev/null +++ b/pkg/ot/metrics.go @@ -0,0 +1,39 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ot + +import ( + crprometheus "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/sdk/metric" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +// RegisterMetrics enables prometheus metrics for OpenTelemetry. +func RegisterMetrics() error { + exporter, err := prometheus.New( + prometheus.WithRegisterer(metrics.Registry.(*crprometheus.Registry)), + ) + if err != nil { + return err + } + meterProvider := metric.NewMeterProvider(metric.WithReader(exporter)) + otel.SetMeterProvider(meterProvider) + + return nil +} diff --git a/pkg/ot/traces.go b/pkg/ot/traces.go new file mode 100644 index 0000000000..9a63ba58fc --- /dev/null +++ b/pkg/ot/traces.go @@ -0,0 +1,91 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ot + +import ( + "context" + "fmt" + "time" + + "github.com/pkg/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" + "sigs.k8s.io/cluster-api-provider-aws/v2/version" +) + +// RegisterTracing enables code tracing via OpenTelemetry. +func RegisterTracing(ctx context.Context, log *logger.Logger, telemetryServer string) error { + tp, err := otlpTracerProvider(ctx, telemetryServer) + if err != nil { + log.Error(err, "failed to initialize tracer provider") + return fmt.Errorf("failed to initialize tracer provider: %w", err) + } + otel.SetTracerProvider(tp) + + // Give the tracer provider 5 seconds to shut down when the context closes. + go func() { + <-ctx.Done() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := tp.Shutdown(ctx); err != nil { + log.Error(err, "failed to shut down tracer provider") + } + }() + + return nil +} + +// otlpTracerProvider initializes an OTLP exporter and configures the corresponding tracer provider. +func otlpTracerProvider(ctx context.Context, url string) (*sdktrace.TracerProvider, error) { + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceNameKey.String("capa"), + attribute.String("exporter", "otlp"), + attribute.String("version", version.Get().String()), + attribute.String("awssdk.version", version.Get().AwsSdkVersion), + ), + ) + if err != nil { + return nil, errors.Wrap(err, "failed to create opentelemetry resource") + } + + traceExporter, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithInsecure(), + otlptracegrpc.WithEndpoint(url), + ) + if err != nil { + return nil, errors.Wrap(err, "failed to create otlp trace exporter") + } + + bsp := sdktrace.NewBatchSpanProcessor(traceExporter) + tracerProvider := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.AlwaysSample()), + sdktrace.WithResource(res), + sdktrace.WithSpanProcessor(bsp), + ) + otel.SetTracerProvider(tracerProvider) + otel.SetTextMapPropagator(propagation.TraceContext{}) + + return tracerProvider, nil +} diff --git a/util/tele/composite_logger.go b/util/tele/composite_logger.go new file mode 100644 index 0000000000..1048d0cbd4 --- /dev/null +++ b/util/tele/composite_logger.go @@ -0,0 +1,95 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tele + +import ( + "github.com/go-logr/logr" + + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" +) + +type compositeLogSink struct { + logSinks []logr.LogSink +} + +func (c *compositeLogSink) Init(info logr.RuntimeInfo) { + // we change the depth of the stack so that we can get the real + // line where the log statement was called. We need to do this because the composite logger adds to the + // call stack due to wrapping the internal logger. + info.CallDepth += 2 + for _, l := range c.logSinks { + l.Init(info) + } +} + +func (c *compositeLogSink) Enabled(v int) bool { + for _, l := range c.logSinks { + if !l.Enabled(v) { + return false + } + } + return true +} + +func (c *compositeLogSink) iter(fn func(l logr.LogSink)) { + for _, l := range c.logSinks { + fn(l) + } +} + +func (c *compositeLogSink) Info(level int, msg string, keysAndValues ...interface{}) { + c.iter(func(l logr.LogSink) { + l.Info(level, msg, keysAndValues...) + }) +} + +func (c *compositeLogSink) Error(err error, msg string, keysAndValues ...interface{}) { + c.iter(func(l logr.LogSink) { + l.Error(err, msg, keysAndValues...) + }) +} + +func (c *compositeLogSink) WithValues(keysAndValues ...interface{}) logr.LogSink { + var logSinks = make([]logr.LogSink, len(c.logSinks)) + for i, l := range c.logSinks { + logSinks[i] = l.WithValues(keysAndValues...) + } + + return &compositeLogSink{ + logSinks: logSinks, + } +} + +func (c *compositeLogSink) WithName(name string) logr.LogSink { + var logSinks = make([]logr.LogSink, len(c.logSinks)) + for i, l := range c.logSinks { + logSinks[i] = l.WithName(name) + } + + return &compositeLogSink{ + logSinks: logSinks, + } +} + +// NewCompositeLogger is the main entry-point to this implementation. +func NewCompositeLogger(logSinks []logr.LogSink) *logger.Logger { + sink := &compositeLogSink{ + logSinks: logSinks, + } + + return logger.NewLogger(logr.New(sink)) +} diff --git a/util/tele/corr_id.go b/util/tele/corr_id.go new file mode 100644 index 0000000000..943a12eb53 --- /dev/null +++ b/util/tele/corr_id.go @@ -0,0 +1,94 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tele + +import ( + "context" + + "github.com/go-logr/logr" + "github.com/google/uuid" +) + +// CorrIDKey is the type of the key used to store correlation +// IDs in context.Contexts. +type CorrIDKey string + +// CorrIDKeyVal is the key used to store the correlation ID in +// context.Contexts, HTTP headers, and other similar locations. +const CorrIDKeyVal CorrIDKey = "x-amzn-requestid" + +// CorrID is a correlation ID that the cluster API provider +// sends with all API requests to AWS. Do not create one +// of these manually. Instead, use the CtxWithCorrelationID function +// to create one of these within a context.Context. +type CorrID string + +// ctxWithCorrID creates a CorrID and creates a new context.Context +// with the new CorrID in it. It returns the _new_ context and the +// newly created CorrID. If there was a problem creating the correlation +// ID, the new context will not have the correlation ID in it and the +// returned CorrID will be the empty string.After you call this function, prefer to +// use the newly created context over the old one. Common usage is +// below: +// +// ctx := context.Background() +// ctx, newCorrID := ctxWithCorrID(ctx) +// fmt.Println("new corr ID: ", newCorrID) +// doSomething(ctx) +func ctxWithCorrID(ctx context.Context) (context.Context, CorrID) { + if currentCorrIDIface := ctx.Value(CorrIDKeyVal); currentCorrIDIface != nil { + currentCorrID, ok := currentCorrIDIface.(CorrID) + if ok { + return ctx, currentCorrID + } + } + uid, err := uuid.NewRandom() + if err != nil { + return nil, CorrID("") + } + newCorrID := CorrID(uid.String()) + ctx = context.WithValue(ctx, CorrIDKeyVal, newCorrID) + return ctx, newCorrID +} + +// CorrIDFromCtx attempts to fetch a correlation ID from the given +// context.Context. If none exists, returns an empty CorrID and false. +// Otherwise returns the CorrID value and true. +func CorrIDFromCtx(ctx context.Context) (CorrID, bool) { + currentCorrIDIface := ctx.Value(CorrIDKeyVal) + if currentCorrIDIface == nil { + return CorrID(""), false + } + + if corrID, ok := currentCorrIDIface.(CorrID); ok { + return corrID, ok + } + + return CorrID(""), false +} + +// corrIDLogger attempts to fetch the correlation ID from the +// given ctx using CorrIDFromCtx. If it finds one, this function +// uses lggr.WithValues to return a new logr.Logger with the +// correlation ID in it. +func corrIDLogger(ctx context.Context, lggr logr.Logger) logr.Logger { + corrID, ok := CorrIDFromCtx(ctx) + if ok { + lggr = lggr.WithValues(string(CorrIDKeyVal), string(corrID)) + } + return lggr +} diff --git a/util/tele/span_logger.go b/util/tele/span_logger.go new file mode 100644 index 0000000000..8e32ba8813 --- /dev/null +++ b/util/tele/span_logger.go @@ -0,0 +1,180 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tele + +import ( + "context" + "fmt" + "time" + + "github.com/go-logr/logr" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "sigs.k8s.io/controller-runtime/pkg/log" + + "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" +) + +// spanLogSink is a logr.LogSink implementation that writes log +// data to a span. +type spanLogSink struct { + trace.Span + name string + vals []interface{} +} + +func (*spanLogSink) Init(info logr.RuntimeInfo) { +} + +func (s *spanLogSink) End(opts ...trace.SpanEndOption) { + s.Span.End(opts...) +} + +func (*spanLogSink) Enabled(v int) bool { + return true +} + +func (s *spanLogSink) kvsToAttrs(keysAndValues ...interface{}) []attribute.KeyValue { + var ret []attribute.KeyValue + for i := 0; i < len(keysAndValues); i += 2 { + kv1 := fmt.Sprintf("%s", keysAndValues[i]) + kv2 := fmt.Sprintf("%s", keysAndValues[i+1]) + ret = append(ret, attribute.String(kv1, kv2)) + } + for i := 0; i < len(s.vals); i += 2 { + kv1 := fmt.Sprintf("%s", s.vals[i]) + kv2 := fmt.Sprintf("%s", s.vals[i+1]) + ret = append(ret, attribute.String(kv1, kv2)) + } + return ret +} + +func (s *spanLogSink) evtStr(evtType, msg string) string { + return fmt.Sprintf( + "[%s | %s] %s", + evtType, + s.name, + msg, + ) +} + +func (s *spanLogSink) Info(level int, msg string, keysAndValues ...interface{}) { + attrs := s.kvsToAttrs(keysAndValues...) + s.AddEvent( + s.evtStr("INFO", msg), + trace.WithTimestamp(time.Now()), + trace.WithAttributes(attrs...), + ) +} + +func (s *spanLogSink) Error(err error, msg string, keysAndValues ...interface{}) { + attrs := s.kvsToAttrs(keysAndValues...) + s.AddEvent( + s.evtStr("ERROR", fmt.Sprintf("%s (%s)", msg, err)), + trace.WithTimestamp(time.Now()), + trace.WithAttributes(attrs...), + ) +} + +func (s *spanLogSink) WithValues(keysAndValues ...interface{}) logr.LogSink { + s.vals = append(s.vals, keysAndValues...) + return s +} + +func (s *spanLogSink) WithName(name string) logr.LogSink { + s.name = name + return s +} + +// NewSpanLogSink is the main entry-point to this implementation. +func NewSpanLogSink(span trace.Span) logr.LogSink { + return &spanLogSink{ + Span: span, + } +} + +// Config holds optional, arbitrary configuration information +// to be added to logs and telemetry data. Instances of +// Config get passed to StartSpanWithLogger via the KVP function. +type Config struct { + KVPs map[string]string +} + +func (c Config) teleKeyValues() []attribute.KeyValue { + ret := make([]attribute.KeyValue, len(c.KVPs)) + i := 0 + for k, v := range c.KVPs { + ret[i] = attribute.String(k, v) + i++ + } + return ret +} + +// Option is the modifier function used to configure +// StartSpanWithLogger. Generally speaking, you should +// not create your own option function. Instead, use +// built-in functions (like KVP) that create them. +type Option func(*Config) + +// KVP returns a new Option function that adds a the given +// key-value pair. +func KVP(key, value string) Option { + return func(cfg *Config) { + cfg.KVPs[key] = value + } +} + +// StartSpanWithLogger starts a new span with the global +// tracer returned from Tracer(), then returns a new logger +// implementation that composes both the logger from the +// given ctx and a logger that logs to the newly created span. +// +// Callers should make sure to call the function in the 3rd return +// value to ensure that the span is ended properly. In many cases, +// that can be done with a defer: +// +// ctx, lggr, done := StartSpanWithLogger(ctx, "my-span") +// defer done() +func StartSpanWithLogger( + ctx context.Context, + spanName string, + opts ...Option, +) (context.Context, *logger.Logger, func()) { + cfg := &Config{KVPs: make(map[string]string)} + for _, opt := range opts { + opt(cfg) + } + ctx, span := Tracer().Start( + ctx, + spanName, + trace.WithAttributes(cfg.teleKeyValues()...), + ) + endFn := func() { + span.End() + } + + kvs := make([]interface{}, 0, 2*len(cfg.KVPs)) + for k, v := range cfg.KVPs { + kvs = append(kvs, k, v) + } + + lggr := log.FromContext(ctx, kvs...).WithName(spanName) + return ctx, NewCompositeLogger([]logr.LogSink{ + corrIDLogger(ctx, lggr).GetSink(), + NewSpanLogSink(span), + }), endFn +} diff --git a/util/tele/tele.go b/util/tele/tele.go new file mode 100644 index 0000000000..e185bdd4c6 --- /dev/null +++ b/util/tele/tele.go @@ -0,0 +1,69 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tele + +import ( + "context" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +type tracer struct { + trace.Tracer +} + +// Start creates a new context with a new AWS correlation ID, then +// creates a new trace.Span with that new context. This function then +// returns the new Context and Span. +func (t tracer) Start( + ctx context.Context, + op string, + opts ...trace.SpanStartOption, +) (context.Context, trace.Span) { + ctx, corrID := ctxWithCorrID(ctx) + opts = append( + opts, + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String( + string(CorrIDKeyVal), + string(corrID), + )), + ) + return t.Tracer.Start(ctx, op, opts...) +} + +// Tracer returns an OpenTelemetry Tracer implementation to be used +// to create spans. If you need access to the raw globally-registered +// tracer, use this function. +// +// Most people should not use this function directly, however. +// Instead, consider using StartSpanWithLogger, which uses +// this tracer to start a new span, configures logging, and +// more. +// +// Example usage: +// +// ctx, span := tele.Tracer().Start(ctx, "myFunction") +// defer span.End() +// // use the span and context here +func Tracer() trace.Tracer { + return tracer{ + Tracer: otel.Tracer("capa"), + } +}