Merge pull request #230 from nikhil-thomas/download-install-community…

…-tasks

Add mechanism install non-Red-Hat-supported Tasks

pkg/apis/operator/v1alpha1/config_types.go

@@ -74,10 +74,19 @@ const (
 	// Check details field for additional details
 	InvalidResource InstallStatus = "invalid-resource"
 
+	// AppliedAddons indicates that the pipeline addons
+	// have been applied on the cluster
+	AppliedAddons InstallStatus = "applied-addons"
+
 	// AddonsError indicates that there was an error installing addons
 	// Check details field for additional details
 	AddonsError InstallStatus = "error-addons"
 
+	// CommunityResourcesError indicates that there was an error
+	// installing Community Provided Resources
+	// Check details field for additional details
+	CommunityResourcesError InstallStatus = "error-community-resources"
+
 	// InstalledStatus indicates that all pipeline resources are installed successfully
 	InstalledStatus InstallStatus = "installed"
 )

pkg/controller/config/config_controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -54,6 +55,7 @@ func Add(mgr manager.Manager) error {
 
 // newReconciler returns a new reconcile.Reconciler
 func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
+	log := ctrlLog.WithName("new-reconciler")
 	pipelinePath := filepath.Join(flag.ResourceDir, "pipelines")
 	pipeline, err := mf.NewManifest(pipelinePath, flag.Recursive, mgr.GetClient())
 	if err != nil {
@@ -65,6 +67,12 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
 		return nil, err
 	}
 
+	community, err := fetchCommuntiyResources(mgr)
+	if err != nil {
+		log.Error(err, "error fetching community resources")
+		community = mf.Manifest{}
+	}
+
 	secClient, err := sec.NewForConfig(mgr.GetConfig())
 	if err != nil {
 		return nil, err
@@ -76,9 +84,24 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
 		secClient: secClient,
 		pipeline:  pipeline,
 		addons:    addons,
+		community: community,
 	}, nil
 }
 
+func fetchCommuntiyResources(mgr manager.Manager) (mf.Manifest, error) {
+	if flag.SkipNonRedHatResources {
+		return mf.Manifest{}, nil
+	}
+	//manifestival can take urls/filepaths as input
+	//more that one items can be passed as a comma separated list string
+	urls := strings.Join(flag.CommunityResourceURLs, ",")
+	community, err := mf.NewManifest(urls, flag.Recursive, mgr.GetClient())
+	if err != nil {
+		return mf.Manifest{}, err
+	}
+	return community, nil
+}
+
 // this will read all the addons files
 func readAddons(mgr manager.Manager) (mf.Manifest, error) {
 	// read addons
@@ -170,6 +193,7 @@ type ReconcileConfig struct {
 	scheme    *runtime.Scheme
 	pipeline  mf.Manifest
 	addons    mf.Manifest
+	community mf.Manifest
 }
 
 // Reconcile reads that state of the cluster for a Config object and makes changes based on the state read
@@ -215,9 +239,10 @@ func (r *ReconcileConfig) Reconcile(req reconcile.Request) (reconcile.Result, er
 		return r.applyPipeline(req, cfg)
 	case op.AppliedPipeline, op.PipelineValidateError:
 		return r.validatePipeline(req, cfg)
-
 	case op.ValidatedPipeline, op.AddonsError:
 		return r.applyAddons(req, cfg)
+	case op.AppliedAddons, op.CommunityResourcesError:
+		return r.applyCommunityResources(req, cfg)
 	case op.InstalledStatus:
 		return r.validateVersion(req, cfg)
 	}
@@ -294,7 +319,9 @@ func matchesUUID(target string) bool {
 func (r *ReconcileConfig) applyAddons(req reconcile.Request, cfg *op.Config) (reconcile.Result, error) {
 	log := requestLogger(req, "apply-addons")
 
-	if err := transformManifest(cfg, &r.addons); err != nil {
+	//add TaskProviderType label to ClusterTasks (community, redhat, certified)
+	injectLabel := transform.InjectLabel(flag.LabelProviderType, flag.ProviderTypeRedHat, transform.Overwrite, "ClusterTask")
+	if err := transformManifest(cfg, &r.addons, injectLabel); err != nil {
 		log.Error(err, "failed to apply manifest transformations on pipeline-addons")
 		// ignoring failure to update
 		_ = r.updateStatus(cfg, op.ConfigCondition{
@@ -313,18 +340,53 @@ func (r *ReconcileConfig) applyAddons(req reconcile.Request, cfg *op.Config) (re
 			Version: flag.TektonVersion})
 		return reconcile.Result{}, err
 	}
-	log.Info("successfully applied all resources")
+	log.Info("successfully applied all addon resources")
+
+	err := r.updateStatus(cfg, op.ConfigCondition{Code: op.AppliedAddons, Version: flag.TektonVersion})
+	return reconcile.Result{Requeue: true}, err
+}
+
+func (r *ReconcileConfig) applyCommunityResources(req reconcile.Request, cfg *op.Config) (reconcile.Result, error) {
+	log := requestLogger(req, "apply-non-redhat-resources")
+
+	//add TaskProviderType label to ClusterTasks (community, redhat, certified)
+	addnTfrms := []mf.Transformer{
+		// replace kind: Task, with kind: ClusterTask
+		transform.ReplaceKind("Task", "ClusterTask"),
+		transform.InjectLabel(flag.LabelProviderType, flag.ProviderTypeCommunity, transform.Overwrite),
+	}
+	if err := transformManifest(cfg, &r.community, addnTfrms...); err != nil {
+		log.Error(err, "failed to apply manifest transformations on pipeline-addons")
+		// ignoring failure to update
+		_ = r.updateStatus(cfg, op.ConfigCondition{
+			Code:    op.CommunityResourcesError,
+			Details: err.Error(),
+			Version: flag.TektonVersion})
+		return reconcile.Result{}, err
+	}
+
+	if err := r.community.ApplyAll(); err != nil {
+		log.Error(err, "failed to apply non Red Hat resources yaml manifest")
+		// ignoring failure to update
+		_ = r.updateStatus(cfg, op.ConfigCondition{
+			Code:    op.CommunityResourcesError,
+			Details: err.Error(),
+			Version: flag.TektonVersion})
+		return reconcile.Result{}, err
+	}
+	log.Info("successfully applied all non Red Hat resources")
 
 	err := r.updateStatus(cfg, op.ConfigCondition{Code: op.InstalledStatus, Version: flag.TektonVersion})
 	return reconcile.Result{Requeue: true}, err
 }
 
-func transformManifest(cfg *op.Config, m *mf.Manifest) error {
+func transformManifest(cfg *op.Config, m *mf.Manifest, addnTfrms ...mf.Transformer) error {
 	tfs := []mf.Transformer{
 		mf.InjectOwner(cfg),
 		transform.InjectNamespaceConditional(flag.AnnotationPreserveNS, cfg.Spec.TargetNamespace),
 		transform.InjectDefaultSA(flag.DefaultSA),
 	}
+	tfs = append(tfs, addnTfrms...)
 	return m.Transform(tfs...)
 }
 

pkg/flag/flag.go

@@ -28,7 +28,11 @@ const (
 	TriggerControllerName = "tekton-triggers-controller"
 	TriggerWebhookName    = "tekton-triggers-webhook"
 
-	AnnotationPreserveNS = "operator.tekton.dev/preserve-namespace"
+	AnnotationPreserveNS  = "operator.tekton.dev/preserve-namespace"
+	LabelProviderType     = "operator.tekton.dev/provider-type"
+	ProviderTypeCommunity = "community"
+	ProviderTypeRedHat    = "redhat"
+	ProviderTypeCertified = "certified"
 
 	uuidPath = "deploy/uuid"
 )
@@ -37,14 +41,21 @@ var (
 	flagSet *pflag.FlagSet
 
 	TektonVersion   = "v0.10.1"
-	PipelineSA      string
-	IgnorePattern   string
-	ResourceWatched string
-	ResourceDir     string
-	TargetNamespace string
-	NoAutoInstall   bool
-	Recursive       bool
-	OperatorUUID    string
+	PipelineSA             string
+	IgnorePattern          string
+	ResourceWatched        string
+	ResourceDir            string
+	TargetNamespace        string
+	NoAutoInstall          bool
+	SkipNonRedHatResources bool
+	Recursive              bool
+	OperatorUUID           string
+	CommunityResourceURLs  = []string{
+		"https://raw.githubusercontent.com/tektoncd/catalog/master/jib-maven/jib-maven.yaml",
+		"https://raw.githubusercontent.com/tektoncd/catalog/master/maven/maven.yaml",
+		"https://raw.githubusercontent.com/tektoncd/catalog/master/tkn/tkn.yaml",
+		"https://raw.githubusercontent.com/tektoncd/catalog/master/kn/kn.yaml",
+	}
 )
 
 func init() {
@@ -80,10 +91,13 @@ func init() {
 		"Do not automatically install tekton pipelines, default: false")
 
 	flagSet.BoolVar(
-		&Recursive, "recursive", false,
+		&SkipNonRedHatResources, "skip-non-redhat", false,
+		"If enabled skip adding Tasks/Pipelines not supported/owned by Red Hat")
+
+	flagSet.BoolVar(
+		&Recursive, "recursive", true,
 		"If enabled apply manifest file in resource directory recursively")
 }
-
 func FlagSet() *pflag.FlagSet {
 	return flagSet
 }

pkg/utils/transform/testdata/test-inject-label-kind-set.yaml

@@ -0,0 +1,36 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-pod
+  labels:
+    operator.tekton.dev/provider-type: community
+spec:
+  containers:
+  - name: busy
+    image: busybox
+---
+apiVersion: tekton.dev/v1alpha1
+kind: Task
+metadata:
+  name: echo-hello-world
+spec:
+  steps:
+  - name: echo
+    image: ubuntu
+    command:
+    - echo
+    args:
+    - "hello world"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: my-service
+spec:
+  selector:
+    app: MyApp
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 9376

pkg/utils/transform/testdata/test-inject-label-overwrite.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-pod
+  labels:
+    operator.tekton.dev/provider-type: redhat
+spec:
+  containers:
+  - name: busy
+    image: busybox

pkg/utils/transform/testdata/test-inject-label.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-pod
+spec:
+  containers:
+  - name: busy
+    image: busybox

pkg/utils/transform/testdata/test-replace-kind.yaml

@@ -0,0 +1,12 @@
+apiVersion: tekton.dev/v1alpha1
+kind: Task
+metadata:
+  name: echo-hello-world
+spec:
+  steps:
+  - name: echo
+    image: ubuntu
+    command:
+    - echo
+    args:
+    - "hello world"

pkg/utils/transform/transform.go

@@ -1,6 +1,7 @@
 package transform
 
 import (
+	"fmt"
 	"strings"
 
 	mf "github.com/jcrossley3/manifestival"
@@ -9,6 +10,13 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+type OverwritePolicy int
+
+const (
+	Retain OverwritePolicy = iota
+	Overwrite
+)
+
 // InjectDefaultSA adds default service account into config-defaults configMap
 func InjectDefaultSA(defaultSA string) mf.Transformer {
 	return func(u *unstructured.Unstructured) error {
@@ -36,7 +44,7 @@ func InjectDefaultSA(defaultSA string) mf.Transformer {
 	}
 }
 
-func InjectNamespaceConditional(preserveNamespace string, targetNamespace string) mf.Transformer {
+func InjectNamespaceConditional(preserveNamespace, targetNamespace string) mf.Transformer {
 	tf := mf.InjectNamespace(targetNamespace)
 	return func(u *unstructured.Unstructured) error {
 		annotations := u.GetAnnotations()
@@ -48,3 +56,64 @@ func InjectNamespaceConditional(preserveNamespace string, targetNamespace string
 		return tf(u)
 	}
 }
+
+func ReplaceKind(fromKind, toKind string) mf.Transformer {
+	return func(u *unstructured.Unstructured) error {
+		kind := u.GetKind()
+
+		if kind != fromKind {
+			return nil
+		}
+		err := unstructured.SetNestedField(u.Object, toKind, "kind")
+		if err != nil {
+			return fmt.Errorf(
+				"failed to change resource Name:%s, KIND from %s to %s, %s",
+				u.GetName(),
+				fromKind,
+				toKind,
+				err,
+			)
+		}
+		return nil
+	}
+}
+
+//InjectLabel adds label key:value to a resource
+// overwritePolicy (Retain/Overwrite) decides whehther to overwite an already existing label
+// []kinds specify the Kinds on which the label should be applied
+// if len(kinds) = 0, label will be apllied to all/any resources irrespective of its Kind
+func InjectLabel(key, value string, overwritePolicy OverwritePolicy, kinds ...string) mf.Transformer {
+	return func(u *unstructured.Unstructured) error {
+		kind := u.GetKind()
+		if len(kinds) != 0 && !ItemInSlice(kind, kinds) {
+			return nil
+		}
+		labels, found, err := unstructured.NestedStringMap(u.Object, "metadata", "labels")
+		if err != nil {
+			return fmt.Errorf("could not find labels set, %q", err)
+		}
+		if overwritePolicy == Retain && found {
+			if _, ok := labels[key]; ok {
+				return nil
+			}
+		}
+		if !found {
+			labels = map[string]string{}
+		}
+		labels[key] = value
+		err = unstructured.SetNestedStringMap(u.Object, labels, "metadata", "labels")
+		if err != nil {
+			return fmt.Errorf("error updateing labes for %s:%s, %s", kind, u.GetName(), err)
+		}
+		return nil
+	}
+}
+
+func ItemInSlice(item string, items []string) bool {
+	for _, v := range items {
+		if v == item {
+			return true
+		}
+	}
+	return false
+}