Migrated CatalogClient from propeller to stdlib

Implemented new datacatalog functionality required for cache eviction
Updated to latest unreleased version of flyteidl and flyteplugins

Signed-off-by: Nick Müller <[email protected]>

catalog/client.go

@@ -0,0 +1,26 @@
+package catalog
+
+import (
+	"context"
+	"fmt"
+
+	"google.golang.org/grpc"
+
+	pluginCatalog "github.com/flyteorg/flyteplugins/go/tasks/pluginmachinery/catalog"
+	"github.com/flyteorg/flytestdlib/catalog/datacatalog"
+)
+
+func NewClient(ctx context.Context, authOpt ...grpc.DialOption) (pluginCatalog.Client, error) {
+	catalogConfig := GetConfig()
+
+	switch catalogConfig.Type {
+	case TypeDataCatalog:
+		return datacatalog.NewDataCatalog(ctx, catalogConfig.Endpoint, catalogConfig.Insecure,
+			catalogConfig.MaxCacheAge.Duration, catalogConfig.UseAdminAuth, catalogConfig.DefaultServiceConfig,
+			authOpt...)
+	case TypeNoOp, "":
+		return NOOPCatalog{}, nil
+	}
+
+	return nil, fmt.Errorf("invalid catalog type %q", catalogConfig.Type)
+}

catalog/config.go

@@ -0,0 +1,43 @@
+package catalog
+
+import (
+	"github.com/flyteorg/flytestdlib/config"
+)
+
+//go:generate pflags Config --default-var defaultConfig
+
+const ConfigSectionKey = "catalog-cache"
+
+var (
+	defaultConfig = &Config{
+		Type: TypeNoOp,
+	}
+
+	configSection = config.MustRegisterSection(ConfigSectionKey, defaultConfig)
+)
+
+type Type = string
+
+const (
+	TypeNoOp        Type = "noop"
+	TypeDataCatalog Type = "datacatalog"
+)
+
+type Config struct {
+	Type         Type            `json:"type" pflag:"\"noop\", Catalog Implementation to use"`
+	Endpoint     string          `json:"endpoint" pflag:"\"\", Endpoint for catalog service"`
+	Insecure     bool            `json:"insecure" pflag:"false, Use insecure grpc connection"`
+	MaxCacheAge  config.Duration `json:"max-cache-age" pflag:", Cache entries past this age will incur cache miss. 0 means cache never expires"`
+	UseAdminAuth bool            `json:"use-admin-auth" pflag:"false, Use the same gRPC credentials option as the flyteadmin client"`
+
+	// Set the gRPC service config formatted as a json string https://github.com/grpc/grpc/blob/master/doc/service_config.md
+	// eg. {"loadBalancingConfig": [{"round_robin":{}}], "methodConfig": [{"name":[{"service": "foo", "method": "bar"}, {"service": "baz"}], "timeout": "1.000000001s"}]}
+	// find the full schema here https://github.com/grpc/grpc-proto/blob/master/grpc/service_config/service_config.proto#L625
+	// Note that required packages may need to be preloaded to support certain service config. For example "google.golang.org/grpc/balancer/roundrobin" should be preloaded to have round-robin policy supported.
+	DefaultServiceConfig string `json:"default-service-config" pflag:"\"\", Set the default service config for the catalog gRPC client"`
+}
+
+// GetConfig returns the parsed Catalog configuration
+func GetConfig() *Config {
+	return configSection.GetConfig().(*Config)
+}