From eb84c524aa22409cdaebc88f31f0342a65b9e0ae Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Tue, 3 Oct 2023 21:03:56 -0700 Subject: [PATCH] Fix repository registration (#13425) * Fix failing repository registration in external * Fix failing repository registration in external * Scan classPath to find Repository annotation * Remove additioanl config, register any classes with Repository or Collection annotation --------- Co-authored-by: mohitdeuex --- conf/openmetadata.yaml | 3 - openmetadata-service/pom.xml | 4 +- .../java/org/openmetadata/service/Entity.java | 23 +++-- .../service/OpenMetadataApplication.java | 31 +------ .../OpenMetadataApplicationConfig.java | 4 - .../service/extension/NoOpExtension.java | 30 ------- .../extension/OpenMetadataExtension.java | 10 --- .../service/resources/CollectionRegistry.java | 90 ++++++++----------- .../openmetadata/service/util/ClassUtil.java | 21 +++++ .../service/util/TablesInitializer.java | 2 +- openmetadata-spec/pom.xml | 4 +- .../configuration/extensionConfiguration.json | 47 ---------- pom.xml | 12 ++- 13 files changed, 88 insertions(+), 193 deletions(-) delete mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/extension/NoOpExtension.java delete mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/extension/OpenMetadataExtension.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/util/ClassUtil.java delete mode 100644 openmetadata-spec/src/main/resources/json/schema/configuration/extensionConfiguration.json diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index 98609fae8a33..bea21a20b469 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -347,6 +347,3 @@ web: changeEventConfig: omUri: ${OM_URI:- "http://localhost:8585"} #openmetadata in om uri for eg http://localhost:8585 -extensionConfiguration: - resourcePackage: ${OM_RESOURCE_PACKAGES:-[]} - extensions: ${OM_EXTENSIONS:-[]} diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml index 2ddaaba4c55f..c3783de291a9 100644 --- a/openmetadata-service/pom.xml +++ b/openmetadata-service/pom.xml @@ -233,8 +233,8 @@ flyway-mysql - io.github.resilience4j - resilience4j-retry + io.github.classgraph + classgraph diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 9fa3ee19f550..db3086d3c51d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -17,6 +17,9 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import io.github.classgraph.ClassGraph; +import io.github.classgraph.ClassInfoList; +import io.github.classgraph.ScanResult; import java.lang.reflect.Modifier; import java.net.URI; import java.util.ArrayList; @@ -36,6 +39,7 @@ import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.Jdbi; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.EntityTimeSeriesInterface; import org.openmetadata.schema.entity.services.ServiceType; @@ -58,7 +62,6 @@ import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.util.EntityUtil.Fields; -import org.reflections.Reflections; @Slf4j public final class Entity { @@ -239,13 +242,13 @@ public final class Entity { private Entity() {} - public static void initializeRepositories(CollectionDAO collectionDAO) { + public static void initializeRepositories(Jdbi jdbi, CollectionDAO collectionDAO) { if (!initializedRepositories) { Entity.collectionDAO = collectionDAO; tokenRepository = new TokenRepository(collectionDAO); // Check Collection DAO Objects.requireNonNull(collectionDAO, "CollectionDAO must not be null"); - Set> repositories = getRepositories(); + List> repositories = getRepositories(); for (Class clz : repositories) { if (Modifier.isAbstract(clz.getModifiers())) { continue; // Don't instantiate abstract classes @@ -253,6 +256,11 @@ public static void initializeRepositories(CollectionDAO collectionDAO) { try { clz.getDeclaredConstructor(CollectionDAO.class).newInstance(collectionDAO); } catch (Exception e) { + try { + clz.getDeclaredConstructor(Jdbi.class).newInstance(jdbi); + } catch (Exception ex) { + LOG.warn("Exception encountered", ex); + } LOG.warn("Exception encountered", e); } } @@ -511,9 +519,10 @@ private static void validateEntities(String name, List list) { } /** Compile a list of REST collections based on Resource classes marked with {@code Repository} annotation */ - private static Set> getRepositories() { - // Get classes marked with @Repository annotation - Reflections reflections = new Reflections("org.openmetadata.service.jdbi3"); - return reflections.getTypesAnnotatedWith(Repository.class); + private static List> getRepositories() { + try (ScanResult scanResult = new ClassGraph().enableAnnotationInfo().scan()) { + ClassInfoList classList = scanResult.getClassesWithAnnotation(Repository.class); + return classList.loadClasses(); + } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index c9ef30e01a73..1c8d1a3e52de 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -42,7 +42,6 @@ import java.time.temporal.ChronoUnit; import java.util.EnumSet; import java.util.HashSet; -import java.util.List; import java.util.Optional; import javax.naming.ConfigurationException; import javax.servlet.DispatcherType; @@ -66,8 +65,6 @@ import org.jdbi.v3.core.statement.SqlLogger; import org.jdbi.v3.core.statement.StatementContext; import org.jdbi.v3.sqlobject.SqlObjects; -import org.openmetadata.schema.api.configuration.extension.Extension; -import org.openmetadata.schema.api.configuration.extension.ExtensionConfiguration; import org.openmetadata.schema.api.security.AuthenticationConfiguration; import org.openmetadata.schema.api.security.AuthorizerConfiguration; import org.openmetadata.schema.services.connections.metadata.AuthProvider; @@ -81,7 +78,6 @@ import org.openmetadata.service.exception.ConstraintViolationExceptionMapper; import org.openmetadata.service.exception.JsonMappingExceptionMapper; import org.openmetadata.service.exception.OMErrorPageHandler; -import org.openmetadata.service.extension.OpenMetadataExtension; import org.openmetadata.service.fernet.Fernet; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; @@ -156,7 +152,7 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ searchRepository = new SearchRepository(catalogConfig.getElasticSearchConfiguration(), collectionDAO); // as first step register all the repositories - Entity.initializeRepositories(collectionDAO); + Entity.initializeRepositories(jdbi, collectionDAO); // Init Settings Cache after repositories SettingsCache.initialize(catalogConfig); @@ -238,8 +234,6 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ String pathPattern = "/" + '*'; environment.servlets().addServlet("static", assetServlet).addMapping(pathPattern); - registerExtensions(catalogConfig, environment, jdbi); - // Handle Pipeline Service Client Status job PipelineServiceStatusJobHandler pipelineServiceStatusJobHandler = PipelineServiceStatusJobHandler.create( @@ -247,25 +241,6 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ pipelineServiceStatusJobHandler.addPipelineServiceStatusJob(); } - private void registerExtensions(OpenMetadataApplicationConfig catalogConfig, Environment environment, Jdbi jdbi) { - ExtensionConfiguration extensionConfiguration = catalogConfig.getExtensionConfiguration(); - if (extensionConfiguration != null) { - for (Extension extension : extensionConfiguration.getExtensions()) { - try { - OpenMetadataExtension omExtension = - Class.forName(extension.getClassName()) - .asSubclass(OpenMetadataExtension.class) - .getConstructor() - .newInstance(); - omExtension.init(extension, catalogConfig, environment, jdbi); - LOG.info("[OmExtension] Registering Extension: {}", extension.getClassName()); - } catch (Exception ex) { - LOG.error("[OmExtension] Failed in registering Extension {}", extension.getClassName()); - } - } - } - } - private void registerSamlHandlers(OpenMetadataApplicationConfig catalogConfig, Environment environment) throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException { if (catalogConfig.getAuthenticationConfiguration() != null @@ -455,9 +430,7 @@ private void registerEventPublisher(OpenMetadataApplicationConfig openMetadataAp private void registerResources( OpenMetadataApplicationConfig config, Environment environment, Jdbi jdbi, CollectionDAO daoObject) { - List extensionResources = - config.getExtensionConfiguration() != null ? config.getExtensionConfiguration().getResourcePackage() : null; - CollectionRegistry.initialize(extensionResources); + CollectionRegistry.initialize(); CollectionRegistry.getInstance() .registerResources(jdbi, environment, config, daoObject, authorizer, authenticatorHandler); environment.jersey().register(new JsonPatchProvider()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java index b95740e40463..3640b1da6872 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java @@ -25,7 +25,6 @@ import org.openmetadata.api.configuration.ApplicationConfiguration; import org.openmetadata.api.configuration.ChangeEventConfiguration; import org.openmetadata.schema.api.configuration.events.EventHandlerConfiguration; -import org.openmetadata.schema.api.configuration.extension.ExtensionConfiguration; import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration; import org.openmetadata.schema.api.fernet.FernetConfiguration; import org.openmetadata.schema.api.security.AuthenticationConfiguration; @@ -94,9 +93,6 @@ public class OpenMetadataApplicationConfig extends Configuration { @JsonProperty("email") private SmtpSettings smtpSettings; - @JsonProperty("extensionConfiguration") - private ExtensionConfiguration extensionConfiguration; - @Valid @NotNull @JsonProperty("web") diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/extension/NoOpExtension.java b/openmetadata-service/src/main/java/org/openmetadata/service/extension/NoOpExtension.java deleted file mode 100644 index 678846a2704b..000000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/extension/NoOpExtension.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2021 Collate - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,m - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.openmetadata.service.extension; - -import io.dropwizard.setup.Environment; -import lombok.extern.slf4j.Slf4j; -import org.jdbi.v3.core.Jdbi; -import org.openmetadata.schema.api.configuration.extension.Extension; -import org.openmetadata.service.OpenMetadataApplicationConfig; - -@Slf4j -public class NoOpExtension implements OpenMetadataExtension { - @Override - public void init( - Extension extension, OpenMetadataApplicationConfig catalogConfig, Environment environment, Jdbi jdbi) { - /* Success */ - LOG.info("Registered NoOp Extension"); - } -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/extension/OpenMetadataExtension.java b/openmetadata-service/src/main/java/org/openmetadata/service/extension/OpenMetadataExtension.java deleted file mode 100644 index 8bab4b280840..000000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/extension/OpenMetadataExtension.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.openmetadata.service.extension; - -import io.dropwizard.setup.Environment; -import org.jdbi.v3.core.Jdbi; -import org.openmetadata.schema.api.configuration.extension.Extension; -import org.openmetadata.service.OpenMetadataApplicationConfig; - -public interface OpenMetadataExtension { - void init(Extension extension, OpenMetadataApplicationConfig catalogConfig, Environment environment, Jdbi jdbi); -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/CollectionRegistry.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/CollectionRegistry.java index 03df3c51869a..305f72d9cb20 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/CollectionRegistry.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/CollectionRegistry.java @@ -15,6 +15,10 @@ import com.google.common.annotations.VisibleForTesting; import io.dropwizard.setup.Environment; +import io.github.classgraph.ClassGraph; +import io.github.classgraph.ClassInfo; +import io.github.classgraph.ClassInfoList; +import io.github.classgraph.ScanResult; import io.swagger.annotations.Api; import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; @@ -27,14 +31,12 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import javax.ws.rs.Path; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.core.Jdbi; -import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.Function; import org.openmetadata.schema.type.CollectionDescriptor; import org.openmetadata.schema.type.CollectionInfo; @@ -42,10 +44,7 @@ import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.auth.AuthenticatorHandler; -import org.reflections.Reflections; -import org.reflections.scanners.MethodAnnotationsScanner; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; +import org.openmetadata.service.util.ClassUtil; /** * Collection registry is a registry of all the REST collections in the catalog. It is used for building REST endpoints @@ -66,19 +65,11 @@ public final class CollectionRegistry { /** Resources used only for testing */ @VisibleForTesting private final List testResources = new ArrayList<>(); - public List getAdditionalResources() { - return additionalResources; - } - - private final List additionalResources; - - private CollectionRegistry(List additionalResources) { - this.additionalResources = additionalResources; - } + private CollectionRegistry() {} public static CollectionRegistry getInstance() { if (!initialized) { - initialize(null); + initialize(); } return instance; } @@ -87,9 +78,9 @@ public List getFunctions(Class clz) { return functionMap.get(clz); } - public static void initialize(List additionalResources) { + public static void initialize() { if (!initialized) { - instance = new CollectionRegistry(additionalResources); + instance = new CollectionRegistry(); initialized = true; instance.loadCollectionDescriptors(); instance.loadConditionFunctions(); @@ -124,29 +115,26 @@ private void loadCollectionDescriptors() { * those conditions and makes it available for listing them over API to author expressions in Rules. */ private void loadConditionFunctions() { - Reflections reflections = - new Reflections( - new ConfigurationBuilder() - .setUrls(ClasspathHelper.forPackage("org.openmetadata.service")) - .setScanners(new MethodAnnotationsScanner())); + try (ScanResult scanResult = new ClassGraph().enableAllInfo().scan()) { + for (ClassInfo classInfo : scanResult.getClassesWithMethodAnnotation(Function.class)) { + List methods = ClassUtil.getMethodsAnnotatedWith(classInfo.loadClass(), Function.class); + for (Method method : methods) { + Function annotation = method.getAnnotation(Function.class); + List functionList = + functionMap.computeIfAbsent(method.getDeclaringClass(), k -> new ArrayList<>()); - // Get classes marked with @Collection annotation - Set methods = reflections.getMethodsAnnotatedWith(Function.class); - for (Method method : methods) { - Function annotation = method.getAnnotation(Function.class); - List functionList = - functionMap.computeIfAbsent(method.getDeclaringClass(), k -> new ArrayList<>()); - - org.openmetadata.schema.type.Function function = - new org.openmetadata.schema.type.Function() - .withName(annotation.name()) - .withInput(annotation.input()) - .withDescription(annotation.description()) - .withExamples(List.of(annotation.examples())) - .withParameterInputType(annotation.paramInputType()); - functionList.add(function); - functionList.sort(Comparator.comparing(org.openmetadata.schema.type.Function::getName)); - LOG.info("Initialized for {} function {}\n", method.getDeclaringClass().getSimpleName(), function); + org.openmetadata.schema.type.Function function = + new org.openmetadata.schema.type.Function() + .withName(annotation.name()) + .withInput(annotation.input()) + .withDescription(annotation.description()) + .withExamples(List.of(annotation.examples())) + .withParameterInputType(annotation.paramInputType()); + functionList.add(function); + functionList.sort(Comparator.comparing(org.openmetadata.schema.type.Function::getName)); + LOG.info("Initialized for {} function {}\n", method.getDeclaringClass().getSimpleName(), function); + } + } } } @@ -210,22 +198,16 @@ private static CollectionDetails getCollection(Class cl) { /** Compile a list of REST collections based on Resource classes marked with {@code Collection} annotation */ private static List getCollections() { - Reflections reflections = new Reflections("org.openmetadata.service.resources"); - // Get classes marked with @Collection annotation - Set> collectionClasses = reflections.getTypesAnnotatedWith(Collection.class); - // Get classes marked in other - if (!CommonUtil.nullOrEmpty(instance.getAdditionalResources())) { - for (String packageName : instance.getAdditionalResources()) { - Reflections packageReflections = new Reflections(packageName); - collectionClasses.addAll(packageReflections.getTypesAnnotatedWith(Collection.class)); + try (ScanResult scanResult = new ClassGraph().enableAnnotationInfo().scan()) { + ClassInfoList classList = scanResult.getClassesWithAnnotation(Collection.class); + List> collectionClasses = classList.loadClasses(); + List collections = new ArrayList<>(); + for (Class cl : collectionClasses) { + CollectionDetails cd = getCollection(cl); + collections.add(cd); } + return collections; } - List collections = new ArrayList<>(); - for (Class cl : collectionClasses) { - CollectionDetails cd = getCollection(cl); - collections.add(cd); - } - return collections; } /** Create a resource class based on dependencies declared in @Collection annotation */ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/ClassUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/ClassUtil.java new file mode 100644 index 000000000000..9b79f5006eea --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/ClassUtil.java @@ -0,0 +1,21 @@ +package org.openmetadata.service.util; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public final class ClassUtil { + public static List getMethodsAnnotatedWith( + final Class clazz, final Class annotation) { + final List methods = new ArrayList<>(); + for (final Method method : clazz.getDeclaredMethods()) { + if (method.isAnnotationPresent(annotation)) { + methods.add(method); + } + } + return methods; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/TablesInitializer.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/TablesInitializer.java index ff788524356c..124e87e61ada 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/TablesInitializer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/TablesInitializer.java @@ -352,7 +352,7 @@ public static void validateAndRunSystemDataMigrations( Jdbi jdbi, ConnectionType connType, String nativeMigrationSQLPath, boolean forceMigrations) { DatasourceConfig.initialize(connType.label); MigrationWorkflow workflow = new MigrationWorkflow(jdbi, nativeMigrationSQLPath, connType, forceMigrations); - Entity.initializeRepositories(jdbi.onDemand(CollectionDAO.class)); + Entity.initializeRepositories(jdbi, jdbi.onDemand(CollectionDAO.class)); workflow.runMigrationWorkflows(); Entity.cleanup(); } diff --git a/openmetadata-spec/pom.xml b/openmetadata-spec/pom.xml index eddd4afdbb90..59bc8f964917 100644 --- a/openmetadata-spec/pom.xml +++ b/openmetadata-spec/pom.xml @@ -12,8 +12,8 @@ openmetadata-spec - 11 - 11 + 17 + 17 2.0.12-1 diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/extensionConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/extensionConfiguration.json deleted file mode 100644 index d1505e3c2bef..000000000000 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/extensionConfiguration.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/entity/configuration/extensionConfiguration.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ExtensionConfiguration", - "description": "This schema defines the OpenMetadata Extensions Configuration.", - "type": "object", - "javaType": "org.openmetadata.schema.api.configuration.extension.ExtensionConfiguration", - "definitions": { - "extension": { - "javaType": "org.openmetadata.schema.api.configuration.extension.Extension", - "description": "Extension Class to Register in OM", - "type": "object", - "properties": { - "className": { - "description": "Class Name for the Extension Service.", - "type": "string" - }, - "parameters": { - "javaType": "org.openmetadata.schema.api.configuration.extension.ExtensionParameters", - "description": "Additional parameters for extension initialization.", - "type": "object", - "additionalProperties": { - ".{1,}": { "type": "string" } - } - } - }, - "required": ["className"], - "additionalProperties": false - } - }, - "properties": { - "resourcePackage" : { - "description": "Resources Package name for Extension.", - "type": "array", - "items": { - "type": "string" - } - }, - "extensions": { - "description": "Extension Class to Register in OM", - "type": "array", - "items": { - "$ref": "#/definitions/extension" - } - } - } -} diff --git a/pom.xml b/pom.xml index 9347cfb8ea7b..4d1c397ec450 100644 --- a/pom.xml +++ b/pom.xml @@ -104,7 +104,6 @@ 2.6 1.18.28 10.1.10 - 1.7.0 @@ -457,9 +456,9 @@ test - io.github.resilience4j - resilience4j-retry - ${resilience4j.version} + io.github.classgraph + classgraph + 4.8.162 org.reflections @@ -500,6 +499,11 @@ disruptor 3.4.4 + + io.github.resilience4j + resilience4j-retry + 2.1.0 +