Fix repository registration (open-metadata#13425)

* Fix failing repository registration in external * Fix failing repository registration in external * Scan classPath to find Repository annotation * Remove additioanl config, register any classes with Repository or Collection annotation --------- Co-authored-by: mohitdeuex <[email protected]>
gorgias · Oct 4, 2023 · eb84c52 · eb84c52
1 parent 8578aee
commit eb84c52
Show file tree

Hide file tree

Showing 13 changed files with 88 additions and 193 deletions.
diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml
@@ -347,6 +347,3 @@ web:
 changeEventConfig:
   omUri: ${OM_URI:- "http://localhost:8585"} #openmetadata in om uri for eg http://localhost:8585
 
-extensionConfiguration:
-  resourcePackage: ${OM_RESOURCE_PACKAGES:-[]}
-  extensions: ${OM_EXTENSIONS:-[]}
diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml
@@ -233,8 +233,8 @@
       <artifactId>flyway-mysql</artifactId>
     </dependency>
     <dependency>
-      <groupId>io.github.resilience4j</groupId>
-      <artifactId>resilience4j-retry</artifactId>
+      <groupId>io.github.classgraph</groupId>
+      <artifactId>classgraph</artifactId>
     </dependency>
     <!-- Dependencies for secret store manager providers -->
     <dependency>

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
@@ -17,6 +17,9 @@
 import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
 
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import io.github.classgraph.ClassGraph;
+import io.github.classgraph.ClassInfoList;
+import io.github.classgraph.ScanResult;
 import java.lang.reflect.Modifier;
 import java.net.URI;
 import java.util.ArrayList;
@@ -36,6 +39,7 @@
 import lombok.NonNull;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.jdbi.v3.core.Jdbi;
 import org.openmetadata.schema.EntityInterface;
 import org.openmetadata.schema.EntityTimeSeriesInterface;
 import org.openmetadata.schema.entity.services.ServiceType;
@@ -58,7 +62,6 @@
 import org.openmetadata.service.resources.feeds.MessageParser.EntityLink;
 import org.openmetadata.service.search.SearchRepository;
 import org.openmetadata.service.util.EntityUtil.Fields;
-import org.reflections.Reflections;
 
 @Slf4j
 public final class Entity {
@@ -239,20 +242,25 @@ public final class Entity {
 
   private Entity() {}
 
-  public static void initializeRepositories(CollectionDAO collectionDAO) {
+  public static void initializeRepositories(Jdbi jdbi, CollectionDAO collectionDAO) {
     if (!initializedRepositories) {
       Entity.collectionDAO = collectionDAO;
       tokenRepository = new TokenRepository(collectionDAO);
       // Check Collection DAO
       Objects.requireNonNull(collectionDAO, "CollectionDAO must not be null");
-      Set<Class<?>> repositories = getRepositories();
+      List<Class<?>> repositories = getRepositories();
       for (Class<?> clz : repositories) {
         if (Modifier.isAbstract(clz.getModifiers())) {
           continue; // Don't instantiate abstract classes
         }
         try {
           clz.getDeclaredConstructor(CollectionDAO.class).newInstance(collectionDAO);
         } catch (Exception e) {
+          try {
+            clz.getDeclaredConstructor(Jdbi.class).newInstance(jdbi);
+          } catch (Exception ex) {
+            LOG.warn("Exception encountered", ex);
+          }
           LOG.warn("Exception encountered", e);
         }
       }
@@ -511,9 +519,10 @@ private static void validateEntities(String name, List<String> list) {
   }
 
   /** Compile a list of REST collections based on Resource classes marked with {@code Repository} annotation */
-  private static Set<Class<?>> getRepositories() {
-    // Get classes marked with @Repository annotation
-    Reflections reflections = new Reflections("org.openmetadata.service.jdbi3");
-    return reflections.getTypesAnnotatedWith(Repository.class);
+  private static List<Class<?>> getRepositories() {
+    try (ScanResult scanResult = new ClassGraph().enableAnnotationInfo().scan()) {
+      ClassInfoList classList = scanResult.getClassesWithAnnotation(Repository.class);
+      return classList.loadClasses();
+    }
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java
@@ -42,7 +42,6 @@
 import java.time.temporal.ChronoUnit;
 import java.util.EnumSet;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Optional;
 import javax.naming.ConfigurationException;
 import javax.servlet.DispatcherType;
@@ -66,8 +65,6 @@
 import org.jdbi.v3.core.statement.SqlLogger;
 import org.jdbi.v3.core.statement.StatementContext;
 import org.jdbi.v3.sqlobject.SqlObjects;
-import org.openmetadata.schema.api.configuration.extension.Extension;
-import org.openmetadata.schema.api.configuration.extension.ExtensionConfiguration;
 import org.openmetadata.schema.api.security.AuthenticationConfiguration;
 import org.openmetadata.schema.api.security.AuthorizerConfiguration;
 import org.openmetadata.schema.services.connections.metadata.AuthProvider;
@@ -81,7 +78,6 @@
 import org.openmetadata.service.exception.ConstraintViolationExceptionMapper;
 import org.openmetadata.service.exception.JsonMappingExceptionMapper;
 import org.openmetadata.service.exception.OMErrorPageHandler;
-import org.openmetadata.service.extension.OpenMetadataExtension;
 import org.openmetadata.service.fernet.Fernet;
 import org.openmetadata.service.jdbi3.CollectionDAO;
 import org.openmetadata.service.jdbi3.EntityRepository;
@@ -156,7 +152,7 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ
     searchRepository = new SearchRepository(catalogConfig.getElasticSearchConfiguration(), collectionDAO);
 
     // as first step register all the repositories
-    Entity.initializeRepositories(collectionDAO);
+    Entity.initializeRepositories(jdbi, collectionDAO);
 
     // Init Settings Cache after repositories
     SettingsCache.initialize(catalogConfig);
@@ -238,34 +234,13 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ
     String pathPattern = "/" + '*';
     environment.servlets().addServlet("static", assetServlet).addMapping(pathPattern);
 
-    registerExtensions(catalogConfig, environment, jdbi);
-
     // Handle Pipeline Service Client Status job
     PipelineServiceStatusJobHandler pipelineServiceStatusJobHandler =
         PipelineServiceStatusJobHandler.create(
             catalogConfig.getPipelineServiceClientConfiguration(), catalogConfig.getClusterName());
     pipelineServiceStatusJobHandler.addPipelineServiceStatusJob();
   }
 
-  private void registerExtensions(OpenMetadataApplicationConfig catalogConfig, Environment environment, Jdbi jdbi) {
-    ExtensionConfiguration extensionConfiguration = catalogConfig.getExtensionConfiguration();
-    if (extensionConfiguration != null) {
-      for (Extension extension : extensionConfiguration.getExtensions()) {
-        try {
-          OpenMetadataExtension omExtension =
-              Class.forName(extension.getClassName())
-                  .asSubclass(OpenMetadataExtension.class)
-                  .getConstructor()
-                  .newInstance();
-          omExtension.init(extension, catalogConfig, environment, jdbi);
-          LOG.info("[OmExtension] Registering Extension: {}", extension.getClassName());
-        } catch (Exception ex) {
-          LOG.error("[OmExtension] Failed in registering Extension {}", extension.getClassName());
-        }
-      }
-    }
-  }
-
   private void registerSamlHandlers(OpenMetadataApplicationConfig catalogConfig, Environment environment)
       throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException {
     if (catalogConfig.getAuthenticationConfiguration() != null
@@ -455,9 +430,7 @@ private void registerEventPublisher(OpenMetadataApplicationConfig openMetadataAp
 
   private void registerResources(
       OpenMetadataApplicationConfig config, Environment environment, Jdbi jdbi, CollectionDAO daoObject) {
-    List<String> extensionResources =
-        config.getExtensionConfiguration() != null ? config.getExtensionConfiguration().getResourcePackage() : null;
-    CollectionRegistry.initialize(extensionResources);
+    CollectionRegistry.initialize();
     CollectionRegistry.getInstance()
         .registerResources(jdbi, environment, config, daoObject, authorizer, authenticatorHandler);
     environment.jersey().register(new JsonPatchProvider());

diff --git a/...etadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java b/...etadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java
@@ -25,7 +25,6 @@
 import org.openmetadata.api.configuration.ApplicationConfiguration;
 import org.openmetadata.api.configuration.ChangeEventConfiguration;
 import org.openmetadata.schema.api.configuration.events.EventHandlerConfiguration;
-import org.openmetadata.schema.api.configuration.extension.ExtensionConfiguration;
 import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration;
 import org.openmetadata.schema.api.fernet.FernetConfiguration;
 import org.openmetadata.schema.api.security.AuthenticationConfiguration;
@@ -94,9 +93,6 @@ public class OpenMetadataApplicationConfig extends Configuration {
   @JsonProperty("email")
   private SmtpSettings smtpSettings;
 
-  @JsonProperty("extensionConfiguration")
-  private ExtensionConfiguration extensionConfiguration;
-
   @Valid
   @NotNull
   @JsonProperty("web")

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/extension/NoOpExtension.java b/openmetadata-service/src/main/java/org/openmetadata/service/extension/NoOpExtension.java
diff --git a/...adata-service/src/main/java/org/openmetadata/service/extension/OpenMetadataExtension.java b/...adata-service/src/main/java/org/openmetadata/service/extension/OpenMetadataExtension.java
diff --git a/...metadata-service/src/main/java/org/openmetadata/service/resources/CollectionRegistry.java b/...metadata-service/src/main/java/org/openmetadata/service/resources/CollectionRegistry.java
@@ -15,6 +15,10 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import io.dropwizard.setup.Environment;
+import io.github.classgraph.ClassGraph;
+import io.github.classgraph.ClassInfo;
+import io.github.classgraph.ClassInfoList;
+import io.github.classgraph.ScanResult;
 import io.swagger.annotations.Api;
 import java.lang.annotation.Annotation;
 import java.lang.reflect.InvocationTargetException;
@@ -27,25 +31,20 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import javax.ws.rs.Path;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.jdbi.v3.core.Jdbi;
-import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.Function;
 import org.openmetadata.schema.type.CollectionDescriptor;
 import org.openmetadata.schema.type.CollectionInfo;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
 import org.openmetadata.service.jdbi3.CollectionDAO;
 import org.openmetadata.service.security.Authorizer;
 import org.openmetadata.service.security.auth.AuthenticatorHandler;
-import org.reflections.Reflections;
-import org.reflections.scanners.MethodAnnotationsScanner;
-import org.reflections.util.ClasspathHelper;
-import org.reflections.util.ConfigurationBuilder;
+import org.openmetadata.service.util.ClassUtil;
 
 /**
  * Collection registry is a registry of all the REST collections in the catalog. It is used for building REST endpoints
@@ -66,19 +65,11 @@ public final class CollectionRegistry {
   /** Resources used only for testing */
   @VisibleForTesting private final List<Object> testResources = new ArrayList<>();
 
-  public List<String> getAdditionalResources() {
-    return additionalResources;
-  }
-
-  private final List<String> additionalResources;
-
-  private CollectionRegistry(List<String> additionalResources) {
-    this.additionalResources = additionalResources;
-  }
+  private CollectionRegistry() {}
 
   public static CollectionRegistry getInstance() {
     if (!initialized) {
-      initialize(null);
+      initialize();
     }
     return instance;
   }
@@ -87,9 +78,9 @@ public List<org.openmetadata.schema.type.Function> getFunctions(Class<?> clz) {
     return functionMap.get(clz);
   }
 
-  public static void initialize(List<String> additionalResources) {
+  public static void initialize() {
     if (!initialized) {
-      instance = new CollectionRegistry(additionalResources);
+      instance = new CollectionRegistry();
       initialized = true;
       instance.loadCollectionDescriptors();
       instance.loadConditionFunctions();
@@ -124,29 +115,26 @@ private void loadCollectionDescriptors() {
    * those conditions and makes it available for listing them over API to author expressions in Rules.
    */
   private void loadConditionFunctions() {
-    Reflections reflections =
-        new Reflections(
-            new ConfigurationBuilder()
-                .setUrls(ClasspathHelper.forPackage("org.openmetadata.service"))
-                .setScanners(new MethodAnnotationsScanner()));
+    try (ScanResult scanResult = new ClassGraph().enableAllInfo().scan()) {
+      for (ClassInfo classInfo : scanResult.getClassesWithMethodAnnotation(Function.class)) {
+        List<Method> methods = ClassUtil.getMethodsAnnotatedWith(classInfo.loadClass(), Function.class);
+        for (Method method : methods) {
+          Function annotation = method.getAnnotation(Function.class);
+          List<org.openmetadata.schema.type.Function> functionList =
+              functionMap.computeIfAbsent(method.getDeclaringClass(), k -> new ArrayList<>());
 
-    // Get classes marked with @Collection annotation
-    Set<Method> methods = reflections.getMethodsAnnotatedWith(Function.class);
-    for (Method method : methods) {
-      Function annotation = method.getAnnotation(Function.class);
-      List<org.openmetadata.schema.type.Function> functionList =
-          functionMap.computeIfAbsent(method.getDeclaringClass(), k -> new ArrayList<>());
-
-      org.openmetadata.schema.type.Function function =
-          new org.openmetadata.schema.type.Function()
-              .withName(annotation.name())
-              .withInput(annotation.input())
-              .withDescription(annotation.description())
-              .withExamples(List.of(annotation.examples()))
-              .withParameterInputType(annotation.paramInputType());
-      functionList.add(function);
-      functionList.sort(Comparator.comparing(org.openmetadata.schema.type.Function::getName));
-      LOG.info("Initialized for {} function {}\n", method.getDeclaringClass().getSimpleName(), function);
+          org.openmetadata.schema.type.Function function =
+              new org.openmetadata.schema.type.Function()
+                  .withName(annotation.name())
+                  .withInput(annotation.input())
+                  .withDescription(annotation.description())
+                  .withExamples(List.of(annotation.examples()))
+                  .withParameterInputType(annotation.paramInputType());
+          functionList.add(function);
+          functionList.sort(Comparator.comparing(org.openmetadata.schema.type.Function::getName));
+          LOG.info("Initialized for {} function {}\n", method.getDeclaringClass().getSimpleName(), function);
+        }
+      }
     }
   }
 
@@ -210,22 +198,16 @@ private static CollectionDetails getCollection(Class<?> cl) {
 
   /** Compile a list of REST collections based on Resource classes marked with {@code Collection} annotation */
   private static List<CollectionDetails> getCollections() {
-    Reflections reflections = new Reflections("org.openmetadata.service.resources");
-    // Get classes marked with @Collection annotation
-    Set<Class<?>> collectionClasses = reflections.getTypesAnnotatedWith(Collection.class);
-    // Get classes marked in other
-    if (!CommonUtil.nullOrEmpty(instance.getAdditionalResources())) {
-      for (String packageName : instance.getAdditionalResources()) {
-        Reflections packageReflections = new Reflections(packageName);
-        collectionClasses.addAll(packageReflections.getTypesAnnotatedWith(Collection.class));
+    try (ScanResult scanResult = new ClassGraph().enableAnnotationInfo().scan()) {
+      ClassInfoList classList = scanResult.getClassesWithAnnotation(Collection.class);
+      List<Class<?>> collectionClasses = classList.loadClasses();
+      List<CollectionDetails> collections = new ArrayList<>();
+      for (Class<?> cl : collectionClasses) {
+        CollectionDetails cd = getCollection(cl);
+        collections.add(cd);
       }
+      return collections;
     }
-    List<CollectionDetails> collections = new ArrayList<>();
-    for (Class<?> cl : collectionClasses) {
-      CollectionDetails cd = getCollection(cl);
-      collections.add(cd);
-    }
-    return collections;
   }
 
   /** Create a resource class based on dependencies declared in @Collection annotation */