remove cyclic repository dependency from core data services (#403)

finos · Oct 2, 2023 · 558310a · 558310a
1 parent eb6ff13
commit 558310a
Show file tree

Hide file tree

Showing 28 changed files with 223 additions and 154 deletions.
diff --git a/...rojectsVersionsReconciliationService.java → ...iation/VersionsReconciliationService.java b/...rojectsVersionsReconciliationService.java → ...iation/VersionsReconciliationService.java
@@ -13,13 +13,13 @@
 //  limitations under the License.
 //
 
-package org.finos.legend.depot.services.api.projects;
+package org.finos.legend.depot.services.api.artifacts.reconciliation;
 
 import org.finos.legend.depot.domain.version.VersionMismatch;
 
 import java.util.List;
 
-public interface ProjectsVersionsReconciliationService
+public interface VersionsReconciliationService
 {
      List<VersionMismatch> findVersionsMismatches();
 }
diff --git a/...pendencies/ManageDependenciesService.java → ...s/refresh/RefreshDependenciesService.java b/...pendencies/ManageDependenciesService.java → ...s/refresh/RefreshDependenciesService.java
@@ -13,19 +13,16 @@
 //  limitations under the License.
 //
 
-package org.finos.legend.depot.services.api.dependencies;
+package org.finos.legend.depot.services.api.artifacts.refresh;
 
 import org.finos.legend.depot.domain.project.ProjectVersion;
 import org.finos.legend.depot.store.model.projects.StoreProjectVersionData;
-import org.finos.legend.depot.domain.project.dependencies.VersionDependencyReport;
 
 import java.util.List;
 
-public interface ManageDependenciesService
+public interface RefreshDependenciesService
 {
-    List<ProjectVersion> calculateDependencies(String groupId, String artifactId, String versionId);
-
-    VersionDependencyReport calculateTransitiveDependencies(List<ProjectVersion> directDependencies);
+    List<ProjectVersion> retrieveDependenciesFromRepository(String groupId, String artifactId, String versionId);
 
     List<String> validateDependencies(List<ProjectVersion> dependencies, String versionId);
 

diff --git a/.../main/java/org/finos/legend/depot/services/artifacts/purge/ArtifactsPurgeServiceImpl.java b/.../main/java/org/finos/legend/depot/services/artifacts/purge/ArtifactsPurgeServiceImpl.java
@@ -24,7 +24,7 @@
 import org.finos.legend.depot.services.api.artifacts.purge.ArtifactsPurgeService;
 import org.finos.legend.depot.services.api.metrics.query.QueryMetricsService;
 import org.finos.legend.depot.services.api.projects.ManageProjectsService;
-import org.finos.legend.depot.services.api.projects.ProjectsVersionsReconciliationService;
+import org.finos.legend.depot.services.api.artifacts.reconciliation.VersionsReconciliationService;
 import org.finos.legend.depot.store.model.projects.StoreProjectData;
 import org.finos.legend.depot.store.model.projects.StoreProjectVersionData;
 import org.finos.legend.depot.tracing.services.TracerFactory;
@@ -60,11 +60,11 @@ public class ArtifactsPurgeServiceImpl implements ArtifactsPurgeService
     private static final String EVICT_OLDEST = "evict_old";
 
     private final ManageProjectsService projects;
-    private final ProjectsVersionsReconciliationService versionsMismatchService;
+    private final VersionsReconciliationService versionsMismatchService;
     private final QueryMetricsService metrics;
 
     @Inject
-    public ArtifactsPurgeServiceImpl(ManageProjectsService projects, ProjectsVersionsReconciliationService versionsMismatchService, QueryMetricsService metrics)
+    public ArtifactsPurgeServiceImpl(ManageProjectsService projects, VersionsReconciliationService versionsMismatchService, QueryMetricsService metrics)
     {
         this.projects = projects;
         this.metrics = metrics;

diff --git a/...pot/services/VersionsMismatchService.java → ...on/VersionsReconciliationServiceImpl.java b/...pot/services/VersionsMismatchService.java → ...on/VersionsReconciliationServiceImpl.java
@@ -13,11 +13,11 @@
 //  limitations under the License.
 //
 
-package org.finos.legend.depot.services;
+package org.finos.legend.depot.services.artifacts.reconciliation;
 
 import org.eclipse.collections.impl.parallel.ParallelIterate;
+import org.finos.legend.depot.services.api.artifacts.reconciliation.VersionsReconciliationService;
 import org.finos.legend.depot.services.api.artifacts.repository.ArtifactRepository;
-import org.finos.legend.depot.services.api.projects.ProjectsVersionsReconciliationService;
 import org.finos.legend.depot.store.model.projects.StoreProjectData;
 import org.finos.legend.depot.store.model.projects.StoreProjectVersionData;
 import org.finos.legend.depot.domain.version.VersionMismatch;
@@ -34,9 +34,9 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
-public class VersionsMismatchService implements ProjectsVersionsReconciliationService
+public class VersionsReconciliationServiceImpl implements VersionsReconciliationService
 {
-    private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(VersionsMismatchService.class);
+    private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(VersionsReconciliationServiceImpl.class);
 
     public static final String REPO_VERSIONS = "repo_versions";
     public static final String STORE_VERSIONS = "store_versions";
@@ -51,7 +51,7 @@ public class VersionsMismatchService implements ProjectsVersionsReconciliationSe
     private final ProjectsService projects;
 
     @Inject
-    public VersionsMismatchService(ArtifactRepository repository, ProjectsService projectsService)
+    public VersionsReconciliationServiceImpl(ArtifactRepository repository, ProjectsService projectsService)
     {
         this.repository = repository;
         this.projects = projectsService;

diff --git a/.../java/org/finos/legend/depot/services/artifacts/refresh/ProjectVersionRefreshHandler.java b/.../java/org/finos/legend/depot/services/artifacts/refresh/ProjectVersionRefreshHandler.java
@@ -26,7 +26,7 @@
 import org.finos.legend.depot.domain.project.ProjectVersionData;
 import org.finos.legend.depot.domain.project.Property;
 import org.finos.legend.depot.domain.version.VersionValidator;
-import org.finos.legend.depot.services.api.dependencies.ManageDependenciesService;
+import org.finos.legend.depot.services.api.artifacts.refresh.RefreshDependenciesService;
 import org.finos.legend.depot.services.api.projects.ManageProjectsService;
 import org.finos.legend.depot.services.artifacts.handlers.ManifestLoader;
 import org.finos.legend.depot.store.api.admin.artifacts.ArtifactsFilesStore;
@@ -84,20 +84,20 @@ public final class ProjectVersionRefreshHandler implements NotificationEventHand
     private final List<String> projectPropertiesInScope;
     private final List<String> manifestPropertiesInScope;
     private final Queue workQueue;
-    private final ManageDependenciesService manageDependenciesService;
+    private final RefreshDependenciesService refreshDependenciesService;
     private final int maximumSnapshotsAllowed;
 
 
     @Inject
-    public ProjectVersionRefreshHandler(ManageProjectsService projects, ArtifactRepository repositoryServices, Queue workQueue, ArtifactsFilesStore artifacts, IncludeProjectPropertiesConfiguration includePropertyConfig, ManageDependenciesService manageDependenciesService, @Named("maximumSnapshotsAllowed") int maximumSnapshotsAllowed)
+    public ProjectVersionRefreshHandler(ManageProjectsService projects, ArtifactRepository repositoryServices, Queue workQueue, ArtifactsFilesStore artifacts, IncludeProjectPropertiesConfiguration includePropertyConfig, RefreshDependenciesService refreshDependenciesService, @Named("maximumSnapshotsAllowed") int maximumSnapshotsAllowed)
     {
         this.projects = projects;
         this.workQueue = workQueue;
         this.artifacts = artifacts;
         this.repositoryServices = repositoryServices;
         this.projectPropertiesInScope = includePropertyConfig != null ? includePropertyConfig.getProperties() : Collections.EMPTY_LIST;
         this.manifestPropertiesInScope = includePropertyConfig != null ? includePropertyConfig.getManifestProperties() : Collections.EMPTY_LIST;
-        this.manageDependenciesService = manageDependenciesService;
+        this.refreshDependenciesService = refreshDependenciesService;
         this.maximumSnapshotsAllowed = maximumSnapshotsAllowed;
 
         try
@@ -230,8 +230,8 @@ MetadataEventResponse doRefresh(MetadataNotification event)
             StoreProjectData project = getProject(event.getGroupId(), event.getArtifactId());
             try
             {
-                List<ProjectVersion> newDependencies = this.manageDependenciesService.calculateDependencies(event.getGroupId(), event.getArtifactId(), event.getVersionId());
-                this.manageDependenciesService.validateDependencies(newDependencies, event.getVersionId()).forEach(error -> response.addError(error));
+                List<ProjectVersion> newDependencies = this.refreshDependenciesService.retrieveDependenciesFromRepository(event.getGroupId(), event.getArtifactId(), event.getVersionId());
+                this.refreshDependenciesService.validateDependencies(newDependencies, event.getVersionId()).forEach(error -> response.addError(error));
                 if (!response.hasErrors())
                 {
                     LOGGER.info("Processing artifacts for [{}-{}-{}]", event.getGroupId(), event.getArtifactId(), event.getVersionId());
@@ -282,7 +282,7 @@ private void updateProjectVersionData(StoreProjectData project, String versionId
         StoreProjectVersionData storeProjectVersionData = projectVersionData.isPresent() ? projectVersionData.get() : new StoreProjectVersionData(project.getGroupId(), project.getArtifactId(), versionId);
         ProjectVersionData versionData = storeProjectVersionData.getVersionData();
         versionData.setDependencies(newDependencies);
-        this.manageDependenciesService.setProjectDataTransitiveDependencies(storeProjectVersionData);
+        this.refreshDependenciesService.setProjectDataTransitiveDependencies(storeProjectVersionData);
 
         if (this.projectPropertiesInScope != null && !this.projectPropertiesInScope.isEmpty())
         {

diff --git a/...encies/ManageDependenciesServiceImpl.java → ...fresh/RefreshDependenciesServiceImpl.java b/...encies/ManageDependenciesServiceImpl.java → ...fresh/RefreshDependenciesServiceImpl.java
@@ -13,13 +13,14 @@
 //  limitations under the License.
 //
 
-package org.finos.legend.depot.services.dependencies;
+package org.finos.legend.depot.services.artifacts.refresh;
 
 import org.finos.legend.depot.domain.project.ProjectVersion;
 import org.finos.legend.depot.domain.project.dependencies.ProjectDependencyWithPlatformVersions;
 import org.finos.legend.depot.domain.project.dependencies.VersionDependencyReport;
 import org.finos.legend.depot.domain.version.VersionValidator;
-import org.finos.legend.depot.services.api.dependencies.ManageDependenciesService;
+import org.finos.legend.depot.services.api.artifacts.refresh.RefreshDependenciesService;
+import org.finos.legend.depot.services.api.dependencies.DependencyOverride;
 import org.finos.legend.depot.services.api.projects.ManageProjectsService;
 import org.finos.legend.depot.services.api.artifacts.repository.ArtifactRepository;
 import org.finos.legend.depot.domain.artifacts.repository.ArtifactDependency;
@@ -35,31 +36,24 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-public class ManageDependenciesServiceImpl implements ManageDependenciesService
+public class RefreshDependenciesServiceImpl implements RefreshDependenciesService
 {
-    private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(ManageDependenciesServiceImpl.class);
+    private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(RefreshDependenciesServiceImpl.class);
 
     private final ManageProjectsService projects;
     private final ArtifactRepository repositoryServices;
-    private final DependencyUtil dependencyUtil;
+    private final DependencyOverride dependencyOverride;
 
     @Inject
-    public ManageDependenciesServiceImpl(ManageProjectsService projects, ArtifactRepository repositoryServices, @Named("dependencyUtil") DependencyUtil dependencyUtil)
+    public RefreshDependenciesServiceImpl(ManageProjectsService projects, ArtifactRepository repositoryServices,@Named("dependencyOverride") DependencyOverride dependencyOverride)
     {
         this.projects = projects;
         this.repositoryServices = repositoryServices;
-        this.dependencyUtil = dependencyUtil;
-    }
-
-    public ManageDependenciesServiceImpl(ManageProjectsService projects, ArtifactRepository repositoryServices)
-    {
-        this.projects = projects;
-        this.repositoryServices = repositoryServices;
-        this.dependencyUtil = new DependencyUtil();
+        this.dependencyOverride = dependencyOverride;
     }
 
     @Override
-    public List<ProjectVersion> calculateDependencies(String groupId, String artifactId, String versionId)
+    public List<ProjectVersion> retrieveDependenciesFromRepository(String groupId, String artifactId, String versionId)
     {
         List<ProjectVersion> versionDependencies = new ArrayList<>();
         LOGGER.info("Finding dependencies for [{}-{}-{}]", groupId, artifactId, versionId);
@@ -69,13 +63,13 @@ public List<ProjectVersion> calculateDependencies(String groupId, String artifac
         return versionDependencies;
     }
 
-    @Override
-    public VersionDependencyReport calculateTransitiveDependencies(List<ProjectVersion> directDependencies)
+
+    private VersionDependencyReport calculateTransitiveDependencies(List<ProjectVersion> projectVersions)
     {
         Set<ProjectVersion> projectDependencies = new HashSet<>();
         try
         {
-            directDependencies.forEach(deps ->
+            projectVersions.forEach(deps ->
             {
                 LOGGER.info(String.format("Finding dependencies for %s-%s-%s", deps.getGroupId(), deps.getArtifactId(), deps.getVersionId()));
                 Optional<StoreProjectVersionData> projectData = this.projects.find(deps.getGroupId(), deps.getArtifactId(), deps.getVersionId());
@@ -95,7 +89,7 @@ else if (!projectData.get().getTransitiveDependenciesReport().isValid())
                 else
                 {
                     LOGGER.info(String.format("Finding dependencies for %s-%s-%s as no data is present in the store", deps.getGroupId(), deps.getArtifactId(), deps.getVersionId()));
-                    List<ProjectVersion> dependencies = this.calculateDependencies(deps.getGroupId(), deps.getArtifactId(), deps.getVersionId());
+                    List<ProjectVersion> dependencies = this.retrieveDependenciesFromRepository(deps.getGroupId(), deps.getArtifactId(), deps.getVersionId());
                     projectDependencies.addAll(dependencies);
                     VersionDependencyReport report = calculateTransitiveDependencies(dependencies);
                     if (!report.isValid())
@@ -116,7 +110,7 @@ else if (!projectData.get().getTransitiveDependenciesReport().isValid())
             return new VersionDependencyReport(new ArrayList<>(), false);
         }
         LOGGER.info("Completed finding dependencies");
-        List<ProjectVersion> finalDependencies = this.dependencyUtil.overrideDependencies(directDependencies, projectDependencies.stream().collect(Collectors.toList()), this::getCalculatedTransitiveDependencies);
+        List<ProjectVersion> finalDependencies = this.dependencyOverride.overrideWith(projectDependencies.stream().collect(Collectors.toList()), projectVersions, this::getCalculatedTransitiveDependencies);
         return new VersionDependencyReport(finalDependencies, true);
     }
 

diff --git a/...services/src/main/java/org/finos/legend/depot/services/guice/ArtifactsServicesModule.java b/...services/src/main/java/org/finos/legend/depot/services/guice/ArtifactsServicesModule.java
@@ -21,7 +21,9 @@
 import com.google.inject.name.Named;
 import org.finos.legend.depot.domain.artifacts.repository.ArtifactType;
 import org.finos.legend.depot.services.api.artifacts.purge.ArtifactsPurgeService;
+import org.finos.legend.depot.services.api.artifacts.reconciliation.VersionsReconciliationService;
 import org.finos.legend.depot.services.api.artifacts.refresh.ArtifactsRefreshService;
+import org.finos.legend.depot.services.api.artifacts.refresh.RefreshDependenciesService;
 import org.finos.legend.depot.services.artifacts.handlers.entities.EntitiesHandlerImpl;
 import org.finos.legend.depot.services.artifacts.handlers.entities.EntityProvider;
 import org.finos.legend.depot.services.artifacts.handlers.entities.VersionedEntitiesHandlerImpl;
@@ -31,6 +33,8 @@
 import org.finos.legend.depot.services.artifacts.purge.ArtifactsPurgeServiceImpl;
 import org.finos.legend.depot.services.artifacts.refresh.ArtifactsRefreshServiceImpl;
 import org.finos.legend.depot.services.artifacts.refresh.ProjectVersionRefreshHandler;
+import org.finos.legend.depot.services.artifacts.reconciliation.VersionsReconciliationServiceImpl;
+import org.finos.legend.depot.services.artifacts.refresh.RefreshDependenciesServiceImpl;
 import org.finos.legend.depot.store.notifications.api.NotificationEventHandler;
 import org.finos.legend.depot.services.api.artifacts.handlers.ProjectArtifactHandlerFactory;
 import org.finos.legend.depot.services.api.artifacts.configuration.ArtifactsRetentionPolicyConfiguration;
@@ -56,6 +60,13 @@ protected void configure()
         configureHandlers();
         configureRefresh();
         configurePurge();
+        configureVersionReconciliation();
+    }
+
+    protected void configureVersionReconciliation()
+    {
+        bind(VersionsReconciliationService.class).to(VersionsReconciliationServiceImpl.class);
+        expose(VersionsReconciliationService.class);
     }
 
     protected void configurePurge()
@@ -68,6 +79,7 @@ protected void configureRefresh()
     {
         bind(ArtifactsRefreshService.class).to(ArtifactsRefreshServiceImpl.class);
         bind(NotificationEventHandler.class).to(ProjectVersionRefreshHandler.class);
+        bind(RefreshDependenciesService.class).to(RefreshDependenciesServiceImpl.class);
         bind(ProjectVersionRefreshHandler.class);
 
         expose(ArtifactsRefreshService.class);

diff --git a/.../guice/ManageCoreDataSchedulesModule.java → ...VersionReconciliationSchedulesModule.java b/.../guice/ManageCoreDataSchedulesModule.java → ...VersionReconciliationSchedulesModule.java
@@ -18,23 +18,22 @@
 import com.google.inject.PrivateModule;
 import com.google.inject.Provides;
 import com.google.inject.Singleton;
-import org.finos.legend.depot.services.api.projects.ProjectsVersionsReconciliationService;
+import com.google.inject.name.Named;
+import org.finos.legend.depot.services.api.artifacts.reconciliation.VersionsReconciliationService;
 import org.finos.legend.depot.services.api.schedules.SchedulesFactory;
 import org.finos.legend.depot.tracing.api.PrometheusMetricsHandler;
 import org.finos.legend.depot.tracing.configuration.PrometheusConfiguration;
 
-import javax.inject.Named;
+import static org.finos.legend.depot.services.artifacts.reconciliation.VersionsReconciliationServiceImpl.MISSING_REPO_VERSIONS;
+import static org.finos.legend.depot.services.artifacts.reconciliation.VersionsReconciliationServiceImpl.MISSING_STORE_VERSIONS;
+import static org.finos.legend.depot.services.artifacts.reconciliation.VersionsReconciliationServiceImpl.PROJECTS;
+import static org.finos.legend.depot.services.artifacts.reconciliation.VersionsReconciliationServiceImpl.REPO_EXCEPTIONS;
+import static org.finos.legend.depot.services.artifacts.reconciliation.VersionsReconciliationServiceImpl.REPO_VERSIONS;
+import static org.finos.legend.depot.services.artifacts.reconciliation.VersionsReconciliationServiceImpl.STORE_VERSIONS;
 
-import static org.finos.legend.depot.services.VersionsMismatchService.MISSING_REPO_VERSIONS;
-import static org.finos.legend.depot.services.VersionsMismatchService.MISSING_STORE_VERSIONS;
-import static org.finos.legend.depot.services.VersionsMismatchService.PROJECTS;
-import static org.finos.legend.depot.services.VersionsMismatchService.REPO_EXCEPTIONS;
-import static org.finos.legend.depot.services.VersionsMismatchService.REPO_VERSIONS;
-import static org.finos.legend.depot.services.VersionsMismatchService.STORE_VERSIONS;
-
-public class ManageCoreDataSchedulesModule extends PrivateModule
+public class VersionReconciliationSchedulesModule extends PrivateModule
 {
-    private static final String REPOSITORY_METRICS_SCHEDULE = "repository-metrics-schedule";
+    public static final String REPOSITORY_METRICS_SCHEDULE = "repository-metrics-schedule";
 
     @Override
     protected void configure()
@@ -44,7 +43,7 @@ protected void configure()
     @Provides
     @Named("repository-metrics")
     @Singleton
-    boolean registerMetrics(PrometheusConfiguration prometheusConfiguration, SchedulesFactory schedulesFactory, ProjectsVersionsReconciliationService versionsMismatchService)
+    boolean registerMetrics(PrometheusConfiguration prometheusConfiguration, SchedulesFactory schedulesFactory, VersionsReconciliationService versionsReconciliationService)
     {
         if (prometheusConfiguration.isEnabled())
         {
@@ -55,8 +54,9 @@ boolean registerMetrics(PrometheusConfiguration prometheusConfiguration, Schedul
             metricsHandler.registerGauge(MISSING_REPO_VERSIONS, MISSING_REPO_VERSIONS);
             metricsHandler.registerGauge(MISSING_STORE_VERSIONS, MISSING_STORE_VERSIONS);
             metricsHandler.registerGauge(REPO_EXCEPTIONS, REPO_EXCEPTIONS);
-            schedulesFactory.register(REPOSITORY_METRICS_SCHEDULE, 5 * SchedulesFactory.MINUTE, 5 * SchedulesFactory.MINUTE, versionsMismatchService::findVersionsMismatches);
+            schedulesFactory.register(REPOSITORY_METRICS_SCHEDULE, 5 * SchedulesFactory.MINUTE, 5 * SchedulesFactory.MINUTE, versionsReconciliationService::findVersionsMismatches);
         }
         return true;
     }
+
 }