From 61ba2f5e0ad98f339574488aaa2fa1bddbedcfa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tib=C3=A9rio=20Baptista?= Date: Fri, 4 Oct 2024 15:19:45 +0100 Subject: [PATCH 1/3] unindex methods for platform interface; minor code refactor --- .../plugins/DicooglePlatformProxy.java | 18 + .../ua/dicoogle/plugins/PluginController.java | 1867 +++++++++-------- .../core/plugins/PlatformInterfaceMock.java | 15 + .../pt/ua/dicoogle/sdk/IndexerInterface.java | 2 +- .../sdk/core/DicooglePlatformInterface.java | 70 +- 5 files changed, 1036 insertions(+), 936 deletions(-) diff --git a/dicoogle/src/main/java/pt/ua/dicoogle/plugins/DicooglePlatformProxy.java b/dicoogle/src/main/java/pt/ua/dicoogle/plugins/DicooglePlatformProxy.java index 6878fa77b..7532e8c67 100644 --- a/dicoogle/src/main/java/pt/ua/dicoogle/plugins/DicooglePlatformProxy.java +++ b/dicoogle/src/main/java/pt/ua/dicoogle/plugins/DicooglePlatformProxy.java @@ -21,6 +21,8 @@ import java.net.URI; import java.util.Collection; import java.util.List; +import java.util.function.Consumer; + import pt.ua.dicoogle.core.settings.ServerSettingsManager; @@ -31,6 +33,7 @@ import pt.ua.dicoogle.sdk.core.DicooglePlatformInterface; import pt.ua.dicoogle.sdk.datastructs.Report; import pt.ua.dicoogle.sdk.datastructs.SearchResult; +import pt.ua.dicoogle.sdk.datastructs.UnindexReport; import pt.ua.dicoogle.sdk.datastructs.dim.DimLevel; import pt.ua.dicoogle.sdk.settings.server.ServerSettingsReader; import pt.ua.dicoogle.sdk.task.JointQueryTask; @@ -157,6 +160,21 @@ public List> index(URI path) { return pluginController.index(path); } + @Override + public void unindex(URI path) { + pluginController.unindex(path); + } + + @Override + public List> unindex(Collection paths) { + return pluginController.unindex(paths, null); + } + + @Override + public List> unindex(Collection paths, Consumer> callbacks) { + return pluginController.unindex(paths, callbacks); + } + @Override public List indexBlocking(URI path) { return pluginController.indexBlocking(path); diff --git a/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java b/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java index 9e8a1341b..bfdf68873 100755 --- a/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java +++ b/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java @@ -1,913 +1,954 @@ -/** - * Copyright (C) 2014 Universidade de Aveiro, DETI/IEETA, Bioinformatics Group - http://bioinformatics.ua.pt/ - * - * This file is part of Dicoogle/dicoogle. - * - * Dicoogle/dicoogle is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Dicoogle/dicoogle is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Dicoogle. If not, see . - */ -package pt.ua.dicoogle.plugins; - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.io.FileUtils; -import org.restlet.resource.ServerResource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pt.ua.dicoogle.core.settings.ServerSettingsManager; -import pt.ua.dicoogle.plugins.webui.WebUIPlugin; -import pt.ua.dicoogle.plugins.webui.WebUIPluginManager; -import pt.ua.dicoogle.sdk.*; -import pt.ua.dicoogle.sdk.datastructs.Report; -import pt.ua.dicoogle.sdk.datastructs.UnindexReport; -import pt.ua.dicoogle.sdk.datastructs.SearchResult; -import pt.ua.dicoogle.sdk.datastructs.dim.DimLevel; -import pt.ua.dicoogle.sdk.settings.ConfigurationHolder; -import pt.ua.dicoogle.sdk.task.JointQueryTask; -import pt.ua.dicoogle.sdk.task.Task; -import pt.ua.dicoogle.server.ControlServices; -import pt.ua.dicoogle.server.PluginRestletApplication; -import pt.ua.dicoogle.server.web.DicoogleWeb; -import pt.ua.dicoogle.taskManager.RunningIndexTasks; -import pt.ua.dicoogle.taskManager.TaskManager; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.zip.ZipFile; - -/** - * - * PluginController is the core of the Plugins architecture. - * - *

- * It loads the plugins, takes care of the list of active plugins and control - * the tasks that are exchanged between plugins and core plugins - * - * @author Carlos Ferreira - * @author Frederico Valente - * @author Luís A. Bastião Silva - * @author Tiago Marques Godinho - * @author Eduardo Pinho - */ -public class PluginController { - - private static final Logger logger = LoggerFactory.getLogger(PluginController.class); - private static PluginController instance; - - public synchronized static PluginController getInstance() { - if (instance == null) { - instance = new PluginController(new File("Plugins")); - } - return instance; - } - - private final Collection pluginSets; - private final Collection deadPluginSets; - private File pluginFolder; - private final PluginPreparer preparer; - - private PluginSet remoteQueryPlugins = null; - private final WebUIPluginManager webUI; - private final DicooglePlatformProxy proxy; - // Task manager for index processes - private TaskManager taskManager = - new TaskManager(Integer.parseInt(System.getProperty("dicoogle.taskManager.nThreads", "4"))); - // Task Managers for Queries - private TaskManager taskManagerQueries = - new TaskManager(Integer.parseInt(System.getProperty("dicoogle.taskManager.nQueryThreads", "4"))); - - /** Whether to shut down Dicoogle when a plugin is marked as dead */ - private static boolean DEAD_PLUGIN_KILL_SWITCH = - System.getProperty("dicoogle.deadPluginKillSwitch", "false").equalsIgnoreCase("true"); - - public PluginController(File pathToPluginDirectory) { - logger.info("Creating PluginController Instance"); - pluginFolder = pathToPluginDirectory; - - // the plugin directory does not exist. lets create it - if (!pathToPluginDirectory.exists()) { - logger.info("Creating new Plugin Folder"); - pathToPluginDirectory.mkdirs(); - } - - this.deadPluginSets = new ArrayList<>(4); - - // loads the plugins - pluginSets = PluginFactory.getPlugins(pathToPluginDirectory); - // load web UI plugins (they are not Java, so the process is delegated to another entity) - this.webUI = new WebUIPluginManager(); - this.loadWebUIPlugins(); - - logger.info("Loaded Local Plugins"); - - this.configurePlugins(); - - pluginSets.add(new DefaultFileStoragePlugin()); - logger.info("Added default storage plugin"); - - this.proxy = new DicooglePlatformProxy(this); - this.preparer = new PluginPreparer(this.proxy); - - initializePlugins(pluginSets); - initRestInterface(pluginSets); - initJettyInterface(pluginSets); - logger.info("Initialized plugins"); - } - - private void loadWebUIPlugins() { - // loadByPluginName all at "WebPlugins" - this.webUI.loadAll(new File("WebPlugins")); - - // go through each jar'd plugin and fetch their WebPlugins - for (File j : FileUtils.listFiles(pluginFolder, new String[] {"jar", "zip"}, false)) { - try { - this.webUI.loadAllFromZip(new ZipFile(j)); - } catch (IOException ex) { - // ignore - logger.warn("Failed to load web UI plugins from {}: {}", j.getName(), ex.getMessage()); - } - } - } - - private void configurePlugins() { - // loads plugins' settings and passes them to the plugin - File settingsFolder = new File(pluginFolder.getPath() + "/settings/"); - if (!settingsFolder.exists()) { - logger.info("Creating Local Settings Folder"); - settingsFolder.mkdir(); - } - - for (Iterator it = pluginSets.iterator(); it.hasNext();) { - PluginSet plugin = it.next(); - try { - final String name = plugin.getName(); - logger.info("Loading plugin: {}", name); - File pluginSettingsFile = new File(settingsFolder + "/" + name.replace('/', '-') + ".xml"); - ConfigurationHolder holder = new ConfigurationHolder(pluginSettingsFile); - if (plugin.getName().equals("RemotePluginSet")) { - this.remoteQueryPlugins = plugin; - // holder.getConfiguration().setProperty("NodeName", ServerSettingsManager.getSettings().getNodeName()); - // holder.getConfiguration().setProperty("TemporaryPath", ServerSettingsManager.getSettings().getPath()); - - logger.info("Started Remote Communications Manager"); - } - applySettings(plugin, holder); - } catch (ConfigurationException e) { - logger.error("Failed to create configuration holder", e); - } catch (RuntimeException e) { - String name; - try { - name = plugin.getName(); - } catch (Exception ex2) { - logger.warn("Plugin set name cannot be retrieved: {}", ex2.getMessage()); - name = "UNKNOWN"; - } - if (DEAD_PLUGIN_KILL_SWITCH) { - logger.error("Unexpected error while loading plugin set {}. Dicoogle will shut down.", name, e); - System.exit(-4); - } else { - logger.error("Unexpected error while loading plugin set {}. Plugin set marked as dead.", name, e); - this.deadPluginSets.add(new DeadPlugin(name, e)); - it.remove(); - } - } - } - logger.debug("Settings pushed to plugins"); - webUI.loadSettings(settingsFolder); - logger.debug("Settings pushed to web UI plugins"); - } - - private void initializePlugins(Collection plugins) { - for (PluginSet set : plugins) { - logger.debug("SetPlugins: {}", set); - - // provide platform to each plugin interface - final Collection> all = Arrays.asList(set.getStoragePlugins(), set.getIndexPlugins(), - set.getQueryPlugins(), set.getJettyPlugins(), set.getRestPlugins()); - for (Collection interfaces : all) { - if (interfaces == null) { - logger.debug("Plugin set {} provided a null collection!"); - continue; - } - for (Object o : interfaces) { - this.preparer.injectPlatform(o); - } - } - - // and to the set itself - this.preparer.setup(set); - } - } - - private void applySettings(PluginSet set, ConfigurationHolder holder) { - // provide platform to each plugin interface - final Collection> all = Arrays.asList(set.getStoragePlugins(), - set.getIndexPlugins(), set.getQueryPlugins(), set.getJettyPlugins()); - for (Collection interfaces : all) { - if (interfaces == null) - continue; - for (DicooglePlugin p : interfaces) { - p.setSettings(holder); - } - } - set.setSettings(holder); - - } - - /** - * Each pluginSet provides a collection of barebone rest interfaces Here we - * check which interfaces are present and create a restlet component to - * handle them. also we export them using common settings and security - * profiles - */ - private void initRestInterface(Collection plugins) { - logger.info("Initializing plugin rest interfaces"); - - ArrayList restInterfaces = new ArrayList<>(); - for (PluginSet set : plugins) { - Collection restInterface = set.getRestPlugins(); - if (restInterface == null) { - continue; - } - restInterfaces.addAll(restInterface); - } - - for (ServerResource resource : restInterfaces) { - PluginRestletApplication.attachRestPlugin(resource); - } - logger.info("Finished initializing rest interfaces"); - } - - private void initJettyInterface(Collection plugins) { - logger.info("Initializing jetty interface"); - - ArrayList jettyInterfaces = new ArrayList<>(); - for (PluginSet set : plugins) { - Collection jettyInterface = set.getJettyPlugins(); - if (jettyInterface == null) - continue; - jettyInterfaces.addAll(jettyInterface); - } - - DicoogleWeb jettyServer = ControlServices.getInstance().getWebServicePlatform(); - for (JettyPluginInterface resource : jettyInterfaces) { - jettyServer.addContextHandlers(resource.getJettyHandlers()); - } - } - - /** - * Stops the plugins and saves the settings - */ - public void shutdown() { - for (PluginSet plugin : pluginSets) { - // TODO(#605) Consider auto-saving settings on shutdown: - /* - Settings settings = plugin.getSettings(); - if (settings != null) { - settings.save(); - } - */ - - // let the plugin know we are shutting down - try { - logger.debug("Plugin set {} is shutting down", plugin.getName()); - plugin.shutdown(); - } catch (Exception ex) { - logger.error("Plugin set {} did not shutdown gracefully", plugin.getName(), ex); - } - } - } - - /** - * stops a pluginset. this could be more efficient, however this is hardly a - * bottleneck TODO: needs more granularity, we should be able to stop only - * the indexers or the queryers - * - * @param pluginName - */ - public void stopPlugin(String pluginName) { - for (PluginSet pluginSet : pluginSets) { - if (pluginSet.getName().compareTo(pluginName) == 0) { - // pluginSet.stop(); - return; - } - } - } - - public void startPlugin(String pluginName) { - for (PluginSet pluginSet : pluginSets) { - if (pluginSet.getName().compareTo(pluginName) == 0) { - // pluginSet.stop(); - return; - } - } - } - - public Collection getIndexingPlugins(boolean onlyEnabled) { - ArrayList indexers = new ArrayList<>(); - for (PluginSet pSet : pluginSets) { - for (IndexerInterface index : pSet.getIndexPlugins()) { - if (!index.isEnabled() && onlyEnabled) { - continue; - } - indexers.add(index); - } - } - return indexers; - } - - public Collection getStoragePlugins(boolean onlyEnabled) { - ArrayList storagePlugins = new ArrayList<>(); - for (PluginSet pSet : pluginSets) { - for (StorageInterface store : pSet.getStoragePlugins()) { - if (!store.isEnabled() && onlyEnabled) { - continue; - } - storagePlugins.add(store); - } - } - return storagePlugins; - } - - public Collection getServletPlugins(boolean onlyEnabled) { - List plugins = new ArrayList<>(); - for (PluginSet pSet : pluginSets) { - for (JettyPluginInterface p : pSet.getJettyPlugins()) { - if (!p.isEnabled() && onlyEnabled) { - continue; - } - plugins.add(p); - } - } - return plugins; - } - - public Collection getServletPlugins() { - return this.getServletPlugins(true); - } - - public Collection getPluginSetNames() { - Collection l = new ArrayList<>(); - for (PluginSet s : this.pluginSets) { - l.add(s.getName()); - } - return l; - } - - public Collection getDeadPluginSets() { - return new ArrayList<>(this.deadPluginSets); - } - - /** - * Resolve a URI to a DicomInputStream - * @param location - * @param args - * @return - */ - public Iterable resolveURI(URI location, Object... args) { - Collection storages = getStoragePlugins(true); - - for (StorageInterface store : storages) { - if (store.handles(location)) { - logger.debug("Resolving URI: {} Storage: {}", location, store.getName()); - return store.at(location, args); - } - } - - logger.error("Could not resolve uri: {}", location); - return Collections.emptyList(); - } - - /** Retrieve a storage interface capable of handling files on a given location. - * - * TODO: this can be heavily improved if we keep a map of scheme->indexer - * However we are not supposed to call this every other cycle. - * - * TODO: we should return a proxy storage that always returns error - * - * @todo "schema" is a typo, should read "scheme" - * - * @param location a URI of the location, only the scheme matters - * @return a storage interface capable of handling the location, null if no suitable plugin is found - */ - public StorageInterface getStorageForSchema(URI location) { - if (location == null) { - logger.warn("URI for retrieving storage interface is null, ignoring"); - return null; - } - Collection storages = getStoragePlugins(false); - - for (StorageInterface store : storages) { - if (store.handles(location)) { - logger.debug("Retrieved storage for scheme: {}", location); - return store; - } - } - logger.warn("Could not get storage for scheme: {}", location); - return null; - } - - /** Retrieve a storage interface capable of handling files with the given scheme. - * - * TODO: this can be heavily improved if we keep a map of scheme->indexer - * However we are not supposed to call this every other cycle. - * - * TODO: we should return a proxy storage that always returns error - * - * @param scheme a URI of the location, only the scheme matters - * @return a storage interface capable of handling the location, null if no suitable plugin is found - */ - public StorageInterface getStorageForSchema(String scheme) { - URI uri = URI.create(scheme + ":/"); - return getStorageForSchema(uri); - } - - public Collection getQueryPlugins(boolean onlyEnabled) { - ArrayList queriers = new ArrayList<>(); - for (PluginSet pSet : pluginSets) { - for (QueryInterface querier : pSet.getQueryPlugins()) { - if (!querier.isEnabled() && onlyEnabled) { - continue; - } - queriers.add(querier); - } - } - return queriers; - } - - public List getQueryProvidersName(boolean enabled) { - Collection plugins = getQueryPlugins(enabled); - List names = new ArrayList<>(plugins.size()); - for (QueryInterface p : plugins) { - names.add(p.getName()); - } - return names; - } - - public QueryInterface getQueryProviderByName(String name, boolean onlyEnabled) { - Collection plugins = getQueryPlugins(onlyEnabled); - for (QueryInterface p : plugins) { - if (p.getName().equalsIgnoreCase(name)) { - return p; - } - } - logger.debug("Could not retrieve query provider {} for onlyEnabled = {}", name, onlyEnabled); - return null; - } - - /** - * Filter a list of query providers, returning the ones that are DICOM and active. - * If the list to filter is empty, return all the active DICOM query providers. - * If there are no DICOM query providers specified in the settings, filter across all the active query providers. - * - * @param providers a list of query providers - * @return the filtered list of active DICOM query providers - */ - public List filterDicomQueryProviders(List providers) { - List baseProviders = ServerSettingsManager.getSettings().getArchiveSettings().getDIMProviders(); - - if (baseProviders == null || baseProviders.isEmpty()) { - baseProviders = PluginController.getInstance().getQueryProvidersName(true); - } - - return baseProviders.stream().filter(p -> providers.isEmpty() || providers.contains(p)) - .collect(Collectors.toList()); - } - - // TODO: CONVENIENCE METHOD - public IndexerInterface getIndexerByName(String name, boolean onlyEnabled) { - Collection plugins = getIndexingPlugins(onlyEnabled); - for (IndexerInterface p : plugins) { - if (p.getName().equalsIgnoreCase(name)) { - return p; - } - } - logger.debug("No indexer matching name {} for onlyEnabled = {}", name, onlyEnabled); - return null; - } - - - public JettyPluginInterface getServletByName(String name, boolean onlyEnabled) { - Collection plugins = getServletPlugins(onlyEnabled); - for (JettyPluginInterface p : plugins) { - if (p.getName().equalsIgnoreCase(name)) { - return p; - } - } - logger.debug("No indexer matching name {} for onlyEnabled = {}", name, onlyEnabled); - return null; - } - - public StorageInterface getStorageByName(String name, boolean onlyEnabled) { - Collection plugins = getStoragePlugins(onlyEnabled); - for (StorageInterface p : plugins) { - if (p.getName().equalsIgnoreCase(name)) { - return p; - } - } - logger.debug("No indexer matching name {} for onlyEnabled = {}", name, onlyEnabled); - return null; - } - - - public JointQueryTask queryAll(JointQueryTask holder, final String query, final Object... parameters) { - List providers = this.getQueryProvidersName(true); - return query(holder, providers, query, parameters); - } - - public JointQueryTask queryAll(JointQueryTask holder, final String query, final DimLevel level, - final Object... parameters) { - List providers = this.getQueryProvidersName(true); - return query(holder, providers, query, level, parameters); - } - - public Task> query(String querySource, final String query, final Object... parameters) { - Task> t = getTaskForQuery(querySource, query, parameters); - taskManagerQueries.dispatch(t); - return t; - - } - - - public Task> query(String querySource, final String query, final DimLevel level, - final Object... parameters) { - Task> t = getTaskForQueryDim(querySource, query, level, parameters); - taskManagerQueries.dispatch(t); - - return t;// returns the handler to obtain the computation results - } - - public JointQueryTask query(JointQueryTask holder, List querySources, final String query, - final Object... parameters) { - if (holder == null) - return null; - - List>> tasks = new ArrayList<>(); - for (String p : querySources) { - Task> task = getTaskForQuery(p, query, parameters); - tasks.add(task); - holder.addTask(task); - } - - // and executes said task asynchronously - for (Task t : tasks) - taskManagerQueries.dispatch(t); - - return holder;// returns the handler to obtain the computation results - } - - public JointQueryTask query(JointQueryTask holder, List querySources, final String query, - final DimLevel level, final Object... parameters) { - if (holder == null) - return null; - - List>> tasks = new ArrayList<>(); - for (String p : querySources) { - Task> task = getTaskForQueryDim(p, query, level, parameters); - tasks.add(task); - holder.addTask(task); - } - - // and executes said task asynchronously - for (Task t : tasks) - taskManagerQueries.dispatch(t); - - return holder;// returns the handler to obtain the computation results - } - - - private Task> getTaskForQuery(final String querySource, final String query, - final Object... parameters) { - - final QueryInterface queryEngine = getQueryProviderByName(querySource, true); - // returns a tasks that runs the query from the selected query engine - String uid = UUID.randomUUID().toString(); - Task> queryTask = new Task<>(uid, querySource, new Callable>() { - @Override - public Iterable call() throws Exception { - if (queryEngine == null) - return Collections.emptyList(); - try { - return queryEngine.query(query, parameters); - } catch (RuntimeException ex) { - logger.warn("Query plugin {} failed unexpectedly", querySource, ex); - return Collections.emptyList(); - } - - } - }); - return queryTask; - } - - - private Task> getTaskForQueryDim(final String querySource, final String query, - final DimLevel level, final Object... parameters) { - - final QueryDimInterface queryEngine; - QueryInterface tmpQueryDimInterface = getQueryProviderByName(querySource, true); - if (tmpQueryDimInterface instanceof QueryDimInterface) { - queryEngine = (QueryDimInterface) tmpQueryDimInterface; - } else { - logger.warn("Query plugin {} is not a DIM query provider", querySource); - queryEngine = null; - } - - // returns a tasks that runs the query from the selected query engine - String uid = UUID.randomUUID().toString(); - Task> queryTask = new Task<>(uid, querySource, new Callable>() { - @Override - public Iterable call() { - if (queryEngine == null) { - logger.warn("Could not query provider {} as a DIM source", querySource); - - return Collections.emptyList(); - } - try { - return queryEngine.query(query, level, parameters); - } catch (RuntimeException ex) { - logger.warn("Query plugin {} failed unexpectedly", querySource, ex); - return Collections.emptyList(); - } - - } - }); - return queryTask; - } - - - - /* - * Given an URI (which may be a path to a dir or file, a web resource or whatever) - * this method creates a task that - * calls the appropriate indexers and instructs them to index the data pointed to by the URI - * it is up to the caller to run the task asynchronously by feeding it to an executor - * or in a blocking way by calling the get() method of the task - */ - public List> index(URI path) { - logger.info("Starting Indexing procedure for {}", path.toString()); - StorageInterface store = getStorageForSchema(path); - - if (store == null) { - logger.error("No storage plugin detected"); - return Collections.emptyList(); - } - - Collection indexers = getIndexingPlugins(true); - // Collection indexers = getIndexingPluginsByMimeType(path); - ArrayList> rettasks = new ArrayList<>(); - final String pathF = path.toString(); - for (IndexerInterface indexer : indexers) { - try { - Task task = indexer.index(store.at(path)); - if (task == null) - continue; - final String taskUniqueID = UUID.randomUUID().toString(); - task.setName(String.format("[%s]index %s", indexer.getName(), path)); - task.onCompletion(() -> { - logger.info("Task [{}] complete on {}", taskUniqueID, pathF); - }); - - taskManager.dispatch(task); - rettasks.add(task); - RunningIndexTasks.getInstance().addTask(task); - } catch (RuntimeException ex) { - logger.warn("Indexer {} failed unexpectedly", indexer.getName(), ex); - } - } - logger.debug("Finished firing all indexing plugins for {}", path); - - return rettasks; - } - - public List> index(String pluginName, URI path) { - logger.info("Starting Indexing procedure for {}", path); - StorageInterface store = getStorageForSchema(path); - - if (store == null) { - logger.error("No storage plugin detected"); - return Collections.emptyList(); - } - - final String taskUniqueID = UUID.randomUUID().toString(); - - IndexerInterface indexer = getIndexerByName(pluginName, true); - ArrayList> rettasks = new ArrayList<>(); - final String pathF = path.toString(); - try { - Task task = indexer.index(store.at(path)); - if (task != null) { - task.setName(String.format("[%s]index %s", pluginName, path)); - task.onCompletion(new Runnable() { - - @Override - public void run() { - logger.info("Task [{}] complete on {}", taskUniqueID, pathF); - } - }); - - taskManager.dispatch(task); - - rettasks.add(task); - logger.debug("Fired indexer {} for URI {}", pluginName, path.toString()); - RunningIndexTasks.getInstance().addTask(task); - } - } catch (RuntimeException ex) { - logger.warn("Indexer {} failed unexpectedly", indexer.getName(), ex); - } - - return rettasks; - } - - public void unindex(URI path) { - logger.info("Starting unindexing procedure for {}", path.toString()); - this.doUnindex(path, this.getIndexingPlugins(true)); - } - - /** Issue the removal of indexed entries in a path from the given indexers. - * - * @param path the URI of the directory or file to unindex - * @param indexProviders a collection of providers - */ - public void unindex(URI path, Collection indexProviders) { - logger.info("Starting unindexing procedure for {}", path); - - if (indexProviders != null) { - List indexers = new ArrayList<>(); - for (String provider : indexProviders) { - indexers.add(this.getIndexerByName(provider, true)); - } - this.doUnindex(path, indexers); - } else { - this.doUnindex(path, this.getIndexingPlugins(true)); - } - } - - /** Issue the removal of indexed entries in bulk. - * - * @param indexProvider the name of the indexer - * @param items a collections of item identifiers to unindex - * @param progressCallback an optional function (can be `null`), - * called for every batch of items successfully unindexed - * to indicate early progress - * and inform consumers that - * it is safe to remove or exclude the unindexed item - * @return an asynchronous task object returning - * a report containing which files were not unindexed, - * and whether some of them were not found in the database - * @throws IOException - */ - public Task unindex(String indexProvider, Collection items, - Consumer> progressCallback) throws IOException { - - IndexerInterface indexer = null; - if (indexProvider != null) { - indexer = this.getIndexerByName(indexProvider, true); - } - if (indexer == null) { - indexer = this.getIndexingPlugins(true).iterator().next(); - } - logger.info("[{}] Starting unindexing procedure for {} items", indexer.getName(), items.size()); - Task task = indexer.unindex(items, progressCallback); - if (task != null) { - final String taskUniqueID = UUID.randomUUID().toString(); - task.setName(String.format("[%s]unindex", indexer.getName())); - task.onCompletion(() -> { - logger.info("Unindexing task [{}] complete", taskUniqueID); - }); - taskManager.dispatch(task); - } - return task; - } - - /** Issue an unindexing procedure to the given indexers. - * - * @param path the URI of the directory or file to unindex - * @param indexers a collection of providers - */ - private void doUnindex(URI path, Collection indexers) { - for (IndexerInterface indexer : indexers) { - indexer.unindex(path); - } - logger.info("Finished unindexing {}", path); - } - - public void remove(URI uri) { - StorageInterface si = getStorageForSchema(uri); - if (si != null) - doRemove(uri, si); - else - logger.error("Could not find storage plugin to handle URI: {}", uri); - } - - public void doRemove(URI uri, StorageInterface si) { - if (Objects.equals(uri.getScheme(), si.getScheme())) { - si.remove(uri); - } else { - logger.warn("Storage Plugin does not handle URI: {},{}", uri, si); - } - logger.info("Finished removing {}", uri); - } - - /* - * Convinience method that calls index(URI) and runs the returned - * tasks on the executing thread - */ - public List indexBlocking(URI path) { - logger.info("Starting indexing blocking procedure for {}", path); - List> ret = index(path); - - ArrayList reports = new ArrayList<>(ret.size()); - for (Task t : ret) { - try { - reports.add(t.get()); - } catch (InterruptedException | ExecutionException e) { - logger.error(e.getMessage(), e); - } - } - logger.info("Finished indexing {}", path); - - return reports; - } - - // Methods for Web UI - - /** Retrieve all web UI plugin descriptors for the given slot id. - * - * @param ids the slot id's for the plugin ("query", "result", "menu", ...), empty or null for any slot - * @return a collection of web UI plugins. - */ - public Collection getWebUIPlugins(String... ids) { - logger.debug("getWebUIPlugins(slot ids: {})", ids != null ? Arrays.asList(ids) : ""); - List plugins = new ArrayList<>(); - Set idSet = Collections.EMPTY_SET; - if (ids != null) { - idSet = new HashSet<>(Arrays.asList(ids)); - } - for (WebUIPlugin plugin : webUI.pluginSet()) { - if (!plugin.isEnabled()) { - continue; - } - if (idSet.isEmpty() || idSet.contains(plugin.getSlotId())) { - plugins.add(plugin); - } - } - return plugins; - } - - /** Retrieve the web UI plugin descriptor of the plugin with the given name. - * - * @param name the unique name of the plugin - * @return a web UI plugin descriptor object, or null if no such plugin exists or is inactive - */ - public WebUIPlugin getWebUIPlugin(String name) { - logger.debug("getWebUIPlugin(name: {})", name); - WebUIPlugin plugin = webUI.get(name); - return plugin == null ? null : plugin.isEnabled() ? plugin : null; - } - - /** Retrieve the web UI plugin descriptor package.json. - * - * @param name the unique name of the plugin - * @return the full contents of the package.json, null if the plugin is not available - */ - public String getWebUIPackageJSON(String name) { - logger.debug("getWebUIPackageJSON(name: {})", name); - try { - Object o = webUI.retrieveJSON(name); - return (o != null) ? o.toString() : null; - } catch (IOException ex) { - logger.error("Failed to retrieve package JSON", ex); - return null; - } - } - - /** Retrieve the web UI plugin module code. - * - * @param name the unique name of the plugin - * @return the full contents of the module file, null if the plugin is not available - */ - public String getWebUIModuleJS(String name) { - logger.debug("getWebUIModuleJS(name: {})", name); - try { - return webUI.retrieveModuleJS(name); - } catch (IOException ex) { - logger.error("Failed to retrieve module", ex); - return null; - } - } -} +/** + * Copyright (C) 2014 Universidade de Aveiro, DETI/IEETA, Bioinformatics Group - http://bioinformatics.ua.pt/ + * + * This file is part of Dicoogle/dicoogle. + * + * Dicoogle/dicoogle is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Dicoogle/dicoogle is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Dicoogle. If not, see . + */ +package pt.ua.dicoogle.plugins; + +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.io.FileUtils; +import org.restlet.resource.ServerResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pt.ua.dicoogle.core.settings.ServerSettingsManager; +import pt.ua.dicoogle.plugins.webui.WebUIPlugin; +import pt.ua.dicoogle.plugins.webui.WebUIPluginManager; +import pt.ua.dicoogle.sdk.*; +import pt.ua.dicoogle.sdk.datastructs.Report; +import pt.ua.dicoogle.sdk.datastructs.UnindexReport; +import pt.ua.dicoogle.sdk.datastructs.SearchResult; +import pt.ua.dicoogle.sdk.datastructs.dim.DimLevel; +import pt.ua.dicoogle.sdk.settings.ConfigurationHolder; +import pt.ua.dicoogle.sdk.task.JointQueryTask; +import pt.ua.dicoogle.sdk.task.Task; +import pt.ua.dicoogle.server.ControlServices; +import pt.ua.dicoogle.server.PluginRestletApplication; +import pt.ua.dicoogle.server.web.DicoogleWeb; +import pt.ua.dicoogle.taskManager.RunningIndexTasks; +import pt.ua.dicoogle.taskManager.TaskManager; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.zip.ZipFile; + +/** + * + * PluginController is the core of the Plugins architecture. + * + *

+ * It loads the plugins, takes care of the list of active plugins and control + * the tasks that are exchanged between plugins and core plugins + * + * @author Carlos Ferreira + * @author Frederico Valente + * @author Luís A. Bastião Silva + * @author Tiago Marques Godinho + * @author Eduardo Pinho + */ +public class PluginController { + + private static final Logger logger = LoggerFactory.getLogger(PluginController.class); + private static PluginController instance; + + public synchronized static PluginController getInstance() { + if (instance == null) { + instance = new PluginController(new File("Plugins")); + } + return instance; + } + + private final Collection pluginSets; + private final Collection deadPluginSets; + private File pluginFolder; + private final PluginPreparer preparer; + + private PluginSet remoteQueryPlugins = null; + private final WebUIPluginManager webUI; + private final DicooglePlatformProxy proxy; + // Task manager for index processes + private TaskManager taskManager = + new TaskManager(Integer.parseInt(System.getProperty("dicoogle.taskManager.nThreads", "4"))); + // Task Managers for Queries + private TaskManager taskManagerQueries = + new TaskManager(Integer.parseInt(System.getProperty("dicoogle.taskManager.nQueryThreads", "4"))); + + /** Whether to shut down Dicoogle when a plugin is marked as dead */ + private static boolean DEAD_PLUGIN_KILL_SWITCH = + System.getProperty("dicoogle.deadPluginKillSwitch", "false").equalsIgnoreCase("true"); + + public PluginController(File pathToPluginDirectory) { + logger.info("Creating PluginController Instance"); + pluginFolder = pathToPluginDirectory; + + // the plugin directory does not exist. lets create it + if (!pathToPluginDirectory.exists()) { + logger.info("Creating new Plugin Folder"); + pathToPluginDirectory.mkdirs(); + } + + this.deadPluginSets = new ArrayList<>(4); + + // loads the plugins + pluginSets = PluginFactory.getPlugins(pathToPluginDirectory); + // load web UI plugins (they are not Java, so the process is delegated to another entity) + this.webUI = new WebUIPluginManager(); + this.loadWebUIPlugins(); + + logger.info("Loaded Local Plugins"); + + this.configurePlugins(); + + pluginSets.add(new DefaultFileStoragePlugin()); + logger.info("Added default storage plugin"); + + this.proxy = new DicooglePlatformProxy(this); + this.preparer = new PluginPreparer(this.proxy); + + initializePlugins(pluginSets); + initRestInterface(pluginSets); + initJettyInterface(pluginSets); + logger.info("Initialized plugins"); + } + + private void loadWebUIPlugins() { + // loadByPluginName all at "WebPlugins" + this.webUI.loadAll(new File("WebPlugins")); + + // go through each jar'd plugin and fetch their WebPlugins + for (File j : FileUtils.listFiles(pluginFolder, new String[] {"jar", "zip"}, false)) { + try { + this.webUI.loadAllFromZip(new ZipFile(j)); + } catch (IOException ex) { + // ignore + logger.warn("Failed to load web UI plugins from {}: {}", j.getName(), ex.getMessage()); + } + } + } + + private void configurePlugins() { + // loads plugins' settings and passes them to the plugin + File settingsFolder = new File(pluginFolder.getPath() + "/settings/"); + if (!settingsFolder.exists()) { + logger.info("Creating Local Settings Folder"); + settingsFolder.mkdir(); + } + + for (Iterator it = pluginSets.iterator(); it.hasNext();) { + PluginSet plugin = it.next(); + try { + final String name = plugin.getName(); + logger.info("Loading plugin: {}", name); + File pluginSettingsFile = new File(settingsFolder + "/" + name.replace('/', '-') + ".xml"); + ConfigurationHolder holder = new ConfigurationHolder(pluginSettingsFile); + if (plugin.getName().equals("RemotePluginSet")) { + this.remoteQueryPlugins = plugin; + // holder.getConfiguration().setProperty("NodeName", ServerSettingsManager.getSettings().getNodeName()); + // holder.getConfiguration().setProperty("TemporaryPath", ServerSettingsManager.getSettings().getPath()); + + logger.info("Started Remote Communications Manager"); + } + applySettings(plugin, holder); + } catch (ConfigurationException e) { + logger.error("Failed to create configuration holder", e); + } catch (RuntimeException e) { + String name; + try { + name = plugin.getName(); + } catch (Exception ex2) { + logger.warn("Plugin set name cannot be retrieved: {}", ex2.getMessage()); + name = "UNKNOWN"; + } + if (DEAD_PLUGIN_KILL_SWITCH) { + logger.error("Unexpected error while loading plugin set {}. Dicoogle will shut down.", name, e); + System.exit(-4); + } else { + logger.error("Unexpected error while loading plugin set {}. Plugin set marked as dead.", name, e); + this.deadPluginSets.add(new DeadPlugin(name, e)); + it.remove(); + } + } + } + logger.debug("Settings pushed to plugins"); + webUI.loadSettings(settingsFolder); + logger.debug("Settings pushed to web UI plugins"); + } + + private void initializePlugins(Collection plugins) { + for (PluginSet set : plugins) { + logger.debug("SetPlugins: {}", set); + + // provide platform to each plugin interface + final Collection> all = Arrays.asList(set.getStoragePlugins(), set.getIndexPlugins(), + set.getQueryPlugins(), set.getJettyPlugins(), set.getRestPlugins()); + for (Collection interfaces : all) { + if (interfaces == null) { + logger.debug("Plugin set {} provided a null collection!"); + continue; + } + for (Object o : interfaces) { + this.preparer.injectPlatform(o); + } + } + + // and to the set itself + this.preparer.setup(set); + } + } + + private void applySettings(PluginSet set, ConfigurationHolder holder) { + // provide platform to each plugin interface + final Collection> all = Arrays.asList(set.getStoragePlugins(), + set.getIndexPlugins(), set.getQueryPlugins(), set.getJettyPlugins()); + for (Collection interfaces : all) { + if (interfaces == null) + continue; + for (DicooglePlugin p : interfaces) { + p.setSettings(holder); + } + } + set.setSettings(holder); + + } + + /** + * Each pluginSet provides a collection of barebone rest interfaces Here we + * check which interfaces are present and create a restlet component to + * handle them. also we export them using common settings and security + * profiles + */ + private void initRestInterface(Collection plugins) { + logger.info("Initializing plugin rest interfaces"); + + ArrayList restInterfaces = new ArrayList<>(); + for (PluginSet set : plugins) { + Collection restInterface = set.getRestPlugins(); + if (restInterface == null) { + continue; + } + restInterfaces.addAll(restInterface); + } + + for (ServerResource resource : restInterfaces) { + PluginRestletApplication.attachRestPlugin(resource); + } + logger.info("Finished initializing rest interfaces"); + } + + private void initJettyInterface(Collection plugins) { + logger.info("Initializing jetty interface"); + + ArrayList jettyInterfaces = new ArrayList<>(); + for (PluginSet set : plugins) { + Collection jettyInterface = set.getJettyPlugins(); + if (jettyInterface == null) + continue; + jettyInterfaces.addAll(jettyInterface); + } + + DicoogleWeb jettyServer = ControlServices.getInstance().getWebServicePlatform(); + for (JettyPluginInterface resource : jettyInterfaces) { + jettyServer.addContextHandlers(resource.getJettyHandlers()); + } + } + + /** + * Stops the plugins and saves the settings + */ + public void shutdown() { + for (PluginSet plugin : pluginSets) { + // TODO(#605) Consider auto-saving settings on shutdown: + /* + Settings settings = plugin.getSettings(); + if (settings != null) { + settings.save(); + } + */ + + // let the plugin know we are shutting down + try { + logger.debug("Plugin set {} is shutting down", plugin.getName()); + plugin.shutdown(); + } catch (Exception ex) { + logger.error("Plugin set {} did not shutdown gracefully", plugin.getName(), ex); + } + } + } + + /** + * stops a pluginset. this could be more efficient, however this is hardly a + * bottleneck TODO: needs more granularity, we should be able to stop only + * the indexers or the queryers + * + * @param pluginName + */ + public void stopPlugin(String pluginName) { + for (PluginSet pluginSet : pluginSets) { + if (pluginSet.getName().compareTo(pluginName) == 0) { + // pluginSet.stop(); + return; + } + } + } + + public void startPlugin(String pluginName) { + for (PluginSet pluginSet : pluginSets) { + if (pluginSet.getName().compareTo(pluginName) == 0) { + // pluginSet.stop(); + return; + } + } + } + + public Collection getIndexingPlugins(boolean onlyEnabled) { + ArrayList indexers = new ArrayList<>(); + for (PluginSet pSet : pluginSets) { + for (IndexerInterface index : pSet.getIndexPlugins()) { + if (!index.isEnabled() && onlyEnabled) { + continue; + } + indexers.add(index); + } + } + return indexers; + } + + public Collection getStoragePlugins(boolean onlyEnabled) { + ArrayList storagePlugins = new ArrayList<>(); + for (PluginSet pSet : pluginSets) { + for (StorageInterface store : pSet.getStoragePlugins()) { + if (!store.isEnabled() && onlyEnabled) { + continue; + } + storagePlugins.add(store); + } + } + return storagePlugins; + } + + public Collection getServletPlugins(boolean onlyEnabled) { + List plugins = new ArrayList<>(); + for (PluginSet pSet : pluginSets) { + for (JettyPluginInterface p : pSet.getJettyPlugins()) { + if (!p.isEnabled() && onlyEnabled) { + continue; + } + plugins.add(p); + } + } + return plugins; + } + + public Collection getServletPlugins() { + return this.getServletPlugins(true); + } + + public Collection getPluginSetNames() { + Collection l = new ArrayList<>(); + for (PluginSet s : this.pluginSets) { + l.add(s.getName()); + } + return l; + } + + public Collection getDeadPluginSets() { + return new ArrayList<>(this.deadPluginSets); + } + + /** + * Resolve a URI to a DicomInputStream + * @param location + * @param args + * @return + */ + public Iterable resolveURI(URI location, Object... args) { + Collection storages = getStoragePlugins(true); + + for (StorageInterface store : storages) { + if (store.handles(location)) { + logger.debug("Resolving URI: {} Storage: {}", location, store.getName()); + return store.at(location, args); + } + } + + logger.error("Could not resolve uri: {}", location); + return Collections.emptyList(); + } + + /** Retrieve a storage interface capable of handling files on a given location. + * + * TODO: this can be heavily improved if we keep a map of scheme->indexer + * However we are not supposed to call this every other cycle. + * + * TODO: we should return a proxy storage that always returns error + * + * @todo "schema" is a typo, should read "scheme" + * + * @param location a URI of the location, only the scheme matters + * @return a storage interface capable of handling the location, null if no suitable plugin is found + */ + public StorageInterface getStorageForSchema(URI location) { + if (location == null) { + logger.warn("URI for retrieving storage interface is null, ignoring"); + return null; + } + Collection storages = getStoragePlugins(false); + + for (StorageInterface store : storages) { + if (store.handles(location)) { + logger.debug("Retrieved storage for scheme: {}", location); + return store; + } + } + logger.warn("Could not get storage for scheme: {}", location); + return null; + } + + /** Retrieve a storage interface capable of handling files with the given scheme. + * + * TODO: this can be heavily improved if we keep a map of scheme->indexer + * However we are not supposed to call this every other cycle. + * + * TODO: we should return a proxy storage that always returns error + * + * @param scheme a URI of the location, only the scheme matters + * @return a storage interface capable of handling the location, null if no suitable plugin is found + */ + public StorageInterface getStorageForSchema(String scheme) { + URI uri = URI.create(scheme + ":/"); + return getStorageForSchema(uri); + } + + public Collection getQueryPlugins(boolean onlyEnabled) { + ArrayList queriers = new ArrayList<>(); + for (PluginSet pSet : pluginSets) { + for (QueryInterface querier : pSet.getQueryPlugins()) { + if (!querier.isEnabled() && onlyEnabled) { + continue; + } + queriers.add(querier); + } + } + return queriers; + } + + public List getQueryProvidersName(boolean enabled) { + Collection plugins = getQueryPlugins(enabled); + List names = new ArrayList<>(plugins.size()); + for (QueryInterface p : plugins) { + names.add(p.getName()); + } + return names; + } + + public QueryInterface getQueryProviderByName(String name, boolean onlyEnabled) { + Collection plugins = getQueryPlugins(onlyEnabled); + for (QueryInterface p : plugins) { + if (p.getName().equalsIgnoreCase(name)) { + return p; + } + } + logger.debug("Could not retrieve query provider {} for onlyEnabled = {}", name, onlyEnabled); + return null; + } + + /** + * Filter a list of query providers, returning the ones that are DICOM and active. + * If the list to filter is empty, return all the active DICOM query providers. + * If there are no DICOM query providers specified in the settings, filter across all the active query providers. + * + * @param providers a list of query providers + * @return the filtered list of active DICOM query providers + */ + public List filterDicomQueryProviders(List providers) { + List baseProviders = ServerSettingsManager.getSettings().getArchiveSettings().getDIMProviders(); + + if (baseProviders == null || baseProviders.isEmpty()) { + baseProviders = PluginController.getInstance().getQueryProvidersName(true); + } + + return baseProviders.stream().filter(p -> providers.isEmpty() || providers.contains(p)) + .collect(Collectors.toList()); + } + + // TODO: CONVENIENCE METHOD + public IndexerInterface getIndexerByName(String name, boolean onlyEnabled) { + Collection plugins = getIndexingPlugins(onlyEnabled); + for (IndexerInterface p : plugins) { + if (p.getName().equalsIgnoreCase(name)) { + return p; + } + } + logger.debug("No indexer matching name {} for onlyEnabled = {}", name, onlyEnabled); + return null; + } + + + public JettyPluginInterface getServletByName(String name, boolean onlyEnabled) { + Collection plugins = getServletPlugins(onlyEnabled); + for (JettyPluginInterface p : plugins) { + if (p.getName().equalsIgnoreCase(name)) { + return p; + } + } + logger.debug("No indexer matching name {} for onlyEnabled = {}", name, onlyEnabled); + return null; + } + + public StorageInterface getStorageByName(String name, boolean onlyEnabled) { + Collection plugins = getStoragePlugins(onlyEnabled); + for (StorageInterface p : plugins) { + if (p.getName().equalsIgnoreCase(name)) { + return p; + } + } + logger.debug("No indexer matching name {} for onlyEnabled = {}", name, onlyEnabled); + return null; + } + + + public JointQueryTask queryAll(JointQueryTask holder, final String query, final Object... parameters) { + List providers = this.getQueryProvidersName(true); + return query(holder, providers, query, parameters); + } + + public JointQueryTask queryAll(JointQueryTask holder, final String query, final DimLevel level, + final Object... parameters) { + List providers = this.getQueryProvidersName(true); + return query(holder, providers, query, level, parameters); + } + + public Task> query(String querySource, final String query, final Object... parameters) { + Task> t = getTaskForQuery(querySource, query, parameters); + taskManagerQueries.dispatch(t); + return t; + + } + + + public Task> query(String querySource, final String query, final DimLevel level, + final Object... parameters) { + Task> t = getTaskForQueryDim(querySource, query, level, parameters); + taskManagerQueries.dispatch(t); + + return t;// returns the handler to obtain the computation results + } + + public JointQueryTask query(JointQueryTask holder, List querySources, final String query, + final Object... parameters) { + if (holder == null) + return null; + + List>> tasks = new ArrayList<>(); + for (String p : querySources) { + Task> task = getTaskForQuery(p, query, parameters); + tasks.add(task); + holder.addTask(task); + } + + // and executes said task asynchronously + for (Task t : tasks) + taskManagerQueries.dispatch(t); + + return holder;// returns the handler to obtain the computation results + } + + public JointQueryTask query(JointQueryTask holder, List querySources, final String query, + final DimLevel level, final Object... parameters) { + if (holder == null) + return null; + + List>> tasks = new ArrayList<>(); + for (String p : querySources) { + Task> task = getTaskForQueryDim(p, query, level, parameters); + tasks.add(task); + holder.addTask(task); + } + + // and executes said task asynchronously + for (Task t : tasks) + taskManagerQueries.dispatch(t); + + return holder;// returns the handler to obtain the computation results + } + + + private Task> getTaskForQuery(final String querySource, final String query, + final Object... parameters) { + + final QueryInterface queryEngine = getQueryProviderByName(querySource, true); + // returns a tasks that runs the query from the selected query engine + String uid = UUID.randomUUID().toString(); + Task> queryTask = new Task<>(uid, querySource, new Callable>() { + @Override + public Iterable call() throws Exception { + if (queryEngine == null) + return Collections.emptyList(); + try { + return queryEngine.query(query, parameters); + } catch (RuntimeException ex) { + logger.warn("Query plugin {} failed unexpectedly", querySource, ex); + return Collections.emptyList(); + } + + } + }); + return queryTask; + } + + + private Task> getTaskForQueryDim(final String querySource, final String query, + final DimLevel level, final Object... parameters) { + + final QueryDimInterface queryEngine; + QueryInterface tmpQueryDimInterface = getQueryProviderByName(querySource, true); + if (tmpQueryDimInterface instanceof QueryDimInterface) { + queryEngine = (QueryDimInterface) tmpQueryDimInterface; + } else { + logger.warn("Query plugin {} is not a DIM query provider", querySource); + queryEngine = null; + } + + // returns a tasks that runs the query from the selected query engine + String uid = UUID.randomUUID().toString(); + Task> queryTask = new Task<>(uid, querySource, new Callable>() { + @Override + public Iterable call() { + if (queryEngine == null) { + logger.warn("Could not query provider {} as a DIM source", querySource); + + return Collections.emptyList(); + } + try { + return queryEngine.query(query, level, parameters); + } catch (RuntimeException ex) { + logger.warn("Query plugin {} failed unexpectedly", querySource, ex); + return Collections.emptyList(); + } + + } + }); + return queryTask; + } + + + + /* + * Given an URI (which may be a path to a dir or file, a web resource or whatever) + * this method creates a task that + * calls the appropriate indexers and instructs them to index the data pointed to by the URI + * it is up to the caller to run the task asynchronously by feeding it to an executor + * or in a blocking way by calling the get() method of the task + */ + public List> index(URI path) { + logger.info("Starting Indexing procedure for {}", path.toString()); + StorageInterface store = getStorageForSchema(path); + + if (store == null) { + logger.error("No storage plugin detected"); + return Collections.emptyList(); + } + + Collection indexers = getIndexingPlugins(true); + // Collection indexers = getIndexingPluginsByMimeType(path); + ArrayList> rettasks = new ArrayList<>(); + final String pathF = path.toString(); + for (IndexerInterface indexer : indexers) { + try { + Task task = indexer.index(store.at(path)); + if (task == null) + continue; + final String taskUniqueID = UUID.randomUUID().toString(); + task.setName(String.format("[%s]index %s", indexer.getName(), path)); + task.onCompletion(() -> { + logger.info("Task [{}] complete on {}", taskUniqueID, pathF); + }); + + taskManager.dispatch(task); + rettasks.add(task); + RunningIndexTasks.getInstance().addTask(task); + } catch (RuntimeException ex) { + logger.warn("Indexer {} failed unexpectedly", indexer.getName(), ex); + } + } + logger.debug("Finished firing all indexing plugins for {}", path); + + return rettasks; + } + + public List> index(String pluginName, URI path) { + logger.info("Starting Indexing procedure for {}", path); + StorageInterface store = getStorageForSchema(path); + + if (store == null) { + logger.error("No storage plugin detected"); + return Collections.emptyList(); + } + + final String taskUniqueID = UUID.randomUUID().toString(); + + IndexerInterface indexer = getIndexerByName(pluginName, true); + ArrayList> rettasks = new ArrayList<>(); + final String pathF = path.toString(); + try { + Task task = indexer.index(store.at(path)); + if (task != null) { + task.setName(String.format("[%s]index %s", pluginName, path)); + task.onCompletion(new Runnable() { + + @Override + public void run() { + logger.info("Task [{}] complete on {}", taskUniqueID, pathF); + } + }); + + taskManager.dispatch(task); + + rettasks.add(task); + logger.debug("Fired indexer {} for URI {}", pluginName, path.toString()); + RunningIndexTasks.getInstance().addTask(task); + } + } catch (RuntimeException ex) { + logger.warn("Indexer {} failed unexpectedly", indexer.getName(), ex); + } + + return rettasks; + } + + public void unindex(URI path) { + logger.info("Starting unindexing procedure for {}", path.toString()); + this.doUnindex(path, this.getIndexingPlugins(true)); + } + + /** Issue the removal of indexed entries in a path from the given indexers. + * + * @param path the URI of the directory or file to unindex + * @param indexProviders a collection of providers + */ + public void unindex(URI path, Collection indexProviders) { + logger.info("Starting unindexing procedure for {}", path); + + if (indexProviders != null) { + List indexers = new ArrayList<>(); + for (String provider : indexProviders) { + indexers.add(this.getIndexerByName(provider, true)); + } + this.doUnindex(path, indexers); + } else { + this.doUnindex(path, this.getIndexingPlugins(true)); + } + } + + /** Issue the removal of indexed entries in bulk. + * + * @param items a collections of item identifiers to unindex + * @param progressCallback an optional function (can be `null`), + * called for every batch of items successfully unindexed + * to indicate early progress + * and inform consumers that + * it is safe to remove or exclude the unindexed item + * @return an asynchronous task object returning + * a report containing which files were not unindexed, + * and whether some of them were not found in the database + */ + public List> unindex(Collection items, Consumer> progressCallback) { + + List> tasks = new ArrayList<>(); + for (IndexerInterface indexer : this.getIndexingPlugins(true)) + tasks.add(createUnindexTask(items, progressCallback, indexer)); + + return tasks; + } + + /** Issue the removal of indexed entries in bulk. + * + * @param indexProvider the name of the indexer + * @param items a collections of item identifiers to unindex + * @param progressCallback an optional function (can be `null`), + * called for every batch of items successfully unindexed + * to indicate early progress + * and inform consumers that + * it is safe to remove or exclude the unindexed item + * @return an asynchronous task object returning + * a report containing which files were not unindexed, + * and whether some of them were not found in the database + */ + public Task unindex(String indexProvider, Collection items, + Consumer> progressCallback) { + + IndexerInterface indexer = null; + if (indexProvider != null) { + indexer = this.getIndexerByName(indexProvider, true); + } + if (indexer == null) { + indexer = this.getIndexingPlugins(true).iterator().next(); + } + return createUnindexTask(items, progressCallback, indexer); + } + + /** Issue an unindexing procedure to the given indexers. + * + * @param path the URI of the directory or file to unindex + * @param indexers a collection of providers + */ + private void doUnindex(URI path, Collection indexers) { + for (IndexerInterface indexer : indexers) { + indexer.unindex(path); + } + logger.info("Finished unindexing {}", path); + } + + public void remove(URI uri) { + StorageInterface si = getStorageForSchema(uri); + if (si != null) + doRemove(uri, si); + else + logger.error("Could not find storage plugin to handle URI: {}", uri); + } + + public void doRemove(URI uri, StorageInterface si) { + if (Objects.equals(uri.getScheme(), si.getScheme())) { + si.remove(uri); + } else { + logger.warn("Storage Plugin does not handle URI: {},{}", uri, si); + } + logger.info("Finished removing {}", uri); + } + + /* + * Convinience method that calls index(URI) and runs the returned + * tasks on the executing thread + */ + public List indexBlocking(URI path) { + logger.info("Starting indexing blocking procedure for {}", path); + List> ret = index(path); + + ArrayList reports = new ArrayList<>(ret.size()); + for (Task t : ret) { + try { + reports.add(t.get()); + } catch (InterruptedException | ExecutionException e) { + logger.error(e.getMessage(), e); + } + } + logger.info("Finished indexing {}", path); + + return reports; + } + + + /** + * Generate and dispatch an unindex task. + * @param items Collection of URIs to unindex + * @param progressCallback Callback raised when URIs are declared + * @param indexer + * @return + */ + private Task createUnindexTask(Collection items, Consumer> progressCallback, + IndexerInterface indexer) { + logger.info("[{}] Starting unindexing procedure for {} items", indexer.getName(), items.size()); + + Task task = null; + try { + task = indexer.unindex(items, progressCallback); + } catch (IOException e) { + logger.error("IO exception ocurred while creating unindex task", e); + } + + if (task != null) { + final String taskUniqueID = UUID.randomUUID().toString(); + task.setName(String.format("[%s]unindex", indexer.getName())); + task.onCompletion(() -> { + logger.info("Unindexing task [{}] complete", taskUniqueID); + }); + taskManager.dispatch(task); + } + return task; + } + + + // Methods for Web UI + + /** Retrieve all web UI plugin descriptors for the given slot id. + * + * @param ids the slot id's for the plugin ("query", "result", "menu", ...), empty or null for any slot + * @return a collection of web UI plugins. + */ + public Collection getWebUIPlugins(String... ids) { + logger.debug("getWebUIPlugins(slot ids: {})", ids != null ? Arrays.asList(ids) : ""); + List plugins = new ArrayList<>(); + Set idSet = Collections.EMPTY_SET; + if (ids != null) { + idSet = new HashSet<>(Arrays.asList(ids)); + } + for (WebUIPlugin plugin : webUI.pluginSet()) { + if (!plugin.isEnabled()) { + continue; + } + if (idSet.isEmpty() || idSet.contains(plugin.getSlotId())) { + plugins.add(plugin); + } + } + return plugins; + } + + /** Retrieve the web UI plugin descriptor of the plugin with the given name. + * + * @param name the unique name of the plugin + * @return a web UI plugin descriptor object, or null if no such plugin exists or is inactive + */ + public WebUIPlugin getWebUIPlugin(String name) { + logger.debug("getWebUIPlugin(name: {})", name); + WebUIPlugin plugin = webUI.get(name); + return plugin == null ? null : plugin.isEnabled() ? plugin : null; + } + + /** Retrieve the web UI plugin descriptor package.json. + * + * @param name the unique name of the plugin + * @return the full contents of the package.json, null if the plugin is not available + */ + public String getWebUIPackageJSON(String name) { + logger.debug("getWebUIPackageJSON(name: {})", name); + try { + Object o = webUI.retrieveJSON(name); + return (o != null) ? o.toString() : null; + } catch (IOException ex) { + logger.error("Failed to retrieve package JSON", ex); + return null; + } + } + + /** Retrieve the web UI plugin module code. + * + * @param name the unique name of the plugin + * @return the full contents of the module file, null if the plugin is not available + */ + public String getWebUIModuleJS(String name) { + logger.debug("getWebUIModuleJS(name: {})", name); + try { + return webUI.retrieveModuleJS(name); + } catch (IOException ex) { + logger.error("Failed to retrieve module", ex); + return null; + } + } +} diff --git a/dicoogle/src/test/java/pt/ua/dicoogle/core/plugins/PlatformInterfaceMock.java b/dicoogle/src/test/java/pt/ua/dicoogle/core/plugins/PlatformInterfaceMock.java index 5b2974dcf..579626655 100644 --- a/dicoogle/src/test/java/pt/ua/dicoogle/core/plugins/PlatformInterfaceMock.java +++ b/dicoogle/src/test/java/pt/ua/dicoogle/core/plugins/PlatformInterfaceMock.java @@ -25,6 +25,7 @@ import pt.ua.dicoogle.sdk.core.DicooglePlatformInterface; import pt.ua.dicoogle.sdk.datastructs.Report; import pt.ua.dicoogle.sdk.datastructs.SearchResult; +import pt.ua.dicoogle.sdk.datastructs.UnindexReport; import pt.ua.dicoogle.sdk.datastructs.dim.DimLevel; import pt.ua.dicoogle.sdk.settings.server.ServerSettingsReader; import pt.ua.dicoogle.sdk.task.JointQueryTask; @@ -33,6 +34,7 @@ import java.net.URI; import java.util.Collection; import java.util.List; +import java.util.function.Consumer; public class PlatformInterfaceMock implements DicooglePlatformInterface { @@ -120,6 +122,19 @@ public List> index(URI path) { return null; } + @Override + public void unindex(URI path) {} + + @Override + public List> unindex(Collection paths) { + return null; + } + + @Override + public List> unindex(Collection paths, Consumer> progressCallback) { + return null; + } + @Override public List indexBlocking(URI path) { return null; diff --git a/sdk/src/main/java/pt/ua/dicoogle/sdk/IndexerInterface.java b/sdk/src/main/java/pt/ua/dicoogle/sdk/IndexerInterface.java index e7df68890..54ebecf8f 100755 --- a/sdk/src/main/java/pt/ua/dicoogle/sdk/IndexerInterface.java +++ b/sdk/src/main/java/pt/ua/dicoogle/sdk/IndexerInterface.java @@ -111,7 +111,7 @@ public default boolean handles(URI path) { * one or more individual operations in batch, * thus becoming faster than unindexing each item individually. * - * Like {@linkplain index}, + * Like {@linkplain #index(Iterable, Object...)}, * this operation is asynchronous. * One can keep track of the unindexing task's progress * by passing a callback function as the second parameter. diff --git a/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java b/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java index ac3fac299..1a2f56dd9 100644 --- a/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java +++ b/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java @@ -21,6 +21,7 @@ import java.net.URI; import java.util.Collection; import java.util.List; +import java.util.function.Consumer; import pt.ua.dicoogle.sdk.IndexerInterface; import pt.ua.dicoogle.sdk.QueryInterface; @@ -28,6 +29,7 @@ import pt.ua.dicoogle.sdk.StorageInterface; import pt.ua.dicoogle.sdk.datastructs.Report; import pt.ua.dicoogle.sdk.datastructs.SearchResult; +import pt.ua.dicoogle.sdk.datastructs.UnindexReport; import pt.ua.dicoogle.sdk.datastructs.dim.DimLevel; import pt.ua.dicoogle.sdk.settings.server.ServerSettingsReader; import pt.ua.dicoogle.sdk.task.JointQueryTask; @@ -53,26 +55,26 @@ public interface DicooglePlatformInterface { * @param name the name of the plugin * @return the plugin that implements this interface */ - public IndexerInterface requestIndexPlugin(String name); + IndexerInterface requestIndexPlugin(String name); /** * Gets an installed query plugin by name. * @param name the name of the plugin * @return the plugin that implements this interface */ - public QueryInterface requestQueryPlugin(String name); + QueryInterface requestQueryPlugin(String name); /** * Obtains a collection of all active index plugins. * @return an unmodifiable collection of active instances of {@link IndexerInterface} */ - public Collection getAllIndexPlugins(); + Collection getAllIndexPlugins(); /** * Obtains a collection of all active query plugins. * @return an unmodifiable collection of active instances of {@link QueryInterface} */ - public Collection getAllQueryPlugins(); + Collection getAllQueryPlugins(); /** Obtains the storage interface for handling storage of the given scheme. * @@ -80,7 +82,7 @@ public interface DicooglePlatformInterface { * @return the storage interface capable of handling that content, or {@code null} if no storage plugin * installed can. */ - public StorageInterface getStoragePluginForSchema(String scheme); + StorageInterface getStoragePluginForSchema(String scheme); /** Obtains the storage interface for handling content in the given location. * @@ -88,7 +90,7 @@ public interface DicooglePlatformInterface { * @return the storage interface capable of handling that content, or {@code null} if no storage plugin * installed can */ - public StorageInterface getStorageForSchema(URI location); + StorageInterface getStorageForSchema(URI location); /** Quickly obtains all storage elements at the given location. * @@ -96,14 +98,14 @@ public interface DicooglePlatformInterface { * @param args a variable list of extra parameters for the retrieve * @return an iterable of storage input streams */ - public Iterable resolveURI(URI location, Object... args); + Iterable resolveURI(URI location, Object... args); /** Obtains all installed storage plugins. * * @param onlyEnabled whether only enabled plugins should be retrieved (all are retrieved if {@code false}) * @return a collection of storage plugin interfaces */ - public Collection getStoragePlugins(boolean onlyEnabled); + Collection getStoragePlugins(boolean onlyEnabled); /** Obtains the storage interface for handling storage of the given scheme. * Same as {@link #getStoragePluginForSchema(java.lang.String)}. @@ -112,21 +114,21 @@ public interface DicooglePlatformInterface { * @return the storage interface capable of handling that content, or {@code null} if no storage plugin * installed can */ - public StorageInterface getStorageForSchema(String scheme); + StorageInterface getStorageForSchema(String scheme); /** Obtains all installed query plugins. * * @param onlyEnabled whether only enabled plugins should be retrieved (all are retrieved if {@code false}) * @return a collection of query plugin interfaces */ - public Collection getQueryPlugins(boolean onlyEnabled); + Collection getQueryPlugins(boolean onlyEnabled); /** Gets the names of all query providers. * * @param enabled whether only enabled plugins should be retrieved (all are retrieved if {@code false}) * @return a collection of unique query provider names */ - public List getQueryProvidersName(boolean enabled); + List getQueryProvidersName(boolean enabled); /** Gets the query provider with the given name. * @@ -134,7 +136,7 @@ public interface DicooglePlatformInterface { * @param onlyEnabled whether only enabled plugins should be retrieved (all are retrieved if {@code false}) * @return a collection of unique query provider names */ - public QueryInterface getQueryProviderByName(String name, boolean onlyEnabled); + QueryInterface getQueryProviderByName(String name, boolean onlyEnabled); /** * Easily performs a query over all query providers. This operation is asynchronous and returns immediately. @@ -143,7 +145,7 @@ public interface DicooglePlatformInterface { * @param parameters a variable list of extra parameters for the query * @return a join query task */ - public JointQueryTask queryAll(JointQueryTask holder, String query, Object... parameters); + JointQueryTask queryAll(JointQueryTask holder, String query, Object... parameters); /** * Easily performs a query over all query providers. This operation is asynchronous and returns immediately. @@ -153,7 +155,7 @@ public interface DicooglePlatformInterface { * @param parameters a variable list of extra parameters for the query * @return a join query task */ - public JointQueryTask queryAll(JointQueryTask holder, String query, DimLevel level, Object... parameters); + JointQueryTask queryAll(JointQueryTask holder, String query, DimLevel level, Object... parameters); /** Easily performs a query over a specific provider. This operation is asynchronous and returns immediately. * @@ -162,7 +164,7 @@ public interface DicooglePlatformInterface { * @param parameters a variable list of extra parameters for the query * @return an asynchronous task containing the results */ - public Task> query(String querySource, String query, Object... parameters); + Task> query(String querySource, String query, Object... parameters); /** Easily performs a query over a specific provider. This operation is asynchronous and returns immediately. * @@ -172,7 +174,7 @@ public interface DicooglePlatformInterface { * @param parameters a variable list of extra parameters for the query * @return an asynchronous task containing the results */ - public Task> query(String querySource, DimLevel level, String query, Object... parameters); + Task> query(String querySource, DimLevel level, String query, Object... parameters); /** Easily performs a query over multiple providers. This operation is asynchronous and returns immediately. * @@ -182,7 +184,7 @@ public interface DicooglePlatformInterface { * @param parameters a variable list of extra parameters for the query * @return an asynchronous task containing the results */ - public JointQueryTask query(JointQueryTask holder, List querySources, String query, Object... parameters); + JointQueryTask query(JointQueryTask holder, List querySources, String query, Object... parameters); /** Easily performs a query over multiple providers. This operation is asynchronous and returns immediately. * @@ -193,17 +195,41 @@ public interface DicooglePlatformInterface { * @param parameters a variable list of extra parameters for the query * @return an asynchronous task containing the results */ - public JointQueryTask query(JointQueryTask holder, List querySources, DimLevel level, String query, + JointQueryTask query(JointQueryTask holder, List querySources, DimLevel level, String query, Object... parameters); /** Easily performs an indexation procedure over all active indexers. This operation is asynchronous * and returns immediately. - * + * * @param path the path to index * @return a list of asynchronous tasks, one for each provider */ - public List> index(URI path); + List> index(URI path); + + /** Easily performs an unindex procedure over all active indexers. This operation is synchronous. + * + * @param path the path to index + */ + void unindex(URI path); + + + /** Easily performs an unindex procedure for several paths over all active indexers. This operation is asynchronous + * and returns immediately. + * + * @param paths the path to index + * @return a list of asynchronous tasks from each provider + */ + List> unindex(Collection paths); + + /** Easily performs an unindex procedure for several paths over all active indexers. This operation is asynchronous + * and returns immediately. + * + * @param paths the path to index + * @return a list of asynchronous tasks from each provider + */ + List> unindex(Collection paths, Consumer> progressCallback); + /** Easily performs an indexation procedure over all active indexers. This operation is synchronous * and will wait until all providers have finished indexing. @@ -211,10 +237,10 @@ public JointQueryTask query(JointQueryTask holder, List querySources, Di * @param path the path to index * @return a list of reports, one for each provider */ - public List indexBlocking(URI path); + List indexBlocking(URI path); /** Obtain access to the server's settings. * @return an object for read-only access to the settings */ - public ServerSettingsReader getSettings(); + ServerSettingsReader getSettings(); } From c1368eb12b218fb0d5942d4205971601ac09ecea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tib=C3=A9rio=20Baptista?= Date: Fri, 4 Oct 2024 16:56:04 +0100 Subject: [PATCH 2/3] add >1 URI index method for platform interface; deprecate indexBlocking --- .../plugins/DicooglePlatformProxy.java | 7 ++ .../ua/dicoogle/plugins/PluginController.java | 66 +++++++++++++++---- .../core/plugins/PlatformInterfaceMock.java | 5 ++ .../sdk/core/DicooglePlatformInterface.java | 10 +++ 4 files changed, 75 insertions(+), 13 deletions(-) diff --git a/dicoogle/src/main/java/pt/ua/dicoogle/plugins/DicooglePlatformProxy.java b/dicoogle/src/main/java/pt/ua/dicoogle/plugins/DicooglePlatformProxy.java index 7532e8c67..90cc2a453 100644 --- a/dicoogle/src/main/java/pt/ua/dicoogle/plugins/DicooglePlatformProxy.java +++ b/dicoogle/src/main/java/pt/ua/dicoogle/plugins/DicooglePlatformProxy.java @@ -160,6 +160,12 @@ public List> index(URI path) { return pluginController.index(path); } + @Override + public List> index(Collection paths) { + return pluginController.index(paths); + } + + @Override public void unindex(URI path) { pluginController.unindex(path); @@ -176,6 +182,7 @@ public List> unindex(Collection paths, Consumer indexBlocking(URI path) { return pluginController.indexBlocking(path); } diff --git a/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java b/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java index bfdf68873..2edc61418 100755 --- a/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java +++ b/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java @@ -733,13 +733,54 @@ public void run() { return rettasks; } + public List> index(Collection paths) { + logger.info("Starting indexing procedure for {} items.", paths.size()); + + List objectsToStore = new ArrayList<>(); + ArrayList> rettasks = new ArrayList<>(); + Collection indexers = getIndexingPlugins(true); + + for (URI path : paths) { + StorageInterface store = getStorageForSchema(path); + + if (store == null) { + logger.error("Could not get storage schema for {}", path); + continue; + } + + store.at(path).forEach(objectsToStore::add); + } + + for (IndexerInterface indexer : indexers) { + try { + Task task = indexer.index(objectsToStore); + if (task == null) + continue; + final String taskUniqueID = UUID.randomUUID().toString(); + task.setName(String.format("[%s]index %d items", indexer.getName(), objectsToStore.size())); + task.onCompletion(() -> { + logger.info("Task [{}] complete on {} items", taskUniqueID, objectsToStore.size()); + }); + + taskManager.dispatch(task); + rettasks.add(task); + RunningIndexTasks.getInstance().addTask(task); + } catch (RuntimeException ex) { + logger.warn("Indexer {} failed unexpectedly", indexer.getName(), ex); + } + + } + + return rettasks; + } + public void unindex(URI path) { logger.info("Starting unindexing procedure for {}", path.toString()); this.doUnindex(path, this.getIndexingPlugins(true)); } /** Issue the removal of indexed entries in a path from the given indexers. - * + * * @param path the URI of the directory or file to unindex * @param indexProviders a collection of providers */ @@ -759,7 +800,7 @@ public void unindex(URI path, Collection indexProviders) { /** Issue the removal of indexed entries in bulk. * - * @param items a collections of item identifiers to unindex + * @param paths a collections of item identifiers to unindex * @param progressCallback an optional function (can be `null`), * called for every batch of items successfully unindexed * to indicate early progress @@ -769,11 +810,10 @@ public void unindex(URI path, Collection indexProviders) { * a report containing which files were not unindexed, * and whether some of them were not found in the database */ - public List> unindex(Collection items, Consumer> progressCallback) { - + public List> unindex(Collection paths, Consumer> progressCallback) { List> tasks = new ArrayList<>(); for (IndexerInterface indexer : this.getIndexingPlugins(true)) - tasks.add(createUnindexTask(items, progressCallback, indexer)); + tasks.add(createUnindexTask(paths, progressCallback, indexer)); return tasks; } @@ -781,7 +821,7 @@ public List> unindex(Collection items, Consumer> unindex(Collection items, Consumer unindex(String indexProvider, Collection items, + public Task unindex(String indexProvider, Collection paths, Consumer> progressCallback) { IndexerInterface indexer = null; @@ -801,11 +841,11 @@ public Task unindex(String indexProvider, Collection items, if (indexer == null) { indexer = this.getIndexingPlugins(true).iterator().next(); } - return createUnindexTask(items, progressCallback, indexer); + return createUnindexTask(paths, progressCallback, indexer); } /** Issue an unindexing procedure to the given indexers. - * + * * @param path the URI of the directory or file to unindex * @param indexers a collection of providers */ @@ -888,7 +928,7 @@ private Task createUnindexTask(Collection items, Consumer getWebUIPlugins(String... ids) { } /** Retrieve the web UI plugin descriptor of the plugin with the given name. - * + * * @param name the unique name of the plugin * @return a web UI plugin descriptor object, or null if no such plugin exists or is inactive */ @@ -922,7 +962,7 @@ public WebUIPlugin getWebUIPlugin(String name) { } /** Retrieve the web UI plugin descriptor package.json. - * + * * @param name the unique name of the plugin * @return the full contents of the package.json, null if the plugin is not available */ @@ -938,7 +978,7 @@ public String getWebUIPackageJSON(String name) { } /** Retrieve the web UI plugin module code. - * + * * @param name the unique name of the plugin * @return the full contents of the module file, null if the plugin is not available */ diff --git a/dicoogle/src/test/java/pt/ua/dicoogle/core/plugins/PlatformInterfaceMock.java b/dicoogle/src/test/java/pt/ua/dicoogle/core/plugins/PlatformInterfaceMock.java index 579626655..6c6b41d28 100644 --- a/dicoogle/src/test/java/pt/ua/dicoogle/core/plugins/PlatformInterfaceMock.java +++ b/dicoogle/src/test/java/pt/ua/dicoogle/core/plugins/PlatformInterfaceMock.java @@ -122,6 +122,11 @@ public List> index(URI path) { return null; } + @Override + public List> index(Collection paths) { + return null; + } + @Override public void unindex(URI path) {} diff --git a/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java b/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java index 1a2f56dd9..4872ff101 100644 --- a/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java +++ b/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java @@ -207,6 +207,15 @@ JointQueryTask query(JointQueryTask holder, List querySources, DimLevel */ List> index(URI path); + + /** Easily performs indexation procedures over all active indexers. This operation is asynchronous + * and returns immediately. + * + * @param paths the paths to index + * @return a list of asynchronous tasks, one for each provider + */ + List> index(Collection paths); + /** Easily performs an unindex procedure over all active indexers. This operation is synchronous. * * @param path the path to index @@ -237,6 +246,7 @@ JointQueryTask query(JointQueryTask holder, List querySources, DimLevel * @param path the path to index * @return a list of reports, one for each provider */ + @Deprecated List indexBlocking(URI path); /** Obtain access to the server's settings. From 7243545e29651160c2ff01e8556d242a3655ab53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tib=C3=A9rio=20Baptista?= <38945670+tiberio-baptista@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:29:35 +0100 Subject: [PATCH 3/3] Apply suggestions by @Enet4 Co-authored-by: Eduardo Pinho --- .../src/main/java/pt/ua/dicoogle/plugins/PluginController.java | 2 +- .../pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java b/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java index 2edc61418..2090c9f4b 100755 --- a/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java +++ b/dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java @@ -734,7 +734,7 @@ public void run() { } public List> index(Collection paths) { - logger.info("Starting indexing procedure for {} items.", paths.size()); + logger.info("Starting indexing procedure for {} items", paths.size()); List objectsToStore = new ArrayList<>(); ArrayList> rettasks = new ArrayList<>(); diff --git a/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java b/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java index 4872ff101..45127b436 100644 --- a/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java +++ b/sdk/src/main/java/pt/ua/dicoogle/sdk/core/DicooglePlatformInterface.java @@ -208,7 +208,7 @@ JointQueryTask query(JointQueryTask holder, List querySources, DimLevel List> index(URI path); - /** Easily performs indexation procedures over all active indexers. This operation is asynchronous + /** Easily performs indexing procedures over all active indexers. This operation is asynchronous * and returns immediately. * * @param paths the paths to index @@ -245,6 +245,7 @@ JointQueryTask query(JointQueryTask holder, List querySources, DimLevel * * @param path the path to index * @return a list of reports, one for each provider + * @deprecated Call {@linkplain #index} and get the result instead */ @Deprecated List indexBlocking(URI path);