Merge branch 'feature/externalclip' into external-api-extraction

.gitignore

@@ -69,3 +69,5 @@ resources/
 
 #IIIF
 iiif-media-*/
+
+cineast-api/src/main/python/__pycache__/

cineast-api/build.gradle

@@ -111,6 +111,8 @@ dependencies {
     implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin', version: version_jackson
     implementation group: 'de.svenkubiak', name: 'jBCrypt', version: version_jbcrypt
     implementation group: 'org.vitrivr', name: 'cineast-proto', version: version_cineast_proto
+    implementation group: 'tech.molecules', name: 'external-umap-java', version: '1.0'
+
 
 }
 

cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java

@@ -50,6 +50,7 @@
 import org.vitrivr.cineast.api.rest.handlers.actions.tag.FindTagsAllGetHandler;
 import org.vitrivr.cineast.api.rest.handlers.actions.tag.FindTagsByIdsPostHandler;
 import org.vitrivr.cineast.api.rest.handlers.actions.tag.FindTagsGetHandler;
+import org.vitrivr.cineast.api.rest.handlers.actions.vector.LoadVectorsForIdsPostHandler;
 import org.vitrivr.cineast.api.rest.handlers.interfaces.DeleteRestHandler;
 import org.vitrivr.cineast.api.rest.handlers.interfaces.DocumentedRestHandler;
 import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler;
@@ -427,7 +428,9 @@ private void registerRestOperations() {
         new SelectFromTablePostHandler(),
         new CountRowsGetHandler(),
         /* Status */
-        new StatusInvocationHandler()
+        new StatusInvocationHandler(),
+        /* Vector */
+        new LoadVectorsForIdsPostHandler(Config.sharedConfig().getDatabase().getSelectorSupplier())
     ));
   }
 

cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java

@@ -1,7 +1,14 @@
 package org.vitrivr.cineast.api;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.concurrent.LinkedBlockingDeque;
-import java.util.logging.Logger;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderJob;
 import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderWorker;
 
@@ -13,83 +20,117 @@
 import org.vitrivr.cineast.standalone.monitoring.PrometheusServer;
 import org.vitrivr.cineast.standalone.util.CLI;
 
+
 public class Main {
+    private static final Logger LOGGER = LogManager.getLogger();
 
-  /**
-   * Entrypoint for Cineast API application.
-   *
-   * @param args Program arguments.
-   */
-  public static void main(String[] args) {
-    /* (Force) load application config. */
-    if (args.length == 0) {
-      System.out.println("No config path given, loading default config '" + DEFAULT_CONFIG_PATH + "'");
-      if (Config.loadConfig(DEFAULT_CONFIG_PATH) == null) {
-        System.err.println("Failed to load Cineast configuration from '" + DEFAULT_CONFIG_PATH + "'. Cineast API will shutdown...");
-        System.exit(1);
-      }
-    }
+    /**
+     * Entrypoint for Cineast API application.
+     *
+     * @param args Program arguments.
+     */
+    public static void main(String[] args) {
+        /* (Force) load application config. */
+        if (args.length == 0) {
+            System.out.println("No config path given, loading default config '" + DEFAULT_CONFIG_PATH + "'");
+            if (Config.loadConfig(DEFAULT_CONFIG_PATH) == null) {
+                System.err.println("Failed to load Cineast configuration from '" + DEFAULT_CONFIG_PATH + "'. Cineast API will shutdown...");
+                System.exit(1);
+            }
+        }
 
-    /* (Force) load application config. */
-    if (args.length != 0) {
-      if (Config.loadConfig(args[0]) == null) {
-        System.err.println("Failed to load Cineast configuration from '" + args[0] + "'. Cineast API will shutdown...");
-        System.exit(1);
-      }
-    }
+        /* (Force) load application config. */
+        if (args.length != 0) {
+            if (Config.loadConfig(args[0]) == null) {
+                System.err.println("Failed to load Cineast configuration from '" + args[0] + "'. Cineast API will shutdown...");
+                System.exit(1);
+            }
+        }
 
-    /* Start API endpoint. */
-    try {
-      APIEndpoint.getInstance().start();
-    } catch (Throwable e) {
-      e.printStackTrace();
-      System.err.println("Failed to initialize API endpoint due to an exception: " + e.getMessage());
-    }
+        /* Start API endpoint. */
+        try {
+            APIEndpoint.getInstance().start();
+        } catch (Throwable e) {
+            e.printStackTrace();
+            System.err.println("Failed to initialize API endpoint due to an exception: " + e.getMessage());
+        }
 
-    /* Start gRPC endpoint. */
-    try {
-      GRPCEndpoint.start();
-    } catch (Throwable e) {
-      e.printStackTrace();
-      System.err.println("Failed to initialize gRPC endpoint due to an exception: " + e.getMessage());
-    }
+        /* Start gRPC endpoint. */
+        try {
+            GRPCEndpoint.start();
+        } catch (Throwable e) {
+            e.printStackTrace();
+            System.err.println("Failed to initialize gRPC endpoint due to an exception: " + e.getMessage());
+        }
 
-    /* Initialize Monitoring */
-    try {
-      PrometheusServer.initialize();
-    } catch (Throwable e) {
-      e.printStackTrace();
-      System.err.println("Failed to initialize Monitoring due to an exception: " + e.getMessage());
-    }
-    Runtime.getRuntime().addShutdownHook(new Thread(() -> {
-      System.out.println("Shutting down endpoints...");
-      APIEndpoint.stop();
-      GRPCEndpoint.stop();
-      PrometheusServer.stopServer();
-      if (RenderWorker.getRenderJobQueue() != null) {
-        RenderWorker.getRenderJobQueue().add(new RenderJob(JobControlCommand.SHUTDOWN_WORKER));
-      }
-      System.out.println("Goodbye!");
-    }));
-
-    if (Config.sharedConfig().getExtractor().getEnableRenderWorker()) {
-      /* Initialize Renderer */
-      var renderThread = new Thread(new RenderWorker(new LinkedBlockingDeque<>()), "RenderWorker");
-      renderThread.start();
-    }
+        /* Initialize Monitoring */
+        try {
+            PrometheusServer.initialize();
+        } catch (Throwable e) {
+            e.printStackTrace();
+            System.err.println("Failed to initialize Monitoring due to an exception: " + e.getMessage());
+        }
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            System.out.println("Shutting down endpoints...");
+            APIEndpoint.stop();
+            GRPCEndpoint.stop();
+            PrometheusServer.stopServer();
+            if (RenderWorker.getRenderJobQueue() != null) {
+                RenderWorker.getRenderJobQueue().add(new RenderJob(JobControlCommand.SHUTDOWN_WORKER));
+            }
+            System.out.println("Goodbye!");
+        }));
+
+        if (Config.sharedConfig().getExtractor().getEnableRenderWorker()) {
+            /* Initialize Renderer */
+            var renderThread = new Thread(new RenderWorker(new LinkedBlockingDeque<>()), "RenderWorker");
+            renderThread.start();
+        }
+
+        if (Config.sharedConfig().getApi().getEnableExternalClip()) {
+            /* Startup Clip Python Endpoint */
+            // TODO: Make this configurable
+            Path condaEnvironmentPath = Path.of( "C:/Users/walten0000/.conda/envs/openclip/python.exe");
+            Path scriptPath = Path.of( "./cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py");
+
+            var processParameters = new ArrayList<String>();
+            processParameters.add(condaEnvironmentPath.toString());
+            processParameters.add(scriptPath.toString());
+
+            var processBuilder = new ProcessBuilder(processParameters);
+            //processBuilder.command("python", "./cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py");
+            processBuilder.redirectErrorStream(true);
+
+                var processBuilderThread = new Thread(()->{
+                    Process process = null;
+                    try {
+                        process = processBuilder.start();
+                        process.info();
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                    var reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+                    LOGGER.info("Starting OpenClip Python Endpoint");
+                    reader.lines().forEach(l -> LOGGER.info("External Clip: " + l));
+
+                }
+            );
+            processBuilderThread.start();
+
+        }
 
-    try {
-      /* Start Cineast CLI in interactive mode (blocking). */
-      if (Config.sharedConfig().getApi().getEnableCli()) {
-        CLI.start(CineastCli.class);
-      } else {
-        while (true) {
-          Thread.sleep(100);
+        try {
+            /* Start Cineast CLI in interactive mode (blocking). */
+            if (Config.sharedConfig().getApi().getEnableCli()) {
+                CLI.start(CineastCli.class);
+            } else {
+                while (true) {
+                    Thread.sleep(100);
+                }
+            }
+        } catch (InterruptedException e) {
+            e.printStackTrace();
         }
-      }
-    } catch (InterruptedException e) {
-      e.printStackTrace();
+        System.exit(0);
     }
-    System.exit(0);
-  }
 }

cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/VectorLookup.java

@@ -0,0 +1,8 @@
+package org.vitrivr.cineast.api.messages.query;
+
+import java.util.Map;
+import org.vitrivr.cineast.api.messages.lookup.IdList;
+
+public record VectorLookup(IdList ids, String feature, String projection, Map<String, String> properties) {
+
+}

cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorList.java

@@ -0,0 +1,7 @@
+package org.vitrivr.cineast.api.messages.result;
+
+import java.util.List;
+
+public record IdVectorList(List<IdVectorPair> points) {
+
+}

cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorPair.java

@@ -0,0 +1,5 @@
+package org.vitrivr.cineast.api.messages.result;
+
+public record IdVectorPair(String id, float[] vector){
+
+}

cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/vector/LoadVectorsForIdsPostHandler.java

@@ -0,0 +1,133 @@
+package org.vitrivr.cineast.api.rest.handlers.actions.vector;
+
+import io.javalin.http.Context;
+import io.javalin.plugin.openapi.dsl.OpenApiBuilder;
+import io.javalin.plugin.openapi.dsl.OpenApiDocumentation;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.vitrivr.cineast.api.messages.query.VectorLookup;
+import org.vitrivr.cineast.api.messages.result.IdVectorList;
+import org.vitrivr.cineast.api.messages.result.IdVectorPair;
+import org.vitrivr.cineast.api.rest.handlers.interfaces.ParsingPostRestHandler;
+import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider;
+import org.vitrivr.cineast.core.db.DBSelector;
+import org.vitrivr.cineast.core.db.DBSelectorSupplier;
+import tagbio.umap.Umap;
+
+public class LoadVectorsForIdsPostHandler implements ParsingPostRestHandler<VectorLookup, IdVectorList> {
+
+  private final DBSelectorSupplier selectorSupply;
+
+  public LoadVectorsForIdsPostHandler(DBSelectorSupplier selectorSupply) {
+    this.selectorSupply = selectorSupply;
+  }
+
+  @Override
+  public OpenApiDocumentation docs() {
+    return OpenApiBuilder.document()
+        .operation(op -> {
+          op.summary("Loads the vectors of a particular feature, applies optional projection");
+          op.description("Loads the vectors of a particular feature, applies optional projection");
+          op.operationId("loadVectors");
+          op.addTagsItem("Vectors");
+        })
+        .body(inClass())
+        .json("200", outClass());
+  }
+
+  @Override
+  public IdVectorList performPost(VectorLookup input, Context ctx) {
+
+    DBSelector selector = this.selectorSupply.get();
+    selector.open("feature_" + input.feature());
+    List<Map<String, PrimitiveTypeProvider>> rows = selector.getRows("feature", input.ids().ids(), "vector_lookup");
+
+    List<String> ids = new ArrayList<>(input.ids().ids().size());
+    List<float[]> vectors = new ArrayList<>(input.ids().ids().size());
+
+    for (Map<String, PrimitiveTypeProvider> row : rows) {
+      ids.add(row.get("id").getString());
+      vectors.add(row.get("feature").getFloatArray());
+    }
+
+    selector.close();
+
+    switch (input.projection().toLowerCase()) {
+      case "umap" -> {
+
+        Umap umap = new Umap();
+
+        umap.setMetric(
+            input.properties().getOrDefault("metric", "cosine")
+        );
+
+        umap.setNumberComponents(
+            Integer.parseInt(
+                input.properties().getOrDefault("components", "3")
+            )
+        );
+
+        umap.setNumberNearestNeighbours(
+            Integer.parseInt(
+                input.properties().getOrDefault("nearestNeighbours", "15")
+            )
+        );
+
+        umap.setThreads(
+            Integer.parseInt(
+                input.properties().getOrDefault("threads", Runtime.getRuntime().availableProcessors() + "")
+            )
+        );
+
+        float[][] data = new float[vectors.size()][];
+
+        for (int i = 0; i < vectors.size(); ++i) {
+          data[i] = vectors.get(i);
+        }
+
+        float[][] transformed = umap.fitTransform(data);
+
+        List<IdVectorPair> pairs = new ArrayList<>(ids.size());
+
+        for (int i = 0; i < ids.size(); ++i) {
+          pairs.add(new IdVectorPair(ids.get(i), transformed[i]));
+        }
+
+        return new IdVectorList(pairs);
+
+      }
+      case "tsne" -> throw new IllegalStateException("tsne projection not implemented");
+
+
+      case "raw" -> {
+
+        List<IdVectorPair> pairs = new ArrayList<>(ids.size());
+
+        for (int i = 0; i < ids.size(); ++i) {
+          pairs.add(new IdVectorPair(ids.get(i), vectors.get(i)));
+        }
+
+        return new IdVectorList(pairs);
+
+      }
+    }
+
+    throw new IllegalArgumentException("Projection " + input.projection() + " not known");
+  }
+
+  @Override
+  public Class<VectorLookup> inClass() {
+    return VectorLookup.class;
+  }
+
+  @Override
+  public Class<IdVectorList> outClass() {
+    return IdVectorList.class;
+  }
+
+  @Override
+  public String route() {
+    return "/find/vectors";
+  }
+}