diff --git a/.gitignore b/.gitignore index 47dfa3073..6aa5b837c 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,5 @@ resources/ #IIIF iiif-media-*/ + +cineast-api/src/main/python/__pycache__/ diff --git a/cineast-api/build.gradle b/cineast-api/build.gradle index 82f47d4e5..bf300b77f 100644 --- a/cineast-api/build.gradle +++ b/cineast-api/build.gradle @@ -111,6 +111,8 @@ dependencies { implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin', version: version_jackson implementation group: 'de.svenkubiak', name: 'jBCrypt', version: version_jbcrypt implementation group: 'org.vitrivr', name: 'cineast-proto', version: version_cineast_proto + implementation group: 'tech.molecules', name: 'external-umap-java', version: '1.0' + } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java index 0d54af496..857b0e562 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java @@ -50,6 +50,7 @@ import org.vitrivr.cineast.api.rest.handlers.actions.tag.FindTagsAllGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.tag.FindTagsByIdsPostHandler; import org.vitrivr.cineast.api.rest.handlers.actions.tag.FindTagsGetHandler; +import org.vitrivr.cineast.api.rest.handlers.actions.vector.LoadVectorsForIdsPostHandler; import org.vitrivr.cineast.api.rest.handlers.interfaces.DeleteRestHandler; import org.vitrivr.cineast.api.rest.handlers.interfaces.DocumentedRestHandler; import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; @@ -427,7 +428,9 @@ private void registerRestOperations() { new SelectFromTablePostHandler(), new CountRowsGetHandler(), /* Status */ - new StatusInvocationHandler() + new StatusInvocationHandler(), + /* Vector */ + new LoadVectorsForIdsPostHandler(Config.sharedConfig().getDatabase().getSelectorSupplier()) )); } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java index 2ae194b4c..1158710a1 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java @@ -1,7 +1,14 @@ package org.vitrivr.cineast.api; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.concurrent.LinkedBlockingDeque; -import java.util.logging.Logger; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderJob; import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderWorker; @@ -13,83 +20,117 @@ import org.vitrivr.cineast.standalone.monitoring.PrometheusServer; import org.vitrivr.cineast.standalone.util.CLI; + public class Main { + private static final Logger LOGGER = LogManager.getLogger(); - /** - * Entrypoint for Cineast API application. - * - * @param args Program arguments. - */ - public static void main(String[] args) { - /* (Force) load application config. */ - if (args.length == 0) { - System.out.println("No config path given, loading default config '" + DEFAULT_CONFIG_PATH + "'"); - if (Config.loadConfig(DEFAULT_CONFIG_PATH) == null) { - System.err.println("Failed to load Cineast configuration from '" + DEFAULT_CONFIG_PATH + "'. Cineast API will shutdown..."); - System.exit(1); - } - } + /** + * Entrypoint for Cineast API application. + * + * @param args Program arguments. + */ + public static void main(String[] args) { + /* (Force) load application config. */ + if (args.length == 0) { + System.out.println("No config path given, loading default config '" + DEFAULT_CONFIG_PATH + "'"); + if (Config.loadConfig(DEFAULT_CONFIG_PATH) == null) { + System.err.println("Failed to load Cineast configuration from '" + DEFAULT_CONFIG_PATH + "'. Cineast API will shutdown..."); + System.exit(1); + } + } - /* (Force) load application config. */ - if (args.length != 0) { - if (Config.loadConfig(args[0]) == null) { - System.err.println("Failed to load Cineast configuration from '" + args[0] + "'. Cineast API will shutdown..."); - System.exit(1); - } - } + /* (Force) load application config. */ + if (args.length != 0) { + if (Config.loadConfig(args[0]) == null) { + System.err.println("Failed to load Cineast configuration from '" + args[0] + "'. Cineast API will shutdown..."); + System.exit(1); + } + } - /* Start API endpoint. */ - try { - APIEndpoint.getInstance().start(); - } catch (Throwable e) { - e.printStackTrace(); - System.err.println("Failed to initialize API endpoint due to an exception: " + e.getMessage()); - } + /* Start API endpoint. */ + try { + APIEndpoint.getInstance().start(); + } catch (Throwable e) { + e.printStackTrace(); + System.err.println("Failed to initialize API endpoint due to an exception: " + e.getMessage()); + } - /* Start gRPC endpoint. */ - try { - GRPCEndpoint.start(); - } catch (Throwable e) { - e.printStackTrace(); - System.err.println("Failed to initialize gRPC endpoint due to an exception: " + e.getMessage()); - } + /* Start gRPC endpoint. */ + try { + GRPCEndpoint.start(); + } catch (Throwable e) { + e.printStackTrace(); + System.err.println("Failed to initialize gRPC endpoint due to an exception: " + e.getMessage()); + } - /* Initialize Monitoring */ - try { - PrometheusServer.initialize(); - } catch (Throwable e) { - e.printStackTrace(); - System.err.println("Failed to initialize Monitoring due to an exception: " + e.getMessage()); - } - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - System.out.println("Shutting down endpoints..."); - APIEndpoint.stop(); - GRPCEndpoint.stop(); - PrometheusServer.stopServer(); - if (RenderWorker.getRenderJobQueue() != null) { - RenderWorker.getRenderJobQueue().add(new RenderJob(JobControlCommand.SHUTDOWN_WORKER)); - } - System.out.println("Goodbye!"); - })); - - if (Config.sharedConfig().getExtractor().getEnableRenderWorker()) { - /* Initialize Renderer */ - var renderThread = new Thread(new RenderWorker(new LinkedBlockingDeque<>()), "RenderWorker"); - renderThread.start(); - } + /* Initialize Monitoring */ + try { + PrometheusServer.initialize(); + } catch (Throwable e) { + e.printStackTrace(); + System.err.println("Failed to initialize Monitoring due to an exception: " + e.getMessage()); + } + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + System.out.println("Shutting down endpoints..."); + APIEndpoint.stop(); + GRPCEndpoint.stop(); + PrometheusServer.stopServer(); + if (RenderWorker.getRenderJobQueue() != null) { + RenderWorker.getRenderJobQueue().add(new RenderJob(JobControlCommand.SHUTDOWN_WORKER)); + } + System.out.println("Goodbye!"); + })); + + if (Config.sharedConfig().getExtractor().getEnableRenderWorker()) { + /* Initialize Renderer */ + var renderThread = new Thread(new RenderWorker(new LinkedBlockingDeque<>()), "RenderWorker"); + renderThread.start(); + } + + if (Config.sharedConfig().getApi().getEnableExternalClip()) { + /* Startup Clip Python Endpoint */ + // TODO: Make this configurable + Path condaEnvironmentPath = Path.of( "C:/Users/walten0000/.conda/envs/openclip/python.exe"); + Path scriptPath = Path.of( "./cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py"); + + var processParameters = new ArrayList(); + processParameters.add(condaEnvironmentPath.toString()); + processParameters.add(scriptPath.toString()); + + var processBuilder = new ProcessBuilder(processParameters); + //processBuilder.command("python", "./cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py"); + processBuilder.redirectErrorStream(true); + + var processBuilderThread = new Thread(()->{ + Process process = null; + try { + process = processBuilder.start(); + process.info(); + } catch (IOException e) { + throw new RuntimeException(e); + } + var reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + LOGGER.info("Starting OpenClip Python Endpoint"); + reader.lines().forEach(l -> LOGGER.info("External Clip: " + l)); + + } + ); + processBuilderThread.start(); + + } - try { - /* Start Cineast CLI in interactive mode (blocking). */ - if (Config.sharedConfig().getApi().getEnableCli()) { - CLI.start(CineastCli.class); - } else { - while (true) { - Thread.sleep(100); + try { + /* Start Cineast CLI in interactive mode (blocking). */ + if (Config.sharedConfig().getApi().getEnableCli()) { + CLI.start(CineastCli.class); + } else { + while (true) { + Thread.sleep(100); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); } - } - } catch (InterruptedException e) { - e.printStackTrace(); + System.exit(0); } - System.exit(0); - } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/VectorLookup.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/VectorLookup.java new file mode 100644 index 000000000..a5597dcac --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/VectorLookup.java @@ -0,0 +1,8 @@ +package org.vitrivr.cineast.api.messages.query; + +import java.util.Map; +import org.vitrivr.cineast.api.messages.lookup.IdList; + +public record VectorLookup(IdList ids, String feature, String projection, Map properties) { + +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorList.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorList.java new file mode 100644 index 000000000..9901c030c --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorList.java @@ -0,0 +1,7 @@ +package org.vitrivr.cineast.api.messages.result; + +import java.util.List; + +public record IdVectorList(List points) { + +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorPair.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorPair.java new file mode 100644 index 000000000..ffe46a260 --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorPair.java @@ -0,0 +1,5 @@ +package org.vitrivr.cineast.api.messages.result; + +public record IdVectorPair(String id, float[] vector){ + +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/vector/LoadVectorsForIdsPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/vector/LoadVectorsForIdsPostHandler.java new file mode 100644 index 000000000..f28691a0c --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/vector/LoadVectorsForIdsPostHandler.java @@ -0,0 +1,133 @@ +package org.vitrivr.cineast.api.rest.handlers.actions.vector; + +import io.javalin.http.Context; +import io.javalin.plugin.openapi.dsl.OpenApiBuilder; +import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.vitrivr.cineast.api.messages.query.VectorLookup; +import org.vitrivr.cineast.api.messages.result.IdVectorList; +import org.vitrivr.cineast.api.messages.result.IdVectorPair; +import org.vitrivr.cineast.api.rest.handlers.interfaces.ParsingPostRestHandler; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.db.DBSelector; +import org.vitrivr.cineast.core.db.DBSelectorSupplier; +import tagbio.umap.Umap; + +public class LoadVectorsForIdsPostHandler implements ParsingPostRestHandler { + + private final DBSelectorSupplier selectorSupply; + + public LoadVectorsForIdsPostHandler(DBSelectorSupplier selectorSupply) { + this.selectorSupply = selectorSupply; + } + + @Override + public OpenApiDocumentation docs() { + return OpenApiBuilder.document() + .operation(op -> { + op.summary("Loads the vectors of a particular feature, applies optional projection"); + op.description("Loads the vectors of a particular feature, applies optional projection"); + op.operationId("loadVectors"); + op.addTagsItem("Vectors"); + }) + .body(inClass()) + .json("200", outClass()); + } + + @Override + public IdVectorList performPost(VectorLookup input, Context ctx) { + + DBSelector selector = this.selectorSupply.get(); + selector.open("feature_" + input.feature()); + List> rows = selector.getRows("feature", input.ids().ids(), "vector_lookup"); + + List ids = new ArrayList<>(input.ids().ids().size()); + List vectors = new ArrayList<>(input.ids().ids().size()); + + for (Map row : rows) { + ids.add(row.get("id").getString()); + vectors.add(row.get("feature").getFloatArray()); + } + + selector.close(); + + switch (input.projection().toLowerCase()) { + case "umap" -> { + + Umap umap = new Umap(); + + umap.setMetric( + input.properties().getOrDefault("metric", "cosine") + ); + + umap.setNumberComponents( + Integer.parseInt( + input.properties().getOrDefault("components", "3") + ) + ); + + umap.setNumberNearestNeighbours( + Integer.parseInt( + input.properties().getOrDefault("nearestNeighbours", "15") + ) + ); + + umap.setThreads( + Integer.parseInt( + input.properties().getOrDefault("threads", Runtime.getRuntime().availableProcessors() + "") + ) + ); + + float[][] data = new float[vectors.size()][]; + + for (int i = 0; i < vectors.size(); ++i) { + data[i] = vectors.get(i); + } + + float[][] transformed = umap.fitTransform(data); + + List pairs = new ArrayList<>(ids.size()); + + for (int i = 0; i < ids.size(); ++i) { + pairs.add(new IdVectorPair(ids.get(i), transformed[i])); + } + + return new IdVectorList(pairs); + + } + case "tsne" -> throw new IllegalStateException("tsne projection not implemented"); + + + case "raw" -> { + + List pairs = new ArrayList<>(ids.size()); + + for (int i = 0; i < ids.size(); ++i) { + pairs.add(new IdVectorPair(ids.get(i), vectors.get(i))); + } + + return new IdVectorList(pairs); + + } + } + + throw new IllegalArgumentException("Projection " + input.projection() + " not known"); + } + + @Override + public Class inClass() { + return VectorLookup.class; + } + + @Override + public Class outClass() { + return IdVectorList.class; + } + + @Override + public String route() { + return "/find/vectors"; + } +} diff --git a/cineast-api/src/main/python/open_clip_lion_text_feature_proxy.py b/cineast-api/src/main/python/open_clip_lion_text_feature_proxy.py new file mode 100644 index 000000000..6633b392d --- /dev/null +++ b/cineast-api/src/main/python/open_clip_lion_text_feature_proxy.py @@ -0,0 +1,121 @@ +import base64 +import os.path + +import torch +import open_clip +from flask import Flask, request +import json +from PIL import Image +import base64 +from io import BytesIO +from datetime import datetime +import requests +import argparse + + +# parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +# parser.add_argument('--device', type=str, help='Device to use for feature extraction.', default='cpu') +# parser.add_argument('--port', type=int, help='Port to listen on.', default=8888) +# +# args = parser.parse_args() + +model, preprocess_train , preprocess_val = open_clip.create_model_and_transforms('xlm-roberta-base-ViT-B-32', pretrained='laion5b_s13b_b90k') +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = model.to(device) +tokenizer = open_clip.get_tokenizer('xlm-roberta-base-ViT-B-32') + +app = Flask(__name__) +app.secret_key = 'BAD_SECRET_KEY' + + +@app.route('/heartbeat', methods=['GET']) +def hearthbeat(): + return "Beat dev" + + +@app.route('/', methods=['POST', 'GET']) +def handle_request(): + if request.method == 'POST': + query = request.form['query'] + if query is None: + return "[]" + + return json.dumps(feature(query).tolist()) + + return "requests only supported vis POST" + + +@app.route('/image', methods=['POST']) +def handle_image_request(): + image_data = request.form['image'] + debug_storeimage = False + + dateTimeObj = datetime.now() + file_name_for_base64_data = dateTimeObj.strftime("%d-%b-%Y--(%H-%M-%S)") + + # File naming process for directory form data. + # We are taken the last 8 characters from the url string. + # file_name_for_regular_data = url[-10:-4] + featureVector = None + try: + # Base64 DATA + if "data:image/jpeg;base64," in image_data: + base_string = image_data.replace("data:image/jpeg;base64,", "") + decoded_img = base64.b64decode(base_string) + img = Image.open(BytesIO(decoded_img)) + featureVector = imagefeature(img) + if debug_storeimage == True: + file_name = file_name_for_base64_data + ".jpg" + img.save(f"./debugImages/{file_name}", "jpg") + + + # Base64 DATA + elif "data:image/png;base64," in image_data: + base_string = image_data.replace("data:image/png;base64,", "") + decoded_img = base64.b64decode(base_string) + img = Image.open(BytesIO(decoded_img)) + featureVector = imagefeature(img) + if debug_storeimage == True: + file_name = file_name_for_base64_data + ".png" + img.save(f"./debugImages/{file_name}", "png") + + + # Regular URL Form DATA + else: + response = requests.get(image_data) + img = Image.open(BytesIO(response.content)).convert("RGB") + featureVector = imagefeature(img) + featureVector = imagefeature(img) + if debug_storeimage == True: + file_name = "file_name_for_regular_data" + ".jpg" + img.save(f"./debugImages/{file_name}", "jpeg") + + status = "Image has been succesfully sent to the server." + except Exception as ex: + status = "Error! = " + str(ex) + return status + + + return json.dumps(featureVector.tolist()) + + +def imagefeature(imagequery): + device = "cuda" if torch.cuda.is_available() else "cpu" + image = preprocess_val(imagequery).unsqueeze(0).to(device) + with torch.no_grad(): + image_features = model.encode_image(image) + image_features /= image_features.norm(dim=-1, keepdim=True) + return image_features.cpu().numpy().flatten() + + +def feature(query): + device = "cuda" if torch.cuda.is_available() else "cpu" + text = tokenizer(query).to(device) + with torch.no_grad(): + text_features = model.encode_text(text) + text_features /= text_features.norm(dim=-1, keepdim=True) + return text_features.cpu().numpy().flatten() + + +if __name__ == '__main__': + app.run(port=8888) diff --git a/cineast-api/src/main/python/openclip.yaml b/cineast-api/src/main/python/openclip.yaml new file mode 100644 index 000000000..ca9833b7a --- /dev/null +++ b/cineast-api/src/main/python/openclip.yaml @@ -0,0 +1,128 @@ +name: openclip +channels: + - defaults +dependencies: + - abseil-cpp=20211102.0=hd77b12b_0 + - aiohttp=3.8.5=py311h2bbff1b_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - arrow-cpp=11.0.0=ha81ea56_2 + - async-timeout=4.0.2=py311haa95532_0 + - attrs=23.1.0=py311haa95532_0 + - aws-c-common=0.6.8=h2bbff1b_1 + - aws-c-event-stream=0.1.6=hd77b12b_6 + - aws-checksums=0.1.11=h2bbff1b_2 + - aws-sdk-cpp=1.8.185=hd77b12b_1 + - blas=1.0=mkl + - boost-cpp=1.82.0=h59b6b97_2 + - bottleneck=1.3.5=py311h5bb9823_0 + - brotli-python=1.0.9=py311hd77b12b_7 + - bzip2=1.0.8=he774522_0 + - c-ares=1.19.1=h2bbff1b_0 + - ca-certificates=2023.08.22=haa95532_0 + - certifi=2023.7.22=py311haa95532_0 + - cffi=1.15.1=py311h2bbff1b_3 + - chardet=4.0.0=py311haa95532_1003 + - click=8.1.7=py311haa95532_0 + - colorama=0.4.6=py311haa95532_0 + - cryptography=41.0.3=py311h89fc84f_0 + - datasets=2.12.0=py311haa95532_0 + - dill=0.3.6=py311haa95532_0 + - filelock=3.9.0=py311haa95532_0 + - flask=2.2.2=py311haa95532_0 + - frozenlist=1.3.3=py311h2bbff1b_0 + - gflags=2.2.2=ha925a31_0 + - glog=0.5.0=hd77b12b_0 + - grpc-cpp=1.48.2=hfe90ff0_1 + - huggingface_hub=0.17.3=py311haa95532_0 + - idna=3.4=py311haa95532_0 + - importlib-metadata=6.0.0=py311haa95532_0 + - intel-openmp=2023.1.0=h59b6b97_46319 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jinja2=3.1.2=py311haa95532_0 + - libboost=1.82.0=h3399ecb_2 + - libbrotlicommon=1.0.9=h2bbff1b_7 + - libbrotlidec=1.0.9=h2bbff1b_7 + - libbrotlienc=1.0.9=h2bbff1b_7 + - libcurl=8.1.1=h86230a5_0 + - libevent=2.1.12=h56d1f94_1 + - libffi=3.4.4=hd77b12b_0 + - libprotobuf=3.20.3=h23ce68f_0 + - libssh2=1.10.0=he2ea4bf_2 + - libthrift=0.15.0=h4364b78_2 + - libuv=1.44.2=h2bbff1b_0 + - lz4-c=1.9.4=h2bbff1b_0 + - markupsafe=2.1.1=py311h2bbff1b_0 + - mkl=2023.1.0=h6b88ed4_46357 + - mkl-service=2.4.0=py311h2bbff1b_1 + - mkl_fft=1.3.8=py311h2bbff1b_0 + - mkl_random=1.2.4=py311h59b6b97_0 + - mpmath=1.3.0=py311haa95532_0 + - multidict=6.0.2=py311h2bbff1b_0 + - multiprocess=0.70.14=py311haa95532_0 + - networkx=3.1=py311haa95532_0 + - ninja=1.10.2=haa95532_5 + - ninja-base=1.10.2=h6d14046_5 + - numexpr=2.8.7=py311h1fcbade_0 + - numpy=1.26.0=py311hdab7c0b_0 + - numpy-base=1.26.0=py311hd01c5d8_0 + - openssl=3.0.11=h2bbff1b_2 + - orc=1.7.4=h623e30f_1 + - pandas=2.1.1=py311hf62ec03_0 + - pip=23.3=py311haa95532_0 + - pyarrow=11.0.0=py311h8a3a540_1 + - pycparser=2.21=pyhd3eb1b0_0 + - pyopenssl=23.2.0=py311haa95532_0 + - pysocks=1.7.1=py311haa95532_0 + - python=3.11.5=he1021f5_0 + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python-xxhash=2.0.2=py311h2bbff1b_1 + - pytz=2023.3.post1=py311haa95532_0 + - pyyaml=6.0.1=py311h2bbff1b_0 + - re2=2022.04.01=hd77b12b_0 + - regex=2023.10.3=py311h2bbff1b_0 + - requests=2.31.0=py311haa95532_0 + - responses=0.13.3=pyhd3eb1b0_0 + - safetensors=0.4.0=py311hcbdf901_0 + - setuptools=68.0.0=py311haa95532_0 + - six=1.16.0=pyhd3eb1b0_1 + - snappy=1.1.9=h6c2663c_0 + - sqlite=3.41.2=h2bbff1b_0 + - sympy=1.11.1=py311haa95532_0 + - tbb=2021.8.0=h59b6b97_0 + - tk=8.6.12=h2bbff1b_0 + - tokenizers=0.13.3=py311h49fca51_0 + - transformers=4.32.1=py311haa95532_0 + - typing-extensions=4.7.1=py311haa95532_0 + - typing_extensions=4.7.1=py311haa95532_0 + - tzdata=2023c=h04d1e81_0 + - utf8proc=2.6.1=h2bbff1b_0 + - vc=14.2=h21ff451_1 + - vs2015_runtime=14.27.29016=h5e58377_2 + - waitress=2.0.0=pyhd3eb1b0_0 + - werkzeug=2.2.3=py311haa95532_0 + - wheel=0.41.2=py311haa95532_0 + - win_inet_pton=1.1.0=py311haa95532_0 + - xxhash=0.8.0=h2bbff1b_3 + - xz=5.4.2=h8cc25b3_0 + - yaml=0.2.5=he774522_0 + - yarl=1.8.1=py311h2bbff1b_0 + - zipp=3.11.0=py311haa95532_0 + - zlib=1.2.13=h8cc25b3_0 + - zstd=1.5.5=hd43e919_0 + - pip: + - charset-normalizer==3.3.1 + - fsspec==2023.10.0 + - ftfy==6.1.1 + - huggingface-hub==0.18.0 + - open-clip-torch==2.23.0 + - packaging==23.2 + - pillow==10.1.0 + - protobuf==4.24.4 + - sentencepiece==0.1.99 + - timm==0.9.8 + - torch==2.1.0 + - torchvision==0.16.0 + - tqdm==4.66.1 + - urllib3==2.0.7 + - wcwidth==0.2.8 diff --git a/cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py b/cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py new file mode 100644 index 000000000..a9e0075c6 --- /dev/null +++ b/cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py @@ -0,0 +1,19 @@ +from waitress import serve +import open_clip_lion_text_feature_proxy as app +import argparse + + +def main(args): + print(f"Serving on {args.ip}, listen on {args.port}. Clip on {args.device}") + serve(app.app, host=args.ip, port=args.port) + + +if __name__ == '__main__': + print("start server") + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--device', type=str, help='Device to use for feature extraction.', default='cpu') + parser.add_argument('--port', type=int, help='Port to listen on.', default=8888) + parser.add_argument('--ip', type=str, help='Ip to serve on.', default='127.0.0.1') + args = parser.parse_args() + + main(args) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index ae2ef44a0..e6fe48a4b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -308,6 +308,7 @@ private List> getRowsHelper(String fieldName, return new ArrayList<>(0); } } + @Override public List> getFulltextRows(int rows, String column, ReadableQueryConfig queryConfig, String... terms) { @@ -316,7 +317,7 @@ public List> getFulltextRows(int rows, String /* TODO Cottontail calls this a distance in its documentation, but it's actually a score. See the tests - that's why we order DESC and not ASC */ final Query query = new Query(this.fqn) - .select("id", null) + .select("*", null) .fulltext(column, predicate, DB_DISTANCE_VALUE_QUALIFIER) .queryId(generateQueryId("ft-rows", queryConfig)) .order(DB_DISTANCE_VALUE_QUALIFIER, Direction.DESC) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java index 32c9bdd03..1325f5752 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java @@ -395,6 +395,7 @@ public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheCon this.factory = cacheConfig.sharedCachedDataFactory(); /* Initialize the AVFormatContext. */ + this.pFormatCtx = null; this.pFormatCtx = avformat.avformat_alloc_context(); /* Open video file. */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java new file mode 100644 index 000000000..de66de4f4 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java @@ -0,0 +1,222 @@ +package org.vitrivr.cineast.core.features; + +import com.fasterxml.jackson.databind.json.JsonMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.QueryConfig; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.CorrespondenceFunction; +import org.vitrivr.cineast.core.data.FloatVectorImpl; +import org.vitrivr.cineast.core.data.distance.DistanceElement; +import org.vitrivr.cineast.core.data.distance.SegmentDistanceElement; +import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; +import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.DBSelector; +import org.vitrivr.cineast.core.db.DBSelectorSupplier; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; +import org.vitrivr.cineast.core.util.web.ImageParser; + +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpClient.Version; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; + +public class ExternalOpenClipText extends AbstractFeatureModule { + + private static final Logger LOGGER = LogManager.getLogger(); + private static final String TABLE_NAME = "features_openclip"; + + private static final int EMBEDDING_SIZE = 512; + private static final ReadableQueryConfig.Distance DISTANCE = ReadableQueryConfig.Distance.cosine; + private static final CorrespondenceFunction CORRESPONDENCE = CorrespondenceFunction.linear(1); + + private static final String DEFAULT_API_ENDPOINT = "http://localhost:8888"; + + private static final String API_ENDPOINT_KEY = "api"; + + private final String externalApi; + + private final HttpClient httpClient = HttpClient.newBuilder() + .version(Version.HTTP_1_1) + .build(); + + private final JsonMapper mapper = new JsonMapper(); + private DBSelector selector; + + + public ExternalOpenClipText() { + super(TABLE_NAME, 1f, EMBEDDING_SIZE); + this.externalApi = DEFAULT_API_ENDPOINT; + } + + public ExternalOpenClipText(Map properties) { + super(TABLE_NAME, 1f, EMBEDDING_SIZE); + this.externalApi = properties.getOrDefault(API_ENDPOINT_KEY, DEFAULT_API_ENDPOINT); + } + + @Override + public void processSegment(SegmentContainer sc) { + // Return if already processed + if (phandler.idExists(sc.getId())) { + return; + } + + // Case: segment contains video frames + if (!sc.getVideoFrames().isEmpty() && sc.getVideoFrames().get(0) != VideoFrame.EMPTY_VIDEO_FRAME) { + var frame = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); + + float[] embeddingArray = new float[0]; + try { + embeddingArray = apiRequest(frame); + LOGGER.debug("ML Clip MostRepresentativeFrame from external API url: {}", externalApi); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + this.persist(sc.getId(), new FloatVectorImpl(embeddingArray)); + + return; + } + + // Case: segment contains image + if (sc.getMostRepresentativeFrame() != VideoFrame.EMPTY_VIDEO_FRAME) { + BufferedImage image = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); + + if (image != null) { + float[] embeddingArray = new float[0]; + try { + embeddingArray = apiRequest(image); + LOGGER.debug("ML Clip image from external API url: {}", externalApi); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + this.persist(sc.getId(), new FloatVectorImpl(embeddingArray)); + } + return; + } + + } + + @Override + public void initalizePersistentLayer(Supplier supply) { + supply.get().createFeatureEntity(TABLE_NAME, true, EMBEDDING_SIZE); + } + + @Override + public void dropPersistentLayer(Supplier supply) { + supply.get().dropEntity(TABLE_NAME); + } + + @Override + public void init(DBSelectorSupplier selectorSupply) { + this.selector = selectorSupply.get(); + this.selector.open(TABLE_NAME); + } + + private float[] apiRequest(BufferedImage image) throws IOException, InterruptedException { + // Image encode to base64 + var imageData = ImageParser.BufferedImageToDataURL(image, "png"); + + var builder = new StringBuilder() + .append(URLEncoder.encode("image", StandardCharsets.UTF_8)) + .append("=") + .append(URLEncoder.encode(imageData, StandardCharsets.UTF_8)); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(externalApi + "/image")) + .header("Content-Type", "application/x-www-form-urlencoded") + .POST(HttpRequest.BodyPublishers.ofString(builder.toString())) + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + throw new IllegalStateException("received response code " + response.statusCode()); + } + + return mapper.readValue(response.body(), float[].class); + } + + private float[] apiRequest(String query) throws IOException, InterruptedException { + + String builder = URLEncoder.encode("query", StandardCharsets.UTF_8) + + "=" + + URLEncoder.encode(query, StandardCharsets.UTF_8); + + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(builder)) + .uri(URI.create(externalApi)) + .header("Content-Type", "application/x-www-form-urlencoded") + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + throw new IllegalStateException("received response code " + response.statusCode()); + } + + return mapper.readValue(response.body(), float[].class); + } + + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + String text = sc.getText(); + if (text == null || text.isBlank()) { + return Collections.emptyList(); + } + + try { + float[] arr = apiRequest(text); + return getSimilar(new FloatArrayTypeProvider(arr), qc); + } catch (Exception e) { + LOGGER.error("error during CLIPText execution", e); + return new ArrayList<>(); + } + } + + public List getSimilar(String segmentId, ReadableQueryConfig qc) { + List list = this.selector.getFeatureVectorsGeneric(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER, qc); + if (list.isEmpty()) { + LOGGER.warn("No feature vector for shotId {} found, returning empty result-list", segmentId); + return Collections.emptyList(); + } + return getSimilar(list.get(0), qc); + } + + public List getSimilar(PrimitiveTypeProvider queryProvider, ReadableQueryConfig qc) { + ReadableQueryConfig qcc = QueryConfig.clone(qc).setDistance(DISTANCE); + List distances = this.selector.getNearestNeighboursGeneric(qc.getResultsPerModule(), queryProvider, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, qcc); + CorrespondenceFunction function = qcc.getCorrespondenceFunction().orElse(CORRESPONDENCE); + return DistanceElement.toScore(distances, function); + } + + + @Override + public void finish() { + if (this.selector != null) { + this.selector.close(); + this.selector = null; + } + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java new file mode 100644 index 000000000..33cce07cc --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java @@ -0,0 +1,97 @@ +package org.vitrivr.cineast.core.features; + +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.stream.Collectors; +import org.apache.commons.text.similarity.LevenshteinDistance; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.data.score.SegmentScoreElement; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.abstracts.AbstractTextRetriever; + +public class LevenshteinScoringTextRetriever extends AbstractTextRetriever { + + private static final Logger LOGGER = LogManager.getLogger(); + + public LevenshteinScoringTextRetriever(Map properties) { + super(ProvidedOcrSearch.PROVIDED_OCR_SEARCH_TABLE_NAME, properties); + } + + private static final LevenshteinDistance distance = new LevenshteinDistance(); + + public static float similarity(String query, String candidate) { + + if (query.isBlank() || candidate.isBlank()) { + return 0f; + } + + int levDist = distance.apply(query, candidate); + + if (query.length() < candidate.length()) { + levDist -= (candidate.length() - query.length()); //don't penalize matching substrings + } + + return 1f - ((float)levDist / (float)query.length()); + + } + + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + + String text = sc.getText(); + if (text == null || text.isBlank()) { + return Collections.emptyList(); + } + return this.getSimilar(qc, text); + } + + protected List getSimilar(ReadableQueryConfig qc, String... terms) { + + if (terms.length == 0) { + return Collections.emptyList(); + } + + final Map scoreMap = new HashMap<>(); + + for (String term : terms) { + + if (term.isBlank()) { + continue; + } + + String stripped = term.strip(); + + final List> resultList = this.selector.getFulltextRows(qc.getResultsPerModule(), SimpleFulltextFeatureDescriptor.FIELDNAMES[1], qc, generateQuery(stripped)); + LOGGER.trace("Retrieved {} results for term '{}'", resultList.size(), term); + + for (Map result : resultList) { + String id = result.get(GENERIC_ID_COLUMN_QUALIFIER).getString(); + String text = result.get(FEATURE_COLUMN_QUALIFIER).getString(); + + float score = similarity(stripped, text); + + float bestScore = scoreMap.getOrDefault(id, 0f); + + if (score > bestScore) { + scoreMap.put(id, score); + } + } + + } + + return scoreMap.entrySet().stream().map(entry -> new SegmentScoreElement(entry.getKey(), entry.getValue())).sorted(SegmentScoreElement.SCORE_COMPARATOR).collect(Collectors.toList()); + + } + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java index ec2fd781f..5fe5634ba 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java @@ -43,7 +43,7 @@ public abstract class AbstractTextRetriever implements Retriever, Extractor { /** * Generate a query term which will then be used for retrieval. */ - private static final Pattern regex = Pattern.compile("([^\"]\\S*|\".+?\")\\s*"); + protected static final Pattern regex = Pattern.compile("([^\"]\\S*|\".+?\")\\s*"); /** * Name of the table/entity used to store the data. */ @@ -167,13 +167,13 @@ public List getSimilar(List segmentIds, ReadableQueryConfi */ @Override public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - final String[] terms = generateQuery(sc, qc); + final String[] terms = generateQuery(sc.getText()); return this.getSimilar(qc, terms); } - protected String[] generateQuery(SegmentContainer sc, ReadableQueryConfig qc) { + protected String[] generateQuery(String text) { - Matcher m = regex.matcher(sc.getText()); + Matcher m = regex.matcher(text); ArrayList matches = new ArrayList<>(); while (m.find()) { @@ -183,7 +183,7 @@ protected String[] generateQuery(SegmentContainer sc, ReadableQueryConfig qc) { } } - return matches.toArray(new String[matches.size()]); + return matches.toArray(new String[0]); } /** diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetrieverTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetrieverTest.java index aba92bf63..08cd5b7cf 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetrieverTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetrieverTest.java @@ -30,7 +30,7 @@ public void testNonQuotedStringSplit() { } public void testMatch(String input, String... output) { - org.junit.jupiter.api.Assertions.assertArrayEquals(output, retriever.generateQuery(new TextQueryTermContainer(input), new QueryConfig(null))); + org.junit.jupiter.api.Assertions.assertArrayEquals(output, retriever.generateQuery(input)); } public SegmentContainer generateSegmentContainerFromText(String text) { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java index e535ff32e..5a3b73ac7 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java @@ -13,286 +13,302 @@ @JsonIgnoreProperties(ignoreUnknown = true) public final class APIConfig { - private boolean enableWebsocket = true; - private boolean enableWebsocketSecure = true; - private boolean enableExtractionServer = true; - private boolean enableRest = false; - private boolean enableRestSecure = false; - - private boolean enableGRPC = true; - private String keystore; - private String keystorePassword; - - private boolean enableRestLiveDoc = false; // Defaults to same result as enableRest - private String apiAddress = "http://localhost:4567/"; - - private int httpPort = 4567; - private int httpsPort = 4568; - - - private int grpcPort = 4570; - private int maxMessageSize = 5120 * 1000; /* Maximum size of a single WebSocket message (binary or text). */ - - private boolean allowExtraction = true; - - private boolean enableCLI = false; - private int threadPoolSize = 8; - - private boolean serveContent = false; - - /** - * A hack-flag to prevent the object serving using {@link MediaObjectDescriptor#getPath()} to find the actual media file. If this is true, the {@link MediaObjectDescriptor#getPath()}'s extension is appendet to {@link MediaObjectDescriptor#getObjectId()} and this is resolved to be directly under {@link #objectLocation}. - */ - private boolean objectsFilesAreIDed = false; - /** - * A hack to use mp4 for object servings - */ - private String videoExtension = "mp4"; - private boolean serveUI = false; - private String sessionExtractionConfigLocation = "extraction_config.json"; - private String thumbnailLocation = ""; - private String objectLocation = ""; - private String uiLocation = ""; - - @JsonCreator - public APIConfig() { - } - - @JsonProperty - public String getVideoExtension() { - return videoExtension; - } - - @JsonProperty - public void setVideoExtension(String videoExtension) { - this.videoExtension = videoExtension; - } - - @JsonProperty - public boolean getEnableWebsocket() { - return this.enableWebsocket; - } - - public void setEnableWebsocket(boolean enableWebsocket) { - this.enableWebsocket = enableWebsocket; - } - - @JsonProperty - public boolean getEnableWebsocketSecure() { - return this.enableWebsocketSecure; - } - - public void setEnableWebsocketSecure(boolean enableWebsocket) { - this.enableWebsocketSecure = enableWebsocket; - } - - @JsonProperty - public boolean getEnableRest() { - return this.enableRest; - } - - public void setEnableRest(boolean enableRest) { - this.enableRest = enableRest; - } - - @JsonProperty - public boolean getEnableRestSecure() { - return this.enableRestSecure; - } - - public void setEnableRestSecure(boolean enableRest) { - this.enableRestSecure = enableRest; - } - - @JsonProperty - public boolean getEnableLiveDoc() { - return this.enableRestLiveDoc; - } - - public void setEnableRestLiveDoc(boolean enableRestLiveDoc) { - this.enableRestLiveDoc = enableRestLiveDoc; - } - - @JsonProperty - public String getKeystore() { - return keystore; - } - - public void setKeystore(String keystore) { - this.keystore = keystore; - } - - @JsonProperty - public String getKeystorePassword() { - return keystorePassword; - } - - public void setKeystorePassword(String keystorePassword) { - this.keystorePassword = keystorePassword; - } - - @JsonProperty - public String getApiAddress() { - return apiAddress; - } - - public void setApiAddress(String apiAddress) { - this.apiAddress = apiAddress; - } - - @JsonProperty - public int getHttpPort() { - return httpPort; - } - - public void setHttpPort(int httpPort) { - if (httpPort < 1) { - throw new IllegalArgumentException("httpPort must be > 0"); - } - this.httpPort = httpPort; - } - - @JsonProperty - public int getHttpsPort() { - return httpsPort; - } - - public void setHttpsPort(int httpsPort) { - if (httpsPort < 1) { - throw new IllegalArgumentException("httpPort must be > 0"); - } - this.httpsPort = httpsPort; - } - - @JsonProperty - public int getMaxMessageSize() { - return this.maxMessageSize; - } - - public void setMaxMessageSize(int maxTextMessageSize) { - this.maxMessageSize = maxTextMessageSize; - } - - @JsonProperty - public boolean getAllowExtraction() { - return this.allowExtraction; - } - - public void setAllowExtraction(boolean allowExtraction) { - this.allowExtraction = allowExtraction; - } - - @JsonProperty - public boolean getEnableCli() { - return this.enableCLI; - } - - public void setEnableCLI(boolean enableCLI) { - this.enableCLI = enableCLI; - } - - @JsonProperty - public int getThreadPoolSize() { - return threadPoolSize; - } - - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } - - @JsonProperty - public String getThumbnailLocation() { - return thumbnailLocation; - } - - public void setThumbnailLocation(String thumbnailLocation) { - this.thumbnailLocation = thumbnailLocation; - } - - @JsonProperty - public String getObjectLocation() { - return objectLocation; - } - - public void setObjectLocation(String objectLocation) { - this.objectLocation = objectLocation; - } - - @JsonProperty - public String getUiLocation() { - return uiLocation; - } - - public void setUiLocation(String uiLocation) { - this.uiLocation = uiLocation; - } - - @JsonProperty - public boolean getServeContent() { - return this.serveContent; - } - - public void setServeContent(boolean serveContent) { - this.serveContent = serveContent; - } - - @JsonProperty - public boolean getServeUI() { - return this.serveUI; - } - - public void setServeUI(boolean serveUI) { - this.serveUI = serveUI; - } - - @JsonProperty - public String getSessionExtractionConfigLocation() { - return this.sessionExtractionConfigLocation; - } - - public void setSessionExtractionConfigLocation(String sessionExtractionConfigLocation) { - this.sessionExtractionConfigLocation = sessionExtractionConfigLocation; - } - - @JsonProperty - public boolean getEnableExtractionServer() { - return enableExtractionServer; - } - - public void setEnableExtractionServer(boolean enableExtractionServer) { - this.enableExtractionServer = enableExtractionServer; - } - - @JsonProperty - public boolean getEnableGRPC() { - return enableGRPC; - } - - @JsonProperty - public void setEnableGRPC(boolean enableGRPC) { - this.enableGRPC = enableGRPC; - } - - @JsonProperty - public int getGrpcPort() { - return grpcPort; - } - - @JsonProperty - public void setGrpcPort(int grpcPort) { - this.grpcPort = grpcPort; - } - - /** - * A hack-flag to prevent the object serving using {@link MediaObjectDescriptor#getPath()} to find the actual media file. If this is true, the {@link MediaObjectDescriptor#getPath()}'s extension is appendet to {@link MediaObjectDescriptor#getObjectId()} and this is resolved to be directly under {@link #objectLocation}. - */ - @JsonProperty - public boolean isObjectsFilesAreIDed() { - return objectsFilesAreIDed; - } - - @JsonProperty - public void setObjectsFilesAreIDed(boolean b) { - objectsFilesAreIDed = b; - } + private boolean enableWebsocket = true; + private boolean enableWebsocketSecure = true; + private boolean enableExtractionServer = true; + private boolean enableRest = false; + private boolean enableRestSecure = false; + + private boolean enableCLI = false; + private boolean enableExternalClip = false; + + private boolean enableGRPC = true; + private String keystore; + private String keystorePassword; + + private boolean enableRestLiveDoc = false; // Defaults to same result as enableRest + private String apiAddress = "http://localhost:4567/"; + + private int httpPort = 4567; + private int httpsPort = 4568; + + + private int grpcPort = 4570; + private int maxMessageSize = 5120 * 1000; /* Maximum size of a single WebSocket message (binary or text). */ + + private boolean allowExtraction = true; + + private int threadPoolSize = 8; + + private boolean serveContent = false; + + /** + * A hack-flag to prevent the object serving using {@link MediaObjectDescriptor#getPath()} to find the actual media file. If this is true, the {@link MediaObjectDescriptor#getPath()}'s extension is appendet to {@link MediaObjectDescriptor#getObjectId()} and this is resolved to be directly under {@link #objectLocation}. + */ + private boolean objectsFilesAreIDed = false; + /** + * A hack to use mp4 for object servings + */ + private String videoExtension = "mp4"; + private boolean serveUI = false; + private String sessionExtractionConfigLocation = "extraction_config.json"; + private String thumbnailLocation = ""; + private String objectLocation = ""; + private String uiLocation = ""; + + + + + @JsonCreator + public APIConfig() { + } + + @JsonProperty + public String getVideoExtension() { + return videoExtension; + } + + @JsonProperty + public void setVideoExtension(String videoExtension) { + this.videoExtension = videoExtension; + } + + @JsonProperty + public boolean getEnableWebsocket() { + return this.enableWebsocket; + } + + public void setEnableWebsocket(boolean enableWebsocket) { + this.enableWebsocket = enableWebsocket; + } + + @JsonProperty + public boolean getEnableWebsocketSecure() { + return this.enableWebsocketSecure; + } + + public void setEnableWebsocketSecure(boolean enableWebsocket) { + this.enableWebsocketSecure = enableWebsocket; + } + + @JsonProperty + public boolean getEnableRest() { + return this.enableRest; + } + + public void setEnableRest(boolean enableRest) { + this.enableRest = enableRest; + } + + @JsonProperty + public boolean getEnableRestSecure() { + return this.enableRestSecure; + } + + public void setEnableRestSecure(boolean enableRest) { + this.enableRestSecure = enableRest; + } + + @JsonProperty + public boolean getEnableLiveDoc() { + return this.enableRestLiveDoc; + } + + public void setEnableRestLiveDoc(boolean enableRestLiveDoc) { + this.enableRestLiveDoc = enableRestLiveDoc; + } + + @JsonProperty + public String getKeystore() { + return keystore; + } + + public void setKeystore(String keystore) { + this.keystore = keystore; + } + + @JsonProperty + public String getKeystorePassword() { + return keystorePassword; + } + + public void setKeystorePassword(String keystorePassword) { + this.keystorePassword = keystorePassword; + } + + @JsonProperty + public String getApiAddress() { + return apiAddress; + } + + public void setApiAddress(String apiAddress) { + this.apiAddress = apiAddress; + } + + @JsonProperty + public int getHttpPort() { + return httpPort; + } + + public void setHttpPort(int httpPort) { + if (httpPort < 1) { + throw new IllegalArgumentException("httpPort must be > 0"); + } + this.httpPort = httpPort; + } + + @JsonProperty + public int getHttpsPort() { + return httpsPort; + } + + public void setHttpsPort(int httpsPort) { + if (httpsPort < 1) { + throw new IllegalArgumentException("httpPort must be > 0"); + } + this.httpsPort = httpsPort; + } + + @JsonProperty + public int getMaxMessageSize() { + return this.maxMessageSize; + } + + public void setMaxMessageSize(int maxTextMessageSize) { + this.maxMessageSize = maxTextMessageSize; + } + + @JsonProperty + public boolean getAllowExtraction() { + return this.allowExtraction; + } + + public void setAllowExtraction(boolean allowExtraction) { + this.allowExtraction = allowExtraction; + } + + @JsonProperty + public boolean getEnableCli() { + return this.enableCLI; + } + + public void setEnableCLI(boolean enableCLI) { + this.enableCLI = enableCLI; + } + + @JsonProperty + public int getThreadPoolSize() { + return threadPoolSize; + } + + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } + + @JsonProperty + public String getThumbnailLocation() { + return thumbnailLocation; + } + + public void setThumbnailLocation(String thumbnailLocation) { + this.thumbnailLocation = thumbnailLocation; + } + + @JsonProperty + public String getObjectLocation() { + return objectLocation; + } + + public void setObjectLocation(String objectLocation) { + this.objectLocation = objectLocation; + } + + @JsonProperty + public String getUiLocation() { + return uiLocation; + } + + public void setUiLocation(String uiLocation) { + this.uiLocation = uiLocation; + } + + @JsonProperty + public boolean getServeContent() { + return this.serveContent; + } + + public void setServeContent(boolean serveContent) { + this.serveContent = serveContent; + } + + @JsonProperty + public boolean getServeUI() { + return this.serveUI; + } + + public void setServeUI(boolean serveUI) { + this.serveUI = serveUI; + } + + @JsonProperty + public String getSessionExtractionConfigLocation() { + return this.sessionExtractionConfigLocation; + } + + public void setSessionExtractionConfigLocation(String sessionExtractionConfigLocation) { + this.sessionExtractionConfigLocation = sessionExtractionConfigLocation; + } + + @JsonProperty + public boolean getEnableExtractionServer() { + return enableExtractionServer; + } + + public void setEnableExtractionServer(boolean enableExtractionServer) { + this.enableExtractionServer = enableExtractionServer; + } + + @JsonProperty + public boolean getEnableGRPC() { + return enableGRPC; + } + + @JsonProperty + public void setEnableGRPC(boolean enableGRPC) { + this.enableGRPC = enableGRPC; + } + + @JsonProperty + public int getGrpcPort() { + return grpcPort; + } + + @JsonProperty + public void setGrpcPort(int grpcPort) { + this.grpcPort = grpcPort; + } + + /** + * A hack-flag to prevent the object serving using {@link MediaObjectDescriptor#getPath()} to find the actual media file. If this is true, the {@link MediaObjectDescriptor#getPath()}'s extension is appendet to {@link MediaObjectDescriptor#getObjectId()} and this is resolved to be directly under {@link #objectLocation}. + */ + @JsonProperty + public boolean isObjectsFilesAreIDed() { + return objectsFilesAreIDed; + } + + @JsonProperty + public void setObjectsFilesAreIDed(boolean b) { + objectsFilesAreIDed = b; + } + + @JsonProperty + public boolean getEnableExternalClip() { + return this.enableExternalClip; + } + + @JsonProperty + public boolean setEnableExternalClip(boolean enableExternalClip) { + return this.enableExternalClip = enableExternalClip; + } + } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java index b1ebafd3e..81c27c63e 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java @@ -69,6 +69,11 @@ public void setTaskQueueSize(int taskQueueSize) { public boolean getEnableRenderWorker() { return this.enableRenderWorker; } + @JsonProperty + public void setEnableRenderWorker(boolean enableRenderWorker){ + this.enableRenderWorker = enableRenderWorker; + } + @JsonProperty public File getOutputLocation() { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java index 67f861fb2..f4a7fb78f 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java @@ -3,12 +3,14 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.File; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; @@ -35,7 +37,27 @@ public ExtractionItemContainer(@JsonProperty("object") MediaObjectDescriptor obj @JsonProperty("metadata") List metadata, @JsonProperty("uri") String uri) throws URISyntaxException { - this(object, metadata, Paths.get(new URI(uri))); + + this(object, metadata, convertUriToPathWindowsSafe(uri)); + } + + /** + * Annoying hack to also handle windows file URIs with a drive letter + */ + private static Path convertUriToPathWindowsSafe(String uri) throws URISyntaxException { + Path path; + try{ + path = new File(new URI(uri)).toPath(); + }catch(IllegalArgumentException ex){ + final URI parsedURI = new URI(uri); + if(StringUtils.isNotBlank(parsedURI.getAuthority())){ + path = new File(parsedURI.getAuthority()+parsedURI.getPath()).toPath(); + }else{ + // If for some unknown reasons we land here with a unix path, this should do the trick + path = new File(parsedURI.getPath()).toPath(); + } + } + return path; } @JsonIgnore diff --git a/cineast-runtime/src/main/resources/log4j2.xml b/cineast-runtime/src/main/resources/log4j2.xml index 3df8712ca..bd4e1fffe 100644 --- a/cineast-runtime/src/main/resources/log4j2.xml +++ b/cineast-runtime/src/main/resources/log4j2.xml @@ -8,7 +8,7 @@ - + diff --git a/cineast.json b/cineast.json index 5f58c5b0f..851d8ead7 100644 --- a/cineast.json +++ b/cineast.json @@ -40,6 +40,12 @@ "visualtextcoembedding": [ {"feature": "VisualTextCoEmbedding", "weight": 1.0} ], + "clip": [ + {"feature": "CLIPText", "weight": 1.0} + ], + "mlclip": [ + {"feature": "ExternalOpenClipText", "weight": 1.0} + ], "boolean": [ { "feature": "CollectionBooleanRetriever", "weight": 1.0, diff --git a/gradle.properties b/gradle.properties index bb38cb62b..d1c729356 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,7 +9,7 @@ version_commonslang3=3.12.0 version_commonsmath3=3.6.1 version_commonstext=1.9 version_cottontaildb_proto=0.14.2 -version_ffmpeg=5.1.2-1.5.8 +version_ffmpeg=6.0-1.5.9 version_gson=2.9.0 version_guava=31.1-jre vershion_hppc=0.9.1