From 6dbdc32ee68c83894942dfe61f0dabf660a3783b Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Sun, 11 Dec 2022 16:46:03 +0100 Subject: [PATCH 01/13] Added proxy retriever for external open_clip feature --- .../open_clip_lion_text_feature_proxy.py | 40 +++++ .../core/features/ExternalOpenClipText.java | 139 ++++++++++++++++++ cineast.json | 6 + 3 files changed, 185 insertions(+) create mode 100644 cineast-api/src/main/python/open_clip_lion_text_feature_proxy.py create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java diff --git a/cineast-api/src/main/python/open_clip_lion_text_feature_proxy.py b/cineast-api/src/main/python/open_clip_lion_text_feature_proxy.py new file mode 100644 index 000000000..14c207108 --- /dev/null +++ b/cineast-api/src/main/python/open_clip_lion_text_feature_proxy.py @@ -0,0 +1,40 @@ +import torch +from PIL import Image +import open_clip +from flask import Flask, request +import json +import argparse + +parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--device', type=str, help='Device to use for feature extraction.', default='cpu') +parser.add_argument('--port', type=int, help='Port to listen on.', default=8888) + +args = parser.parse_args() + +model, _, _ = open_clip.create_model_and_transforms('xlm-roberta-base-ViT-B-32', pretrained='laion5b_s13b_b90k') +model = model.to(args.device) +tokenizer = open_clip.get_tokenizer('xlm-roberta-base-ViT-B-32') + +app = Flask(__name__) + +@app.route('/',methods = ['POST', 'GET']) +def handle_request(): + if request.method == 'POST': + query = request.form['query'] + if query is None: + return "[]" + + return json.dumps(feature(query).tolist()) + + return "requests only supported vis POST" + + +def feature(query): + text = tokenizer(query).to(args.device) + with torch.no_grad(): + text_features = model.encode_text(text) + text_features /= text_features.norm(dim=-1, keepdim=True) + return text_features.cpu().numpy().flatten() + +if __name__ == '__main__': + app.run(port = args.port) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java new file mode 100644 index 000000000..0eb237f4a --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java @@ -0,0 +1,139 @@ +package org.vitrivr.cineast.core.features; + +import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpClient.Version; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.QueryConfig; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.CorrespondenceFunction; +import org.vitrivr.cineast.core.data.distance.DistanceElement; +import org.vitrivr.cineast.core.data.distance.SegmentDistanceElement; +import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; +import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.DBSelector; +import org.vitrivr.cineast.core.db.DBSelectorSupplier; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.features.retriever.Retriever; + +public class ExternalOpenClipText implements Retriever { + + private static final Logger LOGGER = LogManager.getLogger(); + private static final String TABLE_NAME = "features_openclip"; + + private static final int EMBEDDING_SIZE = 512; + private static final ReadableQueryConfig.Distance DISTANCE = ReadableQueryConfig.Distance.cosine; + private static final CorrespondenceFunction CORRESPONDENCE = CorrespondenceFunction.linear(1); + + private static final String API_ENDPOINT = "http://localhost:8888"; + + private final HttpClient httpClient = HttpClient.newBuilder() + .version(Version.HTTP_1_1) + .build(); + + private final JsonMapper mapper = new JsonMapper(); + private DBSelector selector; + + + public ExternalOpenClipText() { + + } + + @Override + public void initalizePersistentLayer(Supplier supply) { + supply.get().createFeatureEntity(TABLE_NAME, true, EMBEDDING_SIZE); + } + + @Override + public void dropPersistentLayer(Supplier supply) { + supply.get().dropEntity(TABLE_NAME); + } + + @Override + public void init(DBSelectorSupplier selectorSupply) { + this.selector = selectorSupply.get(); + this.selector.open(TABLE_NAME); + } + + private float[] apiRequest(String query) throws IOException, InterruptedException { + + String builder = URLEncoder.encode("query", StandardCharsets.UTF_8) + + "=" + + URLEncoder.encode(query, StandardCharsets.UTF_8); + + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(builder)) + .uri(URI.create(API_ENDPOINT)) + .header("Content-Type", "application/x-www-form-urlencoded") + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + throw new IllegalStateException("received response code " + response.statusCode()); + } + + return mapper.readValue(response.body(), float[].class); + + } + + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + String text = sc.getText(); + if (text == null || text.isBlank()) { + return Collections.emptyList(); + } + + try { + float[] arr = apiRequest(text); + return getSimilar(new FloatArrayTypeProvider(arr), qc); + } catch (Exception e) { + LOGGER.error("error during CLIPText execution", e); + return new ArrayList<>(); + } + } + + public List getSimilar(String segmentId, ReadableQueryConfig qc) { + List list = this.selector.getFeatureVectorsGeneric(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER, qc); + if (list.isEmpty()) { + LOGGER.warn("No feature vector for shotId {} found, returning empty result-list", segmentId); + return Collections.emptyList(); + } + return getSimilar(list.get(0), qc); + } + + private List getSimilar(PrimitiveTypeProvider queryProvider, ReadableQueryConfig qc) { + ReadableQueryConfig qcc = QueryConfig.clone(qc).setDistance(DISTANCE); + List distances = this.selector.getNearestNeighboursGeneric(qc.getResultsPerModule(), queryProvider, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, qcc); + CorrespondenceFunction function = qcc.getCorrespondenceFunction().orElse(CORRESPONDENCE); + return DistanceElement.toScore(distances, function); + } + + @Override + public void finish() { + if (this.selector != null) { + this.selector.close(); + this.selector = null; + } + } +} diff --git a/cineast.json b/cineast.json index 5cb278488..ea3c45219 100644 --- a/cineast.json +++ b/cineast.json @@ -43,6 +43,12 @@ "visualtextcoembedding": [ {"feature": "VisualTextCoEmbedding", "weight": 1.0} ], + "clip": [ + {"feature": "CLIPText", "weight": 1.0} + ], + "mlclip": [ + {"feature": "ExternalOpenClipText", "weight": 1.0} + ], "boolean": [ { "feature": "CollectionBooleanRetriever", "weight": 1.0, From 1997b984cc4d0a0be55e06c8e2f07ed4cc8605f4 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Sat, 17 Dec 2022 15:27:32 +0100 Subject: [PATCH 02/13] Added LevenshteinScoringTextRetriever --- .../db/cottontaildb/CottontailSelector.java | 2 +- .../LevenshteinScoringTextRetriever.java | 95 +++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index a91f27384..1482c0025 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -346,7 +346,7 @@ public List> getFulltextRows(int rows, String /* TODO Cottontail calls this a distance in its documentation, but it's actually a score. See the tests - that's why we order DESC and not ASC */ final Query query = new Query(this.fqn) - .select("id", null) + .select("*", null) .fulltext(fieldname, predicate, DB_DISTANCE_VALUE_QUALIFIER) .queryId(generateQueryID("ft-rows", queryConfig)) .order(DB_DISTANCE_VALUE_QUALIFIER, Direction.DESC) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java new file mode 100644 index 000000000..e374b7956 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java @@ -0,0 +1,95 @@ +package org.vitrivr.cineast.core.features; + +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.commons.text.similarity.LevenshteinDistance; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.data.score.SegmentScoreElement; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.abstracts.AbstractTextRetriever; + +public class LevenshteinScoringTextRetriever extends AbstractTextRetriever { + + private static final Logger LOGGER = LogManager.getLogger(); + + public LevenshteinScoringTextRetriever(Map properties) { + super(ProvidedOcrSearch.PROVIDED_OCR_SEARCH_TABLE_NAME, properties); + } + + private static final LevenshteinDistance distance = new LevenshteinDistance(); + + private static float similarity(String query, String candidate) { + + if (query.isBlank() || candidate.isBlank()) { + return 0f; + } + + int levDist = distance.apply(query, candidate); + + if (query.length() < candidate.length()) { + levDist -= (candidate.length() - query.length()); //don't penalize matching substrings + } + + return 1f - ((float)levDist / (float)query.length()); + + } + + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + + String text = sc.getText(); + if (text == null || text.isBlank()) { + return Collections.emptyList(); + } + return this.getSimilar(qc, text); + } + + protected List getSimilar(ReadableQueryConfig qc, String... terms) { + + if (terms.length == 0) { + return Collections.emptyList(); + } + + final Map scoreMap = new HashMap<>(); + + for (String term : terms) { + + if (term.isBlank()) { + continue; + } + + String stripped = term.strip(); + + final List> resultList = this.selector.getFulltextRows(qc.getResultsPerModule(), SimpleFulltextFeatureDescriptor.FIELDNAMES[1], qc, "\"" + stripped + "\"~"); + LOGGER.trace("Retrieved {} results for term '{}'", resultList.size(), term); + + for (Map result : resultList) { + String id = result.get(GENERIC_ID_COLUMN_QUALIFIER).getString(); + String text = result.get(FEATURE_COLUMN_QUALIFIER).getString(); + + float score = similarity(stripped, text); + + float bestScore = scoreMap.getOrDefault(id, 0f); + + if (score > bestScore) { + scoreMap.put(id, score); + } + } + + } + + return scoreMap.entrySet().stream().map(entry -> new SegmentScoreElement(entry.getKey(), entry.getValue())).sorted(SegmentScoreElement.SCORE_COMPARATOR).collect(Collectors.toList()); + + } + +} From afc5fe5e1c63f7b9ea1e32e910e0ac74bbf3d6f7 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Sat, 17 Dec 2022 16:06:03 +0100 Subject: [PATCH 03/13] Changed query string encoding in LevenshteinScoringTextRetriever --- .../LevenshteinScoringTextRetriever.java | 19 ++++++++++++++++++- .../abstracts/AbstractTextRetriever.java | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java index e374b7956..f9df9ea8d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java @@ -3,10 +3,12 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; import java.util.stream.Collectors; import org.apache.commons.text.similarity.LevenshteinDistance; import org.apache.logging.log4j.LogManager; @@ -45,6 +47,21 @@ private static float similarity(String query, String candidate) { } + private static String encodeQueryString(String query) { + Matcher m = regex.matcher(query); + StringBuilder sb = new StringBuilder(); + + while (m.find()) { + final String match = m.group(1).trim(); + if (!match.isEmpty()) { + sb.append(match.trim()).append("~ "); + } + } + + return sb.toString().trim(); + + } + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { String text = sc.getText(); @@ -70,7 +87,7 @@ protected List getSimilar(ReadableQueryConfig qc, String... terms) String stripped = term.strip(); - final List> resultList = this.selector.getFulltextRows(qc.getResultsPerModule(), SimpleFulltextFeatureDescriptor.FIELDNAMES[1], qc, "\"" + stripped + "\"~"); + final List> resultList = this.selector.getFulltextRows(qc.getResultsPerModule(), SimpleFulltextFeatureDescriptor.FIELDNAMES[1], qc, encodeQueryString(stripped)); LOGGER.trace("Retrieved {} results for term '{}'", resultList.size(), term); for (Map result : resultList) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java index acdbdb741..7bfa3130a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java @@ -42,7 +42,7 @@ public abstract class AbstractTextRetriever implements Retriever, Extractor { /** * Generate a query term which will then be used for retrieval. */ - private static final Pattern regex = Pattern.compile("([^\"]\\S*|\".+?\")\\s*"); + protected static final Pattern regex = Pattern.compile("([^\"]\\S*|\".+?\")\\s*"); /** * Name of the table/entity used to store the data. */ From 2cc9b81712899e1f2469ed4f1edfe7ec0a4ed809 Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Sat, 17 Dec 2022 16:22:52 +0100 Subject: [PATCH 04/13] Changed query string encoding in LevenshteinScoringTextRetriever again --- .../LevenshteinScoringTextRetriever.java | 19 ++----------------- .../abstracts/AbstractTextRetriever.java | 8 ++++---- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java index f9df9ea8d..33cce07cc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java @@ -31,7 +31,7 @@ public LevenshteinScoringTextRetriever(Map properties) { private static final LevenshteinDistance distance = new LevenshteinDistance(); - private static float similarity(String query, String candidate) { + public static float similarity(String query, String candidate) { if (query.isBlank() || candidate.isBlank()) { return 0f; @@ -47,21 +47,6 @@ private static float similarity(String query, String candidate) { } - private static String encodeQueryString(String query) { - Matcher m = regex.matcher(query); - StringBuilder sb = new StringBuilder(); - - while (m.find()) { - final String match = m.group(1).trim(); - if (!match.isEmpty()) { - sb.append(match.trim()).append("~ "); - } - } - - return sb.toString().trim(); - - } - public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { String text = sc.getText(); @@ -87,7 +72,7 @@ protected List getSimilar(ReadableQueryConfig qc, String... terms) String stripped = term.strip(); - final List> resultList = this.selector.getFulltextRows(qc.getResultsPerModule(), SimpleFulltextFeatureDescriptor.FIELDNAMES[1], qc, encodeQueryString(stripped)); + final List> resultList = this.selector.getFulltextRows(qc.getResultsPerModule(), SimpleFulltextFeatureDescriptor.FIELDNAMES[1], qc, generateQuery(stripped)); LOGGER.trace("Retrieved {} results for term '{}'", resultList.size(), term); for (Map result : resultList) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java index 7bfa3130a..980848949 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java @@ -166,13 +166,13 @@ public List getSimilar(List segmentIds, ReadableQueryConfi */ @Override public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - final String[] terms = generateQuery(sc, qc); + final String[] terms = generateQuery(sc.getText()); return this.getSimilar(qc, terms); } - protected String[] generateQuery(SegmentContainer sc, ReadableQueryConfig qc) { + protected String[] generateQuery(String text) { - Matcher m = regex.matcher(sc.getText()); + Matcher m = regex.matcher(text); ArrayList matches = new ArrayList<>(); while (m.find()) { @@ -182,7 +182,7 @@ protected String[] generateQuery(SegmentContainer sc, ReadableQueryConfig qc) { } } - return matches.toArray(new String[matches.size()]); + return matches.toArray(new String[0]); } /** From 875fe63c26bab1f21ea0704d5ecd08b7d58bd33b Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Mon, 19 Dec 2022 17:10:26 +0100 Subject: [PATCH 05/13] Added a way to specify the host of the external clip text instance via config --- .../core/features/ExternalOpenClipText.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java index 0eb237f4a..0979cdcaa 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java @@ -3,7 +3,6 @@ import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.json.JsonMapper; import java.io.IOException; import java.net.URI; @@ -15,7 +14,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -45,7 +43,11 @@ public class ExternalOpenClipText implements Retriever { private static final ReadableQueryConfig.Distance DISTANCE = ReadableQueryConfig.Distance.cosine; private static final CorrespondenceFunction CORRESPONDENCE = CorrespondenceFunction.linear(1); - private static final String API_ENDPOINT = "http://localhost:8888"; + private static final String DEFAULT_API_ENDPOINT = "http://localhost:8888"; + + private static final String API_ENDPOINT_KEY = "api"; + + private final String externalApi; private final HttpClient httpClient = HttpClient.newBuilder() .version(Version.HTTP_1_1) @@ -56,7 +58,11 @@ public class ExternalOpenClipText implements Retriever { public ExternalOpenClipText() { + this.externalApi = DEFAULT_API_ENDPOINT; + } + public ExternalOpenClipText(Map properties){ + this.externalApi = properties.getOrDefault(API_ENDPOINT_KEY, DEFAULT_API_ENDPOINT); } @Override @@ -83,7 +89,7 @@ private float[] apiRequest(String query) throws IOException, InterruptedExceptio HttpRequest request = HttpRequest.newBuilder() .POST(HttpRequest.BodyPublishers.ofString(builder)) - .uri(URI.create(API_ENDPOINT)) + .uri(URI.create(externalApi)) .header("Content-Type", "application/x-www-form-urlencoded") .build(); From 2dee77fde7b754c9008270041e990be25d39925c Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Tue, 13 Jun 2023 14:31:15 +0200 Subject: [PATCH 06/13] Fixed windows-path issues in extraction item container for live-extraction --- .../run/ExtractionItemContainer.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java index 67f861fb2..f4a7fb78f 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java @@ -3,12 +3,14 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.File; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; @@ -35,7 +37,27 @@ public ExtractionItemContainer(@JsonProperty("object") MediaObjectDescriptor obj @JsonProperty("metadata") List metadata, @JsonProperty("uri") String uri) throws URISyntaxException { - this(object, metadata, Paths.get(new URI(uri))); + + this(object, metadata, convertUriToPathWindowsSafe(uri)); + } + + /** + * Annoying hack to also handle windows file URIs with a drive letter + */ + private static Path convertUriToPathWindowsSafe(String uri) throws URISyntaxException { + Path path; + try{ + path = new File(new URI(uri)).toPath(); + }catch(IllegalArgumentException ex){ + final URI parsedURI = new URI(uri); + if(StringUtils.isNotBlank(parsedURI.getAuthority())){ + path = new File(parsedURI.getAuthority()+parsedURI.getPath()).toPath(); + }else{ + // If for some unknown reasons we land here with a unix path, this should do the trick + path = new File(parsedURI.getPath()).toPath(); + } + } + return path; } @JsonIgnore From 4aacc010aea802af3b0d7e83c6db9fdc1d466b08 Mon Sep 17 00:00:00 2001 From: x4e-jonas <83283569+x4e-jonas@users.noreply.github.com> Date: Wed, 19 Jul 2023 08:28:24 +0200 Subject: [PATCH 07/13] Make log level configurable (#377) (#378) Co-authored-by: x4e-jonas --- cineast-runtime/src/main/resources/log4j2.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cineast-runtime/src/main/resources/log4j2.xml b/cineast-runtime/src/main/resources/log4j2.xml index 3df8712ca..bd4e1fffe 100644 --- a/cineast-runtime/src/main/resources/log4j2.xml +++ b/cineast-runtime/src/main/resources/log4j2.xml @@ -8,7 +8,7 @@ - + From ca26e6e0c588cb37797bf222197dd74812c7513a Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Sun, 8 Oct 2023 16:42:37 +0200 Subject: [PATCH 08/13] Merged dev --- Dockerfile | 4 +- .../java/org/vitrivr/cineast/api/Main.java | 16 + .../api/rest/resolvers/ResolutionResult.java | 1 - .../queries/AbstractQueryMessageHandler.java | 36 +- cineast-core/build.gradle | 48 +++ .../vitrivr/cineast/core/config/IdConfig.java | 8 +- .../vitrivr/cineast/core/data/MediaType.java | 1 + .../core/data/m3d/texturemodel/Entity.java | 186 ++++++++++ .../core/data/m3d/texturemodel/IModel.java | 39 +++ .../core/data/m3d/texturemodel/Material.java | 145 ++++++++ .../core/data/m3d/texturemodel/Mesh.java | 220 ++++++++++++ .../core/data/m3d/texturemodel/Model.java | 126 +++++++ .../data/m3d/texturemodel/ModelLoader.java | 331 ++++++++++++++++++ .../core/data/m3d/texturemodel/Texture.java | 57 +++ .../texturemodel/util/MinimalBoundingBox.java | 235 +++++++++++++ .../core/data/providers/ModelProvider.java | 16 + .../TextureModelQueryTermContainer.java | 114 ++++++ .../core/data/segments/SegmentContainer.java | 2 + .../data/segments/TextureModel3DSegment.java | 62 ++++ .../vitrivr/cineast/core/db/DBSelector.java | 175 ++++----- .../cineast/core/db/ImporterSelector.java | 59 +--- .../vitrivr/cineast/core/db/NoDBSelector.java | 12 +- .../db/cottontaildb/CottontailSelector.java | 115 +++--- .../db/cottontaildb/CottontailWrapper.java | 2 +- .../db/dao/reader/AbstractMetadataReader.java | 20 +- .../core/db/dao/reader/MediaObjectReader.java | 6 +- .../db/dao/reader/MediaSegmentReader.java | 10 +- .../core/db/polypheny/PolyphenySelector.java | 38 +- .../decode/m3d/ITextureModelDecoder.java | 8 + .../m3d/ModularTextureModelDecoder.java | 200 +++++++++++ .../decode/m3d/TextureModelDecoder.java | 131 +++++++ .../cineast/core/features/CLIPImage.java | 5 +- .../cineast/core/features/CLIPText.java | 2 +- .../core/features/InceptionResnetV2.java | 2 +- .../cineast/core/features/Lightfield.java | 116 +++--- .../core/features/RangeBooleanRetriever.java | 41 --- .../core/features/VisualTextCoEmbedding.java | 191 +++++++++- .../exporter/Model3DThumbnailExporter.java | 112 +++--- .../neuralnet/tf/models/Inception5h.java | 3 +- .../neuralnet/tf/models/deeplab/DeepLab.java | 4 +- .../neuralnet/tf/models/yolo/YOLO.java | 2 +- .../core/render/JOGLOffscreenRenderer.java | 56 ++- .../cineast/core/render/MeshOnlyRenderer.java | 21 ++ .../vitrivr/cineast/core/render/Renderer.java | 14 +- .../core/render/lwjgl/engine/Engine.java | 209 +++++++++++ .../core/render/lwjgl/engine/EngineLogic.java | 61 ++++ .../core/render/lwjgl/glmodel/GLMaterial.java | 87 +++++ .../core/render/lwjgl/glmodel/GLMesh.java | 127 +++++++ .../core/render/lwjgl/glmodel/GLModel.java | 86 +++++ .../core/render/lwjgl/glmodel/GLScene.java | 155 ++++++++ .../core/render/lwjgl/glmodel/GLTexture.java | 95 +++++ .../render/lwjgl/glmodel/GLTextureCache.java | 56 +++ .../core/render/lwjgl/glmodel/IGLModel.java | 47 +++ .../core/render/lwjgl/render/Render.java | 65 ++++ .../render/lwjgl/render/RenderOptions.java | 35 ++ .../core/render/lwjgl/render/SceneRender.java | 148 ++++++++ .../render/lwjgl/render/ShaderProgram.java | 149 ++++++++ .../core/render/lwjgl/render/UniformsMap.java | 107 ++++++ .../renderer/LWJGLOffscreenRenderer.java | 296 ++++++++++++++++ .../render/lwjgl/renderer/RenderActions.java | 14 + .../render/lwjgl/renderer/RenderData.java | 16 + .../core/render/lwjgl/renderer/RenderJob.java | 147 ++++++++ .../render/lwjgl/renderer/RenderStates.java | 18 + .../render/lwjgl/renderer/RenderWorker.java | 247 +++++++++++++ .../core/render/lwjgl/scene/Camera.java | 292 +++++++++++++++ .../core/render/lwjgl/scene/Direction.java | 13 + .../render/lwjgl/scene/LightfieldCamera.java | 88 +++++ .../core/render/lwjgl/scene/Projection.java | 59 ++++ .../core/render/lwjgl/scene/Scene.java | 92 +++++ .../render/lwjgl/util/datatype/Variant.java | 75 ++++ .../lwjgl/util/datatype/VariantException.java | 27 ++ .../lwjgl/util/fsm/abstractworker/Job.java | 188 ++++++++++ .../fsm/abstractworker/JobControlCommand.java | 23 ++ .../util/fsm/abstractworker/JobType.java | 29 ++ .../util/fsm/abstractworker/StateEnter.java | 21 ++ .../util/fsm/abstractworker/StateLeave.java | 21 ++ .../fsm/abstractworker/StateProvider.java | 16 + .../StateProviderAnnotationParser.java | 178 ++++++++++ .../StateProviderException.java | 25 ++ .../fsm/abstractworker/StateTransition.java | 23 ++ .../lwjgl/util/fsm/abstractworker/Worker.java | 217 ++++++++++++ .../fsm/controller/FiniteStateMachine.java | 78 +++++ .../FiniteStateMachineException.java | 25 ++ .../render/lwjgl/util/fsm/model/Action.java | 96 +++++ .../render/lwjgl/util/fsm/model/Graph.java | 141 ++++++++ .../render/lwjgl/util/fsm/model/State.java | 60 ++++ .../lwjgl/util/fsm/model/Transition.java | 79 +++++ .../core/render/lwjgl/window/Window.java | 210 +++++++++++ .../render/lwjgl/window/WindowOptions.java | 48 +++ ...Generator.java => DBQueryIdGenerator.java} | 20 +- .../cineast/core/util/MimeTypeHelper.java | 7 +- .../cineast/core/util/math/MathConstants.java | 16 + .../EntopyCalculationMethod.java | 20 ++ .../EntropyOptimizerStrategy.java | 15 + .../ModelEntropyOptimizer.java | 284 +++++++++++++++ .../EntropyOptimizer/OptimizerOptions.java | 39 +++ .../Viewpoint/ViewpointHelper.java | 129 +++++++ .../Viewpoint/ViewpointStrategy.java | 43 +++ .../cineast/core/util/web/ModelParser.java | 63 ++++ .../core/db/DBBooleanIntegrationTest.java | 9 +- .../abstracts/AbstractTextRetrieverTest.java | 2 +- .../org/vitrivr/cineast/standalone/Main.java | 14 + .../cineast/standalone/cli/CineastCli.java | 10 +- .../cli/ThreeDeeTextureTestCommand.java | 51 +++ .../cineast/standalone/config/APIConfig.java | 1 + .../config/ExtractionPipelineConfig.java | 19 +- .../run/GenericExtractionItemHandler.java | 40 ++- .../standalone/runtime/ExtractionTask.java | 5 +- .../util/ContinuousRetrievalLogic.java | 2 +- cineast-runtime/src/main/resources/log4j2.xml | 7 +- cineast.json | 7 +- extraction_config.json | 3 +- fsm.txt | 20 ++ gradle.properties | 6 +- .../renderer/lwjgl/models/default/default.png | Bin 0 -> 68 bytes .../lwjgl/models/unit-cube/Cube_Text.bin | Bin 0 -> 1128 bytes .../lwjgl/models/unit-cube/Cube_Text.gltf | 143 ++++++++ .../renderer/lwjgl/models/unit-cube/cube.png | Bin 0 -> 1195086 bytes resources/renderer/lwjgl/shaders/scene.frag | 18 + resources/renderer/lwjgl/shaders/scene.vert | 16 + 120 files changed, 7769 insertions(+), 521 deletions(-) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Entity.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/IModel.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Material.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Mesh.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Model.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/ModelLoader.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Texture.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/util/MinimalBoundingBox.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/ModelProvider.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/TextureModelQueryTermContainer.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/TextureModel3DSegment.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ITextureModelDecoder.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularTextureModelDecoder.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/TextureModelDecoder.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/MeshOnlyRenderer.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/Engine.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/EngineLogic.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMaterial.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMesh.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLModel.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLScene.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTexture.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTextureCache.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/IGLModel.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/Render.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/RenderOptions.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/SceneRender.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/ShaderProgram.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/UniformsMap.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/LWJGLOffscreenRenderer.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderActions.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderData.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderJob.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderStates.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderWorker.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Camera.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Direction.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/LightfieldCamera.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Projection.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Scene.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/datatype/Variant.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/datatype/VariantException.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/Job.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/JobControlCommand.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/JobType.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/StateEnter.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/StateLeave.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/StateProvider.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/StateProviderAnnotationParser.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/StateProviderException.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/StateTransition.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/Worker.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/controller/FiniteStateMachine.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/controller/FiniteStateMachineException.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/model/Action.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/model/Graph.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/model/State.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/model/Transition.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/window/Window.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/window/WindowOptions.java rename cineast-core/src/main/java/org/vitrivr/cineast/core/util/{DBQueryIDGenerator.java => DBQueryIdGenerator.java} (65%) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/texturemodel/EntropyOptimizer/EntopyCalculationMethod.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/texturemodel/EntropyOptimizer/EntropyOptimizerStrategy.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/texturemodel/EntropyOptimizer/ModelEntropyOptimizer.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/texturemodel/EntropyOptimizer/OptimizerOptions.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/texturemodel/Viewpoint/ViewpointHelper.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/texturemodel/Viewpoint/ViewpointStrategy.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ModelParser.java create mode 100644 cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ThreeDeeTextureTestCommand.java create mode 100644 fsm.txt create mode 100644 resources/renderer/lwjgl/models/default/default.png create mode 100644 resources/renderer/lwjgl/models/unit-cube/Cube_Text.bin create mode 100644 resources/renderer/lwjgl/models/unit-cube/Cube_Text.gltf create mode 100644 resources/renderer/lwjgl/models/unit-cube/cube.png create mode 100644 resources/renderer/lwjgl/shaders/scene.frag create mode 100644 resources/renderer/lwjgl/shaders/scene.vert diff --git a/Dockerfile b/Dockerfile index 6b527ca67..ffcdbf60d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,12 +25,14 @@ COPY --from=build \ /cineast-src/resources \ /opt/cineast/resources +WORKDIR /opt/cineast + RUN printf '#!/bin/bash\n\ if [ "$1" != "api" ] && [ "$1" != "cli" ]; then\n\ echo "Usage: $0 api|cli" >&2\n\ exit 1\n\ fi\n\ -cd /opt/cineast/ && java -jar cineast-$1.jar ${@:2}'\ +java -jar "cineast-$1.jar" "${@:2}"'\ > /opt/cineast/bootstrap.sh RUN chmod +x /opt/cineast/bootstrap.sh diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java index 001412fe3..2ae194b4c 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java @@ -1,7 +1,13 @@ package org.vitrivr.cineast.api; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.logging.Logger; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderJob; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderWorker; + import static org.vitrivr.cineast.core.util.CineastConstants.DEFAULT_CONFIG_PATH; +import org.vitrivr.cineast.core.render.lwjgl.util.fsm.abstractworker.JobControlCommand; import org.vitrivr.cineast.standalone.cli.CineastCli; import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.monitoring.PrometheusServer; @@ -60,8 +66,18 @@ public static void main(String[] args) { APIEndpoint.stop(); GRPCEndpoint.stop(); PrometheusServer.stopServer(); + if (RenderWorker.getRenderJobQueue() != null) { + RenderWorker.getRenderJobQueue().add(new RenderJob(JobControlCommand.SHUTDOWN_WORKER)); + } System.out.println("Goodbye!"); })); + + if (Config.sharedConfig().getExtractor().getEnableRenderWorker()) { + /* Initialize Renderer */ + var renderThread = new Thread(new RenderWorker(new LinkedBlockingDeque<>()), "RenderWorker"); + renderThread.start(); + } + try { /* Start Cineast CLI in interactive mode (blocking). */ if (Config.sharedConfig().getApi().getEnableCli()) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/ResolutionResult.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/ResolutionResult.java index e9c7c50a5..c466f821a 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/ResolutionResult.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/ResolutionResult.java @@ -22,5 +22,4 @@ public ResolutionResult(File file) throws FileNotFoundException { new FileInputStream(file) ); } - } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/websocket/handlers/queries/AbstractQueryMessageHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/websocket/handlers/queries/AbstractQueryMessageHandler.java index 72db0fca6..a481fa9d6 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/websocket/handlers/queries/AbstractQueryMessageHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/websocket/handlers/queries/AbstractQueryMessageHandler.java @@ -134,14 +134,14 @@ public final void handle(Session session, T message) { * @param segmentIds List of segment IDs that should be looked up. * @return List of found {@link MediaSegmentDescriptor} */ - protected List loadSegments(List segmentIds, String queryID) { - LOGGER.trace("Loading segment information for {} segmentIDs, qid {}", segmentIds.size(), queryID); + protected List loadSegments(List segmentIds, String queryId) { + LOGGER.trace("Loading segment information for {} segmentIds, qId {}", segmentIds.size(), queryId); return TimeHelper.timeCall(() -> { - final Map map = this.mediaSegmentReader.lookUpSegments(segmentIds, queryID); + final Map map = this.mediaSegmentReader.lookUpSegments(segmentIds, queryId); final ArrayList sdList = new ArrayList<>(map.size()); segmentIds.stream().filter(map::containsKey).forEach(s -> sdList.add(map.get(s))); return sdList; - }, "loading segment information, qid " + queryID, Level.TRACE); + }, "loading segment information, qid " + queryId, Level.TRACE); } @@ -151,14 +151,14 @@ protected List loadSegments(List segmentIds, Str * @param objectIds List of object IDs that should be looked up. * @return List of found {@link MediaObjectDescriptor} */ - protected List loadObjects(List objectIds, String queryID) { - LOGGER.trace("Loading object information for {} segmentIDs, qid {}", objectIds.size(), queryID); + protected List loadObjects(List objectIds, String queryId) { + LOGGER.trace("Loading object information for {} segmentIds, qid {}", objectIds.size(), queryId); return TimeHelper.timeCall(() -> { - final Map map = this.mediaObjectReader.lookUpObjects(objectIds, queryID); + final Map map = this.mediaObjectReader.lookUpObjects(objectIds, queryId); final ArrayList vdList = new ArrayList<>(map.size()); objectIds.stream().filter(map::containsKey).forEach(s -> vdList.add(map.get(s))); return vdList; - }, "loading object information, qid " + queryID, Level.TRACE); + }, "loading object information, qid " + queryId, Level.TRACE); } /** @@ -247,21 +247,21 @@ synchronized List loadAndWriteSegmentMetadata(Session session, String qu * * @return objectIds retrieved for the segmentIds */ - protected List submitSegmentAndObjectInformation(Session session, String queryID, List segmentIds) { + protected List submitSegmentAndObjectInformation(Session session, String queryId, List segmentIds) { /* Load segment & object information. */ - LOGGER.trace("Loading segment and object information for submission, {} segments, qid {}", segmentIds.size(), queryID); - final List segments = this.loadSegments(segmentIds, queryID); - return submitPrefetchedSegmentAndObjectInformation(session, queryID, segments); + LOGGER.trace("Loading segment and object information for submission, {} segments, qid {}", segmentIds.size(), queryId); + final List segments = this.loadSegments(segmentIds, queryId); + return submitPrefetchedSegmentAndObjectInformation(session, queryId, segments); } - protected List submitPrefetchedSegmentAndObjectInformation(Session session, String queryID, List segments) { + protected List submitPrefetchedSegmentAndObjectInformation(Session session, String queryId, List segments) { final List objectIds = segments.stream().map(MediaSegmentDescriptor::getObjectId).collect(Collectors.toList()); - return submitPrefetchedSegmentandObjectInformationfromIDs(session, queryID, segments, objectIds); + return submitPrefetchedSegmentandObjectInformationfromIDs(session, queryId, segments, objectIds); } - List submitPrefetchedSegmentandObjectInformationfromIDs(Session session, String queryID, List segments, List objectIds) { + List submitPrefetchedSegmentandObjectInformationfromIDs(Session session, String queryId, List segments, List objectIds) { LOGGER.trace("Loading object information"); - final List objects = this.loadObjects(objectIds, queryID); + final List objects = this.loadObjects(objectIds, queryId); if (segments.isEmpty() || objects.isEmpty()) { LOGGER.traceEntry("Segment / Objectlist is Empty, ignoring this iteration"); @@ -270,8 +270,8 @@ List submitPrefetchedSegmentandObjectInformationfromIDs(Session session, LOGGER.trace("Writing results to the websocket"); /* Write segments, objects and similarity search data to stream. */ - this.write(session, new MediaObjectQueryResult(queryID, objects)); - this.write(session, new MediaSegmentQueryResult(queryID, segments)); + this.write(session, new MediaObjectQueryResult(queryId, objects)); + this.write(session, new MediaSegmentQueryResult(queryId, segments)); return objectIds; } diff --git a/cineast-core/build.gradle b/cineast-core/build.gradle index 826128ce7..fd3a3dfd6 100644 --- a/cineast-core/build.gradle +++ b/cineast-core/build.gradle @@ -1,4 +1,5 @@ import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform +import org.gradle.internal.os.OperatingSystem repositories { maven { @@ -80,6 +81,27 @@ signing { sign publishing.publications.mavenJava } +project.ext.lwjglVersion = "3.3.1" + +switch (OperatingSystem.current()) { + case OperatingSystem.LINUX: + def osArch = System.getProperty("os.arch") + project.ext.lwjglNatives = osArch.startsWith("arm") || osArch.startsWith("aarch64") + ? "natives-linux-${osArch.contains("64") || osArch.startsWith("armv8") ? "arm64" : "arm32"}" + : "natives-linux" + break + case OperatingSystem.MAC_OS: + project.ext.lwjglNatives = System.getProperty("os.arch").startsWith("aarch64") ? "natives-macos-arm64" : "natives-macos" + break + case OperatingSystem.WINDOWS: + def osArch = System.getProperty("os.arch") + project.ext.lwjglNatives = osArch.contains("64") + ? "natives-windows${osArch.startsWith("aarch64") ? "-arm64" : ""}" + : "natives-windows-x86" + break +} + + dependencies { /** THE Cottontail DB proto dependency */ @@ -151,6 +173,32 @@ dependencies { api group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' api group: 'javax.activation', name: 'activation', version: '1.1.1' + + /** LWJGL. Minimal OpenGl Configuration from customizer https://www.lwjgl.org/customize **/ + implementation platform("org.lwjgl:lwjgl-bom:$lwjglVersion") + + implementation "org.lwjgl:lwjgl" + implementation "org.lwjgl:lwjgl-assimp" + implementation "org.lwjgl:lwjgl-bgfx" + implementation "org.lwjgl:lwjgl-glfw" + implementation "org.lwjgl:lwjgl-nanovg" + implementation "org.lwjgl:lwjgl-nuklear" + implementation "org.lwjgl:lwjgl-openal" + implementation "org.lwjgl:lwjgl-opengl" + implementation "org.lwjgl:lwjgl-par" + implementation "org.lwjgl:lwjgl-stb" + implementation "org.lwjgl:lwjgl-vulkan" + runtimeOnly "org.lwjgl:lwjgl::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-assimp::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-bgfx::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-glfw::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-nanovg::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-nuklear::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-openal::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-opengl::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-par::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-stb::$lwjglNatives" + if (lwjglNatives == "natives-macos" || lwjglNatives == "natives-macos-arm64") runtimeOnly "org.lwjgl:lwjgl-vulkan::$lwjglNatives" } shadowJar { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java index 21c800910..74a426c37 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/IdConfig.java @@ -35,7 +35,7 @@ public final class IdConfig { * Constructor for default {@link IdConfig}. */ public IdConfig() { - this("UniqueObjectIdGenerator", ExistenceCheck.CHECK_SKIP, new HashMap<>()); + this("UniqueObjectIdGenerator", ExistenceCheck.SKIP_IF_EXISTS, new HashMap<>()); } /** @@ -50,7 +50,7 @@ public IdConfig(@JsonProperty(value = "name", required = true) String name, @JsonProperty(value = "properties") Map properties) { this.name = name; - this.existenceCheckMode = (existenceCheckMode == null ? ExistenceCheck.CHECK_SKIP : existenceCheckMode); + this.existenceCheckMode = (existenceCheckMode == null ? ExistenceCheck.PROCEED_IF_EXISTS : existenceCheckMode); this.properties = (properties == null ? new HashMap<>(0) : properties); } @@ -77,7 +77,7 @@ public ObjectIdGenerator getGenerator() { } public enum ExistenceCheck { - CHECK_SKIP, - CHECK_PROCEED + SKIP_IF_EXISTS, + PROCEED_IF_EXISTS } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java index 790524406..1601f5779 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java @@ -13,6 +13,7 @@ public enum MediaType { IMAGE(1, "i", "image"), AUDIO(2, "a", "audio"), MODEL3D(3, "m", "3dmodel"), + TEXTUREMODEL3D(5, "mt", "3dtexturemodel"), IMAGE_SEQUENCE(4, "is", "imagesequence"), UNKNOWN(99, "u", "unknown"); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Entity.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Entity.java new file mode 100644 index 000000000..ebe19b446 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Entity.java @@ -0,0 +1,186 @@ +package org.vitrivr.cineast.core.data.m3d.texturemodel; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joml.Matrix4f; +import org.joml.Quaternionf; +import org.joml.Vector3f; +import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLScene; + +/** + * An Entity in the context of a {@link Model} describes a position and scale of a model in the scene. + * The Entity is composed of a model matrix that is used to transform the model in the scene. + * The model matrix is calculated from the position, rotation and scale of the entity. + * The Entity influences how the model is rendered in the scene. + * It does not change the mesh of the model. + * Neither does it change the viewpoint of the camera. + */ +public class Entity { + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * ID of entity. + */ + private final String id; + + /** + * ID of associated model. + */ + private final String modelId; + + /** + * Model matrix of entity. + * Used to transform the model in the scene. + * Calculated from position, rotation and scale. + */ + private final Matrix4f modelMatrix; + + /** + * Position of entity. + */ + private final Vector3f position; + + /** + * Rotation of entity. + */ + private final Quaternionf rotation; + + /** + * Scale of entity. + */ + private float scale; + + /** + * Constructs a new Entity. + * Defines an associated model and an id. + * With associated model one is able to add new transformations to the Scene {@link GLScene#addEntity(Entity)}. + * + * @param id ID of entity. + * @param modelId ID of associated model. + */ + public Entity(String id, String modelId) { + this.id = id; + this.modelId = modelId; + this.position = new Vector3f(); + this.rotation = new Quaternionf(); + this.scale = 1f; + this.modelMatrix = new Matrix4f(); + this.updateModelMatrix(); + } + + /** + * @return Unique ID of entity. + */ + public String getId() { + return this.id; + } + + /** + * @return ID of the associated model. + */ + public String getModelId() { + return this.modelId; + } + + /** + * @return Model matrix of entity, describes a rigid transformation of the Model. + */ + public Matrix4f getModelMatrix() { + return this.modelMatrix; + } + + /** + * Translation values, contained in the ModelMatrix + * @return Translativ position of entity in x, y, z. + */ + public Vector3f getPosition() { + return this.position; + } + + /** + * Rotation values, contained in the ModelMatrix + * @return Rotation around x,y,z axes as a quaternion. + */ + public Quaternionf getRotation() { + return this.rotation; + } + + /** + * Scale value, contained in the ModelMatrix + * Scales the associated model. 1.0f is no scaling. + * @return Scale value. + */ + public float getScale() { + return this.scale; + } + + /** + * Sets the as a translation vector from the origin. + * @param x X coordinate of position. + * @param y Y coordinate of position. + * @param z Z coordinate of position. + */ + @SuppressWarnings("unused") + public void setPosition(float x, float y, float z) { + this.position.x = x; + this.position.y = y; + this.position.z = z; + } + + /** + * Sets translation vector from the origin. + * @param position Position of entity. + */ + public void setPosition(Vector3f position) { + this.position.set(position); + } + + /** + * Sets the rotation of the entity. + * @param x X coordinate of axis. + * @param y Y coordinate of axis. + * @param z Z coordinate of axis. + * @param angle Angle of rotation. + */ + public void setRotation(float x, float y, float z, float angle) { + this.rotation.fromAxisAngleRad(x, y, z, angle); + } + + /** + * Sets the rotation of the entity. + * @param axis Axis of rotation. + * @param angle Angle of rotation. + */ + public void setRotation(Vector3f axis, float angle) { + this.rotation.fromAxisAngleRad(axis, angle); + } + + /** + * Sets the scale of the entity. + * set to 1 for no scaling. + * @param scale Scale of entity. + */ + public void setScale(float scale) { + this.scale = scale; + } + + /** + * Updates the model matrix of the entity. + * @implSpec This has to be called after any transformation. + */ + public void updateModelMatrix() { + this.modelMatrix.translationRotateScale(this.position, this.rotation, this.scale); + } + + /** + * Closes the entity. + * Sets the position, rotation to zero and scale to 1. + */ + public void close() { + this.position.zero(); + this.rotation.identity(); + this.scale = 1; + this.updateModelMatrix(); + LOGGER.trace("Entity {} closed", this.id); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/IModel.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/IModel.java new file mode 100644 index 000000000..954f05b98 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/IModel.java @@ -0,0 +1,39 @@ +package org.vitrivr.cineast.core.data.m3d.texturemodel; + +import java.util.List; +import org.joml.Vector3f; +import org.joml.Vector4f; + +public interface IModel { + + /** + * Returns a list of all entities that are associated with this model. + * @return List of {@link Entity} objects. + */ + List getEntities(); + + /** + * Adds an entity to the model. + * @param entity Entity to be added. + */ + void addEntity(Entity entity); + + /** + * Returns the id of the model. + * @return ID of the model. + */ + String getId(); + + /** + * Returns a list of all materials that are associated with this model. + * @return List of {@link Material} objects. + */ + List getMaterials(); + + /** + * Returns a list of all vertices that are associated with this model. + * @return List of {@link Vector3f} objects. + */ + List getAllNormals(); + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Material.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Material.java new file mode 100644 index 000000000..874c68904 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Material.java @@ -0,0 +1,145 @@ +package org.vitrivr.cineast.core.data.m3d.texturemodel; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joml.Vector3f; +import org.joml.Vector4f; +import org.vitrivr.cineast.core.data.m3d.texturemodel.util.MinimalBoundingBox; + +/** + * The Material contains all meshes and the texture that are drawn with on the meshes Further it contains the diffuse color of the material + */ +public class Material { + + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * List of {@link Mesh} objects that define the appearance of the model. + */ + private final List meshes; + + /** + * Texture that drawn on all meshes + */ + private Texture texture; + + /** + * DEFAULT_COLOR is black and 100% opaque + */ + public static final Vector4f DEFAULT_COLOR = new Vector4f(0.0f, 0.0f, 0.0f, 1.0f); + + /** + * diffuseColor is the color that is drawn on the meshes when no texture is present + */ + private Vector4f diffuseColor; + + /** + * Empty material that can be used as a placeholder. + */ + public static final Material EMPTY = new Material(); + + /** + * Constructor for Material. + */ + public Material() { + this.meshes = new ArrayList<>(); + this.texture = new Texture(); + this.diffuseColor = DEFAULT_COLOR; + } + + /** + * @return A MinimalBoundingBox which enclose all MinimalBoundingBoxes from containing meshes + */ + public MinimalBoundingBox getMinimalBoundingBox() { + var mmb = new MinimalBoundingBox(); + for (var mesh : this.meshes) { + mmb.merge(mesh.getMinimalBoundingBox()); + } + return mmb; + } + + /** + * @return the scaling factor to norm 1 size from all containing meshes merged + * @deprecated use {@link #getMinimalBoundingBox()} instead + */ + @Deprecated + public float getMaxNormalizedScalingFactor() { + var min = Float.MAX_VALUE; + for (var mesh : this.meshes) { + min = Math.min(min, mesh.getNormalizedScalingFactor()); + } + return min; + } + + + /** + * @return the translation to origin (0,0,0) from all containing meshes merged + * @deprecated use {@link #getMinimalBoundingBox()} instead + */ + @Deprecated + public Vector3f getMaxNormalizedPosition() { + var min = new Vector3f(0, 0, 0); + for (var mesh : this.meshes) { + min = min.length() > mesh.getNormalizedPosition().length() ? min : mesh.getNormalizedPosition(); + } + return min; + } + + /** + * @return an unmodifiable list of meshes + */ + public List getMeshes() { + return Collections.unmodifiableList(this.meshes); + } + + /** + * @param mesh adds a mesh to the material + */ + public void addMesh(Mesh mesh) { + this.meshes.add(mesh); + } + + /** + * @return the texture to this material + */ + public Texture getTexture() { + return this.texture; + } + + /** + * @param texture sets the texture to this material + */ + public void setTexture(Texture texture) { + this.texture = texture; + } + + /** + * @return the diffuse color of this material + */ + public Vector4f getDiffuseColor() { + return this.diffuseColor; + } + + /** + * @param diffuseColor sets the diffuse color of this material + */ + public void setDiffuseColor(Vector4f diffuseColor) { + this.diffuseColor = diffuseColor; + } + + /** + * closes all resources the material uses + * Calls close on all containing classes + */ + public void close() { + this.meshes.forEach(Mesh::close); + this.meshes.clear(); + this.texture.close(); + this.texture = null; + this.diffuseColor = null; + LOGGER.trace("Closed Material"); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Mesh.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Mesh.java new file mode 100644 index 000000000..e5d67e655 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Mesh.java @@ -0,0 +1,220 @@ +package org.vitrivr.cineast.core.data.m3d.texturemodel; + +import java.util.ArrayList; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joml.Vector3f; +import org.vitrivr.cineast.core.data.m3d.texturemodel.util.MinimalBoundingBox; + +/** + * The Mesh is the geometric representation of a model. + * It contains the vertices, faces, normals and texture coordinates. + * It also constructs the face normals and the minimal bounding box. + */ +public class Mesh { + + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Number of all vertices in the mesh + */ + private final int numVertices; + + /** + * ID of the mesh + */ + private String id; + + /** + * List of all vertices in the mesh + * The positions are flattened vectors + * positions[0] = x + * positions[1] = y + * positions[2] = z + * positions[3] = x + * ... + */ + private final float[] positions; + + /** + * List of all face normals in the mesh + * The length of the normals describes the area of the face + * The direction of the normals describes the direction of the face and points outwards + */ + private final List facenormals; + + /** + * List of all texture coordinates in the mesh + */ + private final float[] textureCoords; + + /** + * Flattered list of all vertices ids. + * A three tuple describes a face. + * e.g 0, 1, 3, 3, 1, 2, + * face1 = (0, 1, 3) + * face2 = (3, 1, 2) + */ + private final int[] idx; + + /** + * List of all vertices normals in the mesh + * TODO: not used yet, will be used for vertex shading + */ + @SuppressWarnings("all") + private final float[] normals; + + /** + * MinimalBoundingBox that encloses the mesh + */ + private final MinimalBoundingBox minimalBoundingBox; + + /** + * Constructor for Mesh. + * Arrays are flattened vectors. + * e.g. positions[0] = x + * positions[1] = y + * positions[2] = z + * positions[3] = x + * ... + * + * @param positions List of all vertices in the mesh + * @param normals List of all vertices normals in the mesh + * @param textureCoordinates List of all texture coordinates in the mesh + * @param idx List of all vertices ids. + */ + public Mesh(float[] positions, float[] normals, float[] textureCoordinates, int[] idx) { + //Stores all the data + this.positions = positions; + this.idx = idx; + this.numVertices = idx.length; + this.normals = normals; + // List to store results of face normals calculation + this.facenormals = new ArrayList<>(this.numVertices / 3); + //this.areas = new ArrayList<>(positions.length / 3); + this.textureCoords = textureCoordinates; + + // Calculate face normals + // ic increments by 3 because a face is defined by 3 vertices + for (var ic = 0; ic < this.idx.length; ic += 3) { + if (normals == null) { + // Add zero vector if there are no vertex normals + this.facenormals.add(new Vector3f(0f, 0f, 0f)); + } else { + // Get the three vertices of the face + var v1 = new Vector3f(positions[idx[ic] * 3], positions[idx[ic] * 3 + 1], positions[idx[ic] * 3 + 2]); + var v2 = new Vector3f(positions[idx[ic + 1] * 3], positions[idx[ic + 1] * 3 + 1], positions[idx[ic + 1] * 3 + 2]); + var v3 = new Vector3f(positions[idx[ic + 2] * 3], positions[idx[ic + 2] * 3 + 1], positions[idx[ic + 2] * 3 + 2]); + // Get the three vertices normals of the face + var vn1 = new Vector3f(normals[idx[ic] * 3], normals[idx[ic] * 3 + 1], normals[idx[ic] * 3 + 2]); + var vn2 = new Vector3f(normals[idx[ic + 1] * 3], normals[idx[ic + 1] * 3 + 1], normals[idx[ic + 1] * 3 + 2]); + var vn3 = new Vector3f(normals[idx[ic + 2] * 3], normals[idx[ic + 2] * 3 + 1], normals[idx[ic + 2] * 3 + 2]); + // Instance the face normal + var fn = new Vector3f(0, 0, 0); + // Calculate the direction of the face normal by averaging the three vertex normals + fn.add(vn1).add(vn2).add(vn3).div(3).normalize(); + // Instance the face area + var fa = new Vector3f(0, 0, 0); + // Calculate the area of the face by calculating the cross product of the two edges and dividing by 2 + v2.sub(v1).cross(v3.sub(v1),fa); + fa.div(2); + // Add the face normal to the list of face normals + this.facenormals.add(fn.mul(fa.length())); + } + } + // Calculate the minimal bounding box + this.minimalBoundingBox = new MinimalBoundingBox(this.positions); + } + + /** + * + * @return returns the number of vertices in the mesh + */ + public int getNumVertices() { + return this.numVertices; + } + +/** + * @return the flattened array of all positions + */ + public float[] getPositions() { + return this.positions; + } + + /** + * @return the flattened array of all texture coordinates + */ + public float[] getTextureCoords() { + return this.textureCoords; + } + + /** + * @return the flattened array of all vertices ids + * A three tuple describes a face. + * e.g 0, 1, 3, 3, 1, 2, + * face1 = (0, 1, 3) + * face2 = (3, 1, 2) + */ + public int[] getIdx() { + return this.idx; + } + + /** + * @return list containing all face normals + */ + public List getNormals() { + return this.facenormals; + } + + /** + * @return the MinimalBoundingBox which contains the scaling factor to norm and the translation to origin (0,0,0) + */ + public MinimalBoundingBox getMinimalBoundingBox() { + return this.minimalBoundingBox; + } + + /** + * @return the scaling factor to norm 1 size + * @deprecated use {@link #getMinimalBoundingBox()} instead + */ + @Deprecated + public float getNormalizedScalingFactor() { + return this.minimalBoundingBox.getScalingFactorToNorm(); + } + + /** + * @return the translation to origin (0,0,0) + * @deprecated use {@link #getMinimalBoundingBox()} instead + */ + @Deprecated + public Vector3f getNormalizedPosition() { + return this.minimalBoundingBox.getTranslationToNorm(); + } + + /** + * @return the id of the mesh + */ + public String getId() { + return this.id; + } + + /** + * @param id sets the id of the mesh + */ + public void setId(int id) { + this.id = Integer.toString(id); + } + + + /** + * closes the mesh + * releases all resources + */ + public void close() { + this.facenormals.clear(); + this.minimalBoundingBox.close(); + this.id = null; + LOGGER.trace("Closing Mesh"); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Model.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Model.java new file mode 100644 index 000000000..797e680fe --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Model.java @@ -0,0 +1,126 @@ +package org.vitrivr.cineast.core.data.m3d.texturemodel; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joml.Vector3f; +import org.joml.Vector4f; +import org.vitrivr.cineast.core.data.m3d.texturemodel.util.MinimalBoundingBox; + +/** + * This class represents a model that can be rendered by the {@link org.vitrivr.cineast.core.render.lwjgl.engine.Engine}. + * The model is composed of a list of + * {@link Entity} objects + * and a list of {@link Material} objects. + * The {@link Entity} objects are used to position and scale the model in the scene. + * The {@link Material} objects are used to define the appearance of the model. + */ +public class Model implements IModel { + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * ID of the model. + */ + private final String id; + + /** + * List of {@link Entity} objects that define the position and scale of the model. + */ + private final List entities; + + /** + * List of {@link Material} objects that define the appearance of the model. + * Contains all Meshes and Textures that are used by the model. + */ + private final List materials; + + /** + * Empty model that can be used as a placeholder. + */ + public static final Model EMPTY = new Model("EmptyModel", new ArrayList<>() {{ + add(Material.EMPTY); + }}); + + /** + * Constructor for Model. + * + * @param id ID of the model. + * @param materials List of {@link Material} objects that define the appearance of the model. + */ + public Model(String id, List materials) { + this.id = id; + this.entities = new ArrayList<>(); + this.materials = materials; + } + + /** + * {@inheritDoc} + */ + @Override + public List getEntities() { + return Collections.unmodifiableList(this.entities); + } + + /** + * {@inheritDoc} + */ + @Override + public void addEntity(Entity entity) { + this.entities.add(entity); + } + + /** + * Adds an entity to the model and normalizes the model. + * @param entity Entity to be added. + */ + public void addEntityNorm(Entity entity) { + var mbb = new MinimalBoundingBox(); + + for (var material :this.materials) { + mbb.merge(material.getMinimalBoundingBox()); + } + + entity.setPosition(mbb.getTranslationToNorm().mul(-1)); + entity.setScale(mbb.getScalingFactorToNorm()); + this.entities.add(entity); + } + + /** + * {@inheritDoc} + */ + @Override + public String getId() { + return this.id; + } + + /** + * {@inheritDoc} + */ + @Override + public List getMaterials() { + return Collections.unmodifiableList(this.materials); + } + + /** + * {@inheritDoc} + */ + @Override + public List getAllNormals() { + var normals = new ArrayList(); + this.materials.forEach(m -> m.getMeshes().forEach(mesh -> normals.addAll(mesh.getNormals()))); + return normals; + } + + /** + * Closes the model and releases all resources. + */ + public void close(){ + this.materials.forEach(Material::close); + this.materials.clear(); + this.entities.forEach(Entity::close); + this.entities.clear(); + LOGGER.trace("Closed model {}", this.id); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/ModelLoader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/ModelLoader.java new file mode 100644 index 000000000..b550226ff --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/ModelLoader.java @@ -0,0 +1,331 @@ +package org.vitrivr.cineast.core.data.m3d.texturemodel; + + +import static org.lwjgl.assimp.Assimp.AI_MATKEY_COLOR_DIFFUSE; +import static org.lwjgl.assimp.Assimp.aiGetMaterialColor; +import static org.lwjgl.assimp.Assimp.aiGetMaterialTexture; +import static org.lwjgl.assimp.Assimp.aiImportFile; +import static org.lwjgl.assimp.Assimp.aiProcess_CalcTangentSpace; +import static org.lwjgl.assimp.Assimp.aiProcess_FixInfacingNormals; +import static org.lwjgl.assimp.Assimp.aiProcess_GlobalScale; +import static org.lwjgl.assimp.Assimp.aiProcess_JoinIdenticalVertices; +import static org.lwjgl.assimp.Assimp.aiProcess_LimitBoneWeights; +import static org.lwjgl.assimp.Assimp.aiProcess_PreTransformVertices; +import static org.lwjgl.assimp.Assimp.aiProcess_Triangulate; +import static org.lwjgl.assimp.Assimp.aiReleaseImport; +import static org.lwjgl.assimp.Assimp.aiReturn_SUCCESS; +import static org.lwjgl.assimp.Assimp.aiTextureType_DIFFUSE; +import static org.lwjgl.assimp.Assimp.aiTextureType_NONE; + +import java.io.File; +import java.nio.IntBuffer; +import java.util.ArrayList; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joml.Vector4f; +import org.lwjgl.assimp.AIColor4D; +import org.lwjgl.assimp.AIFace; +import org.lwjgl.assimp.AIMaterial; +import org.lwjgl.assimp.AIMesh; +import org.lwjgl.assimp.AIString; +import org.lwjgl.assimp.AIVector3D; +import org.lwjgl.system.MemoryStack; + +public final class ModelLoader { + + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Loads a model from a file. Generates all the standard flags for Assimp. For more details see Assimp. + *
    + *
  • aiProcess_GenSmoothNormals: + * This is ignored if normals are already there at the time this flag + * is evaluated. Model importers try to load them from the source file, so + * they're usually already there. + * This flag may not be specified together with + * #aiProcess_GenNormals. There's a configuration option, + * #AI_CONFIG_PP_GSN_MAX_SMOOTHING_ANGLE which allows you to specify + * an angle maximum for the normal smoothing algorithm. Normals exceeding + * this limit are not smoothed, resulting in a 'hard' seam between two faces. + * Using a decent angle here (e.g. 80 degrees) results in very good visual + * appearance. + *
    • + *
  • aiProcess_JoinIdenticalVertices:
    • + *
  • aiProcess_Triangulate By default the imported mesh data might contain faces with more than 3 + * indices. For rendering you'll usually want all faces to be triangles. + * This post processing step splits up faces with more than 3 indices into + * triangles. Line and point primitives are *not* modified! If you want + * 'triangles only' with no other kinds of primitives, try the following + * solution: + *
      + *
    • Specify both #aiProcess_Triangulate and #aiProcess_SortByPType
      • + * Ignore all point and line meshes when you process assimp's output + *
    + *
    • + *
  • aiProcess_FixInf acingNormals: + * This step tries to determine which meshes have normal vectors that are facing inwards and inverts them. + * The algorithm is simple but effective: the bounding box of all vertices + their normals is compared against + * the volume of the bounding box of all vertices without their normals. This works well for most objects, problems might occur with + * planar surfaces. However, the step tries to filter such cases. + * The step inverts all in-facing normals. Generally it is recommended to enable this step, although the result is not always correct. + *
    • + *
  • aiProcess_CalcTangentSpace: + * Calculates the tangents and bi tangents for the imported meshes + * Does nothing if a mesh does not have normals. + * You might want this post processing step to be executed if you plan to use tangent space calculations such as normal mapping applied to the meshes. + * There's an importer property, AI_CONFIG_PP_CT_MAX_SMOOTHING_ANGLE, which allows you to specify a maximum smoothing angle for the algorithm. + * However, usually you'll want to leave it at the default value. + *
    • + *
  • aiProcess_LimitBoneWeights: + * Limits the number of bones simultaneously affecting a single vertex to a maximum value. + * If any vertex is affected by more than the maximum number of bones, + * the least important vertex weights are removed and the remaining vertex weights are normalized so that the weights still sum up to 1. + * The default bone weight limit is 4 (defined as AI_LBW_MAX_WEIGHTS in config.h), + * but you can use the AI_CONFIG_PP_LBW_MAX_WEIGHTS importer property to supply your own limit to the post processing step. + * If you intend to perform the skinning in hardware, this post processing step might be of interest to you. + *
    • + *
  • aiProcess_PreTransformVertices: + * Removes the node graph and pre-transforms all vertices with the local transformation matrices of their nodes. + * If the resulting scene can be reduced to a single mesh, with a single material, no lights, and no cameras, + * then the output scene will contain only a root node (with no children) that references the single mesh. + * Otherwise, the output scene will be reduced to a root node with a single level of child nodes, each one referencing one mesh, + * and each mesh referencing one material + * In either case, for rendering, you can simply render all meshes in order - you don't need to pay attention to local transformations and the node hierarchy. + * Animations are removed during this step + * This step is intended for applications without a scenegraph. + * The step CAN cause some problems: if e.g. a mesh of the asset contains normals and another, using the same material index, + * does not, they will be brought together, but the first mesh's part of the normal list is zeroed. However, these artifacts are rare. + *
    • + *
+ * + * @param modelId The ID of the model. + * @param modelPath Path to the model file. + * @return Model object. + */ + public static Model loadModel(String modelId, String modelPath) { + var model = loadModel(modelId, modelPath, + aiProcess_JoinIdenticalVertices | + aiProcess_GlobalScale | + aiProcess_FixInfacingNormals | + aiProcess_Triangulate | + aiProcess_CalcTangentSpace | + aiProcess_LimitBoneWeights | + aiProcess_PreTransformVertices); + LOGGER.trace("Try return Model 2"); + return model; + } + + /** + * Loads a model from a file. 1. Loads the model file to an aiScene. 2. Process all Materials. 3. Process all Meshes. 3.1 Process all Vertices. 3.2 Process all Normals. 3.3 Process all Textures. 3.4 Process all Indices. + * + * @param modelId Arbitrary unique ID of the model. + * @param modelPath Path to the model file. + * @param flags Flags for the model loading process. + * @return Model object. + */ + @SuppressWarnings("NullAway") + public static Model loadModel(String modelId, String modelPath, int flags) { + LOGGER.trace("Try loading file {} from {}", modelId, modelPath); + + var file = new File(modelPath); + if (!file.exists()) { + throw new RuntimeException("Model path does not exist [" + modelPath + "]"); + } + var modelDir = file.getParent(); + + LOGGER.trace("Loading aiScene"); + + // DO NOT USE AUTOCLOSEABLE TRY CATCH FOR AI-SCENE!!! THIS WILL CAUSE A FATAL ERROR ON NTH (199) ITERATION! + // RAPHAEL WALTENSPUEL 2023-01-20 + var aiScene = aiImportFile(modelPath, flags); + if (aiScene == null) { + throw new RuntimeException("Error loading model [modelPath: " + modelPath + "]"); + } + + var numMaterials = aiScene.mNumMaterials(); + List materialList = new ArrayList<>(); + for (var ic = 0; ic < numMaterials; ic++) { + //TODO: Warning + var aiMaterial = AIMaterial.create(aiScene.mMaterials().get(ic)); + LOGGER.trace("Try processing material {}", ic); + materialList.add(ModelLoader.processMaterial(aiMaterial, modelDir)); + } + + var numMeshes = aiScene.mNumMeshes(); + var aiMeshes = aiScene.mMeshes(); + var defaultMaterial = new Material(); + for (var ic = 0; ic < numMeshes; ic++) { + LOGGER.trace("Try create AI Mesh {}", ic); + //TODO: Warning + var aiMesh = AIMesh.create(aiMeshes.get(ic)); + var mesh = ModelLoader.processMesh(aiMesh); + LOGGER.trace("Try get Material idx"); + var materialIdx = aiMesh.mMaterialIndex(); + Material material; + if (materialIdx >= 0 && materialIdx < materialList.size()) { + material = materialList.get(materialIdx); + } else { + material = defaultMaterial; + } + LOGGER.trace("Try add Material to Mesh"); + material.addMesh(mesh); + } + + if (!defaultMaterial.getMeshes().isEmpty()) { + LOGGER.trace("Try add default Material"); + materialList.add(defaultMaterial); + } + + LOGGER.trace("Try instantiate Model"); + aiReleaseImport(aiScene); + + var model = new Model(modelId, materialList); + LOGGER.trace("Try return Model"); + return model; + } + + /** + * Convert indices from aiMesh to int array. + * + * @param aiMesh aiMesh to process. + * @return flattened int array of indices. + */ + private static int[] processIndices(AIMesh aiMesh) { + LOGGER.trace("Start processing indices"); + List indices = new ArrayList<>(); + var numFaces = aiMesh.mNumFaces(); + var aiFaces = aiMesh.mFaces(); + for (var ic = 0; ic < numFaces; ic++) { + AIFace aiFace = aiFaces.get(ic); + IntBuffer buffer = aiFace.mIndices(); + while (buffer.remaining() > 0) { + indices.add(buffer.get()); + } + } + LOGGER.trace("End processing indices"); + return indices.stream().mapToInt(Integer::intValue).toArray(); + } + + /** + * Convert an AIMaterial to a Material. Loads the diffuse color and texture. + * + * @param aiMaterial aiMaterial to process. + * @param modelDir Path to the model file. + * @return flattened float array of vertices. + */ + private static Material processMaterial(AIMaterial aiMaterial, String modelDir) { + LOGGER.trace("Start processing material"); + var material = new Material(); + try (MemoryStack stack = MemoryStack.stackPush()) { + AIColor4D color = AIColor4D.create(); + + //** Diffuse color if no texture is present + int result = aiGetMaterialColor( + aiMaterial, AI_MATKEY_COLOR_DIFFUSE, aiTextureType_NONE, 0, color); + if (result == aiReturn_SUCCESS) { + material.setDiffuseColor(new Vector4f(color.r(), color.g(), color.b(), color.a())); + } + + //** Try load texture + AIString aiTexturePath = AIString.calloc(stack); + aiGetMaterialTexture(aiMaterial, aiTextureType_DIFFUSE, 0, aiTexturePath, (IntBuffer) null, + null, null, null, null, null); + var texturePath = aiTexturePath.dataString(); + //TODO: Warning + if (texturePath != null && texturePath.length() > 0) { + material.setTexture(new Texture(modelDir + File.separator + new File(texturePath).toPath())); + material.setDiffuseColor(Material.DEFAULT_COLOR); + } + + return material; + } + } + + /** + * Convert aiMesh to a Mesh. Loads the vertices, normals, texture coordinates and indices. + * Instantiates a new Mesh object. + * @param aiMesh aiMesh to process. + * @return flattened float array of normals. + */ + private static Mesh processMesh(AIMesh aiMesh) { + LOGGER.trace("Start processing mesh"); + var vertices = processVertices(aiMesh); + var normals = processNormals(aiMesh); + var textCoords = processTextCoords(aiMesh); + var indices = processIndices(aiMesh); + + // Texture coordinates may not have been populated. We need at least the empty slots + if (textCoords.length == 0) { + var numElements = (vertices.length / 3) * 2; + textCoords = new float[numElements]; + } + LOGGER.trace("End processing mesh"); + return new Mesh(vertices, normals, textCoords, indices); + } + + /** + * Convert normals from aiMesh to float array. + * + * @param aiMesh aiMesh to process. + * @return flattened float array of normals. + */ + private static float[] processNormals(AIMesh aiMesh) { + LOGGER.trace("Start processing Normals"); + var buffer = aiMesh.mNormals(); + if (buffer == null) { + return null; + } + var data = new float[buffer.remaining() * 3]; + var pos = 0; + while (buffer.remaining() > 0) { + var normal = buffer.get(); + data[pos++] = normal.x(); + data[pos++] = normal.y(); + data[pos++] = normal.z(); + } + return data; + } + + /** + * Convert texture coordinates from aiMesh to float array. + * @param aiMesh aiMesh to process. + * @return flattened float array of texture coordinates. + */ + private static float[] processTextCoords(AIMesh aiMesh) { + LOGGER.trace("Start processing Coordinates"); + var buffer = aiMesh.mTextureCoords(0); + if (buffer == null) { + return new float[]{}; + } + float[] data = new float[buffer.remaining() * 2]; + int pos = 0; + while (buffer.remaining() > 0) { + AIVector3D textCoord = buffer.get(); + data[pos++] = textCoord.x(); + data[pos++] = 1 - textCoord.y(); + } + return data; + } + + /** + * Convert vertices from aiMesh to float array. + * + * @param aiMesh aiMesh to process. + * @return flattened float array of vertices. + */ + private static float[] processVertices(AIMesh aiMesh) { + LOGGER.trace("Start processing Vertices"); + AIVector3D.Buffer buffer = aiMesh.mVertices(); + float[] data = new float[buffer.remaining() * 3]; + int pos = 0; + while (buffer.remaining() > 0) { + AIVector3D textCoord = buffer.get(); + data[pos++] = textCoord.x(); + data[pos++] = textCoord.y(); + data[pos++] = textCoord.z(); + } + + return data; + } +} \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Texture.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Texture.java new file mode 100644 index 000000000..ce4e9c392 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Texture.java @@ -0,0 +1,57 @@ +package org.vitrivr.cineast.core.data.m3d.texturemodel; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * This class represents a texture. + * In the context free 3D model, a texture is basically a path to a texture file. + */ +public class Texture { + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Path to the texture file. + */ + private final String texturePath; + + /** + * Default texture path. + * Points to a png with one white pixel with 100% opacity. + */ + public static final String DEFAULT_TEXTURE = "./resources/renderer/lwjgl/models/default/default.png"; + + + /** + * Constructor for the Texture class. + * Sets the texture path to the default texture path. + */ + public Texture() { + this.texturePath = DEFAULT_TEXTURE; + } + + /** + * Constructor for the Texture class. + * Sets the texture path to the given texture path. + * + * @param texturePath Path to the texture file. + */ + public Texture(String texturePath) { + this.texturePath = texturePath; + } + + /** + * @return Path to the texture file. + */ + public String getTexturePath() { + return this.texturePath; + } + + /** + * Releases all resources associated with this Texture. + */ + public void close() { + // Nothing to do here. + LOGGER.trace("Closing Texture"); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/util/MinimalBoundingBox.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/util/MinimalBoundingBox.java new file mode 100644 index 000000000..e7c71b6af --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/util/MinimalBoundingBox.java @@ -0,0 +1,235 @@ +package org.vitrivr.cineast.core.data.m3d.texturemodel.util; + +import java.util.LinkedList; +import java.util.List; +import org.joml.Vector3f; + +/** + * This class represents a minimal bounding box. It can be generated from a list of vertices. It can be merged with another minimal bounding box. + */ +public class MinimalBoundingBox { + + /** + * Constant for the maximum float value. + */ + public static final float MAX = Float.MAX_VALUE; + /** + * Constant for the minimum float value. + */ + public static final float MIN = -1f * Float.MAX_VALUE; + /** + * Initial value for the maximum vector. The maximum vector contain the highest (positive if normalized) values for x, y and z of the bounding box. + */ + private final Vector3f vMax = new Vector3f(MIN, MIN, MIN); + /** + * Initial value for the minimum vector. The minimum vector contain the lowes (negative if normalized) values for x, y and z of the bounding box. + */ + private final Vector3f vMin = new Vector3f(MAX, MAX, MAX); + + /** + * Center of mass of the bounding box as x, y, z vector. + */ + private final Vector3f com = new Vector3f(0f, 0f, 0f); + + /** + * Scaling factor to norm. The scaling factor is the factor to scale the bounding box to the norm. 1 for no scaling. + */ + private float scalingFactorToNorm = 1f; + + /** + * Translation to norm. The translation is the vector to translate com of the bounding box to the origin. (0, 0, 0) for no translation. + */ + private final Vector3f translationToNorm = new Vector3f(0, 0, 0); + + /** + * Empty constructor to initialize an empty bounding box The purpose is to iteratively add bounding boxes. + */ + public MinimalBoundingBox() { + } + + /** + * Constructor to initialize a bounding box from an array of vertices. The resulting bounding box is the minimal bounding box that contains all vertices. The bounding box is aligned with the Cartesian coordinate system. + * + * @param positions List of vertices. + */ + public MinimalBoundingBox(float[] positions) { + this.update(positions); + } + + /** + * Constructor to initialize a bounding box from a list of vertices. The resulting bounding box is the minimal bounding box that contains all vertices. The bounding box is aligned with the Cartesian coordinate system. + * + * @param positions List of vertices. + */ + @SuppressWarnings("unused") + public MinimalBoundingBox(List positions) { + this.update(positions); + } + + /** + * Crate list of vertices from bounding box. List contains the maximum and minimum vector of the bounding box. + * TODO: A better approach would be to return the 8 vertices of the bounding box. But needs to be checked before. + * + * @return List of vertices. + */ + public List toList() { + var vec = new LinkedList(); + if (this.isValidBoundingBox()) { + vec.add(vMax); + vec.add(vMin); + } + return vec; + } + + /** + * Merge this bounding box with another bounding box. The resulting bounding box is the minimal bounding box that contains both bounding boxes. + * + * @param other Bounding box to merge with. + * @return Merged bounding box. + */ + public MinimalBoundingBox merge(MinimalBoundingBox other) { + if (this.equals(other)) { + return this; + } + this.update(other.toList()); + return this; + } + + /** + * Returns the scaling factor to norm size. + * + * @return Scaling factor to norm size. + */ + public float getScalingFactorToNorm() { + return this.scalingFactorToNorm; + } + + /** + * Get translation to Origin. + * + * @return Translation to Origin. + */ + public Vector3f getTranslationToNorm() { + return this.translationToNorm; + } + + /** + * Helper method to add data to the bounding box and recalculate the bounding boxes values. + */ + protected void update(float[] positions) { + var vectors = new LinkedList(); + for (var ic = 0; ic < positions.length; ic += 3) { + vectors.add(new Vector3f(positions[ic], positions[ic + 1], positions[ic + 2])); + } + this.update(vectors); + } + + /** + * Helper method to add data to the bounding box and recalculate the bounding boxes values. Since the calculation of the bounding box is iterative, the calculation is split into several steps. The steps are: + *
    + *
  • 1. Update the center of mass.
    • + *
  • 2. Update the scaling factor to norm.
    • + *
  • 3. Update the translation to norm.
    • + *
+ * These steps had to be exact in this sequence + */ + protected void update(List vec) { + // Has to be exact this sequence + if (this.updateBounds(vec)) { + this.updateCom(); + this.updateScalingFactorToNorm(); + this.updateTranslationToNorm(); + } + } + + /** + * Checks if the bounding box is valid. A bounding box is valid if each component of the maximum vector is greater than the corresponding component of the minimum vector. + * + * @return True if the bounding box is valid, false otherwise. + */ + private boolean isValidBoundingBox() { + return this.vMax.x > this.vMin.x && this.vMax.y > this.vMin.y && this.vMax.z > this.vMin.z; + } + + /** + * Update the center of mass. The center of mass is the middle point of the bounding box. + */ + private void updateCom() { + this.com.set(new Vector3f((this.vMax.x + this.vMin.x) / 2f, (this.vMax.y + this.vMin.y) / 2f, (this.vMax.z + this.vMin.z) / 2f)); + } + + /** + * Update the translation to norm. The translation is the vector to translate com of the bounding box to the origin. (0, 0, 0) for no translation. + */ + private void updateTranslationToNorm() { + this.translationToNorm.set(new Vector3f(com.mul(this.scalingFactorToNorm))); + } + + /** + * Update the scaling factor to norm. The scaling factor is the factor to scale the longest vector in the bounding box to the norm. 1 for no scaling. + */ + private void updateScalingFactorToNorm() { + var farthest = new Vector3f(0, 0, 0); + for (var vec : this.toList()) { + var vector = new Vector3f(vec).sub(this.com); + if (vector.length() > farthest.length()) { + farthest = vector; + } + this.scalingFactorToNorm = 1f / (farthest.length() * 2); + } + } + + /** + * Update the bounding box with a new vectors + * + * @return True if the bounding box has changed, false otherwise. + */ + private boolean updateBounds(List positions) { + var changed = false; + for (var vec : positions) { + changed |= this.updateBounds(vec); + } + return changed; + } + + /** + * Update the bounding box with a new vector + * + * @return True if the bounding box has changed, false otherwise. + */ + private boolean updateBounds(Vector3f vec) { + var changed = false; + if (vec.x > this.vMax.x) { + this.vMax.x = vec.x; + changed = true; + } + if (vec.x < this.vMin.x) { + this.vMin.x = vec.x; + changed = true; + } + if (vec.y > this.vMax.y) { + this.vMax.y = vec.y; + changed = true; + } + if (vec.y < this.vMin.y) { + this.vMin.y = vec.y; + changed = true; + } + if (vec.z > this.vMax.z) { + this.vMax.z = vec.z; + changed = true; + } + if (vec.z < this.vMin.z) { + this.vMin.z = vec.z; + changed = true; + } + return changed; + } + + /** + * Release all resources. + */ + public void close() { + + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/ModelProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/ModelProvider.java new file mode 100644 index 000000000..3249b6418 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/providers/ModelProvider.java @@ -0,0 +1,16 @@ +package org.vitrivr.cineast.core.data.providers; + +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Model; + +public interface ModelProvider { + + /** + * Returns a 3D Model. Defaults to the empty Model, if not implemented. + * + * @return Model + */ + default IModel getModel() { + return Model.EMPTY; + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/TextureModelQueryTermContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/TextureModelQueryTermContainer.java new file mode 100644 index 000000000..6a0adbf9b --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/query/containers/TextureModelQueryTermContainer.java @@ -0,0 +1,114 @@ +package org.vitrivr.cineast.core.data.query.containers; + +import java.awt.image.BufferedImage; +import java.util.Objects; +import org.vitrivr.cineast.core.data.raw.CachedDataFactory; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Model; +import org.vitrivr.cineast.core.util.web.ImageParser; +import org.vitrivr.cineast.core.util.web.MeshParser; +import org.vitrivr.cineast.core.util.web.ModelParser; + + +public class TextureModelQueryTermContainer extends AbstractQueryTermContainer { + + /** + * Original Mesh as transferred by the client. + */ + private final Model model; + + /** + * Image containing a 2D sketch of the 3D model in question. + */ + private final MultiImage image; + + /** + * Constructs an {@link TextureModelQueryTermContainer} from base 64 encoded JSON data. The constructor assumes either the JSV4 JSON format for Meshes OR a valid image (for 2D sketch to 3D model lookup). + * + * @param data The 3D model data that should be converted. + * @param factory The {@link CachedDataFactory} used to create images. + */ + public TextureModelQueryTermContainer(String data, CachedDataFactory factory) { + if (MeshParser.isValidThreeJSV4Geometry(data)) { + this.model = ModelParser.parseThreeJSV4Geometry(data); + this.image = MultiImage.EMPTY_MULTIIMAGE; + } else if (ImageParser.isValidImage(data)) { + final BufferedImage img = ImageParser.dataURLtoBufferedImage(data); + this.image = factory.newMultiImage(img); + this.model = Model.EMPTY; + } else { + throw new IllegalArgumentException("The provided data could not be converted to a Mesh."); + } + } + + public TextureModelQueryTermContainer(String data) { + this(data, CachedDataFactory.getDefault()); + } + + /** + * Constructor for {@link TextureModelQueryTermContainer} with a Mesh. Used for Query-by-Example. + * + * @param model Mesh for which to create a {@link TextureModelQueryTermContainer}. + */ + public TextureModelQueryTermContainer(Model model) { + this.model = model; + this.image = MultiImage.EMPTY_MULTIIMAGE; + } + + /** + * Constructor for {@link TextureModelQueryTermContainer} with ab image. Used for Query-by-Sketch (2d sketch to 3d model). + * + * @param image BufferedImage for which to create a {@link TextureModelQueryTermContainer}. + * @param factory The {@link CachedDataFactory} to create the {@link MultiImage} with. + */ + public TextureModelQueryTermContainer(BufferedImage image, CachedDataFactory factory) { + this.image = factory.newMultiImage(image); + this.model = Model.EMPTY; + } + + /** + * Constructor for {@link TextureModelQueryTermContainer} constructor with an image (treated as 2D sketch). Used for Query-by-2D-Sketch. + * + * @param image Image for which to create a {@link TextureModelQueryTermContainer}. + */ + public TextureModelQueryTermContainer(MultiImage image) { + this.image = image; + this.model = Model.EMPTY; + } + + @Override + public Model getModel() { + return this.model; + } + + + @Override + public MultiImage getAvgImg() { + return this.image; + } + + @Override + public MultiImage getMedianImg() { + return this.image; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + TextureModelQueryTermContainer that = (TextureModelQueryTermContainer) o; + return Objects.equals(model, that.model) && Objects.equals(image, that.image); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), this.model,this.image); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java index b05e73118..fa8083b60 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java @@ -11,6 +11,7 @@ import org.vitrivr.cineast.core.data.providers.LocationProvider; import org.vitrivr.cineast.core.data.providers.MedianImgProvider; import org.vitrivr.cineast.core.data.providers.MeshProvider; +import org.vitrivr.cineast.core.data.providers.ModelProvider; import org.vitrivr.cineast.core.data.providers.MostRepresentativeFrameProvider; import org.vitrivr.cineast.core.data.providers.PathProvider; import org.vitrivr.cineast.core.data.providers.SemanticMapProvider; @@ -40,6 +41,7 @@ public interface SegmentContainer AudioFrameProvider, AudioSTFTProvider, MeshProvider, + ModelProvider, VoxelGridProvider, LocationProvider, InstantProvider, diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/TextureModel3DSegment.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/TextureModel3DSegment.java new file mode 100644 index 000000000..83495513c --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/TextureModel3DSegment.java @@ -0,0 +1,62 @@ +package org.vitrivr.cineast.core.data.segments; + +import org.vitrivr.cineast.core.data.m3d.VoxelGrid; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Model; + + +public class TextureModel3DSegment implements SegmentContainer { + + + /** + * The original 3D Mesh as extracted from a model file. + */ + private final Model model; + + /** + * Segment ID of the ModelSegmen. + */ + private String segmentId; + /** + * ID of the multimedia object this AudioSegment belongs to. + */ + private String objectId; + private VoxelGrid grid; + + /** + * Default constructor for Model3DSegment + * + * @param model 3D Mesh associated with the segment. + */ + public TextureModel3DSegment(Model model) { + this.model = model; + } + + /** + * @return a unique id of this + */ + @Override + public final String getId() { + return this.segmentId; + } + + @Override + public final void setId(String id) { + this.segmentId = id; + } + + @Override + public final String getSuperId() { + return this.objectId; + } + + @Override + public final void setSuperId(String id) { + this.objectId = id; + } + + @Override + public IModel getModel() { + return this.model; + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java index e93476782..35d7f4cdc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java @@ -6,7 +6,6 @@ import com.google.common.collect.Lists; import java.io.Closeable; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -31,121 +30,128 @@ public interface DBSelector extends Closeable { Logger LOGGER = LogManager.getLogger(); + /** + * Tells this selector to use a specific entity. This is intended to be called once per selector after it is instantiated, calling it multiple times may cause issues depending on the underlying implementation. + */ boolean open(String name); + /** + * Closes this selector and all associated resources. Calling any method afterwards may cause issues. + */ void close(); /** * Convenience-wrapper to query with float-arrays {@link #getNearestNeighboursGeneric(int, PrimitiveTypeProvider, String, Class, ReadableQueryConfig)} */ - default List getNearestNeighboursGeneric(int k, float[] query, String column, Class distanceElementClass, ReadableQueryConfig config) { - return getNearestNeighboursGeneric(k, new FloatArrayTypeProvider(query), column, distanceElementClass, config); + default List getNearestNeighboursGeneric(int k, float[] query, String column, Class distanceElementClass, ReadableQueryConfig queryConfig) { + return getNearestNeighboursGeneric(k, new FloatArrayTypeProvider(query), column, distanceElementClass, queryConfig); } /** - * * Finds the {@code k}-nearest neighbours of the given {@code queryProvider} in {@code column} using the provided distance function in {@code config}. {@code ScoreElementClass} defines the specific type of {@link DistanceElement} to be created internally and returned by this method. + * * Finds the {@code k}-nearest neighbours of the given {@code queryProvider} in {@code column} using the provided distance function in {@code queryConfig}. {@code ScoreElementClass} defines the specific type of {@link DistanceElement} to be created internally and returned by this method. * * @param k maximum number of results * @param queryProvider query vector - * @param column feature column to do the search - * @param distanceElementClass class of the {@link DistanceElement} type - * @param config query config - * @param type of the {@link DistanceElement} - * @return a list of elements with their distance + * @param column feature column to do the nns on + * @param distanceElementClass class of the {@link DistanceElement} type (e.g., {@link SegmentDistanceElement} + * @param type of the {@link DistanceElement} + * @param queryConfig query config + * @return a list of ids with their distance */ - default List getNearestNeighboursGeneric(int k, PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, ReadableQueryConfig config) { + default List getNearestNeighboursGeneric(int k, PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, ReadableQueryConfig queryConfig) { if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType().equals(ProviderDataType.INT_ARRAY)) { //Default-implementation for backwards compatibility. - return getNearestNeighboursGeneric(k, PrimitiveTypeProvider.getSafeFloatArray(queryProvider), column, distanceElementClass, config); + return getNearestNeighboursGeneric(k, PrimitiveTypeProvider.getSafeFloatArray(queryProvider), column, distanceElementClass, queryConfig); } LogManager.getLogger().error("{} does not support other queries than float-arrays.", this.getClass().getSimpleName()); throw new UnsupportedOperationException(); } - default List getFarthestNeighboursGeneric(int k, PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, ReadableQueryConfig config) { - throw new UnsupportedOperationException("Farthest neighbors are not supported"); - } - - /** * Performs a batched kNN-search with multiple query vectors. That is, the storage engine is tasked to perform the kNN search for each vector in the provided list and returns the union of the results for every query. * - * @param k The number k vectors to return per query. - * @param vectors The list of vectors to use. - * @param column The column to perform the kNN search on. - * @param distanceElementClass class of the {@link DistanceElement} type - * @param configs The query configuration, which may contain distance definitions or query-hints. + * @param k maximum number of results + * @param vectors list of query vectors + * @param column feature column to do the nns on + * @param distanceElementClass class of the {@link DistanceElement} type (e.g., {@link SegmentDistanceElement} * @param The type T of the resulting type of the {@link DistanceElement}. - * @return List of results. + * @param queryConfigs query configs + * @return a list of ids with their distance */ - List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs); + List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List queryConfigs); /** - * In contrast to {@link #getNearestNeighboursGeneric(int, float[], String, Class, ReadableQueryConfig)}, this method returns all elements of a row + * In contrast to {@link #getNearestNeighboursGeneric(int, float[], String, Class, ReadableQueryConfig)}, this method returns all columns per result row */ - List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config); + List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig queryConfig); /** - * SELECT 'vectorname' from entity where 'fieldname' = 'value' + * SELECT 'vectorname' from entity where 'column' = 'value' */ - List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig); + List getFeatureVectors(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig); /** - * for legacy support, takes the float[] method by default + * Conversion to PrimitiveTypeProviders is expensive so underlying classes should feel free to override if they wish to optimize for performance + *

+ * takes the float[] method by default */ - default List getFeatureVectorsGeneric(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig qc) { - return getFeatureVectors(fieldName, value, vectorName, qc).stream().map(FloatArrayTypeProvider::new).collect(Collectors.toList()); - } - - default List> getRows(String fieldName, PrimitiveTypeProvider value) { - return getRows(fieldName, Collections.singleton(value)); + default List getFeatureVectorsGeneric(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig qc) { + return getFeatureVectors(column, value, vectorName, qc).stream().map(FloatArrayTypeProvider::new).collect(Collectors.toList()); } - default List> getRows(String fieldName, PrimitiveTypeProvider... values) { - return getRows(fieldName, Arrays.asList(values)); + /** + * {@link #getRows(String, Iterable, String)} + */ + default List> getRows(String column, PrimitiveTypeProvider value) { + return getRows(column, Collections.singleton(value), null); } /** - * SELECT * where fieldName IN (values) + * {@link #getRows(String, Iterable, String)} */ - List> getRows(String fieldName, Iterable values); + default List> getRows(String column, Iterable values) { + return getRows(column, values, null); + } /** - * By default, the queryID is ignored + * SELECT * where column IN (values) */ - default List> getRows(String fieldName, Iterable values, String dbQueryID) { - return getRows(fieldName, values); - } + List> getRows(String column, Iterable values, String dbQueryId); /** - * Conversion to PrimitiveTypeProviders is expensive so feel free to use & implement extension for generic objects + * Conversion to PrimitiveTypeProviders is expensive so underlying classes should feel free to override if they wish to optimize for performance + *

+ * {@link #getRows(String, Iterable, String)} */ - default List> getRows(String fieldName, List values) { - return getRows(fieldName, values.stream().map(StringTypeProvider::new).collect(Collectors.toList())); + default List> getRows(String column, List values) { + return getRows(column, values.stream().map(StringTypeProvider::new).collect(Collectors.toList()), null); } /** - * Conversion to PrimitiveTypeProviders is expensive so feel free to use & implement extension for generic objects + * Conversion to PrimitiveTypeProviders is expensive so underlying classes should feel free to override if they wish to optimize for performance + *

+ * {@link #getRows(String, Iterable, String)} */ - default List> getRows(String fieldName, List values, String dbQueryID) { - return getRows(fieldName, values.stream().map(StringTypeProvider::new).collect(Collectors.toList())); + default List> getRows(String column, List values, String dbQueryId) { + return getRows(column, values.stream().map(StringTypeProvider::new).collect(Collectors.toList()), null); } /** * Performs a fulltext search with multiple query terms. That is, the storage engine is tasked to lookup for entries in the provided fields that match the provided query terms. * - * @param rows The number of rows that should be returned. - * @param fieldname The field that should be used for lookup. - * @param terms The query terms. Individual terms will be connected by a logical OR. + * @param rows The number of rows that should be returned. + * @param column The field that should be used for lookup. + * @param queryConfig query config + * @param terms The query terms. Individual terms will be connected by a logical OR. * @return List of rows that math the fulltext search. */ - List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms); + List> getFulltextRows(int rows, String column, ReadableQueryConfig queryConfig, String... terms); /** - * {@link #getRows(String, RelationalOperator, Iterable)} + * {@link #getRows(String, RelationalOperator, Iterable, ReadableQueryConfig)} */ - default List> getRows(String fieldName, RelationalOperator operator, PrimitiveTypeProvider value) { - return getRows(fieldName, operator, Collections.singleton(value)); + default List> getRows(String column, RelationalOperator operator, PrimitiveTypeProvider value, ReadableQueryConfig queryConfig) { + return getRows(column, operator, Collections.singleton(value), queryConfig); } /** @@ -153,24 +159,26 @@ default List> getRows(String fieldName, Relat *

* i.e. SELECT * from WHERE A B * - * @param fieldName The name of the database field . - * @param operator The {@link RelationalOperator} that should be used for comparison. - * @param values The values the field should be compared to. + * @param column The name of the database field . + * @param operator The {@link RelationalOperator} that should be used for comparison. + * @param values The values the field should be compared to. + * @param queryConfig query config * @return List of rows (one row is represented by one Map of the field ames and the data contained in the field). */ - List> getRows(String fieldName, RelationalOperator operator, Iterable values); + List> getRows(String column, RelationalOperator operator, Iterable values, ReadableQueryConfig queryConfig); /** * Performs a boolean lookup based on multiple conditions, linked with AND. Each element of the list specifies one of the conditions - left middle right, i.e. id IN (1, 5, 7) * - * @param conditions conditions which will be linked by AND - * @param identifier column upon which the retain operation will be performed if the database layer does not support compound boolean retrieval. - * @param projection Which columns shall be selected + * @param conditions conditions which will be linked by AND + * @param identifier column upon which the retain operation will be performed if the database layer does not support compound boolean retrieval. + * @param projection Which columns shall be selected + * @param queryConfig query config */ - default List> getRowsAND(List>> conditions, String identifier, List projection, ReadableQueryConfig qc) { + default List> getRowsAND(List>> conditions, String identifier, List projection, ReadableQueryConfig queryConfig) { HashMap> relevant = new HashMap<>(); for (Triple> condition : conditions) { - List> rows = this.getRows(condition.getLeft(), condition.getMiddle(), condition.getRight()); + List> rows = this.getRows(condition.getLeft(), condition.getMiddle(), condition.getRight(), queryConfig); if (rows.isEmpty()) { return Collections.emptyList(); } @@ -202,28 +210,33 @@ default List getUniqueValues(String column) { return Lists.newArrayList(uniques); } + /** + * counts how many times each element appears per value in a given column. This can be useful for example to debug duplicates or count occurences of tags + */ default Map countDistinctValues(String column) { Map count = new HashMap<>(); - this.getAll(column).forEach(el -> count.compute(el.getString(), (k, v) -> v == null ? 1 : v++)); + this.getAll(Collections.singletonList(column), -1).forEach(el -> count.compute(el.get(column).getString(), (k, v) -> v == null ? 1 : v++)); return count; } /** - * by default just calls the implementation with a null-list for ids + * Returns all available metadata based on the specification. */ - default List> getMetadataBySpec(List spec, String dbQueryID) { - return this.getMetadataByIdAndSpec(null, spec, null, dbQueryID); - } - - default List> getMetadataByIdAndSpec(List ids, List spec, String idColName) { - return getMetadataByIdAndSpec(ids, spec, idColName, null); + default List> getMetadataBySpec(List spec, String dbQueryId, ReadableQueryConfig queryConfig) { + return this.getMetadataByIdAndSpec(null, spec, null, dbQueryId, queryConfig); } /** - * Horribly slow default implementation which iterates over the whole table + * Retrieves Metadata based on ids, access specification and other parameters + * + * @param ids ids for which to fetch metadata + * @param spec which metadata should be fetched + * @param idColName the name of the column which the id refers to. Can be null, in which case the default behavior is used + * @param dbQueryId query identifier. Can be null + * @param queryConfig query config */ - default List> getMetadataByIdAndSpec(List ids, List spec, String idColName, String dbQueryID) { - LOGGER.trace("fetching metadata with spec, dbQueryID {}", dbQueryID); + default List> getMetadataByIdAndSpec(List ids, List spec, String idColName, String dbQueryId, ReadableQueryConfig queryConfig) { + LOGGER.trace("fetching metadata with spec, dbQueryId {}", dbQueryId); return getAll().stream().filter(tuple -> { // check if there are any elements of the specification which do not work if (spec.stream().noneMatch(el -> { @@ -265,16 +278,8 @@ default List> getMetadataByIdAndSpec(List getAll(String column) { - return getAll().stream().map(el -> el.get(column)).collect(Collectors.toList()); - } - /** - * SELECT columns from the table. Be careful with large entities + * SELECT columns from the table. Be careful with large entities (SELECT columns FROM table) * * @param limit if <= 0, parameter is ignored */ @@ -304,7 +309,7 @@ default List> getAll(String order, int skip, } /** - * Get all rows from all tables + * Get all rows from the tables (SELECT * FROM table) */ List> getAll(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java index 7f06b5bcc..abd5859f6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java @@ -5,6 +5,7 @@ import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -54,12 +55,12 @@ public void close() { } @Override - public List getNearestNeighboursGeneric(int k, PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, ReadableQueryConfig config) { + public List getNearestNeighboursGeneric(int k, PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, ReadableQueryConfig queryConfig) { List> results; if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType().equals(ProviderDataType.INT_ARRAY)) { - results = getNearestNeighbourRows(k, queryProvider.getFloatArray(), column, config); + results = getNearestNeighbourRows(k, queryProvider.getFloatArray(), column, queryConfig); } else { - results = getNearestNeighbourRows(k, queryProvider, column, config); + results = getNearestNeighbourRows(k, queryProvider, column, queryConfig); } return results.stream().map(m -> DistanceElement.create(distanceElementClass, m.get(GENERIC_ID_COLUMN_QUALIFIER).getString(), m.get(DB_DISTANCE_VALUE_QUALIFIER).getDouble())).limit(k).collect(Collectors.toList()); } @@ -114,19 +115,19 @@ private List> getNearestNeighbourRows(int k, @Override - public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { + public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig queryConfig) { - config = QueryConfig.clone(config); + queryConfig = QueryConfig.clone(queryConfig); Importer importer = newImporter(this.file); - FloatArrayDistance distance = FloatArrayDistance.fromQueryConfig(config); + FloatArrayDistance distance = FloatArrayDistance.fromQueryConfig(queryConfig); FixedSizePriorityQueue> knn = FixedSizePriorityQueue.create(k, new PrimitiveTypeMapDistanceComparator(column, vector, distance)); HashSet relevant = null; - if (config.hasRelevantSegmentIds()) { - Set ids = config.getRelevantSegmentIds(); + if (queryConfig.hasRelevantSegmentIds()) { + Set ids = queryConfig.getRelevantSegmentIds(); relevant = new HashSet<>(ids.size()); relevant.addAll(ids); } @@ -158,7 +159,7 @@ public List> getNearestNeighbourRows(int k, f } @Override - public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { + public List getFeatureVectors(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { ArrayList _return = new ArrayList<>(1); if (value == null || value.getString().isEmpty()) { @@ -168,13 +169,13 @@ public List getFeatureVectors(String fieldName, PrimitiveTypeProvider v Importer importer = newImporter(this.file); Map map; while ((map = importer.readNextAsMap()) != null) { - if (!map.containsKey(fieldName)) { + if (!map.containsKey(column)) { continue; } if (!map.containsKey(vectorName)) { continue; } - if (value.equals(map.get(fieldName).getString())) { + if (value.equals(map.get(column).getString())) { _return.add(PrimitiveTypeProvider.getSafeFloatArray(map.get(vectorName))); } } @@ -183,33 +184,7 @@ public List getFeatureVectors(String fieldName, PrimitiveTypeProvider v } @Override - public List> getRows(String fieldName, PrimitiveTypeProvider... values) { - ArrayList> _return = new ArrayList<>(1); - - if (values == null || values.length == 0) { - return _return; - } - - Importer importer = newImporter(this.file); - Map map; - while ((map = importer.readNextAsMap()) != null) { - if (!map.containsKey(fieldName)) { - continue; - } - for (int i = 0; i < values.length; ++i) { - if (values[i].equals(map.get(fieldName).getString())) { - _return.add(map); - break; - } - } - - } - - return _return; - } - - @Override - public List> getRows(String fieldName, Iterable values) { + public List> getRows(String column, Iterable values, String dbQueryId) { if (values == null) { return new ArrayList<>(0); } @@ -222,7 +197,7 @@ public List> getRows(String fieldName, Iterab PrimitiveTypeProvider[] valueArr = new PrimitiveTypeProvider[tmp.size()]; tmp.toArray(valueArr); - return this.getRows(fieldName, valueArr); + return this.getRows(column, Arrays.asList(valueArr)); } @Override @@ -249,17 +224,17 @@ public boolean existsEntity(String name) { protected abstract String getFileExtension(); @Override - public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { + public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List queryConfigs) { // TODO Auto-generated method stub return null; } @Override - public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { + public List> getRows(String column, RelationalOperator operator, Iterable values, ReadableQueryConfig queryConfig) { throw new IllegalStateException("Not implemented."); } - public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { + public List> getFulltextRows(int rows, String column, ReadableQueryConfig queryConfig, String... terms) { throw new IllegalStateException("Not implemented."); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java index 9dad2b9c9..ca1cd35e7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java @@ -22,27 +22,27 @@ public void close() { } @Override - public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { + public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List queryConfigs) { return new ArrayList<>(0); } @Override - public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { + public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig queryConfig) { return new ArrayList<>(0); } @Override - public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { + public List getFeatureVectors(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { return new ArrayList<>(0); } @Override - public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { + public List> getFulltextRows(int rows, String column, ReadableQueryConfig queryConfig, String... terms) { return new ArrayList<>(0); } @Override - public List> getRows(String fieldName, Iterable values) { + public List> getRows(String column, Iterable values, String dbQueryId) { return new ArrayList<>(0); } @@ -62,7 +62,7 @@ public boolean ping() { } @Override - public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { + public List> getRows(String column, RelationalOperator operator, Iterable values, ReadableQueryConfig queryConfig) { return new ArrayList<>(0); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index 1482c0025..931a1fbf0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -4,7 +4,7 @@ import static org.vitrivr.cineast.core.util.CineastConstants.DOMAIN_COL_NAME; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.KEY_COL_NAME; -import static org.vitrivr.cineast.core.util.DBQueryIDGenerator.generateQueryID; +import static org.vitrivr.cineast.core.util.DBQueryIdGenerator.generateQueryId; import io.grpc.Status; import io.grpc.StatusRuntimeException; @@ -27,12 +27,11 @@ import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; import org.vitrivr.cineast.core.data.distance.DistanceElement; -import org.vitrivr.cineast.core.data.distance.SegmentDistanceElement; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; -import org.vitrivr.cineast.core.data.providers.primitive.ProviderDataType; import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.RelationalOperator; import org.vitrivr.cineast.core.db.dao.MetadataAccessSpecification; +import org.vitrivr.cineast.core.util.DBQueryIdGenerator; import org.vitrivr.cottontail.client.iterators.Tuple; import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.client.language.basics.Direction; @@ -223,8 +222,8 @@ public void close() { /* No op. */ } * if {@link ReadableQueryConfig#getRelevantSegmentIds()} is null, the where-clause will be left empty */ @Override - public List getNearestNeighboursGeneric(int k, float[] vector, String column, Class distanceElementClass, ReadableQueryConfig config) { - final Query query = knn(k, vector, column, config); + public List getNearestNeighboursGeneric(int k, float[] vector, String column, Class distanceElementClass, ReadableQueryConfig queryConfig) { + final Query query = knn(k, vector, column, queryConfig); try { return handleNearestNeighbourResponse(this.cottontail.client.query(query), distanceElementClass); } catch (StatusRuntimeException e) { @@ -234,29 +233,13 @@ public List getNearestNeighboursGeneric(int k, fl } @Override - public List getFarthestNeighboursGeneric(int k, PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, ReadableQueryConfig config) { - if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType().equals(ProviderDataType.INT_ARRAY)) { - //Default-implementation for backwards compatibility. - var vector = PrimitiveTypeProvider.getSafeFloatArray(queryProvider); - final var query = kn(k, vector, column, config, Direction.DESC, "id"); - try { - return handleNearestNeighbourResponse(this.cottontail.client.query(query), distanceElementClass); - } catch (StatusRuntimeException e) { - LOGGER.warn("Error occurred during query execution in getNearestNeighboursGeneric(): {}", e.getMessage()); - return new ArrayList<>(0); - } - } - throw new UnsupportedOperationException("other types than float vectors are not supported for farthest neighbors"); - } - - @Override - public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { + public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List queryConfigs) { return new ArrayList<>(0); /* TODO. */ } @Override - public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { - final Query query = knn(k, vector, column, config, "*"); + public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig queryConfig) { + final Query query = knn(k, vector, column, queryConfig, "*"); try { return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { @@ -266,8 +249,8 @@ public List> getNearestNeighbourRows(int k, f } @Override - public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { - final Query query = new Query(this.fqn).select(vectorName, null).where(new Expression(fieldName, "==", value.toObject())).queryId(generateQueryID("get-fv", queryConfig)); + public List getFeatureVectors(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { + final Query query = new Query(this.fqn).select(vectorName, null).where(new Expression(column, "==", value.toObject())).queryId(generateQueryId("get-fv", queryConfig)); try { final TupleIterator results = this.cottontail.client.query(query); final List _return = new LinkedList<>(); @@ -284,8 +267,8 @@ public List getFeatureVectors(String fieldName, PrimitiveTypeProvider v @Override - public List getFeatureVectorsGeneric(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig qc) { - final Query query = new Query(this.fqn).select(vectorName, null).where(new Expression(fieldName, "==", value.toObject())).queryId(generateQueryID("get-fv-gen", qc)); + public List getFeatureVectorsGeneric(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig qc) { + final Query query = new Query(this.fqn).select(vectorName, null).where(new Expression(column, "==", value.toObject())).queryId(generateQueryId("get-fv-gen", qc)); try { return toSingleCol(this.cottontail.client.query(query), vectorName); } catch (StatusRuntimeException e) { @@ -295,34 +278,34 @@ public List getFeatureVectorsGeneric(String fieldName, Pr } @Override - public List> getRows(String fieldName, Iterable values) { + public List> getRows(String column, Iterable values) { final Object[] mapped = StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::toObject).toArray(); - return getRowsHelper(fieldName, "IN", mapped, "get-rows-in-iterable"); + return getRowsHelper(column, "IN", mapped, "get-rows-in-iterable"); } @Override - public List> getRows(String fieldName, List values) { + public List> getRows(String column, List values) { final Object[] mapped = values.toArray(); - return getRowsHelper(fieldName, "IN", mapped, "get-rows-stringlist"); + return getRowsHelper(column, "IN", mapped, "get-rows-stringlist"); } @Override - public List> getRows(String fieldName, Iterable values, String dbQueryID) { + public List> getRows(String column, Iterable values, String dbQueryId) { final Object[] mapped = StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::toObject).toArray(); - return getRowsHelper(fieldName, "IN", mapped, dbQueryID); + return getRowsHelper(column, "IN", mapped, dbQueryId); } @Override - public List> getRows(String fieldName, List values, String dbQueryID) { + public List> getRows(String column, List values, String dbQueryId) { final Object[] mapped = values.toArray(); - return getRowsHelper(fieldName, "IN", mapped, dbQueryID); + return getRowsHelper(column, "IN", mapped, dbQueryId); } @Override - public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { + public List> getRows(String column, RelationalOperator operator, Iterable values, ReadableQueryConfig queryConfig) { final Object[] mapped = StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::toObject).toArray(); final String op = toOperator(operator); - return getRowsHelper(fieldName, op, mapped, "get-rows-" + op + "-iterable"); + return getRowsHelper(column, op, mapped, "get-rows-" + op + "-iterable"); } private List> getRowsHelper(String fieldName, String op, Object[] mapped, String dbQueryID) { @@ -330,6 +313,9 @@ private List> getRowsHelper(String fieldName, LOGGER.debug("empty in-clause, not executing query {}", dbQueryID); return new ArrayList<>(0); } + if (dbQueryID == null) { + dbQueryID = DBQueryIdGenerator.generateQueryId("get-rows-help"); + } final Query query = new Query(this.fqn).select("*", null).where(new Expression(fieldName, op, mapped)).queryId(dbQueryID); try { return processResults(this.cottontail.client.query(query)); @@ -340,15 +326,15 @@ private List> getRowsHelper(String fieldName, } @Override - public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { + public List> getFulltextRows(int rows, String column, ReadableQueryConfig queryConfig, String... terms) { /* Prepare plain query. */ final String predicate = Arrays.stream(terms).map(String::trim).collect(Collectors.joining(" OR ")); /* TODO Cottontail calls this a distance in its documentation, but it's actually a score. See the tests - that's why we order DESC and not ASC */ final Query query = new Query(this.fqn) .select("*", null) - .fulltext(fieldname, predicate, DB_DISTANCE_VALUE_QUALIFIER) - .queryId(generateQueryID("ft-rows", queryConfig)) + .fulltext(column, predicate, DB_DISTANCE_VALUE_QUALIFIER) + .queryId(generateQueryId("ft-rows", queryConfig)) .order(DB_DISTANCE_VALUE_QUALIFIER, Direction.DESC) .limit(rows); @@ -371,9 +357,9 @@ public List> getFulltextRows(int rows, String } @Override - public List> getRowsAND(List>> conditions, String identifier, List projection, ReadableQueryConfig qc) { + public List> getRowsAND(List>> conditions, String identifier, List projection, ReadableQueryConfig queryConfig) { /* Prepare plain query. */ - final Query query = new Query(this.fqn).queryId(generateQueryID("get-rows-and", qc)); + final Query query = new Query(this.fqn).queryId(generateQueryId("get-rows-and", queryConfig)); if (projection.isEmpty()) { query.select("*", null); } else { @@ -390,8 +376,8 @@ public List> getRowsAND(List predicates = atomics.stream().reduce(And::new); - if (qc != null && !qc.getRelevantSegmentIds().isEmpty()) { - final Set relevant = qc.getRelevantSegmentIds(); + if (queryConfig != null && !queryConfig.getRelevantSegmentIds().isEmpty()) { + final Set relevant = queryConfig.getRelevantSegmentIds(); final Expression segmentIds = new Expression(GENERIC_ID_COLUMN_QUALIFIER, "IN", relevant.toArray()); if (predicates.isPresent()) { query.where(new And(segmentIds, predicates.get())); @@ -411,20 +397,26 @@ public List> getRowsAND(List> getMetadataBySpec(List spec, String dbQueryID) { - final Query query = new Query(this.fqn).select("*", null).queryId(dbQueryID); + public List> getMetadataBySpec(List spec, String dbQueryId, ReadableQueryConfig queryConfig) { + if (dbQueryId == null) { + dbQueryId = generateQueryId("md-spec", queryConfig); + } + final Query query = new Query(this.fqn).select("*", null).queryId(dbQueryId); final Optional predicates = generateQueryFromMetadataSpec(spec); predicates.ifPresent(query::where); return processResults(this.cottontail.client.query(query)); } @Override - public List> getMetadataByIdAndSpec(List ids, List spec, String idColName, String dbQueryID) { + public List> getMetadataByIdAndSpec(List ids, List spec, String idColName, String dbQueryId, ReadableQueryConfig queryConfig) { if (ids.isEmpty()) { - LOGGER.trace("No ids specified, not fetching any metadata for query id {}", dbQueryID); + LOGGER.trace("No ids specified, not fetching any metadata for query id {}", dbQueryId); return new ArrayList<>(); } - final Query query = new Query(this.fqn).select("*", null).queryId(dbQueryID == null ? "md-id-spec" : dbQueryID); + if (dbQueryId == null) { + dbQueryId = generateQueryId("md-id-spec", queryConfig); + } + final Query query = new Query(this.fqn).select("*", null).queryId(dbQueryId); final Optional predicates = generateQueryFromMetadataSpec(spec); final Expression segmentIds = new Expression(idColName, "IN", ids.toArray()); if (predicates.isPresent()) { @@ -462,21 +454,10 @@ public Optional generateQueryFromMetadataSpec(List getAll(String column) { - final Query query = new Query(this.fqn).select(column, null).queryId(generateQueryID("all-" + column)); - try { - return toSingleCol(this.cottontail.client.query(query), column); - } catch (StatusRuntimeException e) { - LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); - return new ArrayList<>(0); - } - } - @Override public List> getAll(List columns, int limit) { final Query query = new Query(this.fqn); - query.queryId(generateQueryID("all-cols-limit-" + limit)); + query.queryId(DBQueryIdGenerator.generateQueryId("all-cols-limit-" + limit)); for (String c : columns) { query.select(c, null); } @@ -494,7 +475,7 @@ public List> getAll(List columns, int @Override public List getUniqueValues(String column) { final Query query = new Query(this.fqn).distinct(column, null). - queryId(generateQueryID("unique-" + column)); + queryId(DBQueryIdGenerator.generateQueryId("unique-" + column)); try { return toSingleCol(this.cottontail.client.query(query), column); } catch (StatusRuntimeException e) { @@ -505,7 +486,7 @@ public List getUniqueValues(String column) { public Map countDistinctValues(String column) { final Query query = new Query(this.fqn).select(column, null) - .queryId(generateQueryID("count-distinct-" + column)); + .queryId(DBQueryIdGenerator.generateQueryId("count-distinct-" + column)); final Map count = new HashMap<>(); try { final TupleIterator results = this.cottontail.client.query(query); @@ -523,7 +504,7 @@ public Map countDistinctValues(String column) { @Override public List> getAll() { final Query query = new Query(this.fqn).select("*", null) - .queryId(generateQueryID("get-all-" + this.fqn)); + .queryId(DBQueryIdGenerator.generateQueryId("get-all-" + this.fqn)); try { return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { @@ -535,7 +516,7 @@ public List> getAll() { @Override public List> getAll(String order, int skip, int limit) { final Query query = new Query(this.fqn).select("*", null) - .queryId(generateQueryID("get-all-order-skip-limit-" + this.fqn)) + .queryId(DBQueryIdGenerator.generateQueryId("get-all-order-skip-limit-" + this.fqn)) .order(order, Direction.ASC) .skip(skip) .limit(limit); @@ -618,7 +599,7 @@ private Query kn(int k, float[] vector, String column, ReadableQueryConfig confi .distance(column, vector, distance, DB_DISTANCE_VALUE_QUALIFIER) .order(DB_DISTANCE_VALUE_QUALIFIER, direction) .limit(k) - .queryId(generateQueryID(direction == Direction.ASC ? "knn" : "kfn", config)); + .queryId(generateQueryId(direction == Direction.ASC ? "knn" : "kfn", config)); for (String s : select) { query.select(s, null); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java index c946ae295..86c688314 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java @@ -33,7 +33,7 @@ public CottontailWrapper(String host, int port) { boolean pingSuccessful = this.client.ping(); watch.stop(); if (pingSuccessful) { - LOGGER.info("Connected to Cottontail DB in {} ms at {}:{}", watch.getTime(TimeUnit.MILLISECONDS), host, port); + LOGGER.debug("Connected to Cottontail DB in {} ms at {}:{}", watch.getTime(TimeUnit.MILLISECONDS), host, port); } else { LOGGER.warn("Could not ping Cottontail DB instance at {}:{}", host, port); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java index c8b85f8b8..089cc9af1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java @@ -16,7 +16,7 @@ import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.dao.MetadataAccessSpecification; import org.vitrivr.cineast.core.db.dao.MetadataType; -import org.vitrivr.cineast.core.util.DBQueryIDGenerator; +import org.vitrivr.cineast.core.util.DBQueryIdGenerator; /** * Abstraction layer for segment and object metadata retrieval. @@ -68,7 +68,7 @@ public List findBySpec(List ids, List sp return findBySpec(ids, spec, null); } - public List findBySpec(List ids, List spec, String queryID) { + public List findBySpec(List ids, List spec, String queryId) { if (ids == null || spec == null) { LOGGER.warn("provided id-list {} or spec {} is null, returning empty list", ids, spec); return new ArrayList<>(); @@ -80,8 +80,8 @@ public List findBySpec(List ids, List sp StopWatch watch = StopWatch.createStarted(); ids = sanitizeIds(ids); spec = sanitizeSpec(spec); - String dbQueryID = DBQueryIDGenerator.generateQueryID("find-md-spec-" + tableName, queryID); - List> results = selector.getMetadataByIdAndSpec(ids, spec, idColName, dbQueryID); + String dbQueryId = DBQueryIdGenerator.generateQueryId("find-md-spec-" + tableName, queryId); + List> results = selector.getMetadataByIdAndSpec(ids, spec, idColName, dbQueryId, null); LOGGER.debug("Performed metadata lookup for {} ids in {} ms. {} results.", ids.size(), watch.getTime(TimeUnit.MILLISECONDS), results.size()); return mapToResultList(results); } @@ -90,7 +90,7 @@ public List findBySpec(List spec) { return findBySpec(spec, null); } - public List findBySpec(List spec, String queryID) { + public List findBySpec(List spec, String queryId) { if (spec == null) { LOGGER.warn("Provided spec is null, returning empty list"); return new ArrayList<>(); @@ -101,8 +101,8 @@ public List findBySpec(List spec, String queryID } StopWatch watch = StopWatch.createStarted(); spec = sanitizeSpec(spec); - String dbQueryID = DBQueryIDGenerator.generateQueryID("find-my-spec-" + tableName, queryID); - List> results = selector.getMetadataBySpec(spec, dbQueryID); + String dbQueryId = DBQueryIdGenerator.generateQueryId("find-my-spec-" + tableName, queryId); + List> results = selector.getMetadataBySpec(spec, dbQueryId, null); LOGGER.debug("Performed metadata lookup in {} ms. {} results.", watch.getTime(TimeUnit.MILLISECONDS), results.size()); return mapToResultList(results); } @@ -131,10 +131,10 @@ public List findBySpec(MetadataAccessSpecification... spec) { * Returns metadata according to spec * * @param spec access specification - * @param queryID can be null + * @param queryId can be null */ - public List findBySpec(MetadataAccessSpecification spec, String queryID) { - return this.findBySpec(Lists.newArrayList(spec), queryID); + public List findBySpec(MetadataAccessSpecification spec, String queryId) { + return this.findBySpec(Lists.newArrayList(spec), queryId); } public List sanitizeSpec(List spec) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java index 6208ae0ed..602db69ca 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java @@ -16,7 +16,7 @@ import org.vitrivr.cineast.core.data.providers.primitive.ProviderDataType; import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; import org.vitrivr.cineast.core.db.DBSelector; -import org.vitrivr.cineast.core.util.DBQueryIDGenerator; +import org.vitrivr.cineast.core.util.DBQueryIdGenerator; public class MediaObjectReader extends AbstractEntityReader { @@ -113,7 +113,7 @@ public MediaObjectDescriptor lookUpObjectByPath(String path) { return mapToDescriptor(result.get(0)); } - public Map lookUpObjects(Iterable videoIds, String queryID) { + public Map lookUpObjects(Iterable videoIds, String queryId) { if (videoIds == null) { return new HashMap<>(); } @@ -132,7 +132,7 @@ public Map lookUpObjects(Iterable videoId if (!notCached.isEmpty()) { - String dbQueryID = DBQueryIDGenerator.generateQueryID("load-obj", queryID); + String dbQueryID = DBQueryIdGenerator.generateQueryId("load-obj", queryId); List> results = selector.getRows(MediaObjectDescriptor.FIELDNAMES[0], Lists.newArrayList(videoIds), dbQueryID); results.forEach(el -> { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java index c8c303c08..a3b1b09dd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java @@ -22,7 +22,7 @@ import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; import org.vitrivr.cineast.core.db.DBSelector; -import org.vitrivr.cineast.core.util.DBQueryIDGenerator; +import org.vitrivr.cineast.core.util.DBQueryIdGenerator; public class MediaSegmentReader extends AbstractEntityReader { @@ -84,7 +84,7 @@ public Map lookUpSegments(Iterable segme return lookUpSegments(segmentIds, null); } - public Map lookUpSegments(Iterable segmentIds, String queryID) { + public Map lookUpSegments(Iterable segmentIds, String queryId) { ArrayList notCached = new ArrayList<>(); //this implicitly deduplicates the stream @@ -100,7 +100,7 @@ public Map lookUpSegments(Iterable segme }); if (!notCached.isEmpty()) { - Stream descriptors = this.lookUpSegmentsByField(FIELDNAMES[0], notCached, queryID); + Stream descriptors = this.lookUpSegmentsByField(FIELDNAMES[0], notCached, queryId); descriptors.forEach(msd -> { _return.put(msd.getSegmentId(), msd); @@ -138,8 +138,8 @@ private Stream lookUpSegmentsByField(String fieldName, I return lookUpSegmentsByField(fieldName, fieldValues, null); } - private Stream lookUpSegmentsByField(String fieldName, Iterable fieldValues, String queryID) { - String dbQueryID = DBQueryIDGenerator.generateQueryID("seg-lookup", queryID); + private Stream lookUpSegmentsByField(String fieldName, Iterable fieldValues, String queryId) { + String dbQueryID = DBQueryIdGenerator.generateQueryId("seg-lookup", queryId); Set uniqueFieldValues = new HashSet<>(); fieldValues.forEach(el -> uniqueFieldValues.add(new StringTypeProvider(el))); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java index 672db4011..dac5a2e13 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java @@ -187,8 +187,8 @@ public void close() { } @Override - public List getNearestNeighboursGeneric(int k, float[] vector, String column, Class distanceElementClass, ReadableQueryConfig config) { - final Distance distance = config.getDistance().orElse(Distance.euclidean); + public List getNearestNeighboursGeneric(int k, float[] vector, String column, Class distanceElementClass, ReadableQueryConfig queryConfig) { + final Distance distance = queryConfig.getDistance().orElse(Distance.euclidean); try (final PreparedStatement statement = this.wrapper.connection.prepareStatement("SELECT id, distance(" + column + "," + toVectorString(vector) + ",'" + toName(distance) + "') as dist FROM " + this.fqn + " ORDER BY dist ASC LIMIT " + k)) { /* Execute query and return results. */ try (final ResultSet rs = statement.executeQuery()) { @@ -205,14 +205,14 @@ public List getNearestNeighboursGeneric(int k, fl } @Override - public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { + public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List queryConfigs) { LOGGER.warn("Error occurred during query execution in getBatchedNearestNeighbours(): Not supported"); return new ArrayList<>(0); } @Override - public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { - final Distance distance = config.getDistance().orElse(Distance.euclidean); + public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig queryConfig) { + final Distance distance = queryConfig.getDistance().orElse(Distance.euclidean); try (final PreparedStatement statement = this.wrapper.connection.prepareStatement("SELECT id, distance(" + column + "," + toVectorString(vector) + ",'" + toName(distance) + "') as dist FROM " + this.fqn + " ORDER BY dist ASC LIMIT " + k)) { /* Execute query and return results. */ @@ -226,8 +226,8 @@ public List> getNearestNeighbourRows(int k, f } @Override - public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { - try (final PreparedStatement statement = this.prepareStatement(fieldName, RelationalOperator.EQ, List.of(value))) { + public List getFeatureVectors(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { + try (final PreparedStatement statement = this.prepareStatement(column, RelationalOperator.EQ, List.of(value))) { /* Execute query and return results. */ final List _return = new LinkedList<>(); try (final ResultSet rs = statement.executeQuery()) { @@ -246,12 +246,12 @@ public List getFeatureVectors(String fieldName, PrimitiveTypeProvider v } @Override - public List> getRows(String fieldName, Iterable values) { + public List> getRows(String column, Iterable values, String dbQueryId) { final Object[] mapped = StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::toObject).toArray(); if (mapped.length == 0) { return new ArrayList<>(0); } - try (final PreparedStatement statement = this.prepareInStatement(fieldName, values)) { + try (final PreparedStatement statement = this.prepareInStatement(column, values)) { /* Execute query and return results. */ try (final ResultSet rs = statement.executeQuery()) { return processResults(rs); @@ -263,14 +263,14 @@ public List> getRows(String fieldName, Iterab } @Override - public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { + public List> getFulltextRows(int rows, String column, ReadableQueryConfig queryConfig, String... terms) { LOGGER.warn("Error occurred during query execution in getFulltextRows(): Not supported"); return new ArrayList<>(0); } @Override - public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { - try (final PreparedStatement statement = this.prepareStatement(fieldName, operator, values)) { + public List> getRows(String column, RelationalOperator operator, Iterable values, ReadableQueryConfig queryConfig) { + try (final PreparedStatement statement = this.prepareStatement(column, operator, values)) { try (final ResultSet rs = statement.executeQuery()) { return processResults(rs); } @@ -288,7 +288,7 @@ public List> getRows(String fieldName, Relati * @param projection Which columns shall be selected */ @Override - public List> getRowsAND(List>> conditions, String identifier, List projection, ReadableQueryConfig qc) { + public List> getRowsAND(List>> conditions, String identifier, List projection, ReadableQueryConfig queryConfig) { final StringBuilder conditionBuilder = new StringBuilder(); final LinkedList values = new LinkedList<>(); int i = 0; @@ -352,18 +352,6 @@ public List> getAll(List columns, int } } - @Override - public List getAll(String column) { - try (final Statement statement = this.wrapper.connection.createStatement()) { - try (final ResultSet rs = statement.executeQuery("SELECT " + column + " FROM " + this.fqn)) { - return processSingleColumnResult(rs); - } - } catch (SQLException e) { - LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); - return new ArrayList<>(0); - } - } - @Override public List> getAll() { try (final Statement statement = this.wrapper.connection.createStatement()) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ITextureModelDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ITextureModelDecoder.java new file mode 100644 index 000000000..bdbef88f0 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ITextureModelDecoder.java @@ -0,0 +1,8 @@ +package org.vitrivr.cineast.core.extraction.decode.m3d; + + +import org.vitrivr.cineast.core.extraction.decode.general.Decoder; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Model; + +public interface ITextureModelDecoder extends Decoder { +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularTextureModelDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularTextureModelDecoder.java new file mode 100644 index 000000000..4db32f769 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/ModularTextureModelDecoder.java @@ -0,0 +1,200 @@ +package org.vitrivr.cineast.core.extraction.decode.m3d; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; +import org.vitrivr.cineast.core.data.query.containers.AbstractQueryTermContainer; +import org.vitrivr.cineast.core.data.query.containers.TextureModelQueryTermContainer; +import org.vitrivr.cineast.core.extraction.decode.general.Converter; +import org.vitrivr.cineast.core.extraction.decode.general.Decoder; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Model; +import org.vitrivr.cineast.core.util.MimeTypeHelper; + +public class ModularTextureModelDecoder implements ITextureModelDecoder, Converter { + /** + * Default logging facility. + */ + private static final Logger LOGGER = LogManager.getLogger(); + + + + /** + * HashSet containing all the mime-types supported by this ImageDecoder instance. + * + * Important: The decoderForContenttype() method must return a Decoder instance + * for all mime-types contained in this set! + */ + private final static Set supportedFiles; + + static { + HashSet tmp = new HashSet<>(); + tmp.add("application/3dt-obj"); + tmp.add("application/3dt-gltf"); + supportedFiles = Collections.unmodifiableSet(tmp); + } + + /** + * HashMap containing cached decoder instances + */ + private HashMap> cachedDecoders = new HashMap<>(); + + /** + * Path to the input file. + */ + private Path inputFile; + + /** + * Flag indicating whether or not the Decoder is done decoding and the content has been obtained. + */ + private AtomicBoolean complete = new AtomicBoolean(false); + + /** + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} + * @return True if initialization was successful, false otherwise. + */ + @Override + public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { + this.inputFile = path; + this.complete.set(false); + return true; + } + + + /** + * Closes the Decoder. This method should cleanup and relinquish all resources. + *

+ * Note: It is unsafe to re-use a Decoder after it has been closed. + */ + @Override + public void close() { + + } + + /** + * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. + * + * @return Content of type T. + */ + @Override + public Model getNext() { + final String contenttype = MimeTypeHelper.getContentType(this.inputFile.toFile()); + + /* Try to detach decoder from the list of cached decoders. */ + Decoder decoder = this.cachedDecoders.get(contenttype); + + /* If decoder is null, create a new one. */ + if (decoder == null) { + decoder = decoderForContenttype(contenttype); + } + + /* If decoder is still null, return an emtpy Mesh. */ + if (decoder == null) { + LOGGER.warn("Could not find mesh decoder for provided contenttype {}.", contenttype); + return Model.EMPTY; + } else { + this.cachedDecoders.put(contenttype, decoder); + } + + /* Initialize the decoder and return the decoded mesh. */ + decoder.init(this.inputFile, null, null); + Model model = decoder.getNext(); + this.complete.set(true); + return model; + } + + @Override + public AbstractQueryTermContainer convert(Path path) { + final String contenttype = MimeTypeHelper.getContentType(path.toFile()); + + /* Try to detach decoder from the list of cached decoders. */ + Decoder decoder = this.cachedDecoders.get(contenttype); + + /* If decoder is null, create a new one. */ + if (decoder == null) { + decoder = decoderForContenttype(contenttype); + } + + /* If decoder is still null, return an emtpy Mesh. */ + if (decoder == null) { + LOGGER.warn("Could not find mesh decoder for provided contenttype {}.", contenttype); + return null; + } else { + this.cachedDecoders.put(contenttype, decoder); + } + + /* Initialize the decoder and return the decoded mesh. */ + decoder.init(path, null, null); + Model model = decoder.getNext(); + return new TextureModelQueryTermContainer(model); + } + + /** + * Returns the total number of content pieces T this decoder can return for a given file. + */ + @Override + public int count() { + return 0; + } + + /** + * Indicates whether or not the decoder has more content to return. + * + * @return True if more content can be retrieved, false otherwise. + */ + @Override + public boolean complete() { + return this.complete.get(); + } + + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + @Override + public Set supportedFiles() { + return supportedFiles; + } + + /** + * Indicates whether or not a particular instance of the Decoder interface can be re-used. If this method returns true, it is save to call init() with a new file after the previous file has been fully read. + *

+ * This property can be leveraged to reduce the memory-footprint of the application. + * + * @return True if re-use is possible, false otherwise. + */ + @Override + public boolean canBeReused() { + return true; + } + + /** + * Selects a Decoder implementation based on the provided content type. + * + * @param contenttype Mime-type for which to select a decoder. + * @return Decoder or null if the mime-type is not supported. + */ + private Decoder decoderForContenttype(String contenttype) { + switch (contenttype) { + case "application/3dt-gltf": + return new TextureModelDecoder(); + case "application/3dt-obj": + return new TextureModelDecoder(); + default: + return null; + } + } + + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/TextureModelDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/TextureModelDecoder.java new file mode 100644 index 000000000..6b220cc01 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/TextureModelDecoder.java @@ -0,0 +1,131 @@ +package org.vitrivr.cineast.core.extraction.decode.m3d; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.CacheConfig; +import org.vitrivr.cineast.core.config.DecoderConfig; +import org.vitrivr.cineast.core.extraction.decode.general.Decoder; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Model; +import org.vitrivr.cineast.core.data.m3d.texturemodel.ModelLoader; + +public class TextureModelDecoder implements Decoder { + + /** + * Default logging facility. + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * HashSet containing all the mime-types supported by this ImageDecoder instance. + */ + private static final Set supportedFiles; + + static { + HashSet tmp = new HashSet<>(); + tmp.add("application/3dt-gltf"); + tmp.add("application/3dt-obj"); + supportedFiles = Collections.unmodifiableSet(tmp); + } + + /** + * Path to the input file. + */ + private Path inputFile; + + /** + * Flag indicating whether or not the Decoder is done decoding and the content has been obtained. + */ + private AtomicBoolean complete = new AtomicBoolean(false); + + /** + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} + * @return True if initialization was successful, false otherwise. + */ + @Override + public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { + this.inputFile = path; + this.complete.set(false); + return true; + } + + /** + * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. + * + * @return Content of type T. + */ + @Override + public Model getNext() { + Model model = null; + + try { + model = ModelLoader.loadModel( this.inputFile.toString() ,this.inputFile.toString()); + } catch (NumberFormatException e) { + LOGGER.error("Could not decode OBJ file {} because one of the tokens could not be converted to a valid number.", this.inputFile.toString()); + model = null; + } catch (ArrayIndexOutOfBoundsException e) { + LOGGER.error("Could not decode OBJ file {} because one of the faces points to invalid vertex indices.", this.inputFile.toString()); + model = null; + } finally { + this.complete.set(true); + } + + return model; + } + + /** + * Returns the total number of content pieces T this decoder can return for a given file. + */ + @Override + public int count() { + return 1; + } + + /** + * Closes the Decoder. This method should cleanup and relinquish all resources. + *

+ * Note: It is unsafe to re-use a Decoder after it has been closed. + */ + @Override + public void close() { + } + + /** + * Indicates whether or not a particular instance of the Decoder interface can be re-used or not. This property can be leveraged to reduce the memory-footpring of the application. + * + * @return True if re-use is possible, false otherwise. + */ + @Override + public boolean canBeReused() { + return true; + } + + /** + * Indicates whether or not the current decoder instance is complete i.e. if there is content left that can be obtained. + * + * @return true if there is still content, false otherwise. + */ + @Override + public boolean complete() { + return this.complete.get(); + } + + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + @Override + public Set supportedFiles() { + return supportedFiles; + } +} + diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPImage.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPImage.java index c89b35512..36c679cb9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPImage.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPImage.java @@ -7,6 +7,7 @@ import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.tensorflow.Result; import org.tensorflow.SavedModelBundle; import org.tensorflow.Tensor; import org.tensorflow.ndarray.Shape; @@ -102,8 +103,8 @@ private float[] embedImage(BufferedImage img) { try (TFloat16 imageTensor = TFloat16.tensorOf(Shape.of(1, 3, IMAGE_SIZE, IMAGE_SIZE), DataBuffers.of(rgb))) { HashMap inputMap = new HashMap<>(); inputMap.put(EMBEDDING_INPUT, imageTensor); - Map resultMap = model.call(inputMap); - try (TFloat16 encoding = (TFloat16) resultMap.get(EMBEDDING_OUTPUT)) { + Result resultMap = model.call(inputMap); + try (TFloat16 encoding = (TFloat16) resultMap.get(EMBEDDING_OUTPUT).get()) { var embeddingArray = new float[EMBEDDING_SIZE]; var floatBuffer = DataBuffers.of(embeddingArray); encoding.read(floatBuffer); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPText.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPText.java index fe596fbc3..cda37297b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPText.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPText.java @@ -117,7 +117,7 @@ public float[] embedText(String text) { private static float[] exec(HashMap inputMap) { var resultMap = bundle.call(inputMap); - try (TFloat16 embedding = (TFloat16) resultMap.get(EMBEDDING_OUTPUT)) { + try (TFloat16 embedding = (TFloat16) resultMap.get(EMBEDDING_OUTPUT).get()) { var embeddingArray = new float[EMBEDDING_SIZE]; var floatBuffer = DataBuffers.of(embeddingArray); embedding.read(floatBuffer); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/InceptionResnetV2.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/InceptionResnetV2.java index 332252c0e..31d6af880 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/InceptionResnetV2.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/InceptionResnetV2.java @@ -78,7 +78,7 @@ public static float[] encodeImage(BufferedImage image) { inputMap.put(INPUT, imageTensor); var resultMap = model.call(inputMap); - try (TFloat32 encoding = (TFloat32) resultMap.get(OUTPUT)) { + try (TFloat32 encoding = (TFloat32) resultMap.get(OUTPUT).get()) { var embeddingArray = new float[ENCODING_SIZE]; var floatBuffer = DataBuffers.of(embeddingArray); encoding.read(floatBuffer); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java index 679728559..d415551d9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java @@ -1,11 +1,20 @@ package org.vitrivr.cineast.core.features; +import com.jogamp.opengl.awt.GLCanvas; import com.twelvemonkeys.image.ImageUtil; import java.awt.Image; import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; +import java.util.Stack; +import java.util.concurrent.LinkedBlockingDeque; +import javax.imageio.ImageIO; +import org.joml.Vector3f; +import org.joml.Vector4f; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.CorrespondenceFunction; @@ -13,11 +22,24 @@ import org.vitrivr.cineast.core.data.distance.DistanceElement; import org.vitrivr.cineast.core.data.distance.SegmentDistanceElement; import org.vitrivr.cineast.core.data.m3d.ReadableMesh; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.StagedFeatureModule; import org.vitrivr.cineast.core.render.JOGLOffscreenRenderer; +import org.vitrivr.cineast.core.render.MeshOnlyRenderer; import org.vitrivr.cineast.core.render.Renderer; +import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; +import org.vitrivr.cineast.core.render.lwjgl.renderer.LWJGLOffscreenRenderer; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderActions; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderData; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderJob; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderWorker; +import org.vitrivr.cineast.core.render.lwjgl.util.datatype.Variant; +import org.vitrivr.cineast.core.render.lwjgl.util.fsm.abstractworker.JobControlCommand; +import org.vitrivr.cineast.core.render.lwjgl.util.fsm.abstractworker.JobType; +import org.vitrivr.cineast.core.render.lwjgl.util.fsm.model.Action; +import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; import org.vitrivr.cineast.core.util.LogHelper; /** @@ -44,11 +66,10 @@ public abstract class Lightfield extends StagedFeatureModule { */ private final double[][] camerapositions; + /** * Offscreen rendering environment used to create Lightfield images. */ - private final Renderer renderer; - protected Lightfield(String tableName, float maxDist, int vectorLength, double[][] camerapositions) { super(tableName, maxDist, vectorLength); if (camerapositions.length == 0) { @@ -60,19 +81,6 @@ protected Lightfield(String tableName, float maxDist, int vectorLength, double[] } } this.camerapositions = camerapositions; - - /* - * Instantiate JOGLOffscreenRenderer. - * Handle the case where it cannot be created due to missing OpenGL support. - */ - JOGLOffscreenRenderer renderer = null; - try { - renderer = new JOGLOffscreenRenderer(RENDERING_SIZE, RENDERING_SIZE); - } catch (Exception exception) { - LOGGER.error("Could not instantiate JOGLOffscreenRenderer! This instance of {} will not create any results or features!", this.getClass().getSimpleName()); - } finally { - this.renderer = renderer; - } } @@ -87,22 +95,14 @@ protected Lightfield(String tableName, float maxDist, int vectorLength, double[] */ @Override protected List preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) { - /* Check if renderer could be initialised. */ - if (this.renderer == null) { - LOGGER.error("No renderer found. {} does not return any results.", this.getClass().getSimpleName()); - return new ArrayList<>(0); - } - - /* Extract features from either the provided Mesh (1) or image (2). */ - ReadableMesh mesh = sc.getNormalizedMesh(); + IModel model = sc.getModel(); List features; - if (mesh.isEmpty()) { + if (model == null) { BufferedImage image = ImageUtil.createResampled(sc.getAvgImg().getBufferedImage(), RENDERING_SIZE, RENDERING_SIZE, Image.SCALE_SMOOTH); features = this.featureVectorsFromImage(image, POSEIDX_UNKNOWN); } else { - features = this.featureVectorsFromMesh(mesh); + features = this.featureVectorsFromMesh(model); } - return features; } @@ -150,20 +150,15 @@ protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { */ @Override public void processSegment(SegmentContainer sc) { - /* Check for renderer. */ - if (this.renderer == null) { - LOGGER.error("No renderer found! {} does not create any features.", this.getClass().getSimpleName()); - return; - } /* If Mesh is empty, no feature is persisted. */ - ReadableMesh mesh = sc.getNormalizedMesh(); - if (mesh == null || mesh.isEmpty()) { + IModel model = sc.getModel(); + if (model == null) { return; } /* Extract and persist all features. */ - List features = this.featureVectorsFromMesh(mesh); + List features = this.featureVectorsFromMesh(model); for (float[] feature : features) { this.persist(sc.getId(), new FloatVectorImpl(feature)); } @@ -172,50 +167,35 @@ public void processSegment(SegmentContainer sc) { /** * Extracts the Lightfield Fourier descriptors from a provided Mesh. The returned list contains elements of which each holds a pose-index (relative to the camera-positions used by the feature module) and the associated feature-vector (s). * - * @param mesh Mesh for which to extract the Lightfield Fourier descriptors. + * @param model Model for which to extract the Lightfield Fourier descriptors. * @return List of descriptors for mesh. */ - protected List featureVectorsFromMesh(ReadableMesh mesh) { + protected List featureVectorsFromMesh(IModel model) { /* Prepare empty list of features. */ List features = new ArrayList<>(20); - - /* Retains the renderer and returns if retention fails. */ - if (!this.renderer.retain()) { - return features; + var windowOptions = new WindowOptions(RENDERING_SIZE, RENDERING_SIZE) {{ + this.hideWindow = true; + }}; + var renderOptions = new RenderOptions() {{ + this.showTextures = false; + }}; + var camerapositions = new LinkedList(); + for (double[] cameraposition : this.camerapositions) { + camerapositions.add(new Vector3f((float) + cameraposition[0], (float) cameraposition[1], (float) cameraposition[2]).normalize().mul(0.95f)); } + var images = RenderJob.performStandardRenderJob(RenderWorker.getRenderJobQueue(), + model, camerapositions, windowOptions, renderOptions); - /* Everything happens in the try-catch block so as to make sure, that if any exception occurs, - * the renderer is released again. - */ - try { - /* Clears the renderer and assembles a new Mesh. */ - this.renderer.clear(); - this.renderer.assemble(mesh); - - /* Obtains rendered image from configured perspective. */ - for (int i = 0; i < this.camerapositions.length; i++) { - /* Adjust the camera and render the image. */ - this.renderer.positionCamera((float) this.camerapositions[i][0], (float) this.camerapositions[i][1], (float) this.camerapositions[i][2]); - this.renderer.render(); - BufferedImage image = this.renderer.obtain(); - if (image == null) { - LOGGER.error("Could not generate feature for {} because no image could be obtained from JOGOffscreenRenderer.", this.getClass().getSimpleName()); - return features; - } - features.addAll(this.featureVectorsFromImage(image, i)); - } - - } catch (Exception exception) { - LOGGER.error("Could not generate feature for {} because an unknown exception occurred ({}).", this.getClass().getSimpleName(), LogHelper.getStackTrace(exception)); - } finally { - /* Release the rendering context. */ - this.renderer.release(); + var ic = 0; + for ( var image :images){ + features.addAll(this.featureVectorsFromImage(image, ic)); + ic++; } - - /* Extract and persist the feature descriptors. */ return features; } + protected abstract List featureVectorsFromImage(BufferedImage image, int poseidx); public double[] positionsForPoseidx(int poseidx) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java index 94ebb5f8d..a0bc04ab3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/RangeBooleanRetriever.java @@ -25,9 +25,6 @@ public class RangeBooleanRetriever extends BooleanRetriever { RelationalOperator.LESS, RelationalOperator.IN)); - private final HashMap minimumMap = new HashMap<>(); - private final HashMap maximumMap = new HashMap<>(); - protected RangeBooleanRetriever(String entity, Collection attributes) { super(entity, attributes); } @@ -41,42 +38,4 @@ protected Collection getSupportedOperators() { return SUPPORTED_OPERATORS; } - public PrimitiveTypeProvider getMinimum(String column) { - if (this.attributes.contains(column) && !this.minimumMap.containsKey(column)) { - populateExtremaMap(); - } - return minimumMap.get(column); - } - - public PrimitiveTypeProvider getMaximum(String column) { - if (this.attributes.contains(column) && !this.maximumMap.containsKey(column)) { - populateExtremaMap(); - } - return maximumMap.get(column); - } - - private void populateExtremaMap() { - - PrimitiveProviderComparator comparator = new PrimitiveProviderComparator(); - - for (String column : this.attributes) { - List col = this.selector.getAll(column); - if (col.isEmpty()) { - continue; - } - PrimitiveTypeProvider min = col.get(0), max = col.get(0); - for (PrimitiveTypeProvider t : col) { - if (comparator.compare(t, min) < 0) { - min = t; - } - if (comparator.compare(t, max) > 0) { - max = t; - } - } - this.minimumMap.put(column, min); - this.maximumMap.put(column, max); - } - } - - } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java index 6338c6657..e11f05271 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java @@ -1,13 +1,16 @@ package org.vitrivr.cineast.core.features; +import java.awt.Color; import java.awt.image.BufferedImage; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.tensorflow.Result; import org.tensorflow.SavedModelBundle; import org.tensorflow.Tensor; import org.tensorflow.ndarray.NdArrays; @@ -19,14 +22,27 @@ import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; import org.vitrivr.cineast.core.data.FloatVectorImpl; +import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; +import org.vitrivr.cineast.core.data.raw.CachedDataFactory; import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; +import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderJob; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderWorker; +import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; +import org.vitrivr.cineast.core.util.KMeansPP; +import org.vitrivr.cineast.core.util.math.MathHelper; +import org.vitrivr.cineast.core.util.texturemodel.Viewpoint.ViewpointHelper; +import org.vitrivr.cineast.core.util.texturemodel.Viewpoint.ViewpointStrategy; +; /** * A visual-text co-embedding mapping images and text descriptions to the same embedding space. + *

*/ public class VisualTextCoEmbedding extends AbstractFeatureModule { @@ -106,7 +122,16 @@ public void processSegment(SegmentContainer sc) { this.persist(sc.getId(), new FloatVectorImpl(embeddingArray)); } - // Insert return here if additional cases are added! + return; + } + + // Case: segment contains model + var model = sc.getModel(); + if (model != null) { + float[] embeddingArray = embedModel(model); + this.persist(sc.getId(), new FloatVectorImpl(embeddingArray)); + System.gc(); + return; } } @@ -138,6 +163,14 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc LOGGER.error("Image was provided, but could not be decoded!"); } + // Case: segment contains model + var model = sc.getModel(); + if (model != null) { + LOGGER.debug("Retrieving with MODEL."); + float[] embeddingArray = embedModel(model); + return getSimilar(embeddingArray, queryConfig); + } + LOGGER.error("Could not get similar because no acceptable modality was provided."); return new ArrayList<>(); } @@ -169,6 +202,7 @@ private synchronized static void initializeVisualEmbedding() { } } + private float[] embedText(String text) { initializeTextEmbedding(); @@ -176,13 +210,13 @@ private float[] embedText(String text) { HashMap inputMap = new HashMap<>(); inputMap.put(TEXT_EMBEDDING_INPUT, textTensor); - Map resultMap = textEmbedding.call(inputMap); - try (TFloat32 intermediaryEmbedding = (TFloat32) resultMap.get(TEXT_EMBEDDING_OUTPUT)) { + var resultMap = textEmbedding.call(inputMap); + try (TFloat32 intermediaryEmbedding = (TFloat32) resultMap.get(TEXT_EMBEDDING_OUTPUT).get()) { inputMap.clear(); inputMap.put(TEXT_CO_EMBEDDING_INPUT, intermediaryEmbedding); resultMap = textCoEmbedding.call(inputMap); - try (TFloat32 embedding = (TFloat32) resultMap.get(TEXT_CO_EMBEDDING_OUTPUT)) { + try (TFloat32 embedding = (TFloat32) resultMap.get(TEXT_CO_EMBEDDING_OUTPUT).get()) { var embeddingArray = new float[EMBEDDING_SIZE]; var floatBuffer = DataBuffers.of(embeddingArray); // Beware TensorFlow allows tensor writing to buffers through the function read rather than write @@ -194,6 +228,7 @@ private float[] embedText(String text) { } } + private float[] embedImage(BufferedImage image) { initializeVisualEmbedding(); @@ -203,12 +238,12 @@ private float[] embedImage(BufferedImage image) { HashMap inputMap = new HashMap<>(); inputMap.put(InceptionResnetV2.INPUT, imageTensor); - Map resultMap = visualEmbedding.call(inputMap); - try (TFloat32 intermediaryEmbedding = (TFloat32) resultMap.get(InceptionResnetV2.OUTPUT)) { + Result resultMap = visualEmbedding.call(inputMap); + try (TFloat32 intermediaryEmbedding = (TFloat32) resultMap.get(InceptionResnetV2.OUTPUT).get()) { inputMap.clear(); inputMap.put(VISUAL_CO_EMBEDDING_INPUT, intermediaryEmbedding); resultMap = visualCoEmbedding.call(inputMap); - try (TFloat32 embedding = (TFloat32) resultMap.get(VISUAL_CO_EMBEDDING_OUTPUT)) { + try (TFloat32 embedding = (TFloat32) resultMap.get(VISUAL_CO_EMBEDDING_OUTPUT).get()) { var embeddingArray = new float[EMBEDDING_SIZE]; var floatBuffer = DataBuffers.of(embeddingArray); // Beware TensorFlow allows tensor writing to buffers through the function read rather than write @@ -220,6 +255,144 @@ private float[] embedImage(BufferedImage image) { } } + /** + * Helper to convert images to frames + * + * @param images are converted to frames + * @return List for video embedding + */ + private List framesFromImages(List images) { + var frames = new ArrayList(); + for (BufferedImage image : images) { + var factory = CachedDataFactory.getDefault(); + frames.add(factory.newInMemoryMultiImage(image)); + } + return frames; + } + + /** + * This method takes a list of images and determines, based on {@link ViewpointStrategy}, the embed vector which describes the images most precise. This method can be simplified, once the best strategy is determined. + * + * @param images the list of Images to embed + * @param viewpointStrategy the strategy to find the vector + * @return the embedding vector + */ + private float[] embedMostRepresentativeImages(List images, ViewpointStrategy viewpointStrategy) { + + var retVal = new float[EMBEDDING_SIZE]; + + switch (viewpointStrategy) { + // Strategy for embedding multiple images. Choose mean of the most contained cluster. Project mean to unit hypersphere. + case MULTI_IMAGE_KMEANS -> { + var floatvectors = new ArrayList(); + var vectors = embedMultipleImages(images); + vectors.forEach(v -> floatvectors.add(new FloatVectorImpl(v))); + var kmeans = KMeansPP.bestOfkMeansPP(floatvectors, new FloatVectorImpl(new float[EMBEDDING_SIZE]), 3, -1f, 5); + // Find the index of thr cluster with the most elements + int maxIndex = 0; + for (var ic = 0; ic < kmeans.getPoints().size(); ++ic) { + if (kmeans.getPoints().get(ic).size() > kmeans.getPoints().get(maxIndex).size()) { + maxIndex = ic; + } + if (kmeans.getPoints().get(ic).size() == kmeans.getPoints().get(maxIndex).size()) { + if (kmeans.getDistance(ic) < kmeans.getDistance(maxIndex)) { + maxIndex = ic; + } + } + } + ReadableFloatVector.toArray(kmeans.getCenters().get(maxIndex), retVal); + return MathHelper.normalizeL2InPlace(retVal); + } + // Strategy for embedding multiple images. Calculate mean over all. Project mean to unit hypersphere. + case MULTI_IMAGE_PROJECTEDMEAN -> { + var vectors = embedMultipleImages(images); + var vectorsMean = new float[EMBEDDING_SIZE]; + for (var vector : vectors) { + for (var ic = 0; ic < vector.length; ++ic) { + vectorsMean[ic] += vector[ic] / vectors.size(); + } + } + return MathHelper.normalizeL2InPlace(vectorsMean); + } + // Strategy for embedding multiple images. Create a video from the images and embed the video. + case MULTI_IMAGE_FRAME -> { + var frames = framesFromImages(images); + return embedVideo(frames); + } + // Strategy for embedding an image consisting out of four sub images. + case MULTI_IMAGE_2_2 -> { + assert images.size() == 4; + var sz = images.get(0).getWidth(); + var size = sz * 2; + // Combine the images into a single image. + var canvas = new BufferedImage( + size, + size, + BufferedImage.TYPE_INT_RGB); + var graphics = canvas.getGraphics(); + graphics.setColor(Color.BLACK); + // ic: image counter, idx: x-axis-index, idy: yaxis-index + var ic = 0; + for (var partialImage : images) { + int idx = ic % 2; + int idy = ic < 2 ? 0 : 1; + graphics.drawImage(partialImage, idx * sz, idy * sz, null); + ++ic; + } + retVal = embedImage(canvas); + } + } + return retVal; + } + + /** + * Embeds a list of images + * + * @param images the list of images to embed + * @return the list of embedding vectors + */ + private List embedMultipleImages(List images) { + var vectors = new ArrayList(); + for (BufferedImage image : images) { + float[] embeddingArray = embedImage(image); + vectors.add(embeddingArray); + } + return vectors; + } + + /** + * This method embeds a 3D model and returns the feature vector. + */ + private float[] embedModel(IModel model) { + //Options for window + var windowOptions = new WindowOptions() {{ + this.hideWindow = true; + this.width = 600; + this.height = 600; + }}; + // Options for renderer + var renderOptions = new RenderOptions() {{ + this.showTextures = true; + }}; + // Select the strategy which will be used for model embedding + var viewpointStrategy = ViewpointStrategy.MULTI_IMAGE_KMEANS; + // Get camera viewpoint for chosen strategy + var cameraPositions = ViewpointHelper.getCameraPositions(viewpointStrategy, model); + // Render an image for each camera position + var images = RenderJob.performStandardRenderJob(RenderWorker.getRenderJobQueue(), + model, cameraPositions, windowOptions, renderOptions); + + // Embedding based on strategy return value. Empty if an error occurred + if (images.isEmpty()) { + return new float[EMBEDDING_SIZE]; + } + if (images.size() == 1) { + return embedImage(images.get(0)); + } + return embedMostRepresentativeImages(images, viewpointStrategy); + } + + private float[] embedVideo(List frames) { initializeVisualEmbedding(); @@ -229,8 +402,8 @@ private float[] embedVideo(List frames) { HashMap inputMap = new HashMap<>(); inputMap.put(VISUAL_CO_EMBEDDING_INPUT, encoding); - Map resultMap = visualCoEmbedding.call(inputMap); - try (TFloat32 embedding = (TFloat32) resultMap.get(VISUAL_CO_EMBEDDING_OUTPUT)) { + Result resultMap = visualCoEmbedding.call(inputMap); + try (TFloat32 embedding = (TFloat32) resultMap.get(VISUAL_CO_EMBEDDING_OUTPUT).get()) { var embeddingArray = new float[EMBEDDING_SIZE]; var floatBuffer = DataBuffers.of(embeddingArray); // Beware TensorFlow allows tensor writing to buffers through the function read rather than write diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java index 4031fad2e..67d2cbfad 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java @@ -1,25 +1,32 @@ package org.vitrivr.cineast.core.features.exporter; import java.awt.Color; -import java.awt.Graphics; import java.awt.image.BufferedImage; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.LinkedList; import java.util.Map; import java.util.function.Supplier; import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.data.m3d.WritableMesh; +import org.joml.Vector3f; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.features.extractor.Extractor; -import org.vitrivr.cineast.core.render.JOGLOffscreenRenderer; +import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderJob; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderWorker; +import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; import org.vitrivr.cineast.core.util.LogHelper; -import org.vitrivr.cineast.core.util.mesh.MeshColoringUtil; +import org.vitrivr.cineast.core.util.texturemodel.EntropyOptimizer.EntopyCalculationMethod; +import org.vitrivr.cineast.core.util.texturemodel.EntropyOptimizer.EntropyOptimizerStrategy; +import org.vitrivr.cineast.core.util.texturemodel.EntropyOptimizer.ModelEntropyOptimizer; +import org.vitrivr.cineast.core.util.texturemodel.EntropyOptimizer.OptimizerOptions; public class Model3DThumbnailExporter implements Extractor { @@ -32,20 +39,10 @@ public class Model3DThumbnailExporter implements Extractor { private static final String PROPERTY_NAME_DESTINATION = "destination"; private static final String PROPERTY_NAME_SIZE = "size"; - /** - * List of perspective that should be rendered. Azimuth and polar angles in degree. - */ - private static final float[][] PERSPECTIVES = { - {0.0f, 90.0f}, - {45.0f, 135.0f}, - {-135.0f, -225.0f}, - {0.0f, -90.0f} - }; - /** * Distance of camera from object. */ - private static final float DISTANCE = 2.0f; + private static final float DISTANCE = 1f; //(float) Math.sqrt(3); /** * Destination path; can be set in the AudioWaveformExporter properties. @@ -57,15 +54,11 @@ public class Model3DThumbnailExporter implements Extractor { */ private final int size; - /** - * Offscreen rendering context. - */ - private final JOGLOffscreenRenderer renderer; /** * Background color of the resulting image. */ - private Color backgroundColor = Color.lightGray; + private final Color backgroundColor = Color.lightGray; /** * Default constructor. The AudioWaveformExporter can be configured via named properties in the provided HashMap. Supported parameters: @@ -80,7 +73,6 @@ public class Model3DThumbnailExporter implements Extractor { public Model3DThumbnailExporter(Map properties) { this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); this.size = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_SIZE, "800")); - this.renderer = new JOGLOffscreenRenderer(this.size / 2, this.size / 2); } /** @@ -93,42 +85,62 @@ public void processSegment(SegmentContainer shot) { Path directory = this.destination.resolve(shot.getSuperId()); try { Files.createDirectories(directory); - WritableMesh mesh = shot.copyNormalizedMesh(); - - if (!mesh.isEmpty()) { - /* Colors the mesh. */ - MeshColoringUtil.normalColoring(mesh); - - BufferedImage buffer = null; - BufferedImage image = new BufferedImage(this.size, this.size, BufferedImage.TYPE_INT_RGB); - Graphics graphics = image.getGraphics(); - - if (this.renderer.retain()) { - this.renderer.clear(this.backgroundColor); - this.renderer.assemble(mesh); - - for (int i = 0; i < 4; i++) { - this.renderer.positionCameraPolar(DISTANCE, PERSPECTIVES[i][0], PERSPECTIVES[i][1], 0.0, 0.0, 0.0); - this.renderer.render(); - buffer = this.renderer.obtain(); - - int idx = i % 2; - int idy = i < 2 ? 0 : 1; - int sz = this.size / 2; - - graphics.drawImage(buffer, idx * sz, idy * sz, null); - } - } else { - LOGGER.error("Could not export thumbnail image for model {} because renderer could not be retained by current thread.", shot.getId()); + // Get the model to generate a thumbnail for. + IModel model = shot.getModel(); + if (model.getMaterials().size() > 0) { + // Set options for the renderer. + var windowOptions = new WindowOptions(400, 400) {{ + this.hideWindow = true; + }}; + var renderOptions = new RenderOptions() {{ + this.showTextures = true; + }}; + // Set options for the entropy optimizer. + var opts = new OptimizerOptions() {{ + this.iterations = 100; + this.initialViewVector = new Vector3f(0, 0, 1); + this.method = EntopyCalculationMethod.RELATIVE_TO_TOTAL_AREA_WEIGHTED; + this.optimizer = EntropyOptimizerStrategy.RANDOMIZED; + this.yNegWeight = 0.7f; + this.yPosWeight = 0.8f; + }}; + + // Add a Random View, a front View an Upper Left View and an Entropy Optimized View + var cameraPositions = new LinkedList() {{ + add(new Vector3f( + (float) (Math.random() - 0.5) * 2f, + (float) (Math.random() - 0.5) * 2f, + (float) (Math.random() - 0.5) * 2f) + .normalize().mul(DISTANCE)); + add(new Vector3f(0f, 0f, 1f).normalize().mul(DISTANCE)); + add(new Vector3f(-1f, 1f, 1f).normalize().mul(DISTANCE)); + add(ModelEntropyOptimizer.getViewVectorWithMaximizedEntropy(model, opts)); + }}; + + // Render the model. + var images = RenderJob.performStandardRenderJob(RenderWorker.getRenderJobQueue(), model, cameraPositions, windowOptions, renderOptions); + assert images.size() == 4; + // Combine the images into a single image. + var canvas = new BufferedImage(this.size, this.size, BufferedImage.TYPE_INT_RGB); + var graphics = canvas.getGraphics(); + graphics.setColor(this.backgroundColor); + int sz = this.size / 2; + // ic: sub-image counter, idx: x-axis-index, idy: y-axis-index + var ic = 0; + for (var partialImage : images) { + int idx = ic % 2; + int idy = ic < 2 ? 0 : 1; + graphics.drawImage(partialImage, idx * sz, idy * sz, null); + ++ic; } - ImageIO.write(image, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); + ImageIO.write(canvas, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); } } catch (IOException exception) { LOGGER.fatal("Could not export thumbnail image for model {} due to a serious IO error ({}).", shot.getId(), LogHelper.getStackTrace(exception)); } catch (Exception exception) { LOGGER.error("Could not export thumbnail image for model {} because an unknown exception occurred ({}).", shot.getId(), LogHelper.getStackTrace(exception)); } finally { - this.renderer.release(); + LOGGER.trace("Finished processing thumbnail {}.", shot.getId()); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/Inception5h.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/Inception5h.java index c1853edd6..829fc8912 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/Inception5h.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/Inception5h.java @@ -10,6 +10,7 @@ import java.util.List; import org.tensorflow.Graph; import org.tensorflow.Output; +import org.tensorflow.Result; import org.tensorflow.Session; import org.tensorflow.Session.Runner; import org.tensorflow.Tensor; @@ -287,7 +288,7 @@ public HashMap transform(MultiImage img) { runner.fetch(operation); } - List results = runner.run(); + Result results = runner.run(); HashMap _return = new HashMap<>(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLab.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLab.java index b989f8b97..7eebfa1ec 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLab.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/deeplab/DeepLab.java @@ -88,8 +88,8 @@ public synchronized int[][] processImage(TUint8 input) { TInt64 result = (TInt64) session.runner().feed("ImageTensor", input).fetch("SemanticPredictions").run().get(0); - int w = (int) result.shape().size(2); - int h = (int) result.shape().size(1); + int w = (int) result.shape().get(2); + int h = (int) result.shape().get(1); int[][] resultMatrix = new int[w][h]; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/YOLO.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/YOLO.java index fd87f0463..3c61f3e4f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/YOLO.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/neuralnet/tf/models/yolo/YOLO.java @@ -120,7 +120,7 @@ private static TFloat32 readImage(MultiImage img) { * @return the number of classes */ private int getOutputSizeByShape(TFloat32 result) { - return (int) (result.shape().size(3) * Math.pow(SIZE, 2)); + return (int) (result.shape().get(3) * Math.pow(SIZE, 2)); } /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/JOGLOffscreenRenderer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/JOGLOffscreenRenderer.java index 15a360b57..e82ced5d1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/JOGLOffscreenRenderer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/JOGLOffscreenRenderer.java @@ -9,6 +9,7 @@ import com.jogamp.opengl.GLDrawableFactory; import com.jogamp.opengl.GLOffscreenAutoDrawable; import com.jogamp.opengl.GLProfile; +import com.jogamp.opengl.awt.GLCanvas; import com.jogamp.opengl.fixedfunc.GLMatrixFunc; import com.jogamp.opengl.glu.GLU; import com.jogamp.opengl.util.awt.AWTGLReadBufferUtil; @@ -17,29 +18,34 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.locks.ReentrantLock; +import javax.swing.ImageIcon; +import javax.swing.JFrame; +import javax.swing.JLabel; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.joml.Vector3f; import org.vitrivr.cineast.core.data.m3d.Mesh; import org.vitrivr.cineast.core.data.m3d.ReadableMesh; import org.vitrivr.cineast.core.data.m3d.VoxelGrid; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; /** * This class can be used to render 3D models (Meshes or Voxel-models) using the JOGL rendering environment. It currently has the following features: *

* - Rendering of single Mesh or VoxelGrid - Free positioning of the camera in terms of either cartesian or polar coordinate - Snapshot of the rendered image can be obtained at any time. *

- * The class supports offscreen rendering and can be accessed by multipled Threads. However, the multithreading model of JOGL requires a thread to retain() and release() the JOGLOffscreenRenderer before rendering anything by calling the respective function. + * The class supports offscreen rendering and can be accessed by multipled Threads. + * However, the multithreading model of JOGL requires a thread to retain() and release() the JOGLOffscreenRenderer before rendering anything by calling the respective function. * * @see Mesh * @see VoxelGrid */ -public class JOGLOffscreenRenderer implements Renderer { +public class JOGLOffscreenRenderer implements MeshOnlyRenderer { private static final Logger LOGGER = LogManager.getLogger(); /** - * Default GLProfile to be used. Should be GL2. + * Default GLProfile to be used. Should be GL2. "The desktop OpenGL profile 1.x up to 3.0". */ private static final GLProfile GL_PROFILE = GLProfile.get(GLProfile.GL2); @@ -52,12 +58,13 @@ public class JOGLOffscreenRenderer implements Renderer { * This code-block can be used to configure the off-screen renderer's GL_CAPABILITIES. */ static { - GL_CAPABILITIES.setOnscreen(false); + GL_CAPABILITIES.setOnscreen(true); + //GL_CAPABILITIES.setOnscreen(false); GL_CAPABILITIES.setHardwareAccelerated(true); } /** - * OpenGL Utility Library reference + * OpenGL Utility Library reference. */ private final GLU glu; /** @@ -108,16 +115,21 @@ public JOGLOffscreenRenderer(int width, int height) { /* Initialize GLOffscreenAutoDrawable. */ GLDrawableFactory factory = GLDrawableFactory.getFactory(GL_PROFILE); this.drawable = factory.createOffscreenAutoDrawable(null, GL_CAPABILITIES, null, width, height); + this.drawable.display(); + /* Initialize GLU and GL2. */ this.glu = new GLU(); this.gl = drawable.getGL().getGL2(); /* Set default color. */ gl.glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + + } + /** * Getter for width. * @@ -186,6 +198,12 @@ public void render() { } } + @Override + public void assemble(IModel model) { + this.assemble(Mesh.EMPTY); + } + + /** * Renders a new Mesh object and thereby removes any previously rendered object * @@ -229,6 +247,7 @@ public void assemble(ReadableMesh mesh) { */ @Override public void assemble(VoxelGrid grid) { + int meshList = gl.glGenLists(1); this.objects.add(meshList); gl.glNewList(meshList, GL2.GL_COMPILE); @@ -400,7 +419,9 @@ public final BufferedImage obtain() { return null; } AWTGLReadBufferUtil glReadBufferUtil = new AWTGLReadBufferUtil(gl.getGL2().getGLProfile(), false); - return glReadBufferUtil.readPixelsToBufferedImage(gl.getGL2(), true); + var image = glReadBufferUtil.readPixelsToBufferedImage(gl.getGL2(), true); + //this.showImage(image); + return image; } /** @@ -450,4 +471,27 @@ private boolean checkContext() { return true; } } + + + JFrame jFrame; + + /** + * Shows Image in a JFrame for Debug purpose + * + * @param im BufferedImage to show + */ + private void showImage(BufferedImage im) { + if (this.jFrame == null) { + this.jFrame = new JFrame(); + this.jFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + } + this.jFrame.setVisible(false); + this.jFrame.getContentPane().removeAll(); + this.jFrame.getContentPane().add(new JLabel(new ImageIcon(im))); + this.jFrame.getContentPane().setBackground(Color.black); + ; + this.jFrame.pack(); + this.jFrame.toFront(); + this.jFrame.setVisible(true); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/MeshOnlyRenderer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/MeshOnlyRenderer.java new file mode 100644 index 000000000..34b759718 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/MeshOnlyRenderer.java @@ -0,0 +1,21 @@ +package org.vitrivr.cineast.core.render; + +import org.vitrivr.cineast.core.data.m3d.ReadableMesh; +import org.vitrivr.cineast.core.data.m3d.VoxelGrid; + +public interface MeshOnlyRenderer extends Renderer { + /** + * Assembles a new Mesh object and thereby adds it to the list of objects that should be rendered. + * + * @param mesh Mesh that should be rendered + */ + void assemble(ReadableMesh mesh); + + /** + * Assembles a new VoxelGrid object and thereby adds it to the list of objects that should be rendered. + * + * @param grid VoxelGrid that should be rendered. + */ + void assemble(VoxelGrid grid); + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java index 495f3afd3..d448a391f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java @@ -2,14 +2,15 @@ import java.awt.Color; import java.awt.image.BufferedImage; -import org.vitrivr.cineast.core.data.m3d.ReadableMesh; -import org.vitrivr.cineast.core.data.m3d.VoxelGrid; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; /** * This interface defines methods that a renderer for 3D models (e.g. Meshes or Voxels must implement). It currently provides the following features: *

* - Rendering of single Mesh or VoxelGrid - Free positioning of the camera in terms of either cartesian or polar coordinate - Snapshot of the rendered image can be obtained at any time. + * @deprecated This interface is deprecated it can be simplified if the old JOGL renderer is removed. */ +@Deprecated public interface Renderer { /** @@ -22,14 +23,7 @@ public interface Renderer { * * @param mesh Mesh that should be rendered */ - void assemble(ReadableMesh mesh); - - /** - * Assembles a new VoxelGrid object and thereby adds it to the list of objects that should be rendered. - * - * @param grid VoxelGrid that should be rendered. - */ - void assemble(VoxelGrid grid); + void assemble(IModel mesh); /** * Changes the positionCamera of the camera. diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/Engine.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/Engine.java new file mode 100644 index 000000000..2899bf57f --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/Engine.java @@ -0,0 +1,209 @@ +package org.vitrivr.cineast.core.render.lwjgl.engine; + +import org.vitrivr.cineast.core.render.lwjgl.render.Render; +import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; +import org.vitrivr.cineast.core.render.lwjgl.scene.Camera; +import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLScene; +import org.vitrivr.cineast.core.render.lwjgl.scene.Scene; +import org.vitrivr.cineast.core.render.lwjgl.window.Window; +import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; + +/** + * The engine is the main class of the rendering engine. + * It holds the window, the scene and the render object. + * It provides a render loop for continuous rendering and a runOnce method to render a single frame rendering. + */ +public class Engine { + /** + * The window object. + */ + private final Window window; + /** + * Indicates whether the engine is running in continuous rendering mode. + */ + private boolean running; + /** + * The render object. + */ + private final Render render; + /** + * The scene object. + */ + private final GLScene scene; + /** + * The application logic. Connects the engine to the overlaying application. + * The Engine calls the methods of the appLogic object depending on the engine state. + */ + private final EngineLogic appLogic; + /** + * The target frames per second. + */ + private final int targetFps; + /** + * The target updates per second. (e.g inputs rotation,...) + */ + private final int targetUps; + + /** + * Creates a new engine + * @param windowTitle The title of the window. + * @param opts The window options. + * @param appLogic The application logic. + */ + public Engine(String windowTitle, WindowOptions opts, EngineLogic appLogic) { + this.window = new Window(windowTitle, opts, () -> { + this.resize(); + return null; + }); + this.targetFps = opts.fps; + this.targetUps = opts.ups; + this.appLogic = appLogic; + this.render = new Render(); + this.scene = new GLScene( new Scene(this.window.getWidth(), this.window.getHeight())); + this.appLogic.init(this.window, this.scene, this.render); + this.running = true; + } + + /** + * Sets the render options. + * Must be called before render is called. + * @param options The render options. + */ + public void setRenderOptions(RenderOptions options){ + this.render.setOptions(options); + } + + +/** + * Refreshes the engine. + * Is called when the engine is stopped and has to be ready to start again. + */ + public void refresh() { + this.appLogic.cleanup(); + this.render.cleanup(); + this.scene.cleanup(); + } + + + /** + * Releases all resources and terminates the engine. + * Is called when the engine is stopped and all resources have to be released. + */ + public void clear() { + // Calls the registered app for cleaning up + this.appLogic.cleanup(); + this.render.cleanup(); + this.scene.cleanup(); + this.window.cleanup(); + } + + /** + * Starts the engine in continuous rendering mode. + */ + public void start() { + this.running = true; + this.run(); + } + + /** + * Stops the continuous rendering mode. + */ + public void stop() { + this.running = false; + } + + /** + * Runs a single frame rendering. + */ + public void runOnce() { + this.window.pollEvents(); + this.appLogic.beforeRender(this.window, this.scene, this.render); + this.render.render(this.window, this.scene); + this.appLogic.afterRender(this.window, this.scene, this.render); + this.window.update(); + } + + /** + * Run mode runs permanently until the engine is stopped. + * 1. Poll events + * 2. Input + * 3. Update + * 4. Render + * 5. Update window + */ + public void run() { + var initialTime = System.currentTimeMillis(); + // maximum elapsed time between updates + var timeU = 1000.0f / this.targetUps; + // maximum elapsed time between render calls + var timeR = this.targetFps > 0 ? 1000.0f / this.targetFps : 0; + var deltaUpdate = 0.0f; + var deltaFps = 0.0f; + + var updateTime = initialTime; + + while (this.running && !this.window.windowShouldClose()) { + this.window.pollEvents(); + + var now = System.currentTimeMillis(); + + // relation between actual and elapsed time. 1 if equal. + deltaUpdate += (now - initialTime) / timeU; + deltaFps += (now - initialTime) / timeR; + + // If passed maximum elapsed time for render, process user input + if (this.targetFps <= 0 || deltaFps >= 1) { + this.appLogic.input(this.window, this.scene, now - initialTime); + } + + // If passed maximum elapsed time for update, update the scene + if (deltaUpdate >= 1) { + var diffTimeMillis = now - updateTime; + this.appLogic.update(this.window, this.scene, diffTimeMillis); + updateTime = now; + deltaUpdate--; + } + + // If passed maximum elapsed time for render, render the scene + if (this.targetFps <= 0 || deltaFps >= 1) { + this.appLogic.beforeRender(this.window, this.scene, this.render); + this.render.render(this.window, this.scene); + deltaFps--; + this.window.update(); + this.appLogic.afterRender(this.window, this.scene, this.render); + } + } + this.refresh(); + } + + /** + * Resizes the window. + */ + public void resize() { + this.scene.resize(this.window.getWidth(), this.window.getHeight()); + } + + /** + * Returns the camera object. + * @return The camera object. + */ + public Camera getCamera() { + return this.scene.getCamera(); + } + + /** + * Returns the window object. + * @return The window object. + */ + public Window getWindow() { + return this.window; + } + + /** + * Returns the scene object. + * @return The scene object. + */ + public GLScene getScene() { + return this.scene; + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/EngineLogic.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/EngineLogic.java new file mode 100644 index 000000000..e075f6dfa --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/EngineLogic.java @@ -0,0 +1,61 @@ +package org.vitrivr.cineast.core.render.lwjgl.engine; + + +import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLScene; +import org.vitrivr.cineast.core.render.lwjgl.render.Render; +import org.vitrivr.cineast.core.render.lwjgl.window.Window; + + +/** + * The EngineLogic provides methods to be called by the engine on certain states. + */ +public abstract class EngineLogic { + + /** + * Is called from the engine as first step during refresh and cleanup + * @implSpec DO NOT CALL ENGINE LOGIC METHODS IN THIS METHOD + * DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + protected abstract void cleanup(); + + /** + * Is called once at the initialization of the engine. + * @implSpec DO NOT CALL ENGINE LOGIC METHODS IN THIS METHOD + * DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + protected abstract void init(Window window, GLScene scene, Render render); + + /** + * Is called from the engine before the render method. + * @implSpec DO NOT CALL ENGINE LOGIC METHODS IN THIS METHOD + * DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + protected abstract void beforeRender(Window window, GLScene scene, Render render); + + /** + * Is called from the engine after the render method. + * @implSpec DO NOT CALL ENGINE LOGIC METHODS IN THIS METHOD + * DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + protected abstract void afterRender(Window window, GLScene scene, Render render); + + /** + * This method is called every frame. + * This is only used in continuous rendering. + * The purpose is to do some input handling. + * Could be use for optimize view angles on a fast manner. + * @implSpec DO NOT CALL ENGINE LOGIC METHODS IN THIS METHOD + * DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + protected abstract void input(Window window, GLScene scene, long diffTimeMillis); + + /** + * After Engine run This method is called every frame. + * This is only used in continuous rendering. + * The purpose is to process some life output. + * Could be use for optimize view angles on a fast manner. + * @implSpec DO NOT CALL ENGINE LOGIC METHODS IN THIS METHOD + * DO NOT CALL THIS METHOD FROM EXTENDING CLASS + */ + protected abstract void update(Window window, GLScene scene, long diffTimeMillis); +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMaterial.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMaterial.java new file mode 100644 index 000000000..dd66fe99a --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMaterial.java @@ -0,0 +1,87 @@ +package org.vitrivr.cineast.core.render.lwjgl.glmodel; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joml.Vector4f; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Material; + + +/** + * The GLMaterial class is a wrapper for the {@link Material} class. + *

    + *
  • Material -> GLMaterial( Material )
    • + *
+ *

+ * The purpose is to bring the generic Material in an OpenGl context + * {@link Material} -> {@link GLMaterial} + */ +public class GLMaterial { + + private static final Logger LOGGER = LogManager.getLogger(); + /** + * The contained meshes in gl context + */ + private final List meshes; + /** + * The contained texture in gl context + */ + private final GLTexture texture; + /** + * The material that is wrapped by this gl material. + */ + private final Material material; + + /** + * Creates a new GLMaterial from a material. + * + * @param material The material that is wrapped by this gl material. + */ + public GLMaterial(Material material) { + this.meshes = new ArrayList<>(); + this.material = material; + this.material.getMeshes().forEach(mesh -> this.meshes.add(new GLMesh(mesh))); + this.texture = new GLTexture(this.material.getTexture()); + } + + /** + * Cleans up the gl material and calls all underlying cleanup methods. + * Removes only the references to wrapped generic meshes and texture. + * Hence, the material could be used by another extraction task this method does not close the generic meshes or texture. + */ + public void cleanup() { + this.meshes.forEach(GLMesh::cleanup); + this.meshes.clear(); + this.texture.cleanup(); + LOGGER.trace("Cleaned-up GLMaterial"); + } + + /** + * Returns the gl meshes of this gl material. + * + * @return The unmodifiable list of gl meshes of this gl material. + */ + public List getMeshes() { + return Collections.unmodifiableList(this.meshes); + } + + /** + * Returns the gl texture of this gl material. + * + * @return The gl texture of this gl material. + */ + public GLTexture getTexture() { + return this.texture; + } + + /** + * Returns the color from wrapped generic material. + * + * @return The color from wrapped generic material. (r,g,b,opacity) + */ + public Vector4f getDiffuseColor() { + return this.material.getDiffuseColor(); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMesh.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMesh.java new file mode 100644 index 000000000..3d175e5bf --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMesh.java @@ -0,0 +1,127 @@ +package org.vitrivr.cineast.core.render.lwjgl.glmodel; + +import java.util.ArrayList; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.lwjgl.opengl.GL30; +import org.lwjgl.system.MemoryStack; +import org.lwjgl.system.MemoryUtil; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Mesh; + +/** + * The GLMesh class is a wrapper for the {@link Mesh} class. + *

    + *
  • Mesh -> GLMesh( Mesh )
    • + *
+ *

+ * The purpose is to bring the generic Mesh in an OpenGl context + * {@link Mesh} -> {@link GLMesh} + */ +public class GLMesh { + + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * The wrapped generic mesh in gl context + */ + private final Mesh mesh; + /** + * The list of Vertex Buffer Object (VBO) ids + */ + private final List vboIdList; + /** + * The Vertex Array Object (VAO) id + */ + private final int vaoId; + + + /** + * Creates a new GLMesh from a mesh. + *

    + *
  1. Bind Vertex Array Object
    2. + *
  2. Generate, allocate and initialize Vertex (Positions) Buffer
    3. + *
  3. Generate, allocate and initialize Texture Coordinates Buffer
    4. + *
  4. Generate, allocate and initialize Index Buffer
    5. + *
  5. Unbind Vertex Array Object
    6. + *
+ * @param mesh The mesh that is wrapped by this gl mesh. + */ + public GLMesh(Mesh mesh) { + this.mesh = mesh; + this.vboIdList = new ArrayList<>(); + + try (var memoryStack = MemoryStack.stackPush()) { + + this.vaoId = GL30.glGenVertexArrays(); + GL30.glBindVertexArray(this.vaoId); + + // Positions VBO + int vboId = GL30.glGenBuffers(); + this.vboIdList.add(vboId); + var positionsBuffer = memoryStack.callocFloat(this.mesh.getPositions().length); + positionsBuffer.put(0, this.mesh.getPositions()); + GL30.glBindBuffer(GL30.GL_ARRAY_BUFFER, vboId); + GL30.glBufferData(GL30.GL_ARRAY_BUFFER, positionsBuffer, GL30.GL_STATIC_DRAW); + GL30.glEnableVertexAttribArray(0); + GL30.glVertexAttribPointer(0, 3, GL30.GL_FLOAT, false, 0, 0); + + // Textures VBO (Vertex Buffer Object) + vboId = GL30.glGenBuffers(); + this.vboIdList.add(vboId); + var textureCoordinatesBuffer = MemoryUtil.memAllocFloat(this.mesh.getTextureCoords().length); + textureCoordinatesBuffer.put(0, this.mesh.getTextureCoords()); + GL30.glBindBuffer(GL30.GL_ARRAY_BUFFER, vboId); + GL30.glBufferData(GL30.GL_ARRAY_BUFFER, textureCoordinatesBuffer, GL30.GL_STATIC_DRAW); + GL30.glEnableVertexAttribArray(1); + GL30.glVertexAttribPointer(1, 2, GL30.GL_FLOAT, false, 0, 0); + + // Index VBO (Vertex Buffer Object) + vboId = GL30.glGenBuffers(); + this.vboIdList.add(vboId); + var idxBuffer = memoryStack.callocInt(this.mesh.getIdx().length); + idxBuffer.put(0, this.mesh.getIdx()); + GL30.glBindBuffer(GL30.GL_ELEMENT_ARRAY_BUFFER, vboId); + GL30.glBufferData(GL30.GL_ELEMENT_ARRAY_BUFFER, idxBuffer, GL30.GL_STATIC_DRAW); + + GL30.glBindBuffer(GL30.GL_ARRAY_BUFFER, 0); + GL30.glBindVertexArray(0); + } + } + + /** + * Cleans up the gl mesh and calls all underlying cleanup methods. + * Removes only the references to VBOs and VAOs. + * Removes the Vertex Array Object (VAO) and all Vertex Buffer Object (VBO) ids. + */ + public void cleanup() { + this.vboIdList.forEach(GL30::glDeleteBuffers); + GL30.glDeleteVertexArrays(this.vaoId); + this.vboIdList.clear(); + LOGGER.trace("Cleaned-up GLMesh"); + } + + /** + * Returns the number of vertices of the wrapped generic mesh. + * @return The number of vertices of the wrapped generic mesh. + */ + public int getNumVertices() { + return this.mesh.getNumVertices(); + } + + /** + * Returns the Vertex Array Object (VAO) id. + * @return The Vertex Array Object (VAO) id. + */ + public final int getVaoId() { + return this.vaoId; + } + + /** + * Returns the ID of the wrapped generic mesh. + * @return The ID of the wrapped generic mesh. + */ + public String getId() { + return this.mesh.getId(); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLModel.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLModel.java new file mode 100644 index 000000000..c0ad58cca --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLModel.java @@ -0,0 +1,86 @@ +package org.vitrivr.cineast.core.render.lwjgl.glmodel; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Entity; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; + +/** + * The GLModel class is a wrapper for the {@link IModel} class. + *
    + *
  • IModel -> GLModel( IModel )
    • + *
+ *

+ * The purpose is to bring the generic IModel in an OpenGl context + * {@link IModel} -> {@link GLModel} + */ +public class GLModel implements IGLModel { + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * The model that is wrapped by this gl model. + */ + private final IModel model; + + /** + * The contained materials in gl context + */ + private final List materials; + + /** + * Creates a new GLModel from a model. + * + * @param model The model that is wrapped by this gl model. + */ + public GLModel(IModel model) { + this.model = model; + this.materials = new ArrayList<>(); + this.model.getMaterials().forEach(material -> this.materials.add(new GLMaterial(material))); + } + + /** + * {@inheritDoc} + */ + @Override + public List getEntities() { + return Collections.unmodifiableList(this.model.getEntities()); + } + + /** + * {@inheritDoc} + */ + @Override + public void addEntity(Entity entity) { + this.model.addEntity(entity); + } + + /** + * {@inheritDoc} + */ + @Override + public void cleanup() { + this.materials.forEach(GLMaterial::cleanup); + this.materials.clear(); + LOGGER.debug("GLModel cleaned up"); + } + + /** + * {@inheritDoc} + */ + @Override + public String getId() { + return this.model.getId(); + } + + /** + * {@inheritDoc} + */ + @Override + public List getMaterials() { + return Collections.unmodifiableList(this.materials); + } + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLScene.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLScene.java new file mode 100644 index 000000000..38e82c2dc --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLScene.java @@ -0,0 +1,155 @@ +package org.vitrivr.cineast.core.render.lwjgl.glmodel; + +import java.util.HashMap; +import java.util.Map; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Entity; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; +import org.vitrivr.cineast.core.render.lwjgl.scene.Camera; +import org.vitrivr.cineast.core.render.lwjgl.scene.Projection; +import org.vitrivr.cineast.core.render.lwjgl.scene.Scene; + +/** + * The GLScene class ist the top most class of the gl model hierarchy. + * The gl model hierarchy is used as a wrapper for the model hierarchy. + * Therefore, each gl class has a corresponding model class. + * The generic class has to be provided in the constructor. + *

    + *
  • Scene -> GLScene( Scene )
    • + *
  • Model -> GlModel( IModel )
    • + *
  • Material -> GLMaterial( Material )
    • + *
  • Mesh -> GLMesh( Mesh )
    • + *
  • Texture -> GLTexture( Texture )
    • + *
+ * + * The purpose is to bring the generic model in an OpenGl context + * {@link Scene} -> {@link GLScene} + */ +public class GLScene { + + /** + * The scene that is wrapped by this gl scene. + */ + private final Scene scene; + /** + * The wrapped GlModels that are wrapped by this gl scene. + */ + private final Map models; + + /** + * The texture cache that is used by this gl scene. + * Textures are cached to avoid loading the same texture multiple times. + * Has no corresponding generic class. + */ + private final GLTextureCache textureCache; + + /** + * Creates a new GLScene from a scene. + * + * @param scene The scene that is wrapped by this gl scene. + */ + public GLScene(Scene scene) { + this.models = new HashMap<>(); + this.textureCache = new GLTextureCache(); + this.scene = scene; + this.updateGlSceneFromScene(); + } + + /** + * Adds a model to the scene. + * + * @param model The model that is added to the scene. + */ + public void addModel(IModel model) { + this.scene.addModel(model); + this.updateGlSceneFromScene(); + } + + /** + * Updates the gl scene from the scene. + * It updates the gl scene content to match the scene content. + */ + private void updateGlSceneFromScene() { + this.scene.getModels().forEach((k, v) -> this.models.putIfAbsent(k, new GLModel(v))); + this.models.forEach( + (k, v) -> this.models.get(k).getMaterials() + .forEach(mat -> this.textureCache.addTextureIfAbsent(mat.getTexture()))); + } + + /** + * Adds an entity to the corresponding model. + * @param entity The entity that is added to the model. + */ + public void addEntity(Entity entity) { + var modelId = entity.getModelId(); + var model = this.models.get(modelId); + if (model == null) { + throw new RuntimeException("Model not found: " + modelId); + } + model.addEntity(entity); + } + + /** + * Returns the gl models of the gl scene. + * @return The gl models of the gl scene. + */ + public Map getModels() { + return this.models; + } + + /** + * Returns the texture cache of the gl scene. + * @return The texture cache of the gl scene. + */ + public GLTextureCache getTextureCache() { + return this.textureCache; + } + + /** + * Returns the projection of the wrapped generic scene. + * @return The projection of the wrapped generic scene. + */ + public Projection getProjection() { + return this.scene.getProjection(); + } + + /** + * Returns the camera of the wrapped generic scene. + * @return The camera of the wrapped generic scene. + */ + public Camera getCamera() { + return this.scene.getCamera(); + } + + /** + * Clears the models of the gl scene but not containing resources. + * Removes the references to the wrapped generic models and textures. + * Hence, the models could be used by another extraction task this method does not close the models or textures. + * Can be used to only remove Models temporarily from gl scene. + */ + @SuppressWarnings("unused") + public void clearModels() { + this.cleanup(); + this.models.clear(); + } + + /** + * Cleans up the gl scene and calls all underlying cleanup methods. + * Removes only the references to wrapped generic models and textures. + * Hence, the model could be used by another extraction task this method does not close the generic models or textures. + */ + public void cleanup() { + this.models.values().forEach(IGLModel::cleanup); + this.models.clear(); + this.textureCache.cleanup(); + } + + /** + * Resizes the projection of the wrapped generic scene. + * @param width The new width of the projection. + * @param height The new height of the projection. + */ + public void resize(int width, int height) { + this.scene.getProjection().updateProjMatrix(width, height); + } + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTexture.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTexture.java new file mode 100644 index 000000000..db0fb4409 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTexture.java @@ -0,0 +1,95 @@ +package org.vitrivr.cineast.core.render.lwjgl.glmodel; + +import java.nio.ByteBuffer; +import org.lwjgl.opengl.GL30; +import org.lwjgl.stb.STBImage; +import org.lwjgl.system.MemoryStack; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Texture; + +/** + * The GLTexture class is a wrapper for the {@link Texture} class. + *
    + *
  • Texture -> GLTexture( Texture )
    • + *
+ *

+ * The purpose is to bring the generic Mesh in an OpenGl context + * {@link Texture} -> {@link GLTexture} + */ +public class GLTexture { + + /** + * The id of the texture used to bind the texture to the Gl context + */ + private int textureId; + /** + * The wrapped generic texture in gl context + */ + private final Texture texture; + + /** + * Creates a new GLTexture from a texture. + *

    + *
  1. Load the texture from the file
    2. + *
  2. Allocate the texture buffer
    3. + *
  3. Load the texture into the buffer
    4. + *
+ * @param texture The texture that is wrapped by this gl texture. + */ + public GLTexture(Texture texture) { + this.texture = texture; + try (var memoryStack = MemoryStack.stackPush()) { + var w = memoryStack.mallocInt(1); + var h = memoryStack.mallocInt(1); + var channels = memoryStack.mallocInt(1); + + var imageBuffer = STBImage.stbi_load(this.texture.getTexturePath(), w, h, channels, 4); + if (imageBuffer == null) { + throw new RuntimeException("Could not load texture file: " + this.texture.getTexturePath()); + } + this.generateTexture(w.get(), h.get(), imageBuffer); + STBImage.stbi_image_free(imageBuffer); + } + } + + /** + * Binds the GLTexture to the Gl context + */ + public void bind() { + GL30.glBindTexture(GL30.GL_TEXTURE_2D, this.textureId); + } + + /** + * Cleans the GLTexture + * Does not affect the underlying texture + * Removes the texture from the GPU + */ + public void cleanup() { + GL30.glDeleteTextures(this.textureId); + } + + /** + * Generates the texture in the Gl context + * @param width The width of the texture + * @param height The height of the texture + * @param texture The texture buffer + */ + private void generateTexture(int width, int height, ByteBuffer texture) { + this.textureId = GL30.glGenTextures(); + GL30.glBindTexture(GL30.GL_TEXTURE_2D, this.textureId); + GL30.glPixelStorei(GL30.GL_UNPACK_ALIGNMENT, 1); + GL30.glTexParameteri(GL30.GL_TEXTURE_2D, GL30.GL_TEXTURE_MIN_FILTER, GL30.GL_NEAREST); + GL30.glTexParameteri(GL30.GL_TEXTURE_2D, GL30.GL_TEXTURE_MAG_FILTER, GL30.GL_NEAREST); + GL30.glTexImage2D(GL30.GL_TEXTURE_2D, 0, GL30.GL_RGBA, width, height, 0, GL30.GL_RGBA, GL30.GL_UNSIGNED_BYTE, + texture); + GL30.glGenerateMipmap(GL30.GL_TEXTURE_2D); + } + + /** + * Returns the texture path of the underlying wrapped texture + * @return The texture path of the underlying wrapped texture + */ + public String getTexturePath() { + return this.texture.getTexturePath(); + } +} + diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTextureCache.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTextureCache.java new file mode 100644 index 000000000..d38a3b363 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTextureCache.java @@ -0,0 +1,56 @@ +package org.vitrivr.cineast.core.render.lwjgl.glmodel; + +import java.util.HashMap; +import java.util.Map; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Texture; + +/** + * A cache for textures + * Prevents the same texture from being loaded multiple times + */ +public class GLTextureCache { + + /** + * The cache of textures + */ + private final Map textures; + + /** + * Creates a new texture cache + * Adds a default texture to the cache + */ + public GLTextureCache() { + this.textures = new HashMap<>(); + var texture = new Texture(); + this.textures.put(texture.getTexturePath(), new GLTexture(texture)); + this.textures.put("default", new GLTexture(texture)); + } + + /** + * Cleans up the texture cache + * Cleans the registered textures and clears the cache + */ + public void cleanup() { + this.textures.values().forEach(GLTexture::cleanup); + this.textures.clear(); + } + + /** + * Adds a texture to the cache if it is not already present + * + * @param texture Texture to add + */ + public void addTextureIfAbsent(GLTexture texture) { + this.textures.putIfAbsent(texture.getTexturePath(), texture); + } + + /** + * Returns the gl texture with the given texture path + * + * @param texturePath Path of the texture + * @return The texture with the given texture path + */ + public GLTexture getTexture(String texturePath) { + return this.textures.get(texturePath); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/IGLModel.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/IGLModel.java new file mode 100644 index 000000000..e4bbf177d --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/IGLModel.java @@ -0,0 +1,47 @@ +package org.vitrivr.cineast.core.render.lwjgl.glmodel; + +import java.util.List; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Entity; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; + +/** + * The Interface IGLModel provides functionality for a arbitrary model used in the OpenGl context. + * Itr is the context related counterpart to the {@link IModel} interface. + */ +public interface IGLModel{ + + /** + * Returns the entities of the wrapped generic model. + * + * @return The entities of the wrapped generic model. + */ + List getEntities(); + + /** + * Adds an entity to the wrapped generic model. + * + * @param entity The entity to be added. + */ + void addEntity(Entity entity); + + /** + * Cleans up the gl model and calls all underlying cleanup methods. + * Removes only the references to wrapped generic materials + * Hence, the model could be used by another extraction task this method does not close the generic model. + */ + void cleanup(); + + /** + * Returns the id of the wrapped generic model. + * + * @return The id of the wrapped generic model. + */ + String getId(); + + /** + * Returns the gl materials of the gl model. + * + * @return The gl materials of the gl model. + */ + List getMaterials(); +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/Render.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/Render.java new file mode 100644 index 000000000..77b3c72c0 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/Render.java @@ -0,0 +1,65 @@ +package org.vitrivr.cineast.core.render.lwjgl.render; + +import org.vitrivr.cineast.core.render.lwjgl.window.Window; +import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLScene; +import org.lwjgl.opengl.GL; +import org.lwjgl.opengl.GL30; + +/** + * This class holds the render logic for the LWJGL engine + * Holds the {@link SceneRender} which loads shaders + */ +public class Render { + + /** + * Instance of the scene render + * @see SceneRender + */ + private final SceneRender sceneRender; + + /** + * Instance of the render options + * @see RenderOptions + */ + private RenderOptions options; + + /** + * Create a render instance Set up the Render options for OpenGL + */ + public Render() { + GL.createCapabilities(); + GL30.glEnable(GL30.GL_DEPTH_TEST); + GL30.glEnable(GL30.GL_CULL_FACE); + GL30.glCullFace(GL30.GL_BACK); + this.sceneRender = new SceneRender(); + } + + /** + * Set the render options {@link RenderOptions} + * + * @param options see {@link RenderOptions} + */ + public void setOptions(RenderOptions options) { + this.options = options; + } + + /** + * Releases all resources + */ + public void cleanup() { + this.sceneRender.cleanup(); + this.options = null; + } + + /** + * Renders a given Scene in a Given Window + * + * @param window GL (offscreen) window instance {@link Window} + * @param scene GL Scene (containing all models) {@link GLScene} + */ + public void render(Window window, GLScene scene) { + GL30.glClear(GL30.GL_COLOR_BUFFER_BIT | GL30.GL_DEPTH_BUFFER_BIT); + GL30.glViewport(0, 0, window.getWidth(), window.getHeight()); + this.sceneRender.render(scene, this.options); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/RenderOptions.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/RenderOptions.java new file mode 100644 index 000000000..3e5f88d4d --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/RenderOptions.java @@ -0,0 +1,35 @@ + +package org.vitrivr.cineast.core.render.lwjgl.render; + +import java.util.function.Function; +import org.joml.Vector4f; + +/** + * RenderOptions + *
    + *
  • Used to switch on or off the texture rendering
    • + *
  • Used to switch on or off the coloring rendering
    • + *
  • Returns the color for the given value
    • + *
  • Can be used to colorize the model custom
    • + *
+ */ +public class RenderOptions { + + /** + * Used to switch on or off the texture rendering + */ + public boolean showTextures = true; + + /** + * Used to switch on or off the coloring rendering For future face coloring + */ + @SuppressWarnings("unused") + public boolean showColor = false; + + /** + * Returns the color for the given value Can be used to colorize the model custom + */ + public Function colorfunction = + (v) -> new Vector4f(v, v, v, 1f); + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/SceneRender.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/SceneRender.java new file mode 100644 index 000000000..f6d8e0700 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/SceneRender.java @@ -0,0 +1,148 @@ +package org.vitrivr.cineast.core.render.lwjgl.render; + +import java.util.ArrayList; +import org.lwjgl.opengl.GL30; +import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLScene; +import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLTexture; +import org.vitrivr.cineast.core.render.lwjgl.render.ShaderProgram.ShaderModuleData; + +/** + * SceneRender + *
    + *
  • Renders the scene
    • + *
  • Loads the scene shader
    • + *
  • Creates the uniforms
    • + *
  • Binds the Model
    • + *
  • Binds the Texture
    • + *
+ */ +public class SceneRender { + + /** + * Resource path to the scene shader program + */ + private static final String VERTEX_SHADER_PATH = "./resources/renderer/lwjgl/shaders/scene.vert"; + /** + * Resource path to the fragment shader program + */ + private static final String FRAGMENT_SHADER_PATH = "./resources/renderer/lwjgl/shaders/scene.frag"; + /** + * Instance of the scene shader program + */ + private final ShaderProgram shaderProgram; + /** + * Uniforms for the scene shader + */ + private UniformsMap uniformsMap; + + /** + * SceneRender. During construction: Loads the scene shader from the resources + */ + public SceneRender() { + var shaderModuleDataList = new ArrayList(); + shaderModuleDataList.add(new ShaderModuleData(VERTEX_SHADER_PATH, GL30.GL_VERTEX_SHADER)); + shaderModuleDataList.add(new ShaderModuleData(FRAGMENT_SHADER_PATH, GL30.GL_FRAGMENT_SHADER)); + this.shaderProgram = new ShaderProgram(shaderModuleDataList); + this.createUniforms(); + } + + /** + * Creates the uniforms for the scene shader creates the following uniforms: + *
    + *
  • projectionMatrix
    • + *
  • modelMatrix
    • + *
  • viewMatrix
    • + *
  • txtSampler
    • + *
  • material.diffuse
    • + *
+ */ + private void createUniforms() { + this.uniformsMap = new UniformsMap(this.shaderProgram.getProgramId()); + this.uniformsMap.createUniform("projectionMatrix"); + this.uniformsMap.createUniform("modelMatrix"); + this.uniformsMap.createUniform("viewMatrix"); + this.uniformsMap.createUniform("txtSampler"); + this.uniformsMap.createUniform("material.diffuse"); + } + + /** + * Releases all resources + *
    + *
  • Releases the shader program
    • + *
  • Releases the uniforms
    • + *
+ */ + public void cleanup() { + this.shaderProgram.cleanup(); + this.uniformsMap.cleanup(); + this.uniformsMap = null; + } + + /** + * Renders the Models in the scene + * Creates standard render options + * @param scene Scene to render + */ + public void render(GLScene scene) { + this.render(scene, new RenderOptions()); + } + + /** + * Renders the Models in the scene + *
    + *
  • Binds projection matrix
    • + *
  • Binds view matrix
    • + *
  • Binds texture sampler
    • + *
+ * Further, iterate over all models in the scene + *
    + *
  • Iterate over all materials in the model
    • + *
  • Sets texture or color function
    • + *
  • Iterate over all meshes in the material
    • + *
  • Binds the mesh
    • + *
  • Iterate over all entities to draw the mesh
    • + *
  • Binds the model matrix
    • + *
  • Draws the mesh
    • + *
  • Unbinds
    • + *
+ * @param scene Scene to render + * @param opt Render options + */ + public void render(GLScene scene, RenderOptions opt) { + this.shaderProgram.bind(); + + this.uniformsMap.setUniform("projectionMatrix", scene.getProjection().getProjMatrix()); + this.uniformsMap.setUniform("viewMatrix", scene.getCamera().getViewMatrix()); + this.uniformsMap.setUniform("txtSampler", 0); + + var models = scene.getModels().values(); + var textures = scene.getTextureCache(); + + for (var model : models) { + var entities = model.getEntities(); + for (var material : model.getMaterials()) { + GLTexture texture; + // Either draw texture or use color function + if (opt.showTextures) { + this.uniformsMap.setUniform("material.diffuse", material.getDiffuseColor()); + texture = textures.getTexture(material.getTexture().getTexturePath()); + } else { + this.uniformsMap.setUniform("material.diffuse", opt.colorfunction.apply(1f)); + texture = textures.getTexture("default"); + } + GL30.glActiveTexture(GL30.GL_TEXTURE0); + texture.bind(); + for (var mesh : material.getMeshes()) { + GL30.glBindVertexArray(mesh.getVaoId()); + for (var entity : entities) { + this.uniformsMap.setUniform("modelMatrix", entity.getModelMatrix()); + GL30.glDrawElements(GL30.GL_TRIANGLES, mesh.getNumVertices(), GL30.GL_UNSIGNED_INT, 0); + } + } + } + } + GL30.glBindVertexArray(0); + + this.shaderProgram.unbind(); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/ShaderProgram.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/ShaderProgram.java new file mode 100644 index 000000000..7e1acc729 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/ShaderProgram.java @@ -0,0 +1,149 @@ +package org.vitrivr.cineast.core.render.lwjgl.render; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import org.lwjgl.opengl.GL30; + + +/** + * ShaderProgram + * Loads a shader program from the resources to the GL context + */ +public class ShaderProgram { + + /** + * Shader Program ID, is used to bind and release the program from the GL context + */ + private final int programId; + + /** + * Creates a new ShaderProgram + * Takes a list of ShaderModuleData (usually from Scene Renderer which loads the shaders from the resources during construction + * Creates a new ShaderProgram in GL context, links the shaders and validates the program + * For Shader creation, the following steps are performed: + *
    + *
  • Reads the shader file
    • + *
  • Creates a new shader in the GL context {@link ShaderProgram#createShader(String, int)}
    • + *
  • Compiles the shader
    • + *
  • Attaches the shader to the program
    • + *
  • Links the program
    • + *
  • Binds the program to the GL context
    • + *
+ * @param shaderModuleDataList List of ShaderModuleData + */ + public ShaderProgram(List shaderModuleDataList) { + this.programId = GL30.glCreateProgram(); + if (this.programId == 0) { + throw new RuntimeException("Could not Create Shader"); + } + var shaderModules = new ArrayList(); + shaderModuleDataList.forEach(s -> shaderModules.add(this.createShader(ShaderProgram.readShaderFile(s.shaderFile), s.shaderType))); + this.link(shaderModules); + } + + /** + * Binds the ShaderProgram to the GL context + */ + public void bind() { + GL30.glUseProgram(this.programId); + } + + /** + * Unbinds the ShaderProgram from the GL context + */ + public void unbind() { + GL30.glUseProgram(0); + } + + /** + * Unbinds the ShaderProgram from the GL context + * Deletes the ShaderProgram from the GL context + */ + public void cleanup() { + this.unbind(); + if (this.programId != 0) { + GL30.glDeleteProgram(this.programId); + } + } + + /** + * Creates a new Shader in the GL context + * Compiles the shader + * Attaches the shader to the program + * @return the shader id + */ + protected int createShader(String shaderCode, int shaderType) { + int shaderId = GL30.glCreateShader(shaderType); + if (shaderId == 0) { + throw new RuntimeException("Error creating Shader"); + } + GL30.glShaderSource(shaderId, shaderCode); + GL30.glCompileShader(shaderId); + + if (GL30.glGetShaderi(shaderId, GL30.GL_COMPILE_STATUS) == 0) { + throw new RuntimeException("Error Compiling Shader"); + } + GL30.glAttachShader(this.programId, shaderId); + return shaderId; + } + + /** + * Links the program + * Deletes the shaders + * @param shaderModules List of shader ids + */ + private void link(List shaderModules) { + GL30.glLinkProgram(this.programId); + if (GL30.glGetProgrami(this.programId, GL30.GL_LINK_STATUS) == 0) { + throw new RuntimeException("Error linking Shader"); + } + shaderModules.forEach(s -> GL30.glDetachShader(this.programId, s)); + shaderModules.forEach(GL30::glDeleteShader); + this.validate(); + } + + /** + * Validates the program + * Throws an exception if the program is not valid + */ + public void validate() { + GL30.glValidateProgram(this.programId); + if (GL30.glGetProgrami(this.programId, GL30.GL_VALIDATE_STATUS) == 0) { + throw new RuntimeException("Error validate Shader"); + } + } + + /** + * Returns the program id + * @return program id + */ + public int getProgramId() { + return this.programId; + } + + /** + * Reads the shader file + * @param filePath Path to the shader file + * @return String containing the shader code + */ + public static String readShaderFile(String filePath) { + String str; + try { + str = new String(Files.readAllBytes(Paths.get(filePath))); + } catch (IOException ex) { + throw new RuntimeException("Error reading file"); + } + return str; + } + + /** + * RECORD for ShaderModuleData + * @param shaderFile + * @param shaderType + */ + public record ShaderModuleData(String shaderFile, int shaderType) { + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/UniformsMap.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/UniformsMap.java new file mode 100644 index 000000000..798710d4b --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/UniformsMap.java @@ -0,0 +1,107 @@ +package org.vitrivr.cineast.core.render.lwjgl.render; + +import java.util.HashMap; +import org.joml.Matrix4f; +import org.joml.Vector4f; +import org.lwjgl.opengl.GL30; +import org.lwjgl.system.MemoryStack; + +/** + * Holds a hashmap for used uniforms are global variables in the shader e.g. projectionMatrix, modelMatrix, viewMatrix, txtSampler, material.diffuse + * + * @see https://www.khronos.org/opengl/wiki/Uniform_(GLSL) + * @see "./resources/renderer/lwjgl/shaders/scene.vert" + * @see "./resources/renderer/lwjgl/shaders/scene.frag" + */ +public class UniformsMap { + + /** + * HashMap for the uniforms Key: Uniform name Value: Uniform location in the shader + */ + private final HashMap uniforms; + + /** + * Program id of the shader + */ + private final int programId; + + /** + * Instantiate a new UniformsMap + * + * @param programId Program id of the shader {@link ShaderProgram#getProgramId} + */ + public UniformsMap(int programId) { + this.programId = programId; + this.uniforms = new HashMap<>(); + } + + /** + * Creates a new uniform + * + * @param uniformName Name of the uniform + */ + public void createUniform(String uniformName) { + var uniformLocation = GL30.glGetUniformLocation(this.programId, uniformName); + if (uniformLocation < 0) { + throw new RuntimeException("Could not find uniform:" + uniformName); + } + this.uniforms.put(uniformName, uniformLocation); + } + + /** + * Sets the value of a uniform to gl context + * + * @param uniformName Name of the uniform + * @param value Value of the uniform + */ + public void setUniform(String uniformName, int value) { + GL30.glUniform1i(this.getUniformLocation(uniformName), value); + } + + /** + * Returns the location of the uniform from the hashmap + * + * @param uniformName name of the uniform + * @return location of the uniform + */ + private int getUniformLocation(String uniformName) { + var location = this.uniforms.get(uniformName); + if (location == null) { + throw new RuntimeException("Could not find uniform:" + uniformName); + } + return location; + } + + /** + * Sets the value 4 float vector of a uniform to gl context + * + * @param uniformName Name of the uniform + * @param value Value of the uniform + */ + public void setUniform(String uniformName, Vector4f value) { + GL30.glUniform4f(this.getUniformLocation(uniformName), value.x, value.y, value.z, value.w); + } + + /** + * Sets the value 4*4 float matrix of a uniform to gl context + * + * @param uniformName Name of the uniform + * @param value Value of the uniform + */ + public void setUniform(String uniformName, Matrix4f value) { + try (var memoryStack = MemoryStack.stackPush()) { + var location = this.uniforms.get(uniformName); + if (location == null) { + throw new RuntimeException("Could not find uniform:" + uniformName); + } + GL30.glUniformMatrix4fv(location, false, value.get(memoryStack.mallocFloat(16))); + } + } + + /** + * Cleans up the uniforms + */ + public void cleanup() { + this.uniforms.clear(); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/LWJGLOffscreenRenderer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/LWJGLOffscreenRenderer.java new file mode 100644 index 000000000..46026f5b7 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/LWJGLOffscreenRenderer.java @@ -0,0 +1,296 @@ +package org.vitrivr.cineast.core.render.lwjgl.renderer; + +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.util.concurrent.LinkedTransferQueue; + +import org.joml.Vector3f; +import org.vitrivr.cineast.core.render.lwjgl.engine.Engine; +import org.vitrivr.cineast.core.render.lwjgl.engine.EngineLogic; +import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLScene; +import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; +import org.vitrivr.cineast.core.render.lwjgl.window.Window; +import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Entity; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Model; +import org.vitrivr.cineast.core.render.lwjgl.render.Render; +import org.vitrivr.cineast.core.render.lwjgl.scene.LightfieldCamera; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.render.Renderer; + +/** + * This is the top most class of the LWJGL for Java 3D renderer. Its main function is to provide an interface between Engine and the outside world. It sets up the {@link Engine} and provides the interface to the outside world. {@link Renderer} It extends the abstract class {@link EngineLogic} which allows the instanced engine to call methods depending on the engine state. + */ +@SuppressWarnings("deprecation") +public class LWJGLOffscreenRenderer extends EngineLogic implements Renderer { + + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * The (offscreen) window options for the engine. + */ + private WindowOptions windowOptions; + /** + * The engine instance. + */ + private Engine engine; + + /** + * The model queue. From this queue the renderer takes the next model to render. + */ + private final LinkedTransferQueue modelQueue; + + /** + * The image queue. In this queue the renderer puts the rendered images. + */ + private final LinkedTransferQueue imageQueue; + + + /** + * Constructor for the LWJGLOffscreenRenderer. Initializes the model queue and the image queue. + */ + public LWJGLOffscreenRenderer() { + this.modelQueue = new LinkedTransferQueue<>(); + this.imageQueue = new LinkedTransferQueue<>(); + LOGGER.trace("LWJGLOffscreenRenderer created"); + } + + /** + * Sets the window options for the engine. + * + * @param opts The window options. + */ + public void setWindowOptions(WindowOptions opts) { + this.windowOptions = opts; + } + + /** + * Sets the render options for the engine. + * + * @param opts The render options. + */ + public void setRenderOptions(RenderOptions opts) { + this.engine.setRenderOptions(opts); + } + + /** + * Starts the engine with given window options. Registers the LWJGLOffscreenRenderer as the engine logic. + */ + public void startEngine() { + var name = "LWJGLOffscreenRenderer"; + this.engine = new Engine(name, this.windowOptions, this); + } + + /** + * Starts the rendering process. + */ + @Override + public void render() { + this.engine.runOnce(); + LOGGER.trace("LWJGLOffscreenRenderer rendered"); + } + + + /** + * Is called once at the initialization of the engine. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + @Override + protected void init(Window window, GLScene scene, Render render) { + scene.getCamera().setPosition(0, 0, 1); + } + + /** + * Is called from the engine before the render method. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + @Override + protected void beforeRender(Window window, GLScene scene, Render render) { + this.loadNextModelFromQueueToScene(window, scene); + } + + /** + * Is called from the engine after the render method. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + @Override + protected void afterRender(Window window, GLScene scene, Render render) { + var lfc = new LightfieldCamera(this.windowOptions); + this.imageQueue.add(lfc.takeLightfieldImage()); + } + + /** + * Is called from the engine as first step during refresh and cleanup DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + @Override + protected void cleanup() { + LOGGER.trace("LWJGLOffscreenRenderer cleaned"); + } + + + /** + * This method is called every frame. This is only used in continuous rendering. The purpose is to do some input handling. Could be use for optimize view angles on a fast manner. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + @Override + protected void input(Window window, GLScene scene, long diffTimeMillis) { + scene.getModels().forEach((k, v) -> v.getEntities().forEach(Entity::updateModelMatrix)); + } + + /** + * After Engine run This method is called every frame. This is only used in continuous rendering. The purpose is to process some life output. Could be use for optimize view angles on a fast manner. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + @Override + protected void update(Window window, GLScene scene, long diffTimeMillis) { + } + + /** + * This method is called to load the next model into the provided scene. + * + * @param scene The scene to put the model in. + */ + @SuppressWarnings("unused") + private void loadNextModelFromQueueToScene(Window window, GLScene scene) { + if (!this.modelQueue.isEmpty()) { + var model = (Model) this.modelQueue.poll(); + if (model.getEntities().size() == 0) { + var entity = new Entity("cube", model.getId()); + model.addEntityNorm(entity); + } + //cleans all current models from the scene + scene.cleanup(); + //adds the new model to the scene + scene.addModel(model); + } + scene.getModels().forEach((k, v) -> v.getEntities().forEach(Entity::updateModelMatrix)); + } + + /** + * Moves the camera in the scene with given deltas in cartesian coordinates. Look at the origin. + */ + public void moveCameraOrbit(float dx, float dy, float dz) { + this.engine.getCamera().moveOrbit(dx, dy, dz); + } + + /** + * Sets the camera in the scene to cartesian coordinates. Look at the origin. + */ + public void setCameraOrbit(float x, float y, float z) { + this.engine.getCamera().setOrbit(x, y, z); + } + + /** + * Moves the camera in the scene with given deltas in cartesian coordinates. Keep the orientation. + */ + @SuppressWarnings("unused") + public void setCameraPosition(float x, float y, float z) { + this.engine.getCamera().setPosition(x, y, z); + } + + /** + * Set position of the camera and look at the origin. Camera will stay aligned to the y plane. + */ + public void lookFromAtO(float x, float y, float z) { + var lookFrom = new Vector3f(x, y, z); + var lookAt = new Vector3f(0, 0, 0); + + this.engine.getCamera().setPositionAndOrientation(lookFrom, lookAt); + + } + + /** + * Set position and orientation of the camera. + * + * @deprecated Old renderer implementation. Not needed. Use quaternion instead. + */ + @Override + @Deprecated + public void positionCamera(double ex, double ey, double ez, + double cx, double cy, double cz, + double upx, double upy, double upz) { + this.engine.getCamera().setPositionAndOrientation( + new Vector3f((float) ex, (float) ey, (float) ez), + new Vector3f((float) cx, (float) cy, (float) cz), + new Vector3f((float) upx, (float) upy, (float) upz)); + } + + + /** + * Returns the aspect ratio of the window. + */ + @SuppressWarnings("unused") + public float getAspect() { + return (float) this.windowOptions.width / (float) this.windowOptions.height; + } + + /** + * Interface to outside to add a model to the scene. + */ + @Override + public void assemble(IModel model) { + this.modelQueue.add(model); + } + + /** + * Interface to outside to get a rendered image. + */ + @Override + public BufferedImage obtain() { + return this.imageQueue.poll(); + } + + /** + * This method disposes the engine. Window is destroyed and all resources are freed. + */ + @Override + public void clear(Color color) { + this.clear(); + } + + /** + * This method disposes the engine. Window is destroyed and all resources are freed. + */ + @Override + public void clear() { + this.engine.clear(); + this.engine = null; + } + + /** + * Retains control of the Renderer. While a Thread retains a renderer, no other thread should be allowed to use it! + * + * @deprecated Old renderer implementation. Indicates that the renderer should be retained. + */ + @Override + @Deprecated + public boolean retain() { + return true; + } + + /** + * Releases control of the Renderer, making it usable by other Threads again. + * + * @deprecated Old renderer implementation. Indicates that the renderer should be retained. + */ + @Override + @Deprecated + public void release() { + this.engine.clear(); + } + + /** + * Returns the width of the window. + * + * @return The width of the window. (in pixels) + */ + public int getWidth() { + return this.windowOptions.width; + } + + /** + * Returns the height of the window. + * + * @return The height of the window. (in pixels) + */ + public int getHeight() { + return this.windowOptions.height; + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderActions.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderActions.java new file mode 100644 index 000000000..0bf3c2d1c --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderActions.java @@ -0,0 +1,14 @@ +package org.vitrivr.cineast.core.render.lwjgl.renderer; + +/** + * Actions used of the Render Workflow + * {@link RenderWorker} + */ +public final class RenderActions { + + public static final String SETUP = "SETUP"; + public static final String RENDER = "RENDER"; + public static final String ROTATE = "ROTATE"; + public static final String LOOKAT = "LOOKAT"; + public static final String LOOKAT_FROM = "LOOKAT_FROM"; +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderData.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderData.java new file mode 100644 index 000000000..f13db39ce --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderData.java @@ -0,0 +1,16 @@ +package org.vitrivr.cineast.core.render.lwjgl.renderer; + +/** + * Data used of the Render Workflow + * {@link RenderWorker} + */ +public final class RenderData { + + public static final String MODEL = "MODEL"; + public static final String VECTOR = "VECTOR"; + public static final String VECTORS = "VECTORS"; + public static final String IMAGE = "IMAGE"; + public static final String WINDOWS_OPTIONS = "WINDOWS_OPTIONS"; + public static final String RENDER_OPTIONS = "RENDER_OPTIONS"; + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderJob.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderJob.java new file mode 100644 index 000000000..7710a1049 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderJob.java @@ -0,0 +1,147 @@ +package org.vitrivr.cineast.core.render.lwjgl.renderer; + +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.LinkedBlockingDeque; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joml.Vector3f; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; +import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; +import org.vitrivr.cineast.core.render.lwjgl.util.datatype.Variant; +import org.vitrivr.cineast.core.render.lwjgl.util.fsm.abstractworker.Job; +import org.vitrivr.cineast.core.render.lwjgl.util.fsm.abstractworker.JobControlCommand; +import org.vitrivr.cineast.core.render.lwjgl.util.fsm.abstractworker.JobType; +import org.vitrivr.cineast.core.render.lwjgl.util.fsm.model.Action; +import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; + +/** + * The RenderJob is a job which is responsible for rendering a model. + *

+ * This job extends the abstract class Job. + *

+ * It provides constructors for the different types of jobs. + * ORDER Job to render a model. + * COMMAND Job signals caller that the job is done or an error occurred. + * RESULT Job contains the result of the rendering. + */ +public class RenderJob extends Job { + + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Creates a new ORDER RenderJob with the given action sequence and data (containing the model to render). + */ + public RenderJob(BlockingDeque actions, Variant data) { + super(actions, data); + } + + /** + * Creates a new RESPONSE RenderJob with the rendered image. + */ + public RenderJob(Variant data) { + super(data); + } + + /** + * Creates a new CONTROL RenderJob with the given command. + */ + public RenderJob(JobControlCommand command) { + super(command); + } + + + /** + * Static method to create a standard render job. + *

+ * @see RenderJob#performStandardRenderJob(BlockingDeque, IModel, LinkedList, WindowOptions, RenderOptions) + */ + public static List performStandardRenderJob(BlockingDeque renderJobQueue, IModel model, double[][] cameraPositions, WindowOptions windowOptions, RenderOptions renderOptions) { + var cameraPositionVectors = new LinkedList(); + for (double[] cameraPosition : cameraPositions) { + assert cameraPosition.length == 3; + cameraPositionVectors.add(new Vector3f((float) cameraPosition[0], (float) cameraPosition[1], (float) cameraPosition[2])); + } + return performStandardRenderJob(renderJobQueue, model, cameraPositionVectors, windowOptions, renderOptions); + } + + /** + * Static method to create a standard render job. + *

+ *