Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deploy into production #289

Merged
merged 14 commits into from
Jun 25, 2024
Merged

Deploy into production #289

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
189e152
Bump org.mvnpm:lit from 3.1.3 to 3.1.4 in the mvnpm group
dependabot[bot] Jun 17, 2024
a66aef8
Merge pull request #284 from quarkusio/dependabot/maven/mvnpm-e14d2b6420
marko-bekhta Jun 17, 2024
9941eb4
Bump org.apache.maven.plugins:maven-surefire-plugin
dependabot[bot] Jun 17, 2024
d1ad998
Merge pull request #282 from quarkusio/dependabot/maven/maven-plugins…
marko-bekhta Jun 17, 2024
8db522d
Bump the quarkus group with 5 updates
dependabot[bot] Jun 17, 2024
95b62ca
Merge pull request #283 from quarkusio/dependabot/maven/quarkus-45f10…
marko-bekhta Jun 17, 2024
e898a19
Bump the quarkus group with 4 updates
dependabot[bot] Jun 24, 2024
fd97377
Merge pull request #287 from quarkusio/dependabot/maven/quarkus-d40cd…
yrodiere Jun 24, 2024
5dc5085
Use standalone mapper instead of the ORM one
marko-bekhta Jun 19, 2024
0381d0d
Bump the worker execute time from 60s to 180s
marko-bekhta Jun 19, 2024
ad5f18e
Add a copy of PojoMassIndexingLoggingMonitor for stream data
marko-bekhta Jun 24, 2024
f580f6e
Merge pull request #286 from marko-bekhta/convert-to-standalone-mapper
yrodiere Jun 24, 2024
50fea80
Fix env variable for ES hosts
marko-bekhta Jun 24, 2024
59533eb
Merge pull request #288 from marko-bekhta/fix/standalone-followup
yrodiere Jun 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 7 additions & 15 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,17 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
<quarkus.platform.group-id>io.quarkus</quarkus.platform.group-id>
<quarkus.version>3.11.1</quarkus.version>
<quarkus.version>3.12.0</quarkus.version>
<revision>999-SNAPSHOT</revision>
<skipITs>true</skipITs>
<surefire-plugin.version>3.2.5</surefire-plugin.version>
<surefire-plugin.version>3.3.0</surefire-plugin.version>
<test.jvm.args>-Xms2g -Xmx2g</test.jvm.args>
<version.docker.plugin>0.44.0</version.docker.plugin>
<version.formatter.plugin>2.24.1</version.formatter.plugin>
<version.impsort-maven-plugin>1.10.0</version.impsort-maven-plugin>
<!-- This version needs to match the version in src/main/docker/opensearch-custom.Dockerfile -->
<version.opensearch>2.14</version.opensearch>
<version.quarkus-web-bundler>1.5.2</version.quarkus-web-bundler>
<version.quarkus-web-bundler>1.6.0</version.quarkus-web-bundler>
</properties>
<dependencyManagement>
<dependencies>
Expand All @@ -58,7 +58,7 @@
<dependency>
<groupId>io.quarkiverse.jgit</groupId>
<artifactId>quarkus-jgit</artifactId>
<version>3.1.1</version>
<version>3.1.2</version>
</dependency>
<dependency>
<groupId>io.quarkiverse.githubapi</groupId>
Expand Down Expand Up @@ -93,7 +93,7 @@
<dependency>
<groupId>org.mvnpm</groupId>
<artifactId>lit</artifactId>
<version>3.1.3</version>
<version>3.1.4</version>
</dependency>
<dependency>
<groupId>org.mvnpm</groupId>
Expand Down Expand Up @@ -151,15 +151,7 @@
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-hibernate-search-orm-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-hibernate-orm</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-jdbc-h2</artifactId>
<artifactId>quarkus-hibernate-search-standalone-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
Expand Down Expand Up @@ -444,7 +436,7 @@
<dependency>
<groupId>org.mvnpm</groupId>
<artifactId>lit</artifactId>
<version>3.1.3</version>
<version>3.1.4</version>
<scope>provided</scope>
</dependency>
<dependency>
Expand Down
18 changes: 10 additions & 8 deletions src/main/java/io/quarkus/search/app/ReferenceService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import io.quarkus.cache.CacheResult;

import org.hibernate.search.engine.search.aggregation.AggregationKey;
import org.hibernate.search.mapper.orm.session.SearchSession;
import org.hibernate.search.mapper.pojo.standalone.mapping.SearchMapping;

import org.eclipse.microprofile.openapi.annotations.Operation;

Expand All @@ -37,7 +37,7 @@ public class ReferenceService {
Cache cache;

@Inject
SearchSession session;
SearchMapping searchMapping;

@GET
@Produces(MediaType.APPLICATION_JSON)
Expand Down Expand Up @@ -71,11 +71,13 @@ public void invalidateCaches() {
}

private List<String> listAllValues(String fieldName, Comparator<String> comparator) {
var aggKey = AggregationKey.<Map<String, Long>> of("versions");
var result = session.search(Guide.class)
.where(f -> f.matchAll())
.aggregation(aggKey, f -> f.terms().field(fieldName, String.class))
.fetch(0);
return result.aggregation(aggKey).keySet().stream().sorted(comparator).toList();
try (var session = searchMapping.createSession()) {
var aggKey = AggregationKey.<Map<String, Long>> of("versions");
var result = session.search(Guide.class)
.where(f -> f.matchAll())
.aggregation(aggKey, f -> f.terms().field(fieldName, String.class))
.fetch(0);
return result.aggregation(aggKey).keySet().stream().sorted(comparator).toList();
}
}
}
123 changes: 63 additions & 60 deletions src/main/java/io/quarkus/search/app/SearchService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,10 @@

import io.quarkus.runtime.LaunchMode;

import org.hibernate.Length;
import org.hibernate.search.engine.search.common.BooleanOperator;
import org.hibernate.search.engine.search.common.ValueConvert;
import org.hibernate.search.engine.search.predicate.dsl.SimpleQueryFlag;
import org.hibernate.search.mapper.orm.session.SearchSession;
import org.hibernate.search.mapper.pojo.standalone.mapping.SearchMapping;

import org.eclipse.microprofile.openapi.annotations.Operation;
import org.jboss.resteasy.reactive.RestQuery;
Expand All @@ -37,12 +36,13 @@
@Path("/")
public class SearchService {

private static final int NO_MATCH_SIZE = 32_600;
private static final int PAGE_SIZE = 50;
private static final long TOTAL_HIT_COUNT_THRESHOLD = 100;
private static final String MAX_FOR_PERF_MESSAGE = "{jakarta.validation.constraints.Max.message} for performance reasons";

@Inject
SearchSession session;
SearchMapping searchMapping;

public void init(@Observes Router router) {
if (LaunchMode.current().isDevOrTest()) {
Expand All @@ -67,65 +67,68 @@ public SearchResult<GuideSearchHit> search(@RestQuery @DefaultValue(QuarkusVersi
@RestQuery @DefaultValue("0") @Min(0) int page,
@RestQuery @DefaultValue("1") @Min(0) @Max(value = 10, message = MAX_FOR_PERF_MESSAGE) int contentSnippets,
@RestQuery @DefaultValue("100") @Min(0) @Max(value = 200, message = MAX_FOR_PERF_MESSAGE) int contentSnippetsLength) {
var result = session.search(Guide.class)
.select(f -> f.composite().from(
f.id(),
f.field("type"),
f.field("origin"),
f.highlight(language.addSuffix("title")),
f.highlight(language.addSuffix("summary")),
f.highlight(language.addSuffix("fullContent")).highlighter("highlighter_content"))
.asList(GuideSearchHit::new))
.where((f, root) -> {
// Match all documents by default
root.add(f.matchAll());
try (var session = searchMapping.createSession()) {
var result = session.search(Guide.class)
.select(f -> f.composite().from(
f.id(),
f.field("type"),
f.field("origin"),
f.highlight(language.addSuffix("title")),
f.highlight(language.addSuffix("summary")),
f.highlight(language.addSuffix("fullContent")).highlighter("highlighter_content"))
.asList(GuideSearchHit::new))
.where((f, root) -> {
// Match all documents by default
root.add(f.matchAll());

if (categories != null && !categories.isEmpty()) {
root.add(f.terms().field("categories").matchingAny(categories));
}
if (categories != null && !categories.isEmpty()) {
root.add(f.terms().field("categories").matchingAny(categories));
}

if (q != null && !q.isBlank()) {
root.add(f.bool().must(f.simpleQueryString()
.field(language.addSuffix("title")).boost(10.0f)
.field(language.addSuffix("topics")).boost(10.0f)
.field(language.addSuffix("keywords")).boost(10.0f)
.field(language.addSuffix("summary")).boost(5.0f)
.field(language.addSuffix("fullContent"))
.field(language.addSuffix("keywords_autocomplete")).boost(1.0f)
.field(language.addSuffix("title_autocomplete")).boost(1.0f)
.field(language.addSuffix("summary_autocomplete")).boost(0.5f)
.field(language.addSuffix("fullContent_autocomplete")).boost(0.1f)
.matching(q)
// See: https://github.com/elastic/elasticsearch/issues/39905#issuecomment-471578025
// while the issue is about stopwords the same problem is observed for synonyms on search-analyzer side.
// we also add phrase flag so that entire phrases could be searched as well, e.g.: "hibernate search"
.flags(SimpleQueryFlag.AND, SimpleQueryFlag.OR, SimpleQueryFlag.PHRASE)
.defaultOperator(BooleanOperator.AND))
.should(f.match().field("origin").matching("quarkus").boost(50.0f))
.should(f.not(f.match().field(language.addSuffix("topics"))
.matching("compatibility", ValueConvert.NO))
.boost(50.0f)));
}
})
// * Highlighters are going to use spans-with-classes so that we will have more control over styling the visual on the search results screen.
// * We give control to the caller on the content snippet length and the number of these fragments
// * No match size is there to make sure that we are still going to get the text even if the field didn't have a match in it.
// * The title in the Guide entity is `Length.LONG` long, so we use that as a max value for no-match size, but hopefully nobody writes a title that long...
.highlighter(
f -> f.unified().noMatchSize(Length.LONG).fragmentSize(0)
.orderByScore(true)
.numberOfFragments(1)
.tag("<span class=\"" + highlightCssClass + "\">", "</span>")
.boundaryScanner().sentence().end())
// * If there's no match in the full content we don't want to return anything.
// * Also content is really huge, so we want to only get small parts of the sentences. We are allowing caller to pick the number of sentences and their length:
.highlighter("highlighter_content",
f -> f.unified().noMatchSize(0).numberOfFragments(contentSnippets).fragmentSize(contentSnippetsLength))
.sort(f -> f.score().then().field(language.addSuffix("title_sort")))
.routing(QuarkusVersionAndLanguageRoutingBinder.searchKeys(version, language))
.totalHitCountThreshold(TOTAL_HIT_COUNT_THRESHOLD + (page + 1) * PAGE_SIZE)
.fetch(page * PAGE_SIZE, PAGE_SIZE);
return new SearchResult<>(result);
if (q != null && !q.isBlank()) {
root.add(f.bool().must(f.simpleQueryString()
.field(language.addSuffix("title")).boost(10.0f)
.field(language.addSuffix("topics")).boost(10.0f)
.field(language.addSuffix("keywords")).boost(10.0f)
.field(language.addSuffix("summary")).boost(5.0f)
.field(language.addSuffix("fullContent"))
.field(language.addSuffix("keywords_autocomplete")).boost(1.0f)
.field(language.addSuffix("title_autocomplete")).boost(1.0f)
.field(language.addSuffix("summary_autocomplete")).boost(0.5f)
.field(language.addSuffix("fullContent_autocomplete")).boost(0.1f)
.matching(q)
// See: https://github.com/elastic/elasticsearch/issues/39905#issuecomment-471578025
// while the issue is about stopwords the same problem is observed for synonyms on search-analyzer side.
// we also add phrase flag so that entire phrases could be searched as well, e.g.: "hibernate search"
.flags(SimpleQueryFlag.AND, SimpleQueryFlag.OR, SimpleQueryFlag.PHRASE)
.defaultOperator(BooleanOperator.AND))
.should(f.match().field("origin").matching("quarkus").boost(50.0f))
.should(f.not(f.match().field(language.addSuffix("topics"))
.matching("compatibility", ValueConvert.NO))
.boost(50.0f)));
}
})
// * Highlighters are going to use spans-with-classes so that we will have more control over styling the visual on the search results screen.
// * We give control to the caller on the content snippet length and the number of these fragments
// * No match size is there to make sure that we are still going to get the text even if the field didn't have a match in it.
// * The title in the Guide entity is `Length.LONG` long, so we use that as a max value for no-match size, but hopefully nobody writes a title that long...
.highlighter(
f -> f.unified().noMatchSize(NO_MATCH_SIZE).fragmentSize(0)
.orderByScore(true)
.numberOfFragments(1)
.tag("<span class=\"" + highlightCssClass + "\">", "</span>")
.boundaryScanner().sentence().end())
// * If there's no match in the full content we don't want to return anything.
// * Also content is really huge, so we want to only get small parts of the sentences. We are allowing caller to pick the number of sentences and their length:
.highlighter("highlighter_content",
f -> f.unified().noMatchSize(0).numberOfFragments(contentSnippets)
.fragmentSize(contentSnippetsLength))
.sort(f -> f.score().then().field(language.addSuffix("title_sort")))
.routing(QuarkusVersionAndLanguageRoutingBinder.searchKeys(version, language))
.totalHitCountThreshold(TOTAL_HIT_COUNT_THRESHOLD + (page + 1) * PAGE_SIZE)
.fetch(page * PAGE_SIZE, PAGE_SIZE);
return new SearchResult<>(result);
}
}

}
24 changes: 5 additions & 19 deletions src/main/java/io/quarkus/search/app/entity/Guide.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,12 @@
import java.util.Objects;
import java.util.Set;

import jakarta.persistence.ElementCollection;
import jakarta.persistence.Embedded;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.Id;
import jakarta.persistence.Transient;

import io.quarkus.search.app.hibernate.AnalysisConfigurer;
import io.quarkus.search.app.hibernate.I18nFullTextField;
import io.quarkus.search.app.hibernate.I18nKeywordField;
import io.quarkus.search.app.hibernate.InputProvider;
import io.quarkus.search.app.hibernate.InputProviderHtmlBodyTextBridge;
import io.quarkus.search.app.hibernate.URIType;

import org.hibernate.annotations.JavaType;
import org.hibernate.search.engine.backend.types.Aggregable;
import org.hibernate.search.engine.backend.types.Highlightable;
import org.hibernate.search.engine.backend.types.Projectable;
Expand All @@ -30,18 +20,19 @@
import org.hibernate.search.mapper.pojo.automaticindexing.ReindexOnUpdate;
import org.hibernate.search.mapper.pojo.bridge.mapping.annotation.RoutingBinderRef;
import org.hibernate.search.mapper.pojo.bridge.mapping.annotation.ValueBridgeRef;
import org.hibernate.search.mapper.pojo.loading.mapping.annotation.EntityLoadingBinderRef;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.Indexed;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.IndexingDependency;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.KeywordField;
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.SearchEntity;

@Entity
@SearchEntity(loadingBinder = @EntityLoadingBinderRef(name = "guideLoadingBinder"))
@Indexed(routingBinder = @RoutingBinderRef(type = QuarkusVersionAndLanguageRoutingBinder.class))
public class Guide {
@Id
@JavaType(URIType.class)
@DocumentId
public URI url;

@Enumerated(EnumType.STRING)
@KeywordField(searchable = Searchable.NO, aggregable = Aggregable.YES)
public Language language;

Expand All @@ -57,22 +48,18 @@ public class Guide {
@I18nFullTextField(highlightable = Highlightable.UNIFIED, termVector = TermVector.WITH_POSITIONS_OFFSETS, analyzerPrefix = AnalysisConfigurer.DEFAULT, searchAnalyzerPrefix = AnalysisConfigurer.DEFAULT_SEARCH)
@I18nFullTextField(name = "title_autocomplete", analyzerPrefix = AnalysisConfigurer.AUTOCOMPLETE, searchAnalyzerPrefix = AnalysisConfigurer.DEFAULT_SEARCH)
@I18nKeywordField(name = "title_sort", normalizerPrefix = AnalysisConfigurer.SORT, searchable = Searchable.NO, sortable = Sortable.YES)
@Embedded
public I18nData<String> title = new I18nData<>();

@I18nFullTextField(highlightable = Highlightable.UNIFIED, termVector = TermVector.WITH_POSITIONS_OFFSETS, analyzerPrefix = AnalysisConfigurer.DEFAULT, searchAnalyzerPrefix = AnalysisConfigurer.DEFAULT_SEARCH)
@I18nFullTextField(name = "summary_autocomplete", analyzerPrefix = AnalysisConfigurer.AUTOCOMPLETE, searchAnalyzerPrefix = AnalysisConfigurer.DEFAULT_SEARCH)
@Embedded
public I18nData<String> summary = new I18nData<>();

@I18nFullTextField(analyzerPrefix = AnalysisConfigurer.DEFAULT, searchAnalyzerPrefix = AnalysisConfigurer.DEFAULT_SEARCH)
@I18nFullTextField(name = "keywords_autocomplete", analyzerPrefix = AnalysisConfigurer.AUTOCOMPLETE, searchAnalyzerPrefix = AnalysisConfigurer.DEFAULT_SEARCH)
@Embedded
public I18nData<String> keywords = new I18nData<>();

@I18nFullTextField(name = "fullContent", valueBridge = @ValueBridgeRef(type = InputProviderHtmlBodyTextBridge.class), highlightable = Highlightable.UNIFIED, termVector = TermVector.WITH_POSITIONS_OFFSETS, analyzerPrefix = AnalysisConfigurer.DEFAULT, searchAnalyzerPrefix = AnalysisConfigurer.DEFAULT_SEARCH)
@I18nFullTextField(name = "fullContent_autocomplete", valueBridge = @ValueBridgeRef(type = InputProviderHtmlBodyTextBridge.class), analyzerPrefix = AnalysisConfigurer.AUTOCOMPLETE, searchAnalyzerPrefix = AnalysisConfigurer.DEFAULT_SEARCH)
@Transient
@IndexingDependency(reindexOnUpdate = ReindexOnUpdate.NO)
public I18nData<InputProvider> htmlFullContentProvider = new I18nData<>();

Expand All @@ -81,7 +68,6 @@ public class Guide {

@I18nFullTextField(name = "topics", analyzerPrefix = AnalysisConfigurer.DEFAULT, searchAnalyzerPrefix = AnalysisConfigurer.DEFAULT_SEARCH)
@I18nKeywordField(name = "topics_faceting", searchable = Searchable.YES, projectable = Projectable.YES, aggregable = Aggregable.YES)
@ElementCollection
public List<I18nData<String>> topics = List.of();

@KeywordField(name = "extensions_faceting", searchable = Searchable.YES, projectable = Projectable.YES, aggregable = Aggregable.YES)
Expand Down
11 changes: 0 additions & 11 deletions src/main/java/io/quarkus/search/app/entity/I18nData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,11 @@
import java.util.LinkedHashMap;
import java.util.Map;

import jakarta.persistence.Column;
import jakarta.persistence.Embeddable;

import org.hibernate.Length;

@Embeddable
public class I18nData<T> {
@Column(length = Length.LONG32)
private T en;
@Column(length = Length.LONG32)
private T es;
@Column(length = Length.LONG32)
private T pt;
@Column(length = Length.LONG32)
private T cn;
@Column(length = Length.LONG32)
private T ja;

public I18nData() {
Expand Down
Loading
Loading