diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6024-full-index-search-with-filter-parameters.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6024-full-index-search-with-filter-parameters.yaml new file mode 100644 index 000000000000..d62a971afac8 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_4_0/6024-full-index-search-with-filter-parameters.yaml @@ -0,0 +1,10 @@ +--- +type: fix +issue: 6024 +title: "Fixed a bug in search where requesting a count with HSearch indexing + and FilterParameter enabled and using the _filter parameter would result + in inaccurate results being returned. + This happened because the count query would use an incorrect set of parameters + to find the count, and the regular search when then try and ensure its results + matched the count query (which it couldn't because it had different parameters). +" diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java index f7e1764e0038..1a43905b8227 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/FulltextSearchSvcImpl.java @@ -34,6 +34,7 @@ import ca.uhn.fhir.jpa.dao.search.LastNOperation; import ca.uhn.fhir.jpa.model.dao.JpaPid; import ca.uhn.fhir.jpa.model.entity.ResourceTable; +import ca.uhn.fhir.jpa.model.search.ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams; import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData; import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions; @@ -141,17 +142,17 @@ public ExtendedHSearchIndexData extractLuceneIndexData( } @Override - public boolean supportsSomeOf(SearchParameterMap myParams) { - - // keep this in sync with the guts of doSearch + public boolean canUseHibernateSearch(String theResourceType, SearchParameterMap myParams) { boolean requiresHibernateSearchAccess = myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN(); + // we have to use it - _text and _content searches only use hibernate + if (requiresHibernateSearchAccess) { + return true; + } - requiresHibernateSearchAccess |= - myStorageSettings.isAdvancedHSearchIndexing() && myAdvancedIndexQueryBuilder.isSupportsSomeOf(myParams); - - return requiresHibernateSearchAccess; + return myStorageSettings.isAdvancedHSearchIndexing() + && myAdvancedIndexQueryBuilder.canUseHibernateSearch(theResourceType, myParams, mySearchParamRegistry); } @Override @@ -174,6 +175,7 @@ public ISearchQueryExecutor searchNotScrolled( } // keep this in sync with supportsSomeOf(); + @SuppressWarnings("rawtypes") private ISearchQueryExecutor doSearch( String theResourceType, SearchParameterMap theParams, @@ -208,6 +210,7 @@ private int getMaxFetchSize(SearchParameterMap theParams, Integer theMax) { return DEFAULT_MAX_NON_PAGED_SIZE; } + @SuppressWarnings("rawtypes") private SearchQueryOptionsStep getSearchQueryOptionsStep( String theResourceType, SearchParameterMap theParams, IResourcePersistentId theReferencingPid) { @@ -230,6 +233,7 @@ private int getMaxFetchSize(SearchParameterMap theParams, Integer theMax) { return query; } + @SuppressWarnings("rawtypes") private PredicateFinalStep buildWhereClause( SearchPredicateFactory f, String theResourceType, @@ -271,8 +275,12 @@ private PredicateFinalStep buildWhereClause( * Handle other supported parameters */ if (myStorageSettings.isAdvancedHSearchIndexing() && theParams.getEverythingMode() == null) { - myAdvancedIndexQueryBuilder.addAndConsumeAdvancedQueryClauses( - builder, theResourceType, theParams, mySearchParamRegistry); + ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams params = + new ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams(); + params.setSearchParamRegistry(mySearchParamRegistry) + .setResourceType(theResourceType) + .setSearchParameterMap(theParams); + myAdvancedIndexQueryBuilder.addAndConsumeAdvancedQueryClauses(builder, params); } // DROP EARLY HERE IF BOOL IS EMPTY? }); @@ -283,11 +291,13 @@ private SearchSession getSearchSession() { return Search.session(myEntityManager); } + @SuppressWarnings("rawtypes") private List convertLongsToResourcePersistentIds(List theLongPids) { return theLongPids.stream().map(JpaPid::fromId).collect(Collectors.toList()); } @Override + @SuppressWarnings({"rawtypes", "unchecked"}) public List everything( String theResourceName, SearchParameterMap theParams, @@ -336,6 +346,7 @@ public boolean isDisabled() { @Transactional() @Override + @SuppressWarnings("unchecked") public List search( String theResourceName, SearchParameterMap theParams, RequestDetails theRequestDetails) { validateHibernateSearchIsEnabled(); @@ -347,6 +358,7 @@ public List search( /** * Adapt our async interface to the legacy concrete List */ + @SuppressWarnings("rawtypes") private List toList(ISearchQueryExecutor theSearchResultStream, long theMaxSize) { return StreamSupport.stream(Spliterators.spliteratorUnknownSize(theSearchResultStream, 0), false) .map(JpaPid::fromId) @@ -384,6 +396,7 @@ private void ensureElastic() { } @Override + @SuppressWarnings("rawtypes") public List lastN(SearchParameterMap theParams, Integer theMaximumResults) { ensureElastic(); dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java index 6da76807b17c..6890b9bc26fa 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/IFulltextSearchSvc.java @@ -32,6 +32,7 @@ import java.util.Collection; import java.util.List; +@SuppressWarnings({"rawtypes"}) public interface IFulltextSearchSvc { /** @@ -79,11 +80,18 @@ List everything( ExtendedHSearchIndexData extractLuceneIndexData( IBaseResource theResource, ResourceIndexedSearchParams theNewParams); - boolean supportsSomeOf(SearchParameterMap myParams); + /** + * Returns true if the parameter map can be handled for hibernate search. + * We have to filter out any queries that might use search params + * we only know how to handle in JPA. + * - + * See {@link ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder#addAndConsumeAdvancedQueryClauses} + */ + boolean canUseHibernateSearch(String theResourceType, SearchParameterMap theParameterMap); /** * Re-publish the resource to the full-text index. - * + * - * During update, hibernate search only republishes the entity if it has changed. * During $reindex, we want to force the re-index. * diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchSearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchSearchBuilder.java index b5f2d42ff7d9..9f6b6c882616 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchSearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/ExtendedHSearchSearchBuilder.java @@ -20,6 +20,7 @@ package ca.uhn.fhir.jpa.dao.search; import ca.uhn.fhir.context.RuntimeSearchParam; +import ca.uhn.fhir.jpa.model.search.ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil; import ca.uhn.fhir.model.api.IQueryParameterType; @@ -34,6 +35,7 @@ import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.rest.param.UriParam; import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; +import ca.uhn.fhir.rest.server.util.ResourceSearchParams; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.collections4.CollectionUtils; @@ -44,6 +46,7 @@ import java.util.Collection; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import static ca.uhn.fhir.rest.api.Constants.PARAMQUALIFIER_MISSING; @@ -58,18 +61,56 @@ public class ExtendedHSearchSearchBuilder { */ public static final Set ourUnsafeSearchParmeters = Sets.newHashSet("_id", "_meta"); + /** + * Determine if ExtendedHibernateSearchBuilder can support this parameter + * @param theParamName param name + * @param theActiveParamsForResourceType active search parameters for the desired resource type + * @return whether or not this search parameter is supported in hibernate + */ + public boolean supportsSearchParameter(String theParamName, ResourceSearchParams theActiveParamsForResourceType) { + if (theActiveParamsForResourceType == null) { + return false; + } + if (ourUnsafeSearchParmeters.contains(theParamName)) { + return false; + } + if (!theActiveParamsForResourceType.containsParamName(theParamName)) { + return false; + } + return true; + } + /** * Are any of the queries supported by our indexing? + * - + * If not, do not use hibernate, because the results will + * be inaccurate and wrong. */ - public boolean isSupportsSomeOf(SearchParameterMap myParams) { - return myParams.getSort() != null - || myParams.getLastUpdated() != null - || myParams.entrySet().stream() - .filter(e -> !ourUnsafeSearchParmeters.contains(e.getKey())) - // each and clause may have a different modifier, so split down to the ORs - .flatMap(andList -> andList.getValue().stream()) - .flatMap(Collection::stream) - .anyMatch(this::isParamTypeSupported); + public boolean canUseHibernateSearch( + String theResourceType, SearchParameterMap myParams, ISearchParamRegistry theSearchParamRegistry) { + boolean canUseHibernate = true; + ResourceSearchParams resourceActiveSearchParams = theSearchParamRegistry.getActiveSearchParams(theResourceType); + for (String paramName : myParams.keySet()) { + // is this parameter supported? + if (!supportsSearchParameter(paramName, resourceActiveSearchParams)) { + canUseHibernate = false; + } else { + // are the parameter values supported? + canUseHibernate = + myParams.get(paramName).stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()) + .stream() + .anyMatch(this::isParamTypeSupported); + } + + // if not supported, don't use + if (!canUseHibernate) { + return false; + } + } + + return canUseHibernate; } /** @@ -166,86 +207,91 @@ private boolean isParamTypeSupported(IQueryParameterType param) { } public void addAndConsumeAdvancedQueryClauses( - ExtendedHSearchClauseBuilder builder, - String theResourceType, - SearchParameterMap theParams, - ISearchParamRegistry theSearchParamRegistry) { + ExtendedHSearchClauseBuilder theBuilder, + ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams theMethodParams) { + SearchParameterMap searchParameterMap = theMethodParams.getSearchParameterMap(); + String resourceType = theMethodParams.getResourceType(); + ISearchParamRegistry searchParamRegistry = theMethodParams.getSearchParamRegistry(); + // copy the keys to avoid concurrent modification error - ArrayList paramNames = compileParamNames(theParams); + ArrayList paramNames = compileParamNames(searchParameterMap); + ResourceSearchParams activeSearchParams = searchParamRegistry.getActiveSearchParams(resourceType); for (String nextParam : paramNames) { - if (ourUnsafeSearchParmeters.contains(nextParam)) { - continue; - } - RuntimeSearchParam activeParam = theSearchParamRegistry.getActiveSearchParam(theResourceType, nextParam); - if (activeParam == null) { + if (!supportsSearchParameter(nextParam, activeSearchParams)) { // ignore magic params handled in JPA continue; } + RuntimeSearchParam activeParam = activeSearchParams.get(nextParam); // NOTE - keep this in sync with isParamSupported() above. switch (activeParam.getParamType()) { case TOKEN: List> tokenTextAndOrTerms = - theParams.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_TOKEN_TEXT); - builder.addStringTextSearch(nextParam, tokenTextAndOrTerms); + searchParameterMap.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_TOKEN_TEXT); + theBuilder.addStringTextSearch(nextParam, tokenTextAndOrTerms); List> tokenUnmodifiedAndOrTerms = - theParams.removeByNameUnmodified(nextParam); - builder.addTokenUnmodifiedSearch(nextParam, tokenUnmodifiedAndOrTerms); + searchParameterMap.removeByNameUnmodified(nextParam); + theBuilder.addTokenUnmodifiedSearch(nextParam, tokenUnmodifiedAndOrTerms); break; case STRING: List> stringTextAndOrTerms = - theParams.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_TOKEN_TEXT); - builder.addStringTextSearch(nextParam, stringTextAndOrTerms); + searchParameterMap.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_TOKEN_TEXT); + theBuilder.addStringTextSearch(nextParam, stringTextAndOrTerms); - List> stringExactAndOrTerms = - theParams.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_STRING_EXACT); - builder.addStringExactSearch(nextParam, stringExactAndOrTerms); + List> stringExactAndOrTerms = searchParameterMap.removeByNameAndModifier( + nextParam, Constants.PARAMQUALIFIER_STRING_EXACT); + theBuilder.addStringExactSearch(nextParam, stringExactAndOrTerms); List> stringContainsAndOrTerms = - theParams.removeByNameAndModifier(nextParam, Constants.PARAMQUALIFIER_STRING_CONTAINS); - builder.addStringContainsSearch(nextParam, stringContainsAndOrTerms); + searchParameterMap.removeByNameAndModifier( + nextParam, Constants.PARAMQUALIFIER_STRING_CONTAINS); + theBuilder.addStringContainsSearch(nextParam, stringContainsAndOrTerms); - List> stringAndOrTerms = theParams.removeByNameUnmodified(nextParam); - builder.addStringUnmodifiedSearch(nextParam, stringAndOrTerms); + List> stringAndOrTerms = + searchParameterMap.removeByNameUnmodified(nextParam); + theBuilder.addStringUnmodifiedSearch(nextParam, stringAndOrTerms); break; case QUANTITY: - List> quantityAndOrTerms = theParams.removeByNameUnmodified(nextParam); - builder.addQuantityUnmodifiedSearch(nextParam, quantityAndOrTerms); + List> quantityAndOrTerms = + searchParameterMap.removeByNameUnmodified(nextParam); + theBuilder.addQuantityUnmodifiedSearch(nextParam, quantityAndOrTerms); break; case REFERENCE: - List> referenceAndOrTerms = theParams.removeByNameUnmodified(nextParam); - builder.addReferenceUnchainedSearch(nextParam, referenceAndOrTerms); + List> referenceAndOrTerms = + searchParameterMap.removeByNameUnmodified(nextParam); + theBuilder.addReferenceUnchainedSearch(nextParam, referenceAndOrTerms); break; case DATE: List> dateAndOrTerms = nextParam.equalsIgnoreCase("_lastupdated") - ? getLastUpdatedAndOrList(theParams) - : theParams.removeByNameUnmodified(nextParam); - builder.addDateUnmodifiedSearch(nextParam, dateAndOrTerms); + ? getLastUpdatedAndOrList(searchParameterMap) + : searchParameterMap.removeByNameUnmodified(nextParam); + theBuilder.addDateUnmodifiedSearch(nextParam, dateAndOrTerms); break; case COMPOSITE: - List> compositeAndOrTerms = theParams.removeByNameUnmodified(nextParam); + List> compositeAndOrTerms = + searchParameterMap.removeByNameUnmodified(nextParam); // RuntimeSearchParam only points to the subs by reference. Resolve here while we have // ISearchParamRegistry List subSearchParams = - JpaParamUtil.resolveCompositeComponentsDeclaredOrder(theSearchParamRegistry, activeParam); - builder.addCompositeUnmodifiedSearch(activeParam, subSearchParams, compositeAndOrTerms); + JpaParamUtil.resolveCompositeComponentsDeclaredOrder(searchParamRegistry, activeParam); + theBuilder.addCompositeUnmodifiedSearch(activeParam, subSearchParams, compositeAndOrTerms); break; case URI: List> uriUnmodifiedAndOrTerms = - theParams.removeByNameUnmodified(nextParam); - builder.addUriUnmodifiedSearch(nextParam, uriUnmodifiedAndOrTerms); + searchParameterMap.removeByNameUnmodified(nextParam); + theBuilder.addUriUnmodifiedSearch(nextParam, uriUnmodifiedAndOrTerms); break; case NUMBER: - List> numberUnmodifiedAndOrTerms = theParams.remove(nextParam); - builder.addNumberUnmodifiedSearch(nextParam, numberUnmodifiedAndOrTerms); + List> numberUnmodifiedAndOrTerms = searchParameterMap.remove(nextParam); + theBuilder.addNumberUnmodifiedSearch(nextParam, numberUnmodifiedAndOrTerms); break; default: diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/LastNOperation.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/LastNOperation.java index 1263bf027f5e..823a2893a5fd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/LastNOperation.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/search/LastNOperation.java @@ -22,6 +22,7 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.entity.StorageSettings; +import ca.uhn.fhir.jpa.model.search.ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper; import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; @@ -67,8 +68,12 @@ public List executeLastN(SearchParameterMap theParams, Integer theMaximumR b.must(f.match().field("myResourceType").matching(OBSERVATION_RES_TYPE)); ExtendedHSearchClauseBuilder builder = new ExtendedHSearchClauseBuilder(myFhirContext, myStorageSettings, b, f); - myExtendedHSearchSearchBuilder.addAndConsumeAdvancedQueryClauses( - builder, OBSERVATION_RES_TYPE, theParams.clone(), mySearchParamRegistry); + ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams params = + new ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams(); + params.setResourceType(OBSERVATION_RES_TYPE) + .setSearchParameterMap(theParams.clone()) + .setSearchParamRegistry(mySearchParamRegistry); + myExtendedHSearchSearchBuilder.addAndConsumeAdvancedQueryClauses(builder, params); })) .aggregation(observationsByCodeKey, f -> f.fromJson(lastNAggregation.toAggregation())) .fetch(0); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/search/ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/search/ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams.java new file mode 100644 index 000000000000..3bb80396a0fe --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/model/search/ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams.java @@ -0,0 +1,73 @@ +/*- + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2024 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package ca.uhn.fhir.jpa.model.search; + +import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchClauseBuilder; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; + +/** + * This is a parameter class for the + * {@link ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder#addAndConsumeAdvancedQueryClauses(ExtendedHSearchClauseBuilder, ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams)} + * method, so that we can keep the signature manageable (small) and allow for updates without breaking + * implementers so often. + */ +public class ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams { + /** + * Resource type + */ + private String myResourceType; + /** + * The registered search + */ + private SearchParameterMap mySearchParameterMap; + /** + * Search param registry + */ + private ISearchParamRegistry mySearchParamRegistry; + + public String getResourceType() { + return myResourceType; + } + + public ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams setResourceType(String theResourceType) { + myResourceType = theResourceType; + return this; + } + + public SearchParameterMap getSearchParameterMap() { + return mySearchParameterMap; + } + + public ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams setSearchParameterMap(SearchParameterMap theParams) { + mySearchParameterMap = theParams; + return this; + } + + public ISearchParamRegistry getSearchParamRegistry() { + return mySearchParamRegistry; + } + + public ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams setSearchParamRegistry( + ISearchParamRegistry theSearchParamRegistry) { + mySearchParamRegistry = theSearchParamRegistry; + return this; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaSearchFirstPageBundleProvider.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaSearchFirstPageBundleProvider.java index 9843034bee1f..ec4f7fa16e5c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaSearchFirstPageBundleProvider.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/PersistedJpaSearchFirstPageBundleProvider.java @@ -79,7 +79,7 @@ public List getResources( ourLog.trace("Done fetching search resource PIDs"); int countOfPids = pids.size(); - ; + int maxSize = Math.min(theToIndex - theFromIndex, countOfPids); thePageBuilder.setTotalRequestedResourcesFetched(countOfPids); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java index 076c62176607..a2bc4a4fcf5f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java @@ -95,6 +95,7 @@ import ca.uhn.fhir.util.StopWatch; import ca.uhn.fhir.util.StringUtil; import ca.uhn.fhir.util.UrlUtil; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Streams; import com.healthmarketscience.sqlbuilder.Condition; import jakarta.annotation.Nonnull; @@ -165,7 +166,7 @@ public class SearchBuilder implements ISearchBuilder { public static boolean myUseMaxPageSize50ForTest = false; protected final IInterceptorBroadcaster myInterceptorBroadcaster; protected final IResourceTagDao myResourceTagDao; - String myResourceName; + private String myResourceName; private final Class myResourceType; private final HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory; private final SqlObjectFactory mySqlBuilderFactory; @@ -206,6 +207,7 @@ public class SearchBuilder implements ISearchBuilder { /** * Constructor */ + @SuppressWarnings({"rawtypes", "unchecked"}) public SearchBuilder( IDao theDao, String theResourceName, @@ -240,6 +242,11 @@ public SearchBuilder( myIdHelperService = theIdHelperService; } + @VisibleForTesting + void setResourceName(String theName) { + myResourceName = theName; + } + @Override public void setMaxResultsToFetch(Integer theMaxResultsToFetch) { myMaxResultsToFetch = theMaxResultsToFetch; @@ -265,8 +272,6 @@ private void searchForIdsWithAndOr( attemptComboUniqueSpProcessing(theQueryStack, theParams, theRequest); } - SearchContainedModeEnum searchContainedMode = theParams.getSearchContainedMode(); - // Handle _id and _tag last, since they can typically be tacked onto a different parameter List paramNames = myParams.keySet().stream() .filter(t -> !t.equals(IAnyResource.SP_RES_ID)) @@ -399,7 +404,8 @@ private List createQuery( } if (fulltextExecutor == null) { - fulltextExecutor = SearchQueryExecutors.from(fulltextMatchIds); + fulltextExecutor = + SearchQueryExecutors.from(fulltextMatchIds != null ? fulltextMatchIds : new ArrayList<>()); } if (theSearchRuntimeDetails != null) { @@ -486,7 +492,7 @@ private boolean checkUseHibernateSearch() { return fulltextEnabled && myParams != null && myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE - && myFulltextSearchSvc.supportsSomeOf(myParams) + && myFulltextSearchSvc.canUseHibernateSearch(myResourceName, myParams) && myFulltextSearchSvc.supportsAllSortTerms(myResourceName, myParams); } @@ -538,8 +544,7 @@ private List queryHibernateSearchForEverythingPids(RequestDetails theReq pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue); } - List pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid, theRequestDetails); - return pids; + return myFulltextSearchSvc.everything(myResourceName, myParams, pid, theRequestDetails); } private void doCreateChunkedQueries( @@ -862,13 +867,8 @@ private void createSort(QueryStack theQueryStack, SortSpec theSort, SearchParame theQueryStack.addSortOnLastUpdated(ascending); } else { - - RuntimeSearchParam param = null; - - if (param == null) { - // do we have a composition param defined for the whole chain? - param = mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName()); - } + RuntimeSearchParam param = + mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName()); /* * If we have a sort like _sort=subject.name and we have an @@ -896,9 +896,7 @@ private void createSort(QueryStack theQueryStack, SortSpec theSort, SearchParame mySearchParamRegistry.getActiveSearchParam(myResourceName, referenceParam); if (outerParam == null) { throwInvalidRequestExceptionForUnknownSortParameter(myResourceName, referenceParam); - } - - if (outerParam.hasUpliftRefchain(targetParam)) { + } else if (outerParam.hasUpliftRefchain(targetParam)) { for (String nextTargetType : outerParam.getTargets()) { if (referenceParamTargetType != null && !referenceParamTargetType.equals(nextTargetType)) { continue; @@ -945,6 +943,9 @@ private void createSort(QueryStack theQueryStack, SortSpec theSort, SearchParame throwInvalidRequestExceptionForUnknownSortParameter(getResourceName(), paramName); } + // param will never be null here (the above line throws if it does) + // this is just to prevent the warning + assert param != null; if (isNotBlank(chainName) && param.getParamType() != RestSearchParameterTypeEnum.REFERENCE) { throw new InvalidRequestException( Msg.code(2285) + "Invalid chain, " + paramName + " is not a reference SearchParameter"); @@ -1121,11 +1122,15 @@ private void doLoadPids( resourceType, next, tagMap.get(next.getId()), theForHistoryOperation); } if (resource == null) { - ourLog.warn( - "Unable to find resource {}/{}/_history/{} in database", - next.getResourceType(), - next.getIdDt().getIdPart(), - next.getVersion()); + if (next != null) { + ourLog.warn( + "Unable to find resource {}/{}/_history/{} in database", + next.getResourceType(), + next.getIdDt().getIdPart(), + next.getVersion()); + } else { + ourLog.warn("Unable to find resource in database."); + } continue; } @@ -1196,7 +1201,6 @@ public void loadResourcesByPid( RequestDetails theDetails) { if (thePids.isEmpty()) { ourLog.debug("The include pids are empty"); - // return; } // Dupes will cause a crash later anyhow, but this is expensive so only do it @@ -1256,10 +1260,9 @@ private List loadResourcesFromElasticSearch(Collection th // only impl // to handle lastN? if (myStorageSettings.isAdvancedHSearchIndexing() && myStorageSettings.isStoreResourceInHSearchIndex()) { - List pidList = thePids.stream().map(pid -> (pid).getId()).collect(Collectors.toList()); + List pidList = thePids.stream().map(JpaPid::getId).collect(Collectors.toList()); - List resources = myFulltextSearchSvc.getResources(pidList); - return resources; + return myFulltextSearchSvc.getResources(pidList); } else if (!Objects.isNull(myParams) && myParams.isLastN()) { // legacy LastN implementation return myIElasticsearchSvc.getObservationResources(thePids); @@ -1344,7 +1347,7 @@ public Set loadIncludes(SearchBuilderLoadIncludesParameters theP for (Iterator iter = includes.iterator(); iter.hasNext(); ) { Include nextInclude = iter.next(); - if (nextInclude.isRecurse() == false) { + if (!nextInclude.isRecurse()) { iter.remove(); } @@ -1707,6 +1710,8 @@ private void loadCanonicalUrls( } /** + * Calls Performance Trace Hook + * @param request the request deatils * Sends a raw SQL query to the Pointcut for raw SQL queries. */ private void callRawSqlHookWithCurrentThreadQueries(RequestDetails request) { @@ -1890,7 +1895,7 @@ private void attemptComboUniqueSpProcessing( for (RuntimeSearchParam nextCandidate : candidateComboParams) { List nextCandidateParamNames = JpaParamUtil.resolveComponentParameters(mySearchParamRegistry, nextCandidate).stream() - .map(t -> t.getName()) + .map(RuntimeSearchParam::getName) .collect(Collectors.toList()); if (theParams.keySet().containsAll(nextCandidateParamNames)) { comboParam = nextCandidate; @@ -1902,7 +1907,7 @@ private void attemptComboUniqueSpProcessing( if (comboParam != null) { // Since we're going to remove elements below - theParams.values().forEach(nextAndList -> ensureSubListsAreWritable(nextAndList)); + theParams.values().forEach(this::ensureSubListsAreWritable); StringBuilder sb = new StringBuilder(); sb.append(myResourceName); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/builder/SearchBuilderTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/builder/SearchBuilderTest.java index fd8c3948478f..f6564c8c53ef 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/builder/SearchBuilderTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/search/builder/SearchBuilderTest.java @@ -39,7 +39,7 @@ class SearchBuilderTest { @BeforeEach public void beforeEach() { - mySearchBuilder.myResourceName = "QuestionnaireResponse"; + mySearchBuilder.setResourceName("QuestionnaireResponse"); when(myDaoRegistry.getRegisteredDaoTypes()).thenReturn(ourCtx.getResourceTypes()); } diff --git a/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java b/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java index 947638b7d115..1a8d4178c75c 100644 --- a/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java +++ b/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchWithElasticSearchIT.java @@ -26,6 +26,7 @@ import ca.uhn.fhir.jpa.search.BaseSourceSearchParameterTestCases; import ca.uhn.fhir.jpa.search.CompositeSearchParameterTestCases; import ca.uhn.fhir.jpa.search.QuantitySearchParameterTestCases; +import ca.uhn.fhir.jpa.search.builder.SearchBuilder; import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc; @@ -54,6 +55,9 @@ import ca.uhn.fhir.test.utilities.docker.RequiresDocker; import ca.uhn.fhir.validation.FhirValidator; import ca.uhn.fhir.validation.ValidationResult; +import ca.uhn.test.util.LogbackTestExtension; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; import jakarta.annotation.Nonnull; import jakarta.persistence.EntityManager; import org.hl7.fhir.instance.model.api.IBaseCoding; @@ -118,6 +122,7 @@ import static ca.uhn.fhir.rest.api.Constants.CHARSET_UTF8; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -168,6 +173,9 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest impl TestDaoSearch myTestDaoSearch; @RegisterExtension LogbackLevelOverrideExtension myLogbackLevelOverrideExtension = new LogbackLevelOverrideExtension(); + + @RegisterExtension + LogbackTestExtension myLogbackTestExtension = new LogbackTestExtension(); @Autowired @Qualifier("myCodeSystemDaoR4") private IFhirResourceDao myCodeSystemDao; @@ -742,19 +750,21 @@ private void assertObservationSearchMatches(String theMessage, String theSearch, */ @Test public void testDirectPathWholeResourceNotIndexedWorks() { + // setup + myLogbackLevelOverrideExtension.setLogLevel(SearchBuilder.class, Level.WARN); IIdType id1 = myTestDataBuilder.createObservation(List.of(myTestDataBuilder.withObservationCode("http://example.com/", "theCode"))); // set it after creating resource, so search doesn't find it in the index myStorageSettings.setStoreResourceInHSearchIndex(true); - myCaptureQueriesListener.clear(); - - List result = searchForFastResources("Observation?code=theCode"); - myCaptureQueriesListener.logSelectQueriesForCurrentThread(); + List result = searchForFastResources("Observation?code=theCode&_count=10&_total=accurate"); assertThat(result).hasSize(1); assertEquals(((Observation) result.get(0)).getIdElement().getIdPart(), id1.getIdPart()); - assertThat(myCaptureQueriesListener.getSelectQueriesForCurrentThread().size()).as("JPA search for IDs and for resources").isEqualTo(2); + + List events = myLogbackTestExtension.filterLoggingEventsWithPredicate(e -> e.getLevel() == Level.WARN); + assertFalse(events.isEmpty()); + assertTrue(events.stream().anyMatch(e -> e.getFormattedMessage().contains("Some resources were not found in index. Make sure all resources were indexed. Resorting to database search."))); // restore changed property JpaStorageSettings defaultConfig = new JpaStorageSettings(); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java index 14fb2bb0d7e7..a21183839e66 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java @@ -2120,6 +2120,8 @@ public void testValidateResourceContainingProfileDeclarationDoesntResolve() thro @SuppressWarnings("unused") @Test public void testFullTextSearch() throws Exception { + IParser parser = myFhirContext.newJsonParser(); + Observation obs1 = new Observation(); obs1.getCode().setText("Systolic Blood Pressure"); obs1.setStatus(ObservationStatus.FINAL); @@ -2131,13 +2133,21 @@ public void testFullTextSearch() throws Exception { obs2.setStatus(ObservationStatus.FINAL); obs2.setValue(new Quantity(81)); IIdType id2 = myObservationDao.create(obs2, mySrd).getId().toUnqualifiedVersionless(); + obs2.setId(id2); + + myStorageSettings.setAdvancedHSearchIndexing(true); HttpGet get = new HttpGet(myServerBase + "/Observation?_content=systolic&_pretty=true"); + get.addHeader("Content-Type", "application/json"); try (CloseableHttpResponse response = ourHttpClient.execute(get)) { assertEquals(200, response.getStatusLine().getStatusCode()); String responseString = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); ourLog.info(responseString); - assertThat(responseString).contains(id1.getIdPart()); + Bundle bundle = parser.parseResource(Bundle.class, responseString); + assertEquals(1, bundle.getTotal()); + Resource resource = bundle.getEntry().get(0).getResource(); + assertEquals("Observation", resource.fhirType()); + assertEquals(id1.getIdPart(), resource.getIdPart()); } } diff --git a/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java b/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java index d6c38965bbb0..81e03a692b3b 100644 --- a/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java +++ b/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/BaseJpaR5Test.java @@ -398,7 +398,7 @@ public abstract class BaseJpaR5Test extends BaseJpaTest implements ITestDataBuil private PerformanceTracingLoggingInterceptor myPerformanceTracingLoggingInterceptor; @Autowired - private DaoRegistry myDaoRegistry; + protected DaoRegistry myDaoRegistry; @Autowired private IBulkDataExportJobSchedulingHelper myBulkDataSchedulerHelper; diff --git a/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/UpliftedRefchainsAndChainedSortingR5Test.java b/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/UpliftedRefchainsAndChainedSortingR5Test.java index 575da4a5e31e..f852b379a76b 100644 --- a/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/UpliftedRefchainsAndChainedSortingR5Test.java +++ b/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/dao/r5/UpliftedRefchainsAndChainedSortingR5Test.java @@ -13,16 +13,17 @@ import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import ca.uhn.fhir.util.BundleBuilder; import ca.uhn.fhir.util.HapiExtensions; +import jakarta.annotation.Nonnull; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; -import org.hl7.fhir.r5.model.Composition; -import org.hl7.fhir.r5.model.IdType; import org.hl7.fhir.r5.model.Bundle; import org.hl7.fhir.r5.model.CodeType; +import org.hl7.fhir.r5.model.Composition; import org.hl7.fhir.r5.model.DateType; import org.hl7.fhir.r5.model.Encounter; import org.hl7.fhir.r5.model.Enumerations; import org.hl7.fhir.r5.model.Extension; +import org.hl7.fhir.r5.model.IdType; import org.hl7.fhir.r5.model.Identifier; import org.hl7.fhir.r5.model.Organization; import org.hl7.fhir.r5.model.Patient; @@ -35,7 +36,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; -import jakarta.annotation.Nonnull; import java.util.List; import static org.apache.commons.lang3.StringUtils.countMatches; diff --git a/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5Test.java b/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5Test.java index ba047dc73c58..3870487c81c9 100644 --- a/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5Test.java +++ b/hapi-fhir-jpaserver-test-r5/src/test/java/ca/uhn/fhir/jpa/provider/r5/ResourceProviderR5Test.java @@ -1,11 +1,26 @@ package ca.uhn.fhir.jpa.provider.r5; +import ca.uhn.fhir.batch2.jobs.reindex.ReindexAppCtx; +import ca.uhn.fhir.batch2.jobs.reindex.ReindexJobParameters; +import ca.uhn.fhir.batch2.model.JobInstanceStartRequest; +import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; +import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.model.api.Include; +import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.parser.StrictErrorHandler; import ca.uhn.fhir.rest.api.Constants; +import ca.uhn.fhir.rest.api.SearchTotalModeEnum; +import ca.uhn.fhir.rest.api.SortOrderEnum; +import ca.uhn.fhir.rest.api.SortSpec; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.api.server.SystemRequestDetails; import ca.uhn.fhir.rest.client.interceptor.CapturingInterceptor; +import ca.uhn.fhir.rest.param.StringParam; import ca.uhn.fhir.rest.server.exceptions.NotImplementedOperationException; import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException; import ca.uhn.fhir.util.BundleBuilder; @@ -24,13 +39,18 @@ import org.hl7.fhir.r5.model.CodeableConcept; import org.hl7.fhir.r5.model.Condition; import org.hl7.fhir.r5.model.DateTimeType; +import org.hl7.fhir.r5.model.Extension; import org.hl7.fhir.r5.model.MedicationRequest; +import org.hl7.fhir.r5.model.MedicinalProductDefinition; import org.hl7.fhir.r5.model.Observation; import org.hl7.fhir.r5.model.Observation.ObservationComponentComponent; import org.hl7.fhir.r5.model.Organization; import org.hl7.fhir.r5.model.Parameters; import org.hl7.fhir.r5.model.Patient; import org.hl7.fhir.r5.model.Quantity; +import org.hl7.fhir.r5.model.SearchParameter; +import org.hl7.fhir.r5.model.StringType; +import org.intellij.lang.annotations.Language; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -43,11 +63,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static org.apache.commons.lang3.StringUtils.leftPad; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; @SuppressWarnings("Duplicates") public class ResourceProviderR5Test extends BaseResourceProviderR5Test { @@ -206,7 +228,181 @@ public void testErroredSearchReturnsAppropriateResponse() { assertEquals(501, e.getStatusCode()); assertThat(e.getMessage()).contains("Some Failure Message"); } + } + + @Test + public void searchForNewerResources_fullTextSearchWithFilterAndCount_shouldReturnAccurateResults() { + IParser parser = myFhirContext.newJsonParser(); + int count = 10; + boolean presetFilterParameterEnabled = myStorageSettings.isFilterParameterEnabled(); + boolean presetAdvancedHSearchIndexing = myStorageSettings.isAdvancedHSearchIndexing(); + + try { + // fullTextSearch means Advanced Hibernate Search + myStorageSettings.setFilterParameterEnabled(true); + myStorageSettings.setAdvancedHSearchIndexing(true); + + // create custom search parameters - the _filter and _include are needed + { + @SuppressWarnings("unchecked") + IFhirResourceDao spDao = myDaoRegistry.getResourceDao("SearchParameter"); + SearchParameter sp; + + @Language("JSON") + String includeParam = """ + { + "resourceType": "SearchParameter", + "id": "9905463e-e817-4db0-9a3e-ff6aa3427848", + "meta": { + "versionId": "2", + "lastUpdated": "2024-03-28T12:53:57.874+00:00", + "source": "#7b34a4bfa42fe3ae" + }, + "title": "Medicinal Product Manfacturer", + "status": "active", + "publisher": "MOH-IDMS", + "code": "productmanufacturer", + "base": [ + "MedicinalProductDefinition" + ], + "type": "reference", + "expression": "MedicinalProductDefinition.operation.organization" + } + """; + sp = parser.parseResource(SearchParameter.class, includeParam); + spDao.create(sp, new SystemRequestDetails()); + sp = null; + @Language("JSON") + String filterParam = """ + { + "resourceType": "SearchParameter", + "id": "SEARCH-PARAMETER-MedicinalProductDefinition-SearchableString", + "meta": { + "versionId": "2", + "lastUpdated": "2024-03-27T19:20:25.200+00:00", + "source": "#384dd6bccaeafa6c" + }, + "url": "https://health.gov.on.ca/idms/fhir/SearchParameter/MedicinalProductDefinition-SearchableString", + "version": "1.0.0", + "name": "MedicinalProductDefinitionSearchableString", + "status": "active", + "publisher": "MOH-IDMS", + "description": "Search Parameter for the MedicinalProductDefinition Searchable String Extension", + "code": "MedicinalProductDefinitionSearchableString", + "base": [ + "MedicinalProductDefinition" + ], + "type": "string", + "expression": "MedicinalProductDefinition.extension('https://health.gov.on.ca/idms/fhir/StructureDefinition/SearchableExtraString')", + "target": [ + "MedicinalProductDefinition" + ] + } + """; + sp = parser.parseResource(SearchParameter.class, filterParam); + spDao.create(sp, new SystemRequestDetails()); + } + // create MedicinalProductDefinitions + MedicinalProductDefinition mdr; + { + @Language("JSON") + String mpdstr = """ + { + "resourceType": "MedicinalProductDefinition", + "id": "36fb418b-4b1f-414c-bbb1-731bc8744b93", + "meta": { + "versionId": "17", + "lastUpdated": "2024-06-10T16:52:23.907+00:00", + "source": "#3a309416d5f52c5b", + "profile": [ + "https://health.gov.on.ca/idms/fhir/StructureDefinition/IDMS_MedicinalProductDefinition" + ] + }, + "extension": [ + { + "url": "https://health.gov.on.ca/idms/fhir/StructureDefinition/SearchableExtraString", + "valueString": "zahidbrand0610-2up|genupuu|qwewqe2 111|11111115|DF other des|Biologic|Oncology|Private Label" + } + ], + "status": { + "coding": [ + { + "system": "http://hl7.org/fhir/ValueSet/publication-status", + "code": "active", + "display": "Active" + } + ] + }, + "name": [ + { + "productName": "zahidbrand0610-2up" + } + ] + } + """; + mdr = parser.parseResource(MedicinalProductDefinition.class, mpdstr); + } + IFhirResourceDao mdrdao = myDaoRegistry.getResourceDao(MedicinalProductDefinition.class); + + /* + * We actually want a bunch of non-matching resources in the db + * that won't match the filter before we get to the one that will. + * + * To this end, we're going to insert more than we plan + * on retrieving to ensure the _filter is being used in both the + * count query and the actual db hit + */ + List productNames = mdr.getName(); + mdr.setName(null); + List extensions = mdr.getExtension(); + mdr.setExtension(null); + // we need at least 10 of these; 20 should be good + for (int i = 0; i < 2 * count; i++) { + mdr.addName(new MedicinalProductDefinition.MedicinalProductDefinitionNameComponent("Product " + i)); + mdr.addExtension() + .setUrl("https://health.gov.on.ca/idms/fhir/StructureDefinition/SearchableExtraString") + .setValue(new StringType("Non-matching string " + i)); + mdrdao.create(mdr, new SystemRequestDetails()); + } + mdr.setName(productNames); + mdr.setExtension(extensions); + mdrdao.create(mdr, new SystemRequestDetails()); + + // do a reindex + ReindexJobParameters jobParameters = new ReindexJobParameters(); + jobParameters.setRequestPartitionId(RequestPartitionId.allPartitions()); + JobInstanceStartRequest request = new JobInstanceStartRequest(); + request.setJobDefinitionId(ReindexAppCtx.JOB_REINDEX); + request.setParameters(jobParameters); + Batch2JobStartResponse response = myJobCoordinator.startInstance(new SystemRequestDetails(), request); + + myBatch2JobHelper.awaitJobCompletion(response); + + // query like: + // MedicinalProductDefinition?_getpagesoffset=0&_count=10&_total=accurate&_sort:asc=name&status=active&_include=MedicinalProductDefinition:productmanufacturer&_filter=MedicinalProductDefinitionSearchableString%20co%20%22zah%22 + SearchParameterMap map = new SearchParameterMap(); + map.setCount(10); + map.setSearchTotalMode(SearchTotalModeEnum.ACCURATE); + map.setSort(new SortSpec().setOrder(SortOrderEnum.ASC).setParamName("name")); + map.setIncludes(Set.of( + new Include("MedicinalProductDefinition:productmanufacturer") + )); + map.add("_filter", new StringParam("MedicinalProductDefinitionSearchableString co \"zah\"")); + + // test + IBundleProvider result = mdrdao.search(map, new SystemRequestDetails()); + + // validate + // we expect to find our 1 matching resource + assertEquals(1, result.getAllResources().size()); + assertNotNull(result.size()); + assertEquals(1, result.size()); + } finally { + // reset values + myStorageSettings.setFilterParameterEnabled(presetFilterParameterEnabled); + myStorageSettings.setAdvancedHSearchIndexing(presetAdvancedHSearchIndexing); + } } @Test @@ -609,4 +805,5 @@ protected List toUnqualifiedVersionlessIds(Bundle theFound) { } return retVal; } + }