diff --git a/.travis.yml b/.travis.yml index 54c26374..28afd5ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ addons: packages: - oracle-java8-set-default before_install: - - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch restart + - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.5.4.deb && sudo dpkg -i --force-confnew elasticsearch-6.5.4.deb && sudo service elasticsearch restart - mysql -u root -e 'GRANT ALL ON `typo3_ci_ft%`.* TO travis@127.0.0.1;' language: php @@ -26,7 +26,6 @@ env: - typo3DatabaseUsername="travis" - typo3DatabasePassword="" matrix: - - TYPO3_VERSION="~7.6" - TYPO3_VERSION="~8.7" matrix: diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index e0bdeb94..bed8af75 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -1,4 +1,5 @@ indexerFactory->getIndexer($identifier)->indexAllDocuments(); - $this->outputLine($identifier . ' was indexed.'); - } catch (NoMatchingIndexerException $e) { - $this->outputLine('No indexer found for: ' . $identifier); - } + $this->executeForIdentifier($identifiers, function (IndexerInterface $indexer) { + $indexer->indexAllDocuments(); + $this->outputLine('Documents in index ' . $indexer->getIdentifier() . ' were indexed.'); + }); + } + + /** + * Will delete all indexed documents for the given identifiers. + * + * @param string $identifier Comma separated list of identifiers. + */ + public function deleteDocumentsCommand(string $identifiers) + { + $this->executeForIdentifier($identifiers, function (IndexerInterface $indexer) { + $indexer->deleteAllDocuments(); + $this->outputLine('Documents in index ' . $indexer->getIdentifier() . ' were deleted.'); + }); + } + + /** + * Will delete the index for given identifiers. + * + * @param string $identifier Comma separated list of identifiers. + */ + public function deleteCommand(string $identifiers = 'pages') + { + $this->executeForIdentifier($identifiers, function (IndexerInterface $indexer) { + $indexer->delete(); + $this->outputLine('Index ' . $indexer->getIdentifier() . ' was deleted.'); + }); } /** - * Will delete the given identifier. + * Executes the given callback method for each provided identifier. * - * @param string $identifier + * An indexer is created for each identifier, which is provided as first argument to the callback. */ - public function deleteCommand(string $identifier) + private function executeForIdentifier(string $identifiers, callable $callback) { - try { - $this->indexerFactory->getIndexer($identifier)->delete(); - $this->outputLine($identifier . ' was deleted.'); - } catch (NoMatchingIndexerException $e) { - $this->outputLine('No indexer found for: ' . $identifier); + foreach (GeneralUtility::trimExplode(',', $identifiers, true) as $identifier) { + try { + $callback($this->indexerFactory->getIndexer($identifier)); + } catch (NoMatchingIndexerException $e) { + $this->outputLine('No indexer found for: ' . $identifier . '.'); + } } } } diff --git a/Classes/Compatibility/ImplementationRegistrationService.php b/Classes/Compatibility/ImplementationRegistrationService.php deleted file mode 100644 index fa3faa83..00000000 --- a/Classes/Compatibility/ImplementationRegistrationService.php +++ /dev/null @@ -1,56 +0,0 @@ - - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Core\Utility\VersionNumberUtility; -use TYPO3\CMS\Extbase\Object\Container\Container; - -/** - * Register different concrete implementations, depending on current TYPO3 version. - * This way we can provide working implementations for multiple TYPO3 versions. - */ -class ImplementationRegistrationService -{ - public static function registerImplementations() - { - $container = GeneralUtility::makeInstance(Container::class); - if (VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) { - $container->registerImplementation( - \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, - \Codappix\SearchCore\Compatibility\TypoScriptService::class - ); - $container->registerImplementation( - \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class, - \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService::class - ); - } else { - $container->registerImplementation( - \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, - \Codappix\SearchCore\Compatibility\TypoScriptService76::class - ); - $container->registerImplementation( - \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class, - \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService76::class - ); - } - } -} diff --git a/Classes/Compatibility/TypoScriptService.php b/Classes/Compatibility/TypoScriptService.php deleted file mode 100644 index e5aa7886..00000000 --- a/Classes/Compatibility/TypoScriptService.php +++ /dev/null @@ -1,31 +0,0 @@ - - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -use TYPO3\CMS\Core\TypoScript\TypoScriptService as CoreTypoScriptService; - -/** - * Used since TYPO3 CMS 8.7. - */ -class TypoScriptService extends CoreTypoScriptService implements TypoScriptServiceInterface -{ - -} diff --git a/Classes/Compatibility/TypoScriptService76.php b/Classes/Compatibility/TypoScriptService76.php deleted file mode 100644 index 9df82ea4..00000000 --- a/Classes/Compatibility/TypoScriptService76.php +++ /dev/null @@ -1,31 +0,0 @@ - - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -use TYPO3\CMS\Extbase\Service\TypoScriptService as CoreTypoScriptService; - -/** - * Used before TYPO3 CMS 8.7. - */ -class TypoScriptService76 extends CoreTypoScriptService implements TypoScriptServiceInterface -{ - -} diff --git a/Classes/Compatibility/TypoScriptServiceInterface.php b/Classes/Compatibility/TypoScriptServiceInterface.php deleted file mode 100644 index 8e6a9e46..00000000 --- a/Classes/Compatibility/TypoScriptServiceInterface.php +++ /dev/null @@ -1,30 +0,0 @@ - - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * Allows to use DI configuration to switch concrete implementation, depending - * on current TYPO3 Version. - */ -interface TypoScriptServiceInterface -{ - public function convertPlainArrayToTypoScriptArray(array $plainArray); -} diff --git a/Classes/Configuration/ConfigurationContainer.php b/Classes/Configuration/ConfigurationContainer.php index 88aa414d..9917f12d 100644 --- a/Classes/Configuration/ConfigurationContainer.php +++ b/Classes/Configuration/ConfigurationContainer.php @@ -1,4 +1,5 @@ settings, $path); + $value = $this->getValueByPath($path); if ($value === null) { throw new InvalidArgumentException( @@ -77,6 +78,20 @@ public function get(string $path) */ public function getIfExists(string $path) { - return ArrayUtility::getValueByPath($this->settings, $path); + return $this->getValueByPath($path); + } + + /** + * @param string $path In dot notation. + * @return mixed|null Null if no entry was found. + */ + protected function getValueByPath(string $path) + { + try { + return ArrayUtility::getValueByPath($this->settings, $path, '.'); + } catch (\Exception $e) { + // Catch all exceptions to safely handle path retrieval + } + return null; } } diff --git a/Classes/Configuration/ConfigurationContainerInterface.php b/Classes/Configuration/ConfigurationContainerInterface.php index a5cb9617..6d2caeca 100644 --- a/Classes/Configuration/ConfigurationContainerInterface.php +++ b/Classes/Configuration/ConfigurationContainerInterface.php @@ -1,4 +1,5 @@ $entry) { diff --git a/Classes/Configuration/InvalidArgumentException.php b/Classes/Configuration/InvalidArgumentException.php index 1f5f8f06..449589d3 100644 --- a/Classes/Configuration/InvalidArgumentException.php +++ b/Classes/Configuration/InvalidArgumentException.php @@ -1,4 +1,5 @@ withType( $documentType, - function ($type) use ($document) { - $type->addDocument($this->documentFactory->getDocument($type->getName(), $document)); + function (Type $type, string $documentType) use ($document) { + $type->addDocument($this->documentFactory->getDocument($documentType, $document)); } ); } @@ -127,7 +130,7 @@ public function deleteDocument(string $documentType, string $identifier) try { $this->withType( $documentType, - function ($type) use ($identifier) { + function (Type $type, string $documentType) use ($identifier) { $type->deleteById($identifier); } ); @@ -139,12 +142,35 @@ function ($type) use ($identifier) { } } + public function deleteAllDocuments(string $documentType) + { + $this->deleteDocumentsByQuery($documentType, Query::create([ + 'query' => [ + 'term' => [ + 'search_document_type' => $documentType, + ], + ], + ])); + } + + public function deleteIndex(string $documentType) + { + try { + $this->indexFactory->getIndex($this->connection, $documentType)->delete(); + } catch (\InvalidArgumentException $e) { + $this->logger->notice( + 'Index did not exist, therefore was not deleted.', + [$documentType, $e] + ); + } + } + public function updateDocument(string $documentType, array $document) { $this->withType( $documentType, - function ($type) use ($document) { - $type->updateDocument($this->documentFactory->getDocument($type->getName(), $document)); + function (Type $type, string $documentType) use ($document) { + $type->updateDocument($this->documentFactory->getDocument($documentType, $document)); } ); } @@ -153,63 +179,55 @@ public function addDocuments(string $documentType, array $documents) { $this->withType( $documentType, - function ($type) use ($documents) { - $type->addDocuments($this->documentFactory->getDocuments($type->getName(), $documents)); + function (Type $type, string $documentType) use ($documents) { + $type->addDocuments($this->documentFactory->getDocuments($documentType, $documents)); } ); } - public function deleteIndex(string $documentType) + public function search(SearchRequestInterface $searchRequest): SearchResultInterface { - $index = $this->connection->getClient()->getIndex($this->indexFactory->getIndexName()); + $this->logger->debug('Search for', [$searchRequest->getSearchTerm()]); - if (! $index->exists()) { - $this->logger->notice( - 'Index did not exist, therefore was not deleted.', - [$documentType, $this->indexFactory->getIndexName()] - ); - return; - } + $search = new \Elastica\Search($this->connection->getClient()); + $search->addIndex($this->indexFactory->getIndexName()); + $search->setQuery($this->queryFactory->create($searchRequest)); - $index->delete(); + return $this->objectManager->get(SearchResult::class, $searchRequest, $search->search()); } /** * Execute given callback with Elastica Type based on provided documentType */ - protected function withType(string $documentType, callable $callback) + private function withType(string $documentType, callable $callback) { - $type = $this->getType($documentType); + $type = $this->typeFactory->getType($documentType); // TODO: Check whether it's to heavy to send it so often e.g. for every single document. // Perhaps add command controller to submit mapping?! // Also it's not possible to change mapping without deleting index first. // Mattes told about a solution. // So command looks like the best way so far, except we manage mattes solution. // Still then this should be done once. So perhaps singleton which tracks state and does only once? - $this->mappingFactory->getMapping($type)->send(); - $callback($type); + $this->mappingFactory->getMapping($documentType)->send(); + $callback($type, $documentType); $type->getIndex()->refresh(); } - public function search(SearchRequestInterface $searchRequest) : SearchResultInterface + private function deleteDocumentsByQuery(string $documentType, Query $query) { - $this->logger->debug('Search for', [$searchRequest->getSearchTerm()]); - - $search = new \Elastica\Search($this->connection->getClient()); - $search->addIndex($this->indexFactory->getIndexName()); - $search->setQuery($this->queryFactory->create($searchRequest)); - - return $this->objectManager->get(SearchResult::class, $searchRequest, $search->search()); - } + try { + $index = $this->indexFactory->getIndex($this->connection, $documentType); + $response = $index->deleteByQuery($query); - protected function getType(string $documentType) : \Elastica\Type - { - return $this->typeFactory->getType( - $this->indexFactory->getIndex( - $this->connection, - $documentType - ), - $documentType - ); + if ($response->getData()['deleted'] > 0) { + // Refresh index when delete query is invoked + $index->refresh(); + } + } catch (\InvalidArgumentException $e) { + $this->logger->notice( + 'Index did not exist, therefore items can not be deleted by query.', + [$documentType, $query->getQuery()] + ); + } } } diff --git a/Classes/Connection/Elasticsearch/Connection.php b/Classes/Connection/Elasticsearch/Connection.php index a5e7d0f7..b671f01e 100644 --- a/Classes/Connection/Elasticsearch/Connection.php +++ b/Classes/Connection/Elasticsearch/Connection.php @@ -1,4 +1,5 @@ elasticaClient; } diff --git a/Classes/Connection/Elasticsearch/DocumentFactory.php b/Classes/Connection/Elasticsearch/DocumentFactory.php index beb091a2..9b8fcf0c 100644 --- a/Classes/Connection/Elasticsearch/DocumentFactory.php +++ b/Classes/Connection/Elasticsearch/DocumentFactory.php @@ -1,4 +1,5 @@ getDocument($documentType, $document); diff --git a/Classes/Connection/Elasticsearch/Facet.php b/Classes/Connection/Elasticsearch/Facet.php index 27de0769..44646976 100644 --- a/Classes/Connection/Elasticsearch/Facet.php +++ b/Classes/Connection/Elasticsearch/Facet.php @@ -1,4 +1,5 @@ */ - protected $options = []; + protected $options; public function __construct(string $name, array $aggregation, ConfigurationContainerInterface $configuration) { @@ -60,12 +60,12 @@ public function __construct(string $name, array $aggregation, ConfigurationConta } } - public function getName() : string + public function getName(): string { return $this->name; } - public function getField() : string + public function getField(): string { return $this->field; } @@ -75,7 +75,7 @@ public function getField() : string * * @return array */ - public function getOptions() : array + public function getOptions(): array { $this->initOptions(); @@ -84,10 +84,11 @@ public function getOptions() : array protected function initOptions() { - if ($this->options !== []) { + if (is_array($this->options)) { return; } + $this->options = []; foreach ($this->buckets as $bucket) { $this->options[$bucket['key']] = new FacetOption($bucket); } diff --git a/Classes/Connection/Elasticsearch/FacetOption.php b/Classes/Connection/Elasticsearch/FacetOption.php index 6b544bb9..d661c4c0 100644 --- a/Classes/Connection/Elasticsearch/FacetOption.php +++ b/Classes/Connection/Elasticsearch/FacetOption.php @@ -1,4 +1,5 @@ name = $bucket['key']; @@ -49,17 +47,17 @@ public function __construct(array $bucket) $this->count = $bucket['doc_count']; } - public function getName() : string + public function getName(): string { return $this->name; } - public function getDisplayName() : string + public function getDisplayName(): string { return $this->displayName; } - public function getCount() : int + public function getCount(): int { return $this->count; } diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index d76edc19..718342fa 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -1,4 +1,5 @@ configuration->get('connections.elasticsearch.index'); } /** - * Get an index bases on TYPO3 table name. + * @throws \InvalidArgumentException If index does not exist. */ - public function getIndex(Connection $connection, string $documentType) : \Elastica\Index + public function getIndex(Connection $connection, string $documentType): \Elastica\Index { $index = $connection->getClient()->getIndex($this->getIndexName()); if ($index->exists() === false) { - $config = $this->getConfigurationFor($documentType); - $this->logger->debug(sprintf('Create index %s.', $documentType), [$documentType, $config]); - $index->create($config); - $this->logger->debug(sprintf('Created index %s.', $documentType), [$documentType]); + throw new \InvalidArgumentException('The requested index does not exist.', 1546173102); } return $index; } - protected function getConfigurationFor(string $documentType) : array + public function createIndex(Connection $connection, string $documentType): \Elastica\Index + { + $index = $connection->getClient()->getIndex($this->getIndexName()); + + if ($index->exists() === true) { + return $index; + } + + $config = $this->getConfigurationFor($documentType); + $this->logger->debug(sprintf('Create index %s.', $documentType), [$documentType, $config]); + $index->create($config); + $this->logger->debug(sprintf('Created index %s.', $documentType), [$documentType]); + + return $index; + } + + protected function getConfigurationFor(string $documentType): array { try { $configuration = $this->configuration->get('indexing.' . $documentType . '.index'); @@ -108,7 +120,7 @@ protected function getConfigurationFor(string $documentType) : array } } - protected function prepareAnalyzerConfiguration(array $analyzer) : array + protected function prepareAnalyzerConfiguration(array $analyzer): array { $fieldsToExplode = ['char_filter', 'filter', 'word_list']; diff --git a/Classes/Connection/Elasticsearch/MappingFactory.php b/Classes/Connection/Elasticsearch/MappingFactory.php index 38825566..a1edcff4 100644 --- a/Classes/Connection/Elasticsearch/MappingFactory.php +++ b/Classes/Connection/Elasticsearch/MappingFactory.php @@ -1,4 +1,5 @@ configuration = $configuration; + $this->typeFactory = $typeFactory; } /** * Get an mapping based on type. */ - public function getMapping(\Elastica\Type $type) : \Elastica\Type\Mapping + public function getMapping(string $documentType): \Elastica\Type\Mapping { + $type = $this->typeFactory->getType($documentType); + $mapping = new \Elastica\Type\Mapping(); $mapping->setType($type); - $configuration = $this->getConfiguration($type->getName()); - if (isset($configuration['_all'])) { - $mapping->setAllField($configuration['_all']); - unset($configuration['_all']); - } + $configuration = $this->getConfiguration($documentType); $mapping->setProperties($configuration); return $mapping; } - protected function getConfiguration(string $identifier) : array + private function getConfiguration(string $identifier): array { try { return $this->configuration->get('indexing.' . $identifier . '.mapping'); diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 0f814632..cd3db0a1 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -1,4 +1,5 @@ */ - protected $facets = []; + protected $facets; /** * @var array */ - protected $results = []; + protected $results; /** * For Iterator interface. @@ -77,7 +78,7 @@ public function __construct( /** * @return array */ - public function getResults() : array + public function getResults(): array { $this->initResults(); @@ -89,24 +90,25 @@ public function getResults() : array * * @return array */ - public function getFacets() : array + public function getFacets(): array { $this->initFacets(); return $this->facets; } - public function getCurrentCount() : int + public function getCurrentCount(): int { return $this->result->count(); } protected function initResults() { - if ($this->results !== []) { + if (is_array($this->results)) { return; } + $this->results = []; foreach ($this->result->getResults() as $result) { $this->results[] = new ResultItem($result->getData(), $result->getParam('_type')); } @@ -114,27 +116,50 @@ protected function initResults() protected function initFacets() { - if ($this->facets !== [] || !$this->result->hasAggregations()) { + if (is_array($this->facets)) { + return; + } + + $this->facets = []; + + if ($this->result->hasAggregations() === false) { return; } foreach ($this->result->getAggregations() as $aggregationName => $aggregation) { - $this->facets[$aggregationName] = $this->objectManager->get(Facet::class, $aggregationName, $aggregation); + $this->facets[$aggregationName] = $this->objectManager->get( + Facet::class, + $aggregationName, + $aggregation + ); } } - // Countable - Interface + /** + * Countable - Interface + * + * @return integer + */ public function count() { return $this->result->getTotalHits(); } - // Iterator - Interface + /** + * Iterator - Interface + * + * @return mixed + */ public function current() { return $this->getResults()[$this->position]; } + /** + * Iterator - Interface + * + * @return mixed + */ public function next() { ++$this->position; @@ -142,21 +167,37 @@ public function next() return $this->current(); } + /** + * Iterator - Interface + * + * @return int|mixed + */ public function key() { return $this->position; } + /** + * Iterator - Interface + * + * @return bool + */ public function valid() { return isset($this->getResults()[$this->position]); } + /** + * Iterator - Interface + */ public function rewind() { $this->position = 0; } + /** + * @return SearchRequestInterface + */ public function getQuery() { return $this->searchRequest; diff --git a/Classes/Connection/Elasticsearch/TypeFactory.php b/Classes/Connection/Elasticsearch/TypeFactory.php index e84cdd07..8e52eb25 100644 --- a/Classes/Connection/Elasticsearch/TypeFactory.php +++ b/Classes/Connection/Elasticsearch/TypeFactory.php @@ -1,4 +1,5 @@ indexFactory = $indexFactory; + $this->connection = $connection; + } + + public function getType(string $documentType): \Elastica\Type { - return $index->getType($documentType); + $index = $this->indexFactory->createIndex($this->connection, $documentType); + return $index->getType('document'); } } diff --git a/Classes/Connection/FacetInterface.php b/Classes/Connection/FacetInterface.php index 3ec549d5..de22f62c 100644 --- a/Classes/Connection/FacetInterface.php +++ b/Classes/Connection/FacetInterface.php @@ -1,4 +1,5 @@ */ - public function getOptions() : array; + public function getOptions(): array; } diff --git a/Classes/Connection/FacetOptionInterface.php b/Classes/Connection/FacetOptionInterface.php index bf998db1..7c07a5d6 100644 --- a/Classes/Connection/FacetOptionInterface.php +++ b/Classes/Connection/FacetOptionInterface.php @@ -1,4 +1,5 @@ */ - public function getFacets() : array; + public function getFacets(): array; /** * Workaround for paginate widget support which will * use the request to build another search. - * - * @return void */ public function setConnection(ConnectionInterface $connection); /** * Workaround for paginate widget support which will * use the request to build another search. - * - * @return void */ - public function setSearchService(SearchService $searchService); + public function setSearchService(SearchServiceInterface $searchService); } diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index be698fae..692dae31 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -1,4 +1,5 @@ */ - public function getResults() : array; + public function getResults(): array; /** * Return all facets, if any. * * @return array */ - public function getFacets() : array; + public function getFacets(): array; /** * Returns the number of results in current result */ - public function getCurrentCount() : int; + public function getCurrentCount(): int; } diff --git a/Classes/Controller/SearchController.php b/Classes/Controller/SearchController.php index e484fcda..1cdd7992 100644 --- a/Classes/Controller/SearchController.php +++ b/Classes/Controller/SearchController.php @@ -1,4 +1,5 @@ searchService = $searchService; parent::__construct(); } + /** + * Allow dynamic properties in search request + */ public function initializeSearchAction() { - if (isset($this->settings['searching']['mode']) && $this->settings['searching']['mode'] === 'filter' + if (isset($this->settings['searching']['mode']) + && $this->settings['searching']['mode'] === 'filter' && $this->request->hasArgument('searchRequest') === false ) { $this->request->setArguments(array_merge( $this->request->getArguments(), - [ - 'searchRequest' => $this->objectManager->get(SearchRequest::class), - ] + ['searchRequest' => $this->objectManager->get(SearchRequest::class)] )); } if ($this->arguments->hasArgument('searchRequest')) { $this->arguments->getArgument('searchRequest')->getPropertyMappingConfiguration() - ->allowAllProperties() - ; + ->allowAllProperties(); } } /** - * Process a search and deliver original request and result to view. - * - * @param null|SearchRequest $searchRequest + * Display results and deliver original request and result to view. */ public function searchAction(SearchRequest $searchRequest = null) { diff --git a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php index 2b7f5539..d0425a90 100644 --- a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php +++ b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php @@ -1,4 +1,5 @@ typoScriptService = $typoScriptService; } - public function processData(array $data, array $configuration) : array + public function processData(array $data, array $configuration): array { $dataProcessor = GeneralUtility::makeInstance($configuration['_dataProcessor']); $contentObjectRenderer = GeneralUtility::makeInstance(ContentObjectRenderer::class); diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php index 159701fc..7cdde685 100644 --- a/Classes/DataProcessing/CopyToProcessor.php +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -1,4 +1,5 @@ addArray($target, $from); } else { - $target[] = (string) $from; + $target[] = (string)$from; } $target = array_filter($target); @@ -54,7 +55,7 @@ protected function addArray(array &$target, array $from) continue; } - $target[] = (string) $value; + $target[] = (string)$value; } } } diff --git a/Classes/DataProcessing/GeoPointProcessor.php b/Classes/DataProcessing/GeoPointProcessor.php index 971e2c47..15376ee2 100644 --- a/Classes/DataProcessing/GeoPointProcessor.php +++ b/Classes/DataProcessing/GeoPointProcessor.php @@ -1,4 +1,5 @@ isApplyable($record, $configuration)) { + if (!$this->isApplyable($record, $configuration)) { return $record; } $record[$configuration['to']] = [ - 'lat' => (float) $record[$configuration['lat']], - 'lon' => (float) $record[$configuration['lon']], + 'lat' => (float)$record[$configuration['lat']], + 'lon' => (float)$record[$configuration['lon']], ]; return $record; } - protected function isApplyable(array $record, array $configuration) : bool + protected function isApplyable(array $record, array $configuration): bool { if (!isset($record[$configuration['lat']]) || !is_numeric($record[$configuration['lat']]) diff --git a/Classes/DataProcessing/ProcessorInterface.php b/Classes/DataProcessing/ProcessorInterface.php index f7513f2b..d421fad8 100644 --- a/Classes/DataProcessing/ProcessorInterface.php +++ b/Classes/DataProcessing/ProcessorInterface.php @@ -1,4 +1,5 @@ initializeConfiguration($configuration); + /** @var TcaTableServiceInterface $tcaTableService */ $tcaTableService = $this->objectManager->get( TcaTableServiceInterface::class, $configuration['_table'] @@ -84,7 +86,7 @@ protected function initializeConfiguration(array &$configuration) $configuration['excludeFields'] = GeneralUtility::trimExplode(',', $configuration['excludeFields'], true); } - protected function getRecordToProcess(array $record, array $configuration) : array + protected function getRecordToProcess(array $record, array $configuration): array { if ($configuration['excludeFields'] === []) { return $record; diff --git a/Classes/Database/Doctrine/Join.php b/Classes/Database/Doctrine/Join.php index df1a8c64..928d9632 100644 --- a/Classes/Database/Doctrine/Join.php +++ b/Classes/Database/Doctrine/Join.php @@ -1,4 +1,5 @@ condition = $condition; } - public function getTable() : string + public function getTable(): string { return $this->table; } - public function getCondition() : string + public function getCondition(): string { return $this->condition; } diff --git a/Classes/Database/Doctrine/Where.php b/Classes/Database/Doctrine/Where.php index 6586b8aa..b02feb63 100644 --- a/Classes/Database/Doctrine/Where.php +++ b/Classes/Database/Doctrine/Where.php @@ -1,4 +1,5 @@ parameters = $parameters; } - public function getStatement() : string + public function getStatement(): string { return $this->statement; } - public function getParameters() : array + public function getParameters(): array { return $this->parameters; } diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index b22c0110..0172c12c 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -1,4 +1,5 @@ logger = $logManager->getLogger(__CLASS__); } - public function setIdentifier(string $identifier) - { - $this->identifier = $identifier; - } - public function __construct(ConnectionInterface $connection, ConfigurationContainerInterface $configuration) { $this->connection = $connection; @@ -96,13 +92,13 @@ public function indexDocument(string $identifier) { $this->logger->info('Start indexing single record.', [$identifier]); try { - $record = $this->getRecord((int) $identifier); + $record = $this->getRecord((int)$identifier); $this->prepareRecord($record); $this->connection->addDocument($this->getDocumentName(), $record); } catch (NoRecordFoundException $e) { $this->logger->info('Could not index document. Try to delete it therefore.', [$e->getMessage()]); - $this->connection->deleteDocument($this->getDocumentName(), $identifier); + $this->connection->deleteDocument($this->getDocumentName(), $this->getDocumentIdentifier($identifier)); } $this->logger->info('Finish indexing'); } @@ -114,13 +110,22 @@ public function delete() $this->logger->info('Finish deletion.'); } - protected function getRecordGenerator() : \Generator + public function deleteAllDocuments() + { + $this->logger->info('Start deletion of indexed documents.'); + $this->connection->deleteAllDocuments($this->getDocumentName()); + $this->logger->info('Finish deletion.'); + } + + protected function getRecordGenerator(): \Generator { $offset = 0; $limit = $this->getLimit(); - while (($records = $this->getRecords($offset, $limit)) !== []) { - yield $records; + while (($records = $this->getRecords($offset, $limit)) !== null) { + if (!empty($records)) { + yield $records; + } $offset += $limit; } } @@ -134,10 +139,20 @@ protected function prepareRecord(array &$record) } catch (InvalidArgumentException $e) { // Nothing to do. } - + $this->generateSearchIdentifiers($record); $this->handleAbstract($record); } + protected function generateSearchIdentifiers(array &$record) + { + if (!isset($record['search_document'])) { + $record['search_document_type'] = $this->getDocumentName(); + } + if (!isset($record['search_identifier']) && isset($record['uid'])) { + $record['search_identifier'] = $this->getDocumentIdentifier($record['uid']); + } + } + protected function handleAbstract(array &$record) { $record['search_abstract'] = ''; @@ -148,8 +163,9 @@ protected function handleAbstract(array &$record) $this->configuration->get('indexing.' . $this->identifier . '.abstractFields') ); if ($fieldsToUse === []) { - return; + throw new InvalidArgumentException('No fields to use', 1538487209251); } + foreach ($fieldsToUse as $fieldToUse) { if (isset($record[$fieldToUse]) && trim($record[$fieldToUse])) { $record['search_abstract'] = trim($record[$fieldToUse]); @@ -157,28 +173,40 @@ protected function handleAbstract(array &$record) } } } catch (InvalidArgumentException $e) { - return; + // Nothing to do. } } /** * Returns the limit to use to fetch records. */ - protected function getLimit() : int + protected function getLimit(): int { // TODO: Make configurable. return 50; } + public function setIdentifier(string $identifier) + { + $this->identifier = $identifier; + } + + public function getIdentifier(): string + { + return $this->identifier; + } + /** - * @return array|null + * @return array|null Nullable when no items are found and execution should be stopped */ abstract protected function getRecords(int $offset, int $limit); /** * @throws NoRecordFoundException If record could not be found. */ - abstract protected function getRecord(int $identifier) : array; + abstract protected function getRecord(int $identifier): array; + + abstract protected function getDocumentName(): string; - abstract protected function getDocumentName() : string; + abstract protected function getDocumentIdentifier($identifier): string; } diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index 668111d9..860a4cac 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -1,4 +1,5 @@ buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier); @@ -71,9 +71,9 @@ public function getIndexer(string $identifier) : IndexerInterface } /** - * @throws NoMatchingIndexer + * @throws NoMatchingIndexerException */ - protected function buildIndexer(string $indexerClass, string $identifier) : IndexerInterface + protected function buildIndexer(string $indexerClass, string $identifier): IndexerInterface { $indexer = null; if (is_subclass_of($indexerClass, TcaIndexer\PagesIndexer::class) diff --git a/Classes/Domain/Index/IndexerInterface.php b/Classes/Domain/Index/IndexerInterface.php index 4acfb289..4071ee85 100644 --- a/Classes/Domain/Index/IndexerInterface.php +++ b/Classes/Domain/Index/IndexerInterface.php @@ -1,4 +1,5 @@ tcaTableService->getRecord($identifier); - if ($record === []) { throw new NoRecordFoundException( 'Record could not be fetched from database: "' . $identifier . '". Perhaps record is not active.', @@ -84,8 +84,13 @@ protected function getRecord(int $identifier) : array return $record; } - protected function getDocumentName() : string + protected function getDocumentName(): string { return $this->tcaTableService->getTableName(); } + + protected function getDocumentIdentifier($identifier): string + { + return $this->getDocumentName() . '-' . $identifier; + } } diff --git a/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php b/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php index bc2036fc..dda027a5 100644 --- a/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php +++ b/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php @@ -1,4 +1,5 @@ contentTableService instanceof TcaTableService) { $queryBuilder = $this->contentTableService->getQuery(); @@ -123,17 +118,17 @@ protected function fetchContentForPage(int $uid) : array ]; } - protected function getContentElementImages(int $uidOfContentElement) : array + protected function getContentElementImages(int $uidOfContentElement): array { return $this->fetchSysFileReferenceUids($uidOfContentElement, 'tt_content', 'image'); } - protected function fetchMediaForPage(int $uid) : array + protected function fetchMediaForPage(int $uid): array { return $this->fetchSysFileReferenceUids($uid, 'pages', 'media'); } - protected function fetchSysFileReferenceUids(int $uid, string $tablename, string $fieldname) : array + protected function fetchSysFileReferenceUids(int $uid, string $tablename, string $fieldname): array { $imageRelationUids = []; $imageRelations = $this->fileRepository->findByRelation($tablename, $fieldname, $uid); @@ -145,7 +140,7 @@ protected function fetchSysFileReferenceUids(int $uid, string $tablename, string return $imageRelationUids; } - protected function getContentFromContentElement(array $contentElement) : string + protected function getContentFromContentElement(array $contentElement): string { $content = ''; diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index e992577b..3a33796d 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -1,4 +1,5 @@ getLanguageUidColumn()])) { - $record[$column] = (int) $record[$column]; + $record[$column] = (int)$record[$column]; continue; } @@ -74,25 +75,24 @@ protected function resolveValue($value, array $tcaColumn) return []; } - protected function isRelation(array &$config) : bool + protected function isRelation(array &$config): bool { return isset($config['foreign_table']) || (isset($config['renderType']) && !in_array($config['renderType'], ['selectSingle', 'inputDateTime'])) - || (isset($config['internal_type']) && strtolower($config['internal_type']) === 'db') - ; + || (isset($config['internal_type']) && strtolower($config['internal_type']) === 'db'); } - protected function resolveForeignDbValue(string $value) : array + protected function resolveForeignDbValue(string $value): array { return array_map('trim', explode(';', $value)); } - protected function resolveInlineValue(string $value) : array + protected function resolveInlineValue(string $value): array { return array_map('trim', explode(',', $value)); } - protected function getUtilityForMode() : string + protected function getUtilityForMode(): string { if (TYPO3_MODE === 'BE') { return BackendUtility::class; @@ -104,12 +104,13 @@ protected function getUtilityForMode() : string protected function getColumnValue(array $record, string $column, TcaTableServiceInterface $service): string { $utility = GeneralUtility::makeInstance($this->getUtilityForMode()); - return $utility::getProcessedValueExtra( + $value = $utility::getProcessedValueExtra( $service->getTableName(), $column, $record[$column], 0, $record['uid'] - ) ?? ''; + ); + return $value ? (string)$value : ''; } } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 1330e228..8e5cde9c 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -1,4 +1,5 @@ configuration = $configuration; } - public function getTableName() : string + public function getTableName(): string { return $this->tableName; } - public function getTableClause() : string + public function getTableClause(): string { return $this->tableName; } - public function getRecords(int $offset, int $limit) : array + public function getRecords(int $offset, int $limit): array { $records = $this->getQuery() ->setFirstResult($offset) @@ -123,7 +122,7 @@ public function getRecords(int $offset, int $limit) : array return $records ?: []; } - public function getRecord(int $identifier) : array + public function getRecord(int $identifier): array { $query = $this->getQuery(); $query = $query->andWhere($this->getTableName() . '.uid = ' . $identifier); @@ -137,22 +136,19 @@ public function filterRecordsByRootLineBlacklist(array &$records) $records = array_filter( $records, function ($record) { - return ! $this->isRecordBlacklistedByRootline($record); + return !$this->isRecordBlacklistedByRootline($record); } ); } public function prepareRecord(array &$record) { - if (isset($record['uid']) && !isset($record['search_identifier'])) { - $record['search_identifier'] = $record['uid']; - } if (isset($record[$this->tca['ctrl']['label']]) && !isset($record['search_title'])) { $record['search_title'] = $record[$this->tca['ctrl']['label']]; } } - protected function getWhereClause() : Where + protected function getWhereClause(): Where { $parameters = []; $whereClause = $this->getSystemWhereClause(); @@ -174,17 +170,16 @@ protected function getWhereClause() : Where return new Where($whereClause, $parameters); } - protected function getFields() : array + protected function getFields(): array { $fields = array_merge( - ['uid','pid'], + ['uid', 'pid'], array_filter( array_keys($this->tca['columns']), function ($columnName) { return !$this->isSystemField($columnName) && !$this->isUserField($columnName) - && !$this->isPassthroughField($columnName) - ; + && !$this->isPassthroughField($columnName); } ) ); @@ -197,7 +192,7 @@ function ($columnName) { return $fields; } - protected function getJoins() : array + protected function getJoins(): array { if ($this->tableName === 'pages') { return []; @@ -212,31 +207,36 @@ protected function getJoins() : array * Generate SQL for TYPO3 as a system, to make sure only available records * are fetched. */ - protected function getSystemWhereClause() : string + protected function getSystemWhereClause(): string { $whereClause = '1=1' . BackendUtility::BEenableFields($this->tableName) . BackendUtility::deleteClause($this->tableName) - . ' AND pages.no_search = 0' - ; + . ' AND pages.no_search = 0'; if ($this->tableName !== 'pages') { $whereClause .= BackendUtility::BEenableFields('pages') - . BackendUtility::deleteClause('pages') - ; + . BackendUtility::deleteClause('pages'); } return $whereClause; } - protected function isSystemField(string $columnName) : bool + protected function isSystemField(string $columnName): bool { $systemFields = [ // Versioning fields, // https://docs.typo3.org/typo3cms/TCAReference/Reference/Ctrl/Index.html#versioningws - 't3ver_oid', 't3ver_id', 't3ver_label', 't3ver_wsid', - 't3ver_state', 't3ver_stage', 't3ver_count', 't3ver_tstamp', - 't3ver_move_id', 't3ver_swapmode', + 't3ver_oid', + 't3ver_id', + 't3ver_label', + 't3ver_wsid', + 't3ver_state', + 't3ver_stage', + 't3ver_count', + 't3ver_tstamp', + 't3ver_move_id', + 't3ver_swapmode', $this->tca['ctrl']['transOrigDiffSourceField'], $this->tca['ctrl']['cruser_id'], $this->tca['ctrl']['fe_cruser_id'], @@ -247,22 +247,28 @@ protected function isSystemField(string $columnName) : bool return in_array($columnName, $systemFields); } - protected function isUserField(string $columnName) : bool + /** + * @throws InvalidArgumentException If column does not exist. + */ + protected function isUserField(string $columnName): bool { $config = $this->getColumnConfig($columnName); return isset($config['type']) && $config['type'] === 'user'; } - protected function isPassthroughField(string $columnName) : bool + /** + * @throws InvalidArgumentException If column does not exist. + */ + protected function isPassthroughField(string $columnName): bool { $config = $this->getColumnConfig($columnName); return isset($config['type']) && $config['type'] === 'passthrough'; } /** - * @throws InvalidArgumentException + * @throws InvalidArgumentException If column does not exist. */ - public function getColumnConfig(string $columnName) : array + public function getColumnConfig(string $columnName): array { if (!isset($this->tca['columns'][$columnName])) { throw new InvalidArgumentException( @@ -274,7 +280,7 @@ public function getColumnConfig(string $columnName) : array return $this->tca['columns'][$columnName]['config']; } - public function getLanguageUidColumn() : string + public function getLanguageUidColumn(): string { if (!isset($this->tca['ctrl']['languageField'])) { return ''; @@ -291,7 +297,7 @@ public function getLanguageUidColumn() : string * line exist, is page inside a recycler, is inherited start- endtime * excluded, etc. */ - protected function isRecordBlacklistedByRootline(array &$record) : bool + protected function isRecordBlacklistedByRootline(array &$record): bool { $pageUid = $record['pid']; if ($this->tableName === 'pages') { @@ -325,9 +331,9 @@ protected function isRecordBlacklistedByRootline(array &$record) : bool } if ($pageInRootLine['extendToSubpages'] && ( - ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time()) - || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time()) - )) { + ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time()) + || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time()) + )) { $this->logger->info( sprintf( 'Record %u is black listed due to configured timing of parent page %u.', @@ -346,9 +352,9 @@ protected function isRecordBlacklistedByRootline(array &$record) : bool /** * Checks whether any page uids are black listed. */ - protected function isBlackListedRootLineConfigured() : bool + protected function isBlackListedRootLineConfigured(): bool { - return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'); + return (bool)$this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'); } /** @@ -356,15 +362,16 @@ protected function isBlackListedRootLineConfigured() : bool * * @return array */ - protected function getBlackListedRootLine() : array + protected function getBlackListedRootLine(): array { return GeneralUtility::intExplode( ',', - $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist') + $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'), + true ); } - public function getQuery() : QueryBuilder + public function getQuery(): QueryBuilder { $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($this->getTableName()); $where = $this->getWhereClause(); @@ -381,7 +388,7 @@ public function getQuery() : QueryBuilder return $query; } - protected function getDatabaseConnection() : ConnectionPool + protected function getDatabaseConnection(): ConnectionPool { return GeneralUtility::makeInstance(ConnectionPool::class); } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php deleted file mode 100644 index 4445d6d0..00000000 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php +++ /dev/null @@ -1,378 +0,0 @@ - - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; -use Codappix\SearchCore\Domain\Index\IndexingException; -use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException; -use TYPO3\CMS\Backend\Utility\BackendUtility; -use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Core\Utility\RootlineUtility; -use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; - -/** - * Encapsulate logik related to TCA configuration. - */ -class TcaTableService76 implements TcaTableServiceInterface -{ - /** - * TCA for current table. - * !REFERENCE! To save memory. - * @var array - */ - protected $tca; - - /** - * @var string - */ - protected $tableName; - - /** - * @var ConfigurationContainerInterface - */ - protected $configuration; - - /** - * @var \TYPO3\CMS\Core\Log\Logger - */ - protected $logger; - - /** - * @var ObjectManagerInterface - */ - protected $objectManager; - - /** - * Inject log manager to get concrete logger from it. - * - * @param \TYPO3\CMS\Core\Log\LogManager $logManager - */ - public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager) - { - $this->logger = $logManager->getLogger(__CLASS__); - } - - /** - * @param ObjectManagerInterface $objectManager - */ - public function injectObjectManager(ObjectManagerInterface $objectManager) - { - $this->objectManager = $objectManager; - } - - /** - * @param string $tableName - * @param ConfigurationContainerInterface $configuration - */ - public function __construct( - $tableName, - ConfigurationContainerInterface $configuration - ) { - if (!isset($GLOBALS['TCA'][$tableName])) { - throw new IndexingException( - 'Table "' . $tableName . '" is not configured in TCA.', - IndexingException::CODE_UNKOWN_TCA_TABLE - ); - } - - $this->tableName = $tableName; - $this->tca = &$GLOBALS['TCA'][$this->tableName]; - $this->configuration = $configuration; - } - - public function getTableName() : string - { - return $this->tableName; - } - - public function getTableClause() : string - { - if ($this->tableName === 'pages') { - return $this->tableName; - } - - return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid'; - } - - public function getRecords(int $offset, int $limit) : array - { - $records = $this->getConnection()->exec_SELECTgetRows( - $this->getFields(), - $this->getTableClause(), - $this->getWhereClause(), - '', - '', - (int) $offset . ',' . (int) $limit - ); - - return $records ?: []; - } - - public function getRecord(int $identifier) : array - { - $record = $this->getConnection()->exec_SELECTgetSingleRow( - $this->getFields(), - $this->getTableClause(), - $this->getWhereClause() - . ' AND ' . $this->getTableName() . '.uid = ' . (int) $identifier - ); - - return $record ?: []; - } - - public function filterRecordsByRootLineBlacklist(array &$records) - { - $records = array_filter( - $records, - function ($record) { - return ! $this->isRecordBlacklistedByRootline($record); - } - ); - } - - public function prepareRecord(array &$record) - { - if (isset($record['uid']) && !isset($record['search_identifier'])) { - $record['search_identifier'] = $record['uid']; - } - if (isset($record[$this->tca['ctrl']['label']]) && !isset($record['search_title'])) { - $record['search_title'] = $record[$this->tca['ctrl']['label']]; - } - } - - public function getWhereClause() : string - { - $whereClause = '1=1' - . BackendUtility::BEenableFields($this->tableName) - . BackendUtility::deleteClause($this->tableName) - . ' AND pages.no_search = 0' - ; - - if ($this->tableName !== 'pages') { - $whereClause .= BackendUtility::BEenableFields('pages') - . BackendUtility::deleteClause('pages') - ; - } - - $userDefinedWhere = $this->configuration->getIfExists( - 'indexing.' . $this->getTableName() . '.additionalWhereClause' - ); - if (is_string($userDefinedWhere)) { - $whereClause .= ' AND ' . $userDefinedWhere; - } - if ($this->isBlacklistedRootLineConfigured()) { - $whereClause .= ' AND pages.uid NOT IN (' - . implode(',', $this->getBlacklistedRootLine()) - . ')' - . ' AND pages.pid NOT IN (' - . implode(',', $this->getBlacklistedRootLine()) - . ')'; - } - - $this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]); - return $whereClause; - } - - public function getFields() : string - { - $fields = array_merge( - ['uid','pid'], - array_filter( - array_keys($this->tca['columns']), - function ($columnName) { - return !$this->isSystemField($columnName) - && !$this->isUserField($columnName) - && !$this->isPassthroughField($columnName) - ; - } - ) - ); - - foreach ($fields as $key => $field) { - $fields[$key] = $this->tableName . '.' . $field; - } - - $this->logger->debug('Generated fields.', [$this->tableName, $fields]); - return implode(',', $fields); - } - - - /** - * Generate SQL for TYPO3 as a system, to make sure only available records - * are fetched. - */ - protected function getSystemWhereClause() : string - { - $whereClause = '1=1' - . BackendUtility::BEenableFields($this->tableName) - . BackendUtility::deleteClause($this->tableName) - . ' AND pages.no_search = 0' - ; - - if ($this->tableName !== 'pages') { - $whereClause .= BackendUtility::BEenableFields('pages') - . BackendUtility::deleteClause('pages') - ; - } - - return $whereClause; - } - - protected function isSystemField(string $columnName) : bool - { - $systemFields = [ - // Versioning fields, - // https://docs.typo3.org/typo3cms/TCAReference/Reference/Ctrl/Index.html#versioningws - 't3ver_oid', 't3ver_id', 't3ver_label', 't3ver_wsid', - 't3ver_state', 't3ver_stage', 't3ver_count', 't3ver_tstamp', - 't3ver_move_id', 't3ver_swapmode', - $this->tca['ctrl']['transOrigDiffSourceField'], - $this->tca['ctrl']['cruser_id'], - $this->tca['ctrl']['fe_cruser_id'], - $this->tca['ctrl']['fe_crgroup_id'], - $this->tca['ctrl']['origUid'], - ]; - - return in_array($columnName, $systemFields); - } - - protected function isUserField(string $columnName) : bool - { - $config = $this->getColumnConfig($columnName); - return isset($config['type']) && $config['type'] === 'user'; - } - - protected function isPassthroughField(string $columnName) : bool - { - $config = $this->getColumnConfig($columnName); - return isset($config['type']) && $config['type'] === 'passthrough'; - } - - /** - * @throws InvalidArgumentException - */ - public function getColumnConfig(string $columnName) : array - { - if (!isset($this->tca['columns'][$columnName])) { - throw new InvalidArgumentException( - 'Column does not exist.', - InvalidArgumentException::COLUMN_DOES_NOT_EXIST - ); - } - - return $this->tca['columns'][$columnName]['config']; - } - - public function getLanguageUidColumn() : string - { - if (!isset($this->tca['ctrl']['languageField'])) { - return ''; - } - - return $this->tca['ctrl']['languageField']; - } - - /** - * Checks whether the given record was blacklisted by root line. - * This can be configured by typoscript as whole root lines can be black listed. - * - * Also further TYPO3 mechanics are taken into account. Does a valid root - * line exist, is page inside a recycler, is inherited start- endtime - * excluded, etc. - */ - protected function isRecordBlacklistedByRootline(array &$record) : bool - { - $pageUid = $record['pid']; - if ($this->tableName === 'pages') { - $pageUid = $record['uid']; - } - - try { - $rootline = $this->objectManager->get(RootlineUtility::class, $pageUid)->get(); - } catch (\RuntimeException $e) { - $this->logger->notice( - sprintf('Could not fetch rootline for page %u, because: %s', $pageUid, $e->getMessage()), - [$record, $e] - ); - return true; - } - - foreach ($rootline as $pageInRootLine) { - // Check configured black list if present. - if ($this->isBlackListedRootLineConfigured() - && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine()) - ) { - $this->logger->info( - sprintf( - 'Record %u is black listed due to configured root line configuration of page %u.', - $record['uid'], - $pageInRootLine['uid'] - ), - [$record, $pageInRootLine] - ); - return true; - } - - if ($pageInRootLine['extendToSubpages'] && ( - ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time()) - || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time()) - )) { - $this->logger->info( - sprintf( - 'Record %u is black listed due to configured timing of parent page %u.', - $record['uid'], - $pageInRootLine['uid'] - ), - [$record, $pageInRootLine] - ); - return true; - } - } - - return false; - } - - /** - * Checks whether any page uids are black listed. - */ - protected function isBlackListedRootLineConfigured() : bool - { - return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'); - } - - /** - * Get the list of black listed root line page uids. - * - * @return array - */ - protected function getBlackListedRootLine() : array - { - return GeneralUtility::intExplode( - ',', - $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist') - ); - } - - protected function getConnection() : \TYPO3\CMS\Core\Database\DatabaseConnection - { - return $GLOBALS['TYPO3_DB']; - } -} diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php b/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php index e4ee7c5e..d7184d2c 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php @@ -1,4 +1,5 @@ config = $config; } - public function getIdentifier() : string + public function getIdentifier(): string { return $this->identifier; } - public function getConfig() : array + public function getConfig(): array { return $this->config; } diff --git a/Classes/Domain/Model/QueryResultInterfaceStub.php b/Classes/Domain/Model/QueryResultInterfaceStub.php index 960fd40c..6cd6f4fe 100644 --- a/Classes/Domain/Model/QueryResultInterfaceStub.php +++ b/Classes/Domain/Model/QueryResultInterfaceStub.php @@ -1,4 +1,5 @@ type = $type; } - public function getType() : string + public function getType(): string { return $this->type; } - public function getPlainData() : array + public function getPlainData(): array { return $this->data; } @@ -60,11 +61,17 @@ public function offsetGet($offset) return $this->data[$offset]; } + /** + * @throws \BadMethodCallException + */ public function offsetSet($offset, $value) { throw new \BadMethodCallException('It\'s not possible to change the search result.', 1499179077); } + /** + * @throws \BadMethodCallException + */ public function offsetUnset($offset) { throw new \BadMethodCallException('It\'s not possible to change the search result.', 1499179077); diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 3c888712..183463e0 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -1,4 +1,5 @@ query = $query; } - public function getQuery() : string + public function getQuery(): string { return $this->query; } - public function getSearchTerm() : string + public function getSearchTerm(): string { return $this->query; } /** + * Type hint necessary for extbase! + * * @param array $filter */ public function setFilter(array $filter) { - $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, ''); - $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter); + $filter = ArrayUtility::removeArrayEntryByValue($filter, ''); + $this->filter = ArrayUtility::removeEmptyElementsRecursively($filter); } - public function hasFilter() : bool + public function hasFilter(): bool { return count($this->filter) > 0; } - public function getFilter() : array + public function getFilter(): array { return $this->filter; } @@ -117,7 +121,7 @@ public function addFacet(FacetRequestInterface $facet) /** * Returns all configured facets to fetch in this search request. */ - public function getFacets() : array + public function getFacets(): array { return $this->facets; } @@ -131,22 +135,26 @@ public function setConnection(ConnectionInterface $connection) $this->connection = $connection; } - public function setSearchService(SearchService $searchService) + public function setSearchService(SearchServiceInterface $searchService) { $this->searchService = $searchService; } // Extbase QueryInterface // Current implementation covers only paginate widget support. + + /** + * @throws \InvalidArgumentException + */ public function execute($returnRawQueryResult = false) { - if (! ($this->connection instanceof ConnectionInterface)) { + if (!($this->connection instanceof ConnectionInterface)) { throw new \InvalidArgumentException( 'Connection was not set before, therefore execute can not work. Use `setConnection` before.', 1502197732 ); } - if (! ($this->searchService instanceof SearchService)) { + if (!($this->searchService instanceof SearchServiceInterface)) { throw new \InvalidArgumentException( 'SearchService was not set before, therefore execute can not work. Use `setSearchService` before.', 1520325175 @@ -158,14 +166,14 @@ public function execute($returnRawQueryResult = false) public function setLimit($limit) { - $this->limit = (int) $limit; + $this->limit = (int)$limit; return $this; } public function setOffset($offset) { - $this->offset = (int) $offset; + $this->offset = (int)$offset; return $this; } @@ -180,116 +188,199 @@ public function getOffset() return $this->offset; } + /** + * Used, e.g. by caching to determine identifier. + */ + public function __sleep() + { + return [ + 'query', + 'filter', + 'facets', + 'offset', + 'limit', + ]; + } + + /** + * @throws \BadMethodCallException + */ public function getSource() { throw new \BadMethodCallException('Method is not implemented yet.', 1502196146); } + /** + * @throws \BadMethodCallException + */ public function setOrderings(array $orderings) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196163); } + /** + * @throws \BadMethodCallException + */ public function matching($constraint) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196197); } + /** + * @throws \BadMethodCallException + */ public function logicalAnd($constraint1) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196166); } + /** + * @throws \BadMethodCallException + */ public function logicalOr($constraint1) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196198); } + /** + * @throws \BadMethodCallException + */ public function logicalNot(\TYPO3\CMS\Extbase\Persistence\Generic\Qom\ConstraintInterface $constraint) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196166); } + /** + * @throws \BadMethodCallException + */ public function equals($propertyName, $operand, $caseSensitive = true) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196199); } + /** + * @throws \BadMethodCallException + */ public function like($propertyName, $operand, $caseSensitive = true) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196167); } + /** + * @throws \BadMethodCallException + */ public function contains($propertyName, $operand) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196200); } + /** + * @throws \BadMethodCallException + */ public function in($propertyName, $operand) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196167); } + /** + * @throws \BadMethodCallException + */ public function lessThan($propertyName, $operand) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196201); } + /** + * @throws \BadMethodCallException + */ public function lessThanOrEqual($propertyName, $operand) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196168); } + /** + * @throws \BadMethodCallException + */ public function greaterThan($propertyName, $operand) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196202); } + /** + * @throws \BadMethodCallException + */ public function greaterThanOrEqual($propertyName, $operand) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196168); } + /** + * @throws \BadMethodCallException + */ public function getType() { throw new \BadMethodCallException('Method is not implemented yet.', 1502196203); } + /** + * @throws \BadMethodCallException + */ public function setQuerySettings(\TYPO3\CMS\Extbase\Persistence\Generic\QuerySettingsInterface $querySettings) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196168); } + /** + * @throws \BadMethodCallException + */ public function getQuerySettings() { throw new \BadMethodCallException('Method is not implemented yet.', 1502196205); } + /** + * @throws \BadMethodCallException + */ public function count() { throw new \BadMethodCallException('Method is not implemented yet.', 1502196169); } + /** + * @throws \BadMethodCallException + */ public function getOrderings() { throw new \BadMethodCallException('Method is not implemented yet.', 1502196206); } + /** + * @throws \BadMethodCallException + */ public function getConstraint() { throw new \BadMethodCallException('Method is not implemented yet.', 1502196171); } + /** + * @throws \BadMethodCallException + */ public function isEmpty($propertyName) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196207); } + /** + * @throws \BadMethodCallException + */ public function setSource(\TYPO3\CMS\Extbase\Persistence\Generic\Qom\SourceInterface $source) { throw new \BadMethodCallException('Method is not implemented yet.', 1502196172); } + /** + * @throws \BadMethodCallException + */ public function getStatement() { throw new \BadMethodCallException('Method is not implemented yet.', 1502196208); diff --git a/Classes/Domain/Model/SearchResult.php b/Classes/Domain/Model/SearchResult.php index 516c333e..e3697cb2 100644 --- a/Classes/Domain/Model/SearchResult.php +++ b/Classes/Domain/Model/SearchResult.php @@ -1,4 +1,5 @@ */ - public function getResults() : array + public function getResults(): array { $this->initResults(); @@ -71,21 +71,22 @@ public function getResults() : array protected function initResults() { - if ($this->results !== []) { + if (is_array($this->results)) { return; } + $this->results = []; foreach ($this->resultItems as $item) { $this->results[] = new ResultItem($item['data'], $item['type']); } } - public function getFacets() : array + public function getFacets(): array { return $this->originalSearchResult->getFacets(); } - public function getCurrentCount() : int + public function getCurrentCount(): int { return $this->originalSearchResult->getCurrentCount(); } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 98e33247..c7565b14 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -1,4 +1,5 @@ logger = $logManager->getLogger(__CLASS__); $this->configuration = $configuration; $this->configurationUtility = $configurationUtility; + $this->objectManager = $objectManager; } /** @@ -59,18 +69,18 @@ public function __construct( * \Elastica\Query, but decide to use a more specific QueryFactory like * ElasticaQueryFactory, once the second query is added? */ - public function create(SearchRequestInterface $searchRequest) : \Elastica\Query + public function create(SearchRequestInterface $searchRequest): \Elastica\Query { return $this->createElasticaQuery($searchRequest); } - protected function createElasticaQuery(SearchRequestInterface $searchRequest) : \Elastica\Query + protected function createElasticaQuery(SearchRequestInterface $searchRequest): \Elastica\Query { $query = []; $this->addSize($searchRequest, $query); $this->addSearch($searchRequest, $query); $this->addBoosts($searchRequest, $query); - $this->addFilter($searchRequest, $query); + $this->addFilters($searchRequest, $query); $this->addFacets($searchRequest, $query); $this->addFields($searchRequest, $query); $this->addSort($searchRequest, $query); @@ -85,7 +95,7 @@ protected function createElasticaQuery(SearchRequestInterface $searchRequest) : protected function addSize(SearchRequestInterface $searchRequest, array &$query) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + ArrayUtility::mergeRecursiveWithOverrule($query, [ 'from' => $searchRequest->getOffset(), 'size' => $searchRequest->getLimit(), ]); @@ -100,15 +110,27 @@ protected function addSearch(SearchRequestInterface $searchRequest, array &$quer $matchExpression = [ 'type' => 'most_fields', 'query' => $searchRequest->getSearchTerm(), - 'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.query')), ]; + try { + $fieldsToQuery = GeneralUtility::trimExplode( + ',', + $this->configuration->get('searching.fields.query'), + true + ); + if ($fieldsToQuery !== []) { + $matchExpression['fields'] = $fieldsToQuery; + } + } catch (InvalidArgumentException $e) { + // Nothing configured + } + $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch'); if ($minimumShouldMatch) { $matchExpression['minimum_should_match'] = $minimumShouldMatch; } - $query = ArrayUtility::setValueByPath($query, 'query.bool.must.0.multi_match', $matchExpression); + $query = ArrayUtility::setValueByPath($query, 'query.bool.must.0.multi_match', $matchExpression, '.'); } protected function addBoosts(SearchRequestInterface $searchRequest, array &$query) @@ -137,7 +159,7 @@ protected function addBoosts(SearchRequestInterface $searchRequest, array &$quer } if (!empty($boostQueryParts)) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + ArrayUtility::mergeRecursiveWithOverrule($query, [ 'query' => [ 'bool' => [ 'should' => $boostQueryParts, @@ -164,7 +186,7 @@ protected function addFactorBoost(array &$query) protected function addFields(SearchRequestInterface $searchRequest, array &$query) { try { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + ArrayUtility::mergeRecursiveWithOverrule($query, [ 'stored_fields' => GeneralUtility::trimExplode( ',', $this->configuration->get('searching.fields.stored_fields'), @@ -183,7 +205,7 @@ protected function addFields(SearchRequestInterface $searchRequest, array &$quer ); $scriptFields = $this->configurationUtility->filterByCondition($scriptFields); if ($scriptFields !== []) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + ArrayUtility::mergeRecursiveWithOverrule($query, ['script_fields' => $scriptFields]); } } catch (InvalidArgumentException $e) { // Nothing configured @@ -196,46 +218,39 @@ protected function addSort(SearchRequestInterface $searchRequest, array &$query) $sorting = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $sorting); $sorting = $this->configurationUtility->filterByCondition($sorting); if ($sorting !== []) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); + ArrayUtility::mergeRecursiveWithOverrule($query, ['sort' => $sorting]); } } - protected function addFilter(SearchRequestInterface $searchRequest, array &$query) + protected function addFilters(SearchRequestInterface $searchRequest, array &$query) { - if (! $searchRequest->hasFilter()) { + if (!$searchRequest->hasFilter()) { return; } - $filter = []; foreach ($searchRequest->getFilter() as $name => $value) { - $filter[] = $this->buildFilter( + $this->addFilter( $name, $value, - $this->configuration->getIfExists('searching.mapping.filter.' . $name) ?: [] + $this->configuration->getIfExists('searching.mapping.filter.' . $name) ?: [], + $query ); } - - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'query' => [ - 'bool' => [ - 'filter' => $filter, - ], - ], - ]); } - protected function buildFilter(string $name, $value, array $config) : array + protected function addFilter(string $name, $value, array $config, array &$query): array { - if ($config === []) { - return [ + if (empty($config)) { + // Fallback on default term query when no added configuration + $query['query']['bool']['filter'][] = [ 'term' => [ - $name => $value, - ], + $name => $value + ] ]; + return $query; } $filter = []; - if (isset($config['fields'])) { foreach ($config['fields'] as $elasticField => $inputField) { $filter[$elasticField] = $value[$inputField]; @@ -247,20 +262,23 @@ protected function buildFilter(string $name, $value, array $config) : array } if ($config['type'] === 'range') { - return [ + $query['query']['bool']['filter'][] = [ 'range' => [ - $config['field'] => $filter, - ], + $config['field'] => $filter + ] + ]; + } else { + $query['query']['bool']['filter'][] = [ + $config['field'] => $filter ]; } - - return [$config['field'] => $filter]; + return $query; } protected function addFacets(SearchRequestInterface $searchRequest, array &$query) { foreach ($searchRequest->getFacets() as $facet) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + ArrayUtility::mergeRecursiveWithOverrule($query, [ 'aggs' => [ $facet->getIdentifier() => $facet->getConfig(), ], diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 7c8beeb0..e5b81d58 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -1,4 +1,5 @@ connection = $connection; $this->configuration = $configuration; $this->objectManager = $objectManager; $this->dataProcessorService = $dataProcessorService; + $this->cache = $cacheManager->getCache('search_core'); } - public function search(SearchRequestInterface $searchRequest) : SearchResultInterface + public function search(SearchRequestInterface $searchRequest): SearchResultInterface { $this->addSize($searchRequest); $this->addConfiguredFacets($searchRequest); @@ -84,7 +95,15 @@ public function search(SearchRequestInterface $searchRequest) : SearchResultInte $searchRequest->setConnection($this->connection); $searchRequest->setSearchService($this); - return $this->processResult($this->connection->search($searchRequest)); + $identifier = sha1('search' . serialize($searchRequest)); + if ($this->cache->has($identifier)) { + return $this->cache->get($identifier); + } + + $result = $this->processResult($this->connection->search($searchRequest)); + $this->cache->set($identifier, $result); + + return $result; } /** @@ -138,7 +157,7 @@ protected function addConfiguredFilters(SearchRequestInterface $searchRequest) /** * Processes the result, e.g. applies configured data processing to result. */ - public function processResult(SearchResultInterface $searchResult) : SearchResultInterface + public function processResult(SearchResultInterface $searchResult): SearchResultInterface { try { $newSearchResultItems = []; diff --git a/Classes/Domain/Search/SearchServiceInterface.php b/Classes/Domain/Search/SearchServiceInterface.php new file mode 100644 index 00000000..977a6f60 --- /dev/null +++ b/Classes/Domain/Search/SearchServiceInterface.php @@ -0,0 +1,41 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; + +/** + * Service to process a search request. + */ +interface SearchServiceInterface +{ + /** + * Fetches result for provided search request. + */ + public function search(SearchRequestInterface $searchRequest): SearchResultInterface; + + /** + * Processes the result, e.g. applies configured data processing to result. + */ + public function processResult(SearchResultInterface $searchResult): SearchResultInterface; +} diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index de226b90..f085a365 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -1,4 +1,5 @@ logger = $logManager->getLogger(__CLASS__); } - /** - * @param ConfigurationContainerInterface $configuration - * @param IndexerFactory $indexerFactory - */ public function __construct(ConfigurationContainerInterface $configuration, IndexerFactory $indexerFactory) { $this->configuration = $configuration; $this->indexerFactory = $indexerFactory; } + /** + * @throws NoMatchingIndexerException + */ public function update(string $table, array $record) { $this->logger->debug('Record received for update.', [$table, $record]); @@ -97,12 +97,12 @@ public function delete(string $table, string $identifier) /** * @throws NoMatchingIndexerException */ - protected function getIndexer(string $table) : IndexerInterface + protected function getIndexer(string $table): IndexerInterface { return $this->indexerFactory->getIndexer($table); } - public function supportsTable(string $table) : bool + public function supportsTable(string $table): bool { try { $this->getIndexer($table); diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index 6c3da5c8..74584ae4 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -1,4 +1,5 @@ shouldProcessHookForTable($table)) { + if (!$this->shouldProcessHookForTable($table)) { $this->logger->debug('Delete not processed.', [$table, $uid]); return false; } - $this->dataHandler->delete($table, (string) $uid); + $this->dataHandler->delete($table, $uid); return true; } + /** + * @throws NoMatchingIndexerException + */ public function processDatamap_afterAllOperations(CoreDataHandler $dataHandler) { foreach ($dataHandler->datamap as $table => $record) { @@ -103,6 +108,9 @@ public function processDatamap_afterAllOperations(CoreDataHandler $dataHandler) } } + /** + * @throws NoMatchingIndexerException + */ public function clearCachePostProc(array $parameters, CoreDataHandler $dataHandler) { $pageUid = 0; @@ -117,13 +125,16 @@ public function clearCachePostProc(array $parameters, CoreDataHandler $dataHandl } if ($pageUid > 0) { - $this->processRecord('pages', (int) $pageUid); + $this->processRecord('pages', (int)$pageUid); } } - protected function processRecord(string $table, int $uid) : bool + /** + * @throws NoMatchingIndexerException + */ + protected function processRecord(string $table, int $uid): bool { - if (! $this->shouldProcessHookForTable($table)) { + if (!$this->shouldProcessHookForTable($table)) { $this->logger->debug('Indexing of record not processed.', [$table, $uid]); return false; } @@ -138,13 +149,13 @@ protected function processRecord(string $table, int $uid) : bool return false; } - protected function shouldProcessHookForTable(string $table) : bool + protected function shouldProcessHookForTable(string $table): bool { if ($this->dataHandler === null) { $this->logger->debug('Datahandler could not be setup.'); return false; } - if (! $this->dataHandler->supportsTable($table)) { + if (!$this->dataHandler->supportsTable($table)) { $this->logger->debug('Table is not allowed.', [$table]); return false; } diff --git a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php index 6a90e015..1751cd6a 100644 --- a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php +++ b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php @@ -1,4 +1,5 @@ '', ]; + /** + * @throws FinisherException + * @throws NoMatchingIndexerException + */ protected function executeInternal() { $action = $this->parseOption('action'); - $record = ['uid' => (int) $this->parseOption('recordUid')]; + $record = ['uid' => (int)$this->parseOption('recordUid')]; $tableName = $this->parseOption('indexIdentifier'); if ($action === '' || $tableName === '' || !is_string($tableName) || $record['uid'] === 0) { @@ -62,7 +68,7 @@ protected function executeInternal() $this->dataHandler->update($tableName, $record); break; case 'delete': - $this->dataHandler->delete($tableName, (string) $record['uid']); + $this->dataHandler->delete($tableName, (string)$record['uid']); break; } } diff --git a/Classes/Utility/ArrayUtility.php b/Classes/Utility/ArrayUtility.php new file mode 100644 index 00000000..29815b15 --- /dev/null +++ b/Classes/Utility/ArrayUtility.php @@ -0,0 +1,48 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\Utility\ArrayUtility as Typo3ArrayUtility; + +class ArrayUtility extends Typo3ArrayUtility +{ + /** + * Recursively removes empty array elements. + * + * @see \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively Removed in TYPO3 v9 + */ + public static function removeEmptyElementsRecursively(array $array): array + { + $result = $array; + foreach ($result as $key => $value) { + if (is_array($value)) { + $result[$key] = self::removeEmptyElementsRecursively($value); + if ($result[$key] === []) { + unset($result[$key]); + } + } elseif ($value === null) { + unset($result[$key]); + } + } + return $result; + } +} diff --git a/Classes/Utility/FrontendUtility.php b/Classes/Utility/FrontendUtility.php index ffdbb6d4..383ae74a 100644 --- a/Classes/Utility/FrontendUtility.php +++ b/Classes/Utility/FrontendUtility.php @@ -1,4 +1,5 @@ + + + Add opening for possible different partials based on Document types: + + + {f:render(partial: 'resultItem-{result.search_document_type}', arguments: {result: result)} + + + // Render pages + + + + // Render custom "documentType" + + + diff --git a/Documentation/source/changelog/0.1.0/20181227-rename-of-configuration-files.rst b/Documentation/source/changelog/0.1.0/20181227-rename-of-configuration-files.rst new file mode 100644 index 00000000..2aa71d8d --- /dev/null +++ b/Documentation/source/changelog/0.1.0/20181227-rename-of-configuration-files.rst @@ -0,0 +1,6 @@ +Breaking Change "Configuration files were renamed" +================================================== + +TypoScript configuration files now end with ``.typoscript`` instead of ``.txt``. + +If you require these files via include statements with full file name, these need to be adjusted. diff --git a/Documentation/source/concepts.rst b/Documentation/source/concepts.rst index f81121b5..d203f878 100644 --- a/Documentation/source/concepts.rst +++ b/Documentation/source/concepts.rst @@ -4,7 +4,8 @@ Concepts ======== The main concept is to provide a foundation where other developers can profit from, to provide -integrations into search services like Elasticsearch, Algolia, ... . +integrations into search services like Elasticsearch, Algolia, …. But also to provide +an ETL Framework. Our code contains the following concepts which should be understand: @@ -18,6 +19,9 @@ interfaces. The main purpose is to provide a stable API between TYPO3 and concre For information about implementing a new connection, take a look at :ref:`development_connection`. +These are equivalent to "Load" of ETL while "indexing", and equivalent to +"Extraction" in frontend mode. + .. _concepts_indexing: Indexing @@ -31,6 +35,8 @@ Currently :ref:`TcaIndexer` and :ref:`PagesIndexer` are provided. For information about implementing a new indexer, take a look at :ref:`development_indexer`. +This is the process of "loading" data inside the ETL. + .. _concepts_dataprocessing: DataProcessing @@ -47,3 +53,5 @@ flexible as integrators are able to configure DataProcessors and change their or Configuration is done through TypoScript, see :ref:`dataprocessors`. For information about implementing a new DataProcessor, take a look at :ref:`development_dataprocessor`. + +This is the "transforming" step of ETL. diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index c559ba33..f339f20a 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -304,6 +304,7 @@ intersphinx_mapping = { 't3tcaref': ('https://docs.typo3.org/typo3cms/TCAReference/', None), 't3tsref': ('https://docs.typo3.org/typo3cms/TyposcriptReference/', None), + 't3explained': ('https://docs.typo3.org/typo3cms/CoreApiReference/', None), } extlinks = { 'project': ('https://github.com/Codappix/search_core/projects/%s', 'Github project: '), diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 53ac469b..78910086 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -24,11 +24,11 @@ The structure is following TYPO3 Extbase conventions. All settings are placed in Here is the example default configuration that's provided through static include: -.. literalinclude:: ../../Configuration/TypoScript/constants.txt +.. literalinclude:: ../../Configuration/TypoScript/constants.typoscript :language: typoscript :caption: Static TypoScript Constants -.. literalinclude:: ../../Configuration/TypoScript/setup.txt +.. literalinclude:: ../../Configuration/TypoScript/setup.typoscript :language: typoscript :caption: Static TypoScript Setup diff --git a/Documentation/source/development.rst b/Documentation/source/development.rst index e65dcdfb..54fb7c9c 100644 --- a/Documentation/source/development.rst +++ b/Documentation/source/development.rst @@ -10,6 +10,7 @@ DataProcessor and Connection. The other is how to contribute. :maxdepth: 1 :glob: + development/configuration development/indexer development/dataProcessor development/connection diff --git a/Documentation/source/development/configuration.rst b/Documentation/source/development/configuration.rst new file mode 100644 index 00000000..e9827ff8 --- /dev/null +++ b/Documentation/source/development/configuration.rst @@ -0,0 +1,53 @@ +.. _development_configuration: + +Using custom (non-typoscript) configuration +=========================================== + +When you are in need of your own non-typoscript configuration, you can create your own +Configuration Container using the TYPO3 Dependency Injection handler. + +Example: Configuration through LocalConfiguration.php +----------------------------------------------------- + +Configure your custom ext_localconf.php:: + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class) + ->registerImplementation( + \Codappix\SearchCore\Configuration\ConfigurationContainerInterface::class, + \YourNamespace\Configuration\SearchCoreConfigurationContainer::class + ); + +SearchCoreConfigurationContainer.php:: + + settings, $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['search_core']); + } + // Or manipulate it your own custom way. + } + } + diff --git a/Documentation/source/development/contribution.rst b/Documentation/source/development/contribution.rst index f65bb214..783c5b95 100644 --- a/Documentation/source/development/contribution.rst +++ b/Documentation/source/development/contribution.rst @@ -41,8 +41,8 @@ If all tests are okay, start your work. If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and `TYPO3_VERSION` in your environment like:: - export typo3DatabaseName="searchcoretest76" - export TYPO3_VERSION="~7.6" + export typo3DatabaseName="searchcoretest87" + export TYPO3_VERSION="~8.7" Also run the install command for each version before running any tests. Only this will make sure you are testing against the actual TYPO3 Version and database scheme. diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index 57014bdf..1cb5a660 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -46,6 +46,16 @@ output. See :ref:`concepts_dataprocessing` in :ref:`concepts` section. +.. _feature_up_to_you: + +Up to you +--------- + +The following is not provided by default, but possible with custom PHP code: + +* Integrating Social Media Feed imports via custom indexer to display feed content + inside of TYPO3 or publishing via custom connections. + .. _features_planned: Planned diff --git a/Documentation/source/readme.rst b/Documentation/source/readme.rst index f4ad9179..6f26c433 100644 --- a/Documentation/source/readme.rst +++ b/Documentation/source/readme.rst @@ -7,22 +7,28 @@ Introduction What does it do? ---------------- -The goal of this extension is to provide search integrations into TYPO3 CMS. The extension will -provide a convenient API to allow developers to provide concrete implementations of backends like -Elasticsearch, Algolia or Solr. +Contrary to most search solutions, search_core is an ETL (=Extract, Transform, Load) +Framework. This allows to extract data from one source, transform it, and load them +into an target system. Focusing on search solutions, but not limited to them. -The extension provides integration into TYPO3 like a frontend plugin for searches and hooks to -update search indexes on updates. Also a command line interface is provided for interactions like -re-indexing. +The provided process is to extract data from TYPO3 database storage using TCA, to +transform those data using data processors, and to load them into some search +storage like Elasticsearch. This is done via Hooks and CLI. + +Also the process is to extract data from some storage like Elasticsearch, transform +the data using data processors and to load them into the TYPO3 frontend. This is done +via a Frontend Plugin. Current state ------------- -This is still a very early beta version. More information can be taken from Github at -`current issues`_. +The basic necessary features are already implemented. Still features like workspaces +or multi language are not provided out of the box. -We are also focusing on Code Quality and Testing through `travis ci`_, ``phpcs``, ``phpunit`` and -``phpstan``. +Also only Elasticsearch is provided out of the box as a storage backend. But an +implementation for Algolia is already available via 3rd Party: +https://github.com/martinhummer/search_algolia -.. _current issues: https://github.com/Codappix/search_core/issues -.. _travis ci: https://travis-ci.org/Codappix/search_core +As the initial intend was to provide a common API and implementation for arbitrary +search implementations for TYPO3, the API is not fully implemented for ETL right now. +Also that's the reason for using "search_core" as extension name. diff --git a/Documentation/source/usage.rst b/Documentation/source/usage.rst index fc6d08a3..b385c556 100644 --- a/Documentation/source/usage.rst +++ b/Documentation/source/usage.rst @@ -11,12 +11,13 @@ Manual indexing You can trigger indexing from CLI:: - ./typo3/cli_dispatch.phpsh extbase index:index --identifier 'pages' - ./bin/typo3cms index:index --identifier 'pages' + ./typo3/cli_dispatch.phpsh extbase index:index --identifiers 'pages' + ./bin/typo3cms index:index --identifiers 'pages' This will index the table ``pages`` using the :ref:`TcaIndexer`. -Only one index per call is available, to run multiple indexers, just make multiple calls. +Multiple indexer can be called by providing a comma separated list of identifiers as +a single argument. Spaces before and after commas are ignored. The indexers have to be defined in TypoScript via :ref:`configuration_options_index`. .. _usage_manual_deletion: @@ -24,14 +25,31 @@ The indexers have to be defined in TypoScript via :ref:`configuration_options_in Manual deletion --------------- -You can trigger deletion for a single index from CLI:: +You can trigger deletion for indexes from CLI:: - ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'pages' - ./bin/typo3cms index:delete --identifier 'pages' + ./typo3/cli_dispatch.phpsh extbase index:delete --identifiers 'pages' + ./bin/typo3cms index:delete --identifiers 'pages' -This will delete the index for the table ``pages``. +This will delete the index for the table ``pages``. Deletion means removing all +documents from the index. -Only one delete per call is available, to run multiple deletions, just make multiple calls. +Multiple indexes can be called by providing a comma separated list of identifiers as +a single argument. Spaces before and after commas are ignored. + +.. _usage_manual_delete_all_documents: + +Manual delete all documents +--------------------------- + +You can trigger deletion of all documents for indexes from CLI:: + + ./typo3/cli_dispatch.phpsh extbase index:deletedocuments --identifiers 'pages' + ./bin/typo3cms index:deletedocuments --identifiers 'pages' + +This will delete all documents within the index for the table ``pages``. + +Multiple indexes can be called by providing a comma separated list of identifiers as +a single argument. Spaces before and after commas are ignored. .. _usage_auto_indexing: diff --git a/Makefile b/Makefile index 21f6c315..2646a3cc 100644 --- a/Makefile +++ b/Makefile @@ -11,16 +11,9 @@ typo3DatabasePassword ?= "dev" typo3DatabaseHost ?= "127.0.0.1" sourceOrDist=--prefer-dist -ifeq ($(TYPO3_VERSION),~7.6) - sourceOrDist=--prefer-source -endif .PHONY: install install: clean - if [ $(TYPO3_VERSION) = ~7.6 ]; then \ - patch composer.json Tests/InstallPatches/composer.json.patch; \ - fi - COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev $(sourceOrDist) typo3/cms="$(TYPO3_VERSION)" git checkout composer.json diff --git a/Resources/Private/Language/locallang.xlf b/Resources/Private/Language/locallang.xlf new file mode 100644 index 00000000..eac33ce8 --- /dev/null +++ b/Resources/Private/Language/locallang.xlf @@ -0,0 +1,14 @@ + + + +
+ + + Please enter your search term in the box above. + + + No results found with given search query (%s). + + + + diff --git a/Resources/Private/Language/locallang_be.xlf b/Resources/Private/Language/locallang_be.xlf new file mode 100644 index 00000000..524d5e09 --- /dev/null +++ b/Resources/Private/Language/locallang_be.xlf @@ -0,0 +1,14 @@ + + + +
+ + + Search Core: Search + + + List results from search + + + + diff --git a/Resources/Private/Layouts/Default.html b/Resources/Private/Layouts/Default.html new file mode 100644 index 00000000..4866c205 --- /dev/null +++ b/Resources/Private/Layouts/Default.html @@ -0,0 +1,6 @@ + +
+ +
+ diff --git a/Resources/Private/Partials/Form/SearchRequest.html b/Resources/Private/Partials/Form/SearchRequest.html new file mode 100644 index 00000000..3348de13 --- /dev/null +++ b/Resources/Private/Partials/Form/SearchRequest.html @@ -0,0 +1,9 @@ + + + + + + + + diff --git a/Resources/Private/Partials/Results/Item/Default.html b/Resources/Private/Partials/Results/Item/Default.html new file mode 100644 index 00000000..a4b5fdc3 --- /dev/null +++ b/Resources/Private/Partials/Results/Item/Default.html @@ -0,0 +1,8 @@ + + + + [{result.search_document_type}:{result.uid}] - {result.search_title} + + + diff --git a/Resources/Private/Partials/Results/ListItem.html b/Resources/Private/Partials/Results/ListItem.html new file mode 100644 index 00000000..34e5f8b9 --- /dev/null +++ b/Resources/Private/Partials/Results/ListItem.html @@ -0,0 +1,6 @@ + + + + + diff --git a/Resources/Private/Templates/Search/Results.html b/Resources/Private/Templates/Search/Results.html new file mode 100644 index 00000000..095d2751 --- /dev/null +++ b/Resources/Private/Templates/Search/Results.html @@ -0,0 +1,29 @@ + + + + +
+ +
+ + + +
    + +
  • + +
  • +
    +
+
+ + + {f:translate(key: 'no_results_found_with_search', arguments: '{0: searchRequest.searchTerm}')} + {f:translate(key: 'no_results_found')} + + +
+
+ + diff --git a/Resources/Private/Templates/Search/Search.html b/Resources/Private/Templates/Search/Search.html deleted file mode 100644 index dc9e6b26..00000000 --- a/Resources/Private/Templates/Search/Search.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - {result.id} [{result.type}] - {result.hit._source.search_title} - -
-
diff --git a/Resources/Public/Icons/Elastic.svg b/Resources/Public/Icons/Elastic.svg new file mode 100644 index 00000000..927e60be --- /dev/null +++ b/Resources/Public/Icons/Elastic.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/Resources/Public/Icons/Extension.svg b/Resources/Public/Icons/Extension.svg new file mode 100644 index 00000000..69fa8e87 --- /dev/null +++ b/Resources/Public/Icons/Extension.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/Resources/Public/Icons/PluginForm.svg b/Resources/Public/Icons/PluginForm.svg new file mode 100644 index 00000000..23adb080 --- /dev/null +++ b/Resources/Public/Icons/PluginForm.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/Resources/Public/Icons/PluginSearch.svg b/Resources/Public/Icons/PluginSearch.svg new file mode 100644 index 00000000..14a0eaed --- /dev/null +++ b/Resources/Public/Icons/PluginSearch.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index d0154576..2c04cdf6 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -1,4 +1,5 @@ get(IndexerFactory::class) ->getIndexer('tt_content') - ->indexAllDocuments() - ; + ->indexAllDocuments(); $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(SearchService::class); diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index 894fb3f6..2a93f1c5 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -1,4 +1,5 @@ get(IndexerFactory::class) ->getIndexer('tt_content') - ->indexAllDocuments() - ; + ->indexAllDocuments(); $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(SearchService::class); @@ -55,7 +55,7 @@ public function itsPossibleToFilterResultsByASingleField() $searchRequest->setFilter(['CType' => 'HTML']); $result = $searchService->search($searchRequest); - $this->assertSame(5, (int) $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); + $this->assertSame(5, (int)$result->getResults()[0]['uid'], 'Did not get the expected result entry.'); $this->assertSame(1, count($result), 'Did not receive the single filtered element.'); } } diff --git a/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php b/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php index 8297a2af..ae277378 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php @@ -1,4 +1,5 @@ get(IndexerFactory::class) ->getIndexer('tt_content') - ->delete() - ; + ->delete(); $this->assertFalse( $this->client->getIndex('typo3content')->exists(), 'Index could not be deleted through command controller.' ); } + + /** + * @test + */ + public function documentsAreDeleted() + { + $index = $this->client->getIndex('typo3content'); + $index->create(); + $this->assertTrue( + $index->exists(), + 'Could not create index for test.' + ); + + $contentIndexer = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content'); + $pageIndexer = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('pages'); + + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 0, 'Index should be empty.'); + + $contentIndexer->indexAllDocuments(); + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents are in index.'); + + $pageIndexer->indexAllDocuments(); + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 5, 'Not exactly 5 documents are in index.'); + + $contentIndexer->deleteAllDocuments(); + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents are in index.'); + + $pageIndexer->deleteAllDocuments(); + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 0, 'Index should be empty.'); + } } diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index ae671dda..0c629c33 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -1,4 +1,5 @@ get(IndexerFactory::class) ->getIndexer('tt_content') - ->indexAllDocuments() - ; + ->indexAllDocuments(); $response = $this->client->request('typo3content/_search?q=*:*'); @@ -53,7 +56,7 @@ public function indexBasicTtContent() $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); $this->assertSame( 'indexed content element', - $response->getData()['hits']['hits'][2]['_source']['header'], + $response->getData()['hits']['hits'][0]['_source']['header'], 'Record was not indexed.' ); } @@ -66,8 +69,7 @@ public function indexSingleBasicTtContent() \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) ->getIndexer('tt_content') - ->indexDocument(6) - ; + ->indexDocument(6); $response = $this->client->request('typo3content/_search?q=*:*'); @@ -89,8 +91,7 @@ public function indexingNonConfiguredTableWillThrowException() { \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) - ->getIndexer('non_existing_table') - ; + ->getIndexer('non_existing_table'); } /** @@ -101,8 +102,7 @@ public function canHandleExistingIndex() { $indexer = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) - ->getIndexer('tt_content') - ; + ->getIndexer('tt_content'); $indexer->indexAllDocuments(); @@ -124,13 +124,12 @@ public function indexingRespectsUserWhereClause() parent::getTypoScriptFilesForFrontendRootPage(), ['EXT:search_core/Tests/Functional/Fixtures/Indexing/UserWhereClause.ts'] )); - $this->importDataSet('Tests/Functional/Fixtures/Indexing/UserWhereClause.xml'); + $this->importDataSet('EXT:search_core/Tests/Functional/Fixtures/Indexing/UserWhereClause.xml'); \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) ->getIndexer('tt_content') - ->indexAllDocuments() - ; + ->indexAllDocuments(); $response = $this->client->request('typo3content/_search?q=*:*'); @@ -157,13 +156,12 @@ public function indexingRespectsUserWhereClause() */ public function resolvesRelations() { - $this->importDataSet('Tests/Functional/Fixtures/Indexing/ResolveRelations.xml'); + $this->importDataSet('EXT:search_core/Tests/Functional/Fixtures/Indexing/ResolveRelations.xml'); \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) ->getIndexer('tt_content') - ->indexAllDocuments() - ; + ->indexAllDocuments(); $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); @@ -171,11 +169,13 @@ public function resolvesRelations() $response = $this->client->request('typo3content/_search?q=uid:11'); $this->assertArraySubset( - ['_source' => [ - 'uid' => '11', - 'CType' => 'Header', // Testing items - 'categories' => ['Category 2', 'Category 1'], // Testing mm - ]], + [ + '_source' => [ + 'uid' => '11', + 'CType' => 'Header', // Testing items + 'categories' => ['Category 2', 'Category 1'], // Testing mm + ] + ], $response->getData()['hits']['hits'][0], false, 'Record was not indexed with resolved category relations to multiple values.' @@ -183,11 +183,13 @@ public function resolvesRelations() $response = $this->client->request('typo3content/_search?q=uid:12'); $this->assertArraySubset( - ['_source' => [ - 'uid' => '12', - 'CType' => 'Header', - 'categories' => ['Category 2'], - ]], + [ + '_source' => [ + 'uid' => '12', + 'CType' => 'Header', + 'categories' => ['Category 2'], + ] + ], $response->getData()['hits']['hits'][0], false, 'Record was not indexed with resolved category relations to a single value.' @@ -195,10 +197,12 @@ public function resolvesRelations() $response = $this->client->request('typo3content/_search?q=uid:6'); $this->assertArraySubset( - ['_source' => [ - 'uid' => '6', - 'categories' => null, - ]], + [ + '_source' => [ + 'uid' => '6', + 'categories' => null, + ] + ], $response->getData()['hits']['hits'][0], false, 'Record was indexed with resolved category relation, but should not have any.' @@ -213,8 +217,7 @@ public function indexingDeletedRecordIfRecordShouldBeIndexedButIsNoLongerAvailab \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) ->getIndexer('tt_content') - ->indexAllDocuments() - ; + ->indexAllDocuments(); $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); @@ -234,8 +237,7 @@ public function indexingDeletedRecordIfRecordShouldBeIndexedButIsNoLongerAvailab \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) ->getIndexer('tt_content') - ->indexDocument(10) - ; + ->indexDocument(10); $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 document is in index.'); diff --git a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php index bb997cf5..acfe570c 100644 --- a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php +++ b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php @@ -1,4 +1,5 @@ ['value1', 'value2'], ]; - if ($this->isLegacyVersion()) { - $typoScriptService = new TypoScriptService76(); - } else { - $typoScriptService = new TypoScriptService(); - } + $typoScriptService = new TypoScriptService(); $subject = new ContentObjectDataProcessorAdapterProcessor($typoScriptService); $processedData = $subject->processData($record, $configuration); diff --git a/Tests/Functional/DataProcessing/TcaRelationResolvingProcessorTest.php b/Tests/Functional/DataProcessing/TcaRelationResolvingProcessorTest.php index c5011df9..b2e5b775 100644 --- a/Tests/Functional/DataProcessing/TcaRelationResolvingProcessorTest.php +++ b/Tests/Functional/DataProcessing/TcaRelationResolvingProcessorTest.php @@ -1,4 +1,5 @@ importDataSet('Tests/Functional/Fixtures/Indexing/TcaIndexer/RelationResolver/InlineRelation.xml'); + $this->importDataSet( + 'EXT:search_core/Tests/Functional/Fixtures/Indexing/TcaIndexer/RelationResolver/InlineRelation.xml' + ); $objectManager = GeneralUtility::makeInstance(ObjectManager::class); $table = 'sys_file'; @@ -55,7 +58,9 @@ public function resolveInlineRelation() */ public function resolveStaticSelectItems() { - $this->importDataSet('Tests/Functional/Fixtures/Indexing/TcaIndexer/RelationResolver/StaticSelectItems.xml'); + $this->importDataSet( + 'EXT:search_core/Tests/Functional/Fixtures/Indexing/TcaIndexer/RelationResolver/StaticSelectItems.xml' + ); $objectManager = GeneralUtility::makeInstance(ObjectManager::class); $table = 'tt_content'; @@ -74,7 +79,9 @@ public function resolveStaticSelectItems() */ public function resolveForeignDb() { - $this->importDataSet('Tests/Functional/Fixtures/Indexing/TcaIndexer/RelationResolver/ForeignDb.xml'); + $this->importDataSet( + 'EXT:search_core/Tests/Functional/Fixtures/Indexing/TcaIndexer/RelationResolver/ForeignDb.xml' + ); $objectManager = GeneralUtility::makeInstance(ObjectManager::class); $table = 'tt_content'; @@ -96,7 +103,9 @@ public function resolveForeignDb() */ public function resolveForeignMmSelect() { - $this->importDataSet('Tests/Functional/Fixtures/Indexing/TcaIndexer/RelationResolver/ForeignMmSelect.xml'); + $this->importDataSet( + 'EXT:search_core/Tests/Functional/Fixtures/Indexing/TcaIndexer/RelationResolver/ForeignMmSelect.xml' + ); $objectManager = GeneralUtility::makeInstance(ObjectManager::class); $table = 'tt_content'; diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index e2cc456d..ef47830b 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -45,12 +45,6 @@ plugin { } } } - - searching { - fields { - query = _all - } - } } } } diff --git a/Tests/Functional/Fixtures/Indexing/IndexDeletion.xml b/Tests/Functional/Fixtures/Indexing/IndexDeletion.xml new file mode 100644 index 00000000..409e46cd --- /dev/null +++ b/Tests/Functional/Fixtures/Indexing/IndexDeletion.xml @@ -0,0 +1,129 @@ + + + + 6 + 1 + 1480686370 + 1480686370 + 0 + 72 + header +
indexed content element
+ this is the content of header content element that should get indexed + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 +
+ + + 7 + 1 + 1480686371 + 1480686370 + 0 + 72 + header +
endtime hidden record
+ + 0 + 0 + 0 + 0 + 0 + 1481305963 + 0 + 0 +
+ + + 8 + 1 + 1480686370 + 1480686370 + 1 + 72 + header +
Hidden record
+ + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 +
+ + + 9 + 1 + 1480686370 + 1480686370 + 0 + 72 + div +
not indexed due to ctype
+ this is the content of div content element that should not get indexed + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 +
+ + + 10 + 1 + 1480686370 + 1480686370 + 0 + 72 + html +
Indexed without html tags
+ Some text in paragraph

]]>
+ 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 +
+ + + 100 + 2 + 1480686370 + 1480686370 + 0 + 72 + header +
Indexed on page 2
+ This element is on a different page + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 +
+ + + 2 + 1 + Second page with content + Used to check whether content is indexed only for parent page. + +
diff --git a/Tests/Functional/FunctionalTests.xml b/Tests/Functional/FunctionalTests.xml index e4f78fef..4429e696 100644 --- a/Tests/Functional/FunctionalTests.xml +++ b/Tests/Functional/FunctionalTests.xml @@ -1,7 +1,7 @@ isLegacyVersion()) { return isset($record['uid']) && $record['uid'] === '1' && isset($record['pid']) && $record['pid'] === '1' - && isset($record['colPos']) && $record['colPos'] === '1' - ; + && isset($record['colPos']) && $record['colPos'] === '1'; } return isset($record['uid']) && $record['uid'] === 1 && isset($record['pid']) && $record['pid'] === 1 - && isset($record['colPos']) && $record['colPos'] === 1 - ; + && isset($record['colPos']) && $record['colPos'] === 1; }) ], [ @@ -132,14 +128,12 @@ public function updateWillBeTriggeredForNewTtContent() if ($this->isLegacyVersion()) { return isset($record['uid']) && $record['uid'] === '2' && isset($record['pid']) && $record['pid'] === '1' - && isset($record['header']) && $record['header'] === 'a new record' - ; + && isset($record['header']) && $record['header'] === 'a new record'; } return isset($record['uid']) && $record['uid'] === 2 && isset($record['pid']) && $record['pid'] === 1 - && isset($record['header']) && $record['header'] === 'a new record' - ; + && isset($record['header']) && $record['header'] === 'a new record'; }) ], [ diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php index c0b0aa39..e16ade4e 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php @@ -1,4 +1,5 @@ importDataSet('Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml'); + $this->importDataSet('EXT:search_core/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml'); $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class); $tableName = 'pages'; @@ -54,8 +54,7 @@ public function pagesContainAllAdditionalInformation() ' this is the content of header content element that should get indexed' . ' Indexed without html tags Some text in paragraph' && isset($documents[0]['search_abstract']) && $documents[0]['search_abstract'] === - 'Used as abstract as no abstract is defined.' - ; + 'Used as abstract as no abstract is defined.'; }) ); @@ -95,12 +94,21 @@ public function rootLineIsRespectedDuringIndexing($dataSetPath) $indexer->indexAllDocuments(); } + /** + * @return array + */ public function rootLineDataSets() { return [ - 'Broken root line' => ['Tests/Functional/Fixtures/Indexing/PagesIndexer/BrokenRootLine.xml'], - 'Recycler doktype' => ['Tests/Functional/Fixtures/Indexing/PagesIndexer/Recycler.xml'], - 'Extended timing to sub pages' => ['Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml'], + 'Broken root line' => [ + 'EXT:search_core/Tests/Functional/Fixtures/Indexing/PagesIndexer/BrokenRootLine.xml' + ], + 'Recycler doktype' => [ + 'EXT:search_core/Tests/Functional/Fixtures/Indexing/PagesIndexer/Recycler.xml' + ], + 'Extended timing to sub pages' => [ + 'EXT:search_core/Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml' + ], ]; } } diff --git a/Tests/Functional/Indexing/TcaIndexerTest.php b/Tests/Functional/Indexing/TcaIndexerTest.php index 472cc154..5a2328b1 100644 --- a/Tests/Functional/Indexing/TcaIndexerTest.php +++ b/Tests/Functional/Indexing/TcaIndexerTest.php @@ -1,4 +1,5 @@ importDataSet('Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.xml'); + $this->importDataSet( + 'EXT:search_core/Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.xml' + ); $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class); $tableName = 'tt_content'; $tableService = $objectManager->get( @@ -66,7 +72,7 @@ public function respectRootLineBlacklist() foreach ($documents as $document) { // Page uids 1 and 2 are allowed while 3 and 4 are not allowed. // Therefore only documents with page uid 1 and 2 should exist. - if (! isset($document['pid']) || ! in_array($document['pid'], [1, 2])) { + if (!isset($document['pid']) || !in_array($document['pid'], [1, 2])) { return false; } } diff --git a/Tests/InstallPatches/composer.json.patch b/Tests/InstallPatches/composer.json.patch deleted file mode 100644 index 6f11cdc7..00000000 --- a/Tests/InstallPatches/composer.json.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/composer.json b/composer.json -index 83e5f47..e9fa296 100644 ---- a/composer.json -+++ b/composer.json -@@ -21,8 +21,7 @@ - "ruflin/elastica": "~3.2" - }, - "require-dev": { -- "phpunit/phpunit": "~6.4.4", -- "typo3/testing-framework": "~1.1.5", -+ "phpunit/phpunit": "~5.7.0", - "squizlabs/php_codesniffer": "~3.1.1" - }, - "config": { diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index 384236c5..b895d01f 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -1,4 +1,5 @@ [ diff --git a/Tests/Unit/Bootstrap.php b/Tests/Unit/Bootstrap.php deleted file mode 100644 index 3ed0a1cb..00000000 --- a/Tests/Unit/Bootstrap.php +++ /dev/null @@ -1,11 +0,0 @@ -subject->expects($this->once()) ->method('outputLine') - ->with('No indexer found for: nonAllowedTable'); + ->with('No indexer found for: nonAllowedTable.'); $this->indexerFactory->expects($this->once()) ->method('getIndexer') ->with('nonAllowedTable') @@ -79,11 +78,14 @@ public function indexerExecutesForAllowedTable() $indexerMock = $this->getMockBuilder(TcaIndexer::class) ->disableOriginalConstructor() ->getMock(); + $indexerMock->expects($this->any()) + ->method('getIdentifier') + ->willReturn('allowedTable'); $this->subject->expects($this->never()) ->method('quit'); $this->subject->expects($this->once()) ->method('outputLine') - ->with('allowedTable was indexed.'); + ->with('Documents in index allowedTable were indexed.'); $this->indexerFactory->expects($this->once()) ->method('getIndexer') ->with('allowedTable') @@ -95,22 +97,49 @@ public function indexerExecutesForAllowedTable() /** * @test */ - public function deletionIsPossible() + public function deletionOfDocumentsIsPossible() { $indexerMock = $this->getMockBuilder(TcaIndexer::class) ->disableOriginalConstructor() ->getMock(); + $indexerMock->expects($this->any()) + ->method('getIdentifier') + ->willReturn('allowedTable'); $this->subject->expects($this->once()) ->method('outputLine') - ->with('allowedTable was deleted.'); + ->with('Documents in index allowedTable were deleted.'); $this->indexerFactory->expects($this->once()) ->method('getIndexer') ->with('allowedTable') ->will($this->returnValue($indexerMock)); + $indexerMock->expects($this->once()) + ->method('deleteAllDocuments'); + $this->subject->deleteDocumentsCommand('allowedTable'); + } + + /** + * @test + */ + public function deletionOfIndexIsPossible() + { + $indexerMock = $this->getMockBuilder(TcaIndexer::class) + ->disableOriginalConstructor() + ->getMock(); + $indexerMock->expects($this->any()) + ->method('getIdentifier') + ->willReturn('pages'); + $this->subject->expects($this->once()) + ->method('outputLine') + ->with('Index pages was deleted.'); + $this->indexerFactory->expects($this->once()) + ->method('getIndexer') + ->with('pages') + ->will($this->returnValue($indexerMock)); + $indexerMock->expects($this->once()) ->method('delete'); - $this->subject->deleteCommand('allowedTable'); + $this->subject->deleteCommand('pages'); } /** @@ -120,7 +149,7 @@ public function deletionForNonExistingIndexerDoesNotWork() { $this->subject->expects($this->once()) ->method('outputLine') - ->with('No indexer found for: nonAllowedTable'); + ->with('No indexer found for: nonAllowedTable.'); $this->indexerFactory->expects($this->once()) ->method('getIndexer') ->with('nonAllowedTable') @@ -128,4 +157,88 @@ public function deletionForNonExistingIndexerDoesNotWork() $this->subject->deleteCommand('nonAllowedTable'); } + + // As all methods use the same code base, we test the logic for multiple + // identifiers only for indexing. + + /** + * @test + */ + public function indexerExecutesForAllowedTables() + { + $indexerMock = $this->getMockBuilder(TcaIndexer::class) + ->disableOriginalConstructor() + ->getMock(); + $indexerMock->expects($this->any()) + ->method('getIdentifier') + ->will($this->onConsecutiveCalls('allowedTable', 'anotherTable')); + $this->subject->expects($this->never()) + ->method('quit'); + $this->subject->expects($this->exactly(2)) + ->method('outputLine') + ->withConsecutive( + ['Documents in index allowedTable were indexed.'], + ['Documents in index anotherTable were indexed.'] + ); + $this->indexerFactory->expects($this->exactly(2)) + ->method('getIndexer') + ->withConsecutive(['allowedTable'], ['anotherTable']) + ->will($this->returnValue($indexerMock)); + + $this->subject->indexCommand('allowedTable, anotherTable'); + } + + /** + * @test + */ + public function indexerSkipsEmptyIdentifier() + { + $indexerMock = $this->getMockBuilder(TcaIndexer::class) + ->disableOriginalConstructor() + ->getMock(); + $indexerMock->expects($this->any()) + ->method('getIdentifier') + ->willReturn('allowedTable'); + $this->subject->expects($this->never()) + ->method('quit'); + $this->subject->expects($this->once()) + ->method('outputLine') + ->with('Documents in index allowedTable were indexed.'); + $this->indexerFactory->expects($this->once()) + ->method('getIndexer') + ->with('allowedTable') + ->will($this->returnValue($indexerMock)); + + $this->subject->indexCommand('allowedTable, '); + } + + /** + * @test + */ + public function indexerSkipsAndOutputsNonExistingIdentifier() + { + $indexerMock = $this->getMockBuilder(TcaIndexer::class) + ->disableOriginalConstructor() + ->getMock(); + $indexerMock->expects($this->any()) + ->method('getIdentifier') + ->willReturn('allowedTable'); + $this->subject->expects($this->never()) + ->method('quit'); + $this->subject->expects($this->exactly(2)) + ->method('outputLine') + ->withConsecutive( + ['No indexer found for: nonExisting.'], + ['Documents in index allowedTable were indexed.'] + ); + $this->indexerFactory->expects($this->exactly(2)) + ->method('getIndexer') + ->withConsecutive(['nonExisting'], ['allowedTable']) + ->will($this->onConsecutiveCalls( + $this->throwException(new NoMatchingIndexerException), + $this->returnValue($indexerMock) + )); + + $this->subject->indexCommand('nonExisting, allowedTable'); + } } diff --git a/Tests/Unit/Configuration/ConfigurationUtilityTest.php b/Tests/Unit/Configuration/ConfigurationUtilityTest.php index 4db367c8..36bca82b 100644 --- a/Tests/Unit/Configuration/ConfigurationUtilityTest.php +++ b/Tests/Unit/Configuration/ConfigurationUtilityTest.php @@ -1,4 +1,5 @@ [ @@ -92,6 +96,8 @@ public function possibleRequestAndConfigurationForFluidtemplate() : array /** * @test * @dataProvider possibleConditionEntries + * @param array $entries + * @param array $expected */ public function conditionsAreHandledAsExpected(array $entries, array $expected) { @@ -104,7 +110,7 @@ public function conditionsAreHandledAsExpected(array $entries, array $expected) ); } - public function possibleConditionEntries() : array + public function possibleConditionEntries(): array { return [ 'Nothing in array' => [ diff --git a/Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php b/Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php index 696762c7..9b4d822c 100644 --- a/Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php +++ b/Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php @@ -1,4 +1,5 @@ subject->getIndex($connection, 'someIndex'); + $this->subject->createIndex($connection, 'someIndex'); } } diff --git a/Tests/Unit/Connection/Elasticsearch/MappingFactoryTest.php b/Tests/Unit/Connection/Elasticsearch/MappingFactoryTest.php index 8dde1cbf..d2c89c98 100644 --- a/Tests/Unit/Connection/Elasticsearch/MappingFactoryTest.php +++ b/Tests/Unit/Connection/Elasticsearch/MappingFactoryTest.php @@ -1,4 +1,5 @@ configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); - $this->subject = new MappingFactory($this->configuration); + $this->configurationMock = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->typeFactoryMock = $this->getMockBuilder(TypeFactory::class)->disableOriginalConstructor()->getMock(); + + $this->subject = new MappingFactory($this->configurationMock, $this->typeFactoryMock); } /** @@ -44,40 +59,27 @@ public function setUp() */ public function typoScriptConfigurationIsProvidedToIndex() { - $indexName = 'someIndex'; + $documentType = 'someDocument'; $configuration = [ - '_all' => [ - 'type' => 'text', - 'analyzer' => 'ngram4', - ], 'channel' => [ 'type' => 'keyword', ], ]; - $type = $this->getMockBuilder(\Elastica\Type::class) - ->disableOriginalConstructor() - ->getMock(); - $type->expects($this->any()) - ->method('getName') - ->willReturn($indexName); - $this->configuration->expects($this->once()) + + $typeMock = $this->getMockBuilder(Type::class)->disableOriginalConstructor()->getMock(); + + $this->typeFactoryMock->expects($this->any()) + ->method('getType') + ->with($documentType) + ->willReturn($typeMock); + $this->configurationMock->expects($this->once()) ->method('get') - ->with('indexing.' . $indexName . '.mapping') + ->with('indexing.' . $documentType . '.mapping') ->willReturn($configuration); - $mapping = $this->subject->getMapping($type)->toArray()[$indexName]; - $this->assertArraySubset( - [ - '_all' => $configuration['_all'] - ], - $mapping, - true, - 'Configuration of _all field was not set for mapping.' - ); + $mapping = $this->subject->getMapping($documentType)->toArray()['']; $this->assertArraySubset( - [ - 'channel' => $configuration['channel'] - ], + $configuration, $mapping['properties'], true, 'Configuration for properties was not set for mapping.' diff --git a/Tests/Unit/Controller/SearchControllerTest.php b/Tests/Unit/Controller/SearchControllerTest.php index 67c6d98f..6c9fed64 100644 --- a/Tests/Unit/Controller/SearchControllerTest.php +++ b/Tests/Unit/Controller/SearchControllerTest.php @@ -1,4 +1,5 @@ dataProcessorService->expects($this->any()) ->method('executeDataProcessor') @@ -122,8 +124,7 @@ public function executesConfiguredDataProcessingWithConfiguration() $this->subject->expects($this->once()) ->method('getRecord') ->with(1) - ->willReturn($record) - ; + ->willReturn($record); $this->connection->expects($this->once())->method('addDocument')->with('testTable', $expectedRecord); $this->subject->indexDocument(1); @@ -137,6 +138,7 @@ public function executesNoDataProcessingForMissingConfiguration() $record = ['field 1' => 'test']; $expectedRecord = $record; $expectedRecord['search_abstract'] = ''; + $expectedRecord['search_document_type'] = 'testTable'; $this->configuration->expects($this->exactly(2)) ->method('get') @@ -145,8 +147,7 @@ public function executesNoDataProcessingForMissingConfiguration() $this->subject->expects($this->once()) ->method('getRecord') ->with(1) - ->willReturn($record) - ; + ->willReturn($record); $this->connection->expects($this->once())->method('addDocument')->with('testTable', $expectedRecord); $this->subject->indexDocument(1); diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index f4706138..c01ece14 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -1,4 +1,5 @@ databaseConnection = $this->getMockBuilder(DatabaseConnection::class)->getMock(); $className = TcaTableService::class; - if ($this->isLegacyVersion()) { - $className = TcaTableService76::class; - } $this->subject = $this->getMockBuilder($className) ->disableOriginalConstructor() ->setMethods(['getConnection', 'getSystemWhereClause']) diff --git a/Tests/Unit/Domain/Model/ResultItemTest.php b/Tests/Unit/Domain/Model/ResultItemTest.php index a4c18fd3..9e73ee71 100644 --- a/Tests/Unit/Domain/Model/ResultItemTest.php +++ b/Tests/Unit/Domain/Model/ResultItemTest.php @@ -1,4 +1,5 @@ setSearchService( - $this->getMockBuilder(SearchService::class) + $this->getMockBuilder(SearchServiceInterface::class) ->disableOriginalConstructor() ->getMock() ); @@ -112,8 +118,7 @@ public function exceptionIsThrownIfConnectionWasNotSet() */ public function executionMakesUseOfProvidedConnectionAndSearchService() { - $searchServiceMock = $this->getMockBuilder(SearchService::class) - ->disableOriginalConstructor() + $searchServiceMock = $this->getMockBuilder(SearchServiceInterface::class) ->getMock(); $connectionMock = $this->getMockBuilder(ConnectionInterface::class) ->getMock(); diff --git a/Tests/Unit/Domain/Model/SearchResultTest.php b/Tests/Unit/Domain/Model/SearchResultTest.php index db5ccb50..ba4dab39 100644 --- a/Tests/Unit/Domain/Model/SearchResultTest.php +++ b/Tests/Unit/Domain/Model/SearchResultTest.php @@ -1,4 +1,5 @@ configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); $configurationUtility = new ConfigurationUtility(); - $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration, $configurationUtility); + $this->objectManager = $this->getMockBuilder(ObjectManagerInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->subject = new QueryFactory( + $this->getMockedLogger(), + $this->configuration, + $configurationUtility, + $this->objectManager + ); } /** @@ -232,9 +247,6 @@ public function searchTermIsAddedToQuery() 'multi_match' => [ 'type' => 'most_fields', 'query' => 'SearchWord', - 'fields' => [ - '_all', - ], ], ], ], @@ -272,9 +284,6 @@ public function minimumShouldMatchIsAddedToQuery() 'multi_match' => [ 'type' => 'most_fields', 'query' => 'SearchWord', - 'fields' => [ - '_all', - ], 'minimum_should_match' => '50%', ], ], @@ -303,7 +312,7 @@ public function boostsAreAddedToQuery() ['searching.fieldValueFactor'] ) ->will($this->onConsecutiveCalls( - '_all', + '', [ 'search_title' => 3, 'search_abstract' => 1.5, @@ -360,7 +369,7 @@ public function factorBoostIsAddedToQuery() ['searching.fieldValueFactor'] ) ->will($this->onConsecutiveCalls( - '_all', + '', $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), @@ -378,9 +387,6 @@ public function factorBoostIsAddedToQuery() 'multi_match' => [ 'type' => 'most_fields', 'query' => 'SearchWord', - 'fields' => [ - '_all', - ], ], ], ], @@ -405,7 +411,7 @@ public function emptySearchStringWillNotAddSearchToQuery() $query = $this->subject->create($searchRequest); $this->assertInstanceOf( - stdClass, + 'stdClass', $query->toArray()['query']['match_all'], 'Empty search request does not create expected query.' ); @@ -428,7 +434,7 @@ public function configuredQueryFieldsAreAddedToQuery() ['searching.fieldValueFactor'] ) ->will($this->onConsecutiveCalls( - '_all, field1, field2', + 'field1, field2', $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), @@ -445,7 +451,6 @@ public function configuredQueryFieldsAreAddedToQuery() 'type' => 'most_fields', 'query' => 'SearchWord', 'fields' => [ - '_all', 'field1', 'field2', ], @@ -535,7 +540,7 @@ public function scriptFieldsAreAddedToQuery() ['searching.fieldValueFactor'] ) ->will($this->onConsecutiveCalls( - '_all', + '', $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), [ @@ -662,13 +667,16 @@ public function sortIsNotAddedToQuery() ); } + /** + * @return void + */ protected function configureConfigurationMockWithDefault() { $this->configuration->expects($this->any()) ->method('get') ->will($this->returnCallback(function ($configName) { if ($configName === 'searching.fields.query') { - return '_all'; + return ''; } throw new InvalidArgumentException(); diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index 3b880768..49e821d0 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -1,4 +1,5 @@ getMockBuilder(FrontendInterface::class)->getMock(); + $cacheManagerMock = $this->getMockBuilder(CacheManager::class)->getMock(); + $cacheManagerMock->expects($this->any()) + ->method('getCache') + ->with('search_core') + ->willReturn($cacheMock); + $this->result = $this->getMockBuilder(SearchResultInterface::class) ->disableOriginalConstructor() ->getMock(); @@ -87,7 +97,8 @@ public function setUp() $this->connection, $this->configuration, $this->objectManager, - $this->dataProcessorService + $this->dataProcessorService, + $cacheManagerMock ); } @@ -100,7 +111,7 @@ public function sizeIsAddedFromConfiguration() ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(45, null)); - $this->configuration->expects($this->any()) + $this->configuration->expects($this->any()) ->method('get') ->will($this->throwException(new InvalidArgumentException)); $this->connection->expects($this->once()) @@ -211,9 +222,9 @@ public function configuredFilterAreAddedToRequestWithExistingFilter() ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getFilter() === [ - 'anotherProperty' => 'anything', - 'property' => 'something', - ]; + 'anotherProperty' => 'anything', + 'property' => 'something', + ]; })) ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); diff --git a/Tests/Unit/Hook/DataHandlerTest.php b/Tests/Unit/Hook/DataHandlerTest.php index aaf3935c..5cb0bdcc 100644 --- a/Tests/Unit/Hook/DataHandlerTest.php +++ b/Tests/Unit/Hook/DataHandlerTest.php @@ -1,4 +1,5 @@ clearCachePostProc($parameters, $coreDataHandlerMock); } - public function getPossibleCallCombinations() : array + public function getPossibleCallCombinations(): array { return [ 'Editor triggered cache clear of page manual' => [ @@ -75,7 +78,7 @@ public function getPossibleCallCombinations() : array ], 'Editor changed records on a page' => [ 'parameters' => [ - 'uid_page' =>10, + 'uid_page' => 10, ], 'expectCall' => true, ], @@ -110,6 +113,7 @@ public function indexingIsNotCalledForCacheClearIfDataIsInvalid() 'cacheCmd' => 'NEW343', ], $coreDataHandlerMock); } + /** * @test */ diff --git a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php index 30745cd1..a82c52fe 100644 --- a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php +++ b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php @@ -1,4 +1,5 @@ subject->execute($this->finisherContextMock); } - public function possibleFinisherSetup() : array + public function possibleFinisherSetup(): array { return [ 'valid update configuration' => [ @@ -101,6 +105,7 @@ public function possibleFinisherSetup() : array * @test * @requires function \TYPO3\CMS\Form\Domain\Finishers\AbstractFinisher::setOptions * @dataProvider invalidFinisherSetup + * @param array $options */ public function nothingHappensIfUnknownActionIsConfigured(array $options) { @@ -114,7 +119,7 @@ public function nothingHappensIfUnknownActionIsConfigured(array $options) $this->subject->execute($this->finisherContextMock); } - public function invalidFinisherSetup() : array + public function invalidFinisherSetup(): array { return [ 'missing options' => [ diff --git a/Tests/Unit/UnitTests.xml b/Tests/Unit/UnitTests.xml index d8285be8..6486594e 100644 --- a/Tests/Unit/UnitTests.xml +++ b/Tests/Unit/UnitTests.xml @@ -1,7 +1,7 @@ =7.0.0", - "typo3/cms": ">= 7.6.0 < 9.0.0", - "ruflin/elastica": "~3.2" + "typo3/cms": ">= 8.7.0 < 9.0.0", + "ruflin/elastica": "^6.1.0" }, "require-dev": { "phpunit/phpunit": "~6.4.4", - "typo3/testing-framework": "~1.1.5", + "typo3/testing-framework": "^2.0.0", "squizlabs/php_codesniffer": "~3.1.1" }, "config": { diff --git a/ext_emconf.php b/ext_emconf.php index b0bc5ed9..db3f0781 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -7,7 +7,7 @@ 'clearCacheOnLoad' => 1, 'constraints' => [ 'depends' => [ - 'typo3' => '7.6.0-8.7.99', + 'typo3' => '8.7.0-8.7.99', 'php' => '7.0.0-7.2.99' ], 'conflicts' => [], @@ -18,7 +18,7 @@ ], ], 'state' => 'beta', - 'version' => '0.0.7', + 'version' => '0.1.0', 'author' => 'Daniel Siepmann', 'author_email' => 'coding@daniel-siepmann.de', ]; diff --git a/ext_localconf.php b/ext_localconf.php index 658d6834..2934bc4e 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -1,59 +1,77 @@ to handle records modified through - // Frontend and backend modules not using datahandler +(function ($extension, $configuration) { + if (is_string($configuration)) { + $configuration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$extension]); + } - $GLOBALS['TYPO3_CONF_VARS'] = TYPO3\CMS\Extbase\Utility\ArrayUtility::arrayMergeRecursiveOverrule( - $GLOBALS['TYPO3_CONF_VARS'], - [ - 'SC_OPTIONS' => [ - 'extbase' => [ - 'commandControllers' => [ - Codappix\SearchCore\Command\IndexCommandController::class, - ], + $iconRegistry = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Core\Imaging\IconRegistry::class); + $iconRegistry->registerIcon( + 'plugin-' . $extension . '-form', + \TYPO3\CMS\Core\Imaging\IconProvider\SvgIconProvider::class, + ['source' => 'EXT:search_core/Resources/Public/Icons/PluginForm.svg'] + ); + $iconRegistry->registerIcon( + 'plugin-' . $extension . '-search', + \TYPO3\CMS\Core\Imaging\IconProvider\SvgIconProvider::class, + ['source' => 'EXT:search_core/Resources/Public/Icons/PluginSearch.svg'] + ); + + // TODO: Add hook for Extbase -> to handle records modified through + // Frontend and backend modules not using datahandler + \TYPO3\CMS\Core\Utility\ArrayUtility::mergeRecursiveWithOverrule( + $GLOBALS['TYPO3_CONF_VARS'], + [ + 'SC_OPTIONS' => [ + 'extbase' => [ + 'commandControllers' => [ + $extension => Codappix\SearchCore\Command\IndexCommandController::class, + ], + ], + 't3lib/class.t3lib_tcemain.php' => [ + 'clearCachePostProc' => [ + $extension => \Codappix\SearchCore\Hook\DataHandler::class . '->clearCachePostProc', ], - 't3lib/class.t3lib_tcemain.php' => [ - 'clearCachePostProc' => [ - $extensionKey => \Codappix\SearchCore\Hook\DataHandler::class . '->clearCachePostProc', - ], - 'processCmdmapClass' => [ - $extensionKey => \Codappix\SearchCore\Hook\DataHandler::class, - ], - 'processDatamapClass' => [ - $extensionKey => \Codappix\SearchCore\Hook\DataHandler::class, - ], + 'processCmdmapClass' => [ + $extension => \Codappix\SearchCore\Hook\DataHandler::class, + ], + 'processDatamapClass' => [ + $extension => \Codappix\SearchCore\Hook\DataHandler::class, ], ], - ] - ); - - TYPO3\CMS\Extbase\Utility\ExtensionUtility::configurePlugin( - 'Codappix.' . $extensionKey, - 'search', - [ - 'Search' => 'search' ], - [ - 'Search' => 'search' - ] - ); + ] + ); + + TYPO3\CMS\Extbase\Utility\ExtensionUtility::configurePlugin( + 'Codappix.' . $extension, + 'Search', + ['Search' => 'search'], + ['Search' => 'search'] + ); + + \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::addPageTSConfig( + '' + ); - \Codappix\SearchCore\Compatibility\ImplementationRegistrationService::registerImplementations(); + if (!is_array($GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations']['search_core'])) { + $GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations']['search_core'] = []; + } + if (!isset($GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations']['search_core']['backend'])) { + $GLOBALS['TYPO3_CONF_VARS']['SYS']['caching']['cacheConfigurations']['search_core']['backend'] = + \TYPO3\CMS\Core\Cache\Backend\NullBackend::class; + } - // API does make use of object manager, therefore use GLOBALS - $extensionConfiguration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$extensionKey]); - if ($extensionConfiguration === false - || !isset($extensionConfiguration['disable.']['elasticsearch']) - || $extensionConfiguration['disable.']['elasticsearch'] !== '1' - ) { - \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class) - ->registerImplementation( - \Codappix\SearchCore\Connection\ConnectionInterface::class, - \Codappix\SearchCore\Connection\Elasticsearch::class - ); - } - }, - $_EXTKEY -); + if (empty($configuration) || + (isset($configuration['disable.']['elasticsearch']) && + filter_var($configuration['disable.']['elasticsearch'], FILTER_VALIDATE_BOOLEAN) === false) + ) { + $container = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance( + \TYPO3\CMS\Extbase\Object\Container\Container::class + ); + $container->registerImplementation( + \Codappix\SearchCore\Connection\ConnectionInterface::class, + \Codappix\SearchCore\Connection\Elasticsearch::class + ); + } +})($_EXTKEY, $_EXTCONF); diff --git a/ext_tables.php b/ext_tables.php deleted file mode 100644 index 1a547878..00000000 --- a/ext_tables.php +++ /dev/null @@ -1,13 +0,0 @@ -