NGSTACK-843 page indexing implementation from fina + FieldMapper impl…

.php-cs-fixer.php

@@ -19,16 +19,17 @@
         'method_chaining_indentation' => false,
         'multiline_whitespace_before_semicolons' => false,
         'native_function_invocation' => ['include' => ['@all']],
-        'no_superfluous_phpdoc_tags' => false,
+        'no_superfluous_phpdoc_tags' => true,
         'no_unset_on_property' => false,
         'ordered_imports' => ['imports_order' => ['class', 'function', 'const'], 'sort_algorithm' => 'alpha'],
         'php_unit_internal_class' => false,
         'php_unit_test_case_static_method_calls' => ['call_type' => 'self'],
         'php_unit_test_class_requires_covers' => false,
         'phpdoc_align' => false,
         'phpdoc_types_order' => ['null_adjustment' => 'always_last', 'sort_algorithm' => 'none'],
+        'phpdoc_no_alias_tag' => ['replacements' => ['type' => 'var', 'link' => 'see']],
         'single_line_comment_style' => false,
-        'trailing_comma_in_multiline' => ['elements' => ['arrays', 'arguments']],
+        'trailing_comma_in_multiline' => ['elements' => ['arrays', 'arguments', 'match', 'parameters']],
         'yoda_style' => false,
         'php_unit_strict' => false,
         'php_unit_test_annotation' => false,
@@ -50,7 +51,8 @@
         'static_lambda' => true,
         'ternary_to_null_coalescing' => true,
         'use_arrow_functions' => true,
-   ])
+        'no_alias_language_construct_call' => true,
+    ])
     ->setRiskyAllowed(true)
     ->setFinder($finder)
-;
+    ;

bundle/Command/IndexPageContentCommand.php

@@ -0,0 +1,174 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Netgen\Bundle\IbexaSearchExtraBundle\Command;
+
+use Ibexa\Contracts\Core\Persistence\Handler as PersistenceHandler;
+use Ibexa\Contracts\Core\Repository\ContentService;
+use Ibexa\Contracts\Core\Repository\Exceptions\InvalidArgumentException;
+use Ibexa\Contracts\Core\Repository\Exceptions\NotFoundException;
+use Ibexa\Contracts\Core\Repository\Exceptions\UnauthorizedException;
+use Ibexa\Contracts\Core\Repository\Values\Content\Content;
+use Ibexa\Contracts\Core\Repository\Values\Content\ContentList;
+use Ibexa\Contracts\Core\Repository\Values\Content\Query;
+use Ibexa\Contracts\Core\Repository\Values\Filter\Filter;
+use Ibexa\Contracts\Core\Search\Handler as SearchHandler;
+use Netgen\IbexaSearchExtra\Exception\IndexPageUnavailableException;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+
+use function count;
+use function explode;
+
+class IndexPageContentCommand extends Command
+{
+    protected static $defaultName = 'netgen-search-extra:index-page-content';
+    private SymfonyStyle $style;
+
+    /**
+     * @param array<string, mixed> $sitesConfig
+     */
+    public function __construct(
+        private readonly ContentService $contentService,
+        private readonly SearchHandler $searchHandler,
+        private readonly PersistenceHandler $persistenceHandler,
+        private readonly array $sitesConfig,
+    ) {
+        parent::__construct($this::$defaultName);
+    }
+
+    protected function configure(): void
+    {
+        $this
+            ->setDescription('Index content related through layouts')
+            ->addOption(
+                'content-ids',
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'Comma separated list of content id\'s of content to index.',
+            );
+    }
+
+    protected function initialize(InputInterface $input, OutputInterface $output): void
+    {
+        $this->style = new SymfonyStyle($input, $output);
+    }
+
+    /**
+     * @throws NotFoundException
+     * @throws InvalidArgumentException
+     * @throws UnauthorizedException
+     */
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        foreach ($this->sitesConfig as $site => $siteConfig) {
+            $this->style->info('Indexing for site ' . $site);
+            $this->indexContent($output, $input, $siteConfig);
+        }
+
+        return Command::SUCCESS;
+    }
+
+    private function indexContent(OutputInterface $output, InputInterface $input, array $siteConfig): void
+    {
+        $contentIds = explode(',', $input->getOption('content-ids'));
+
+        $allowedContentTypes = $siteConfig['allowed_content_types'];
+        $offset = 0;
+        $limit = 50;
+        $totalCount = $this->getTotalCount($allowedContentTypes, $contentIds);
+        $progressBar = new ProgressBar($output, $totalCount);
+
+        if ($totalCount <= 0) {
+            $this->style->info('No content found to index, exiting.');
+
+            return;
+        }
+
+        $this->style->info('Found ' . $totalCount . ' content objects...');
+
+        $progressBar->start($totalCount);
+
+        while ($offset < $totalCount) {
+            $chunk = $this->getChunk($limit, $offset, $allowedContentTypes, $contentIds);
+
+            $this->processChunk($chunk, $progressBar);
+
+            $offset += $limit;
+        }
+
+        $progressBar->finish();
+
+        $output->writeln('');
+        $this->style->info('Finished.');
+    }
+
+    /**
+     * @throws InvalidArgumentException
+     */
+    private function getTotalCount(array $allowedContentTypes, array $contentIds): int
+    {
+        $filter = $this->getFilter($allowedContentTypes, $contentIds);
+
+        $filter
+            ->withLimit(0)
+            ->withOffset(0);
+
+        return $this->contentService->find($filter)->getTotalCount() ?? 0;
+    }
+
+    /**
+     * @throws InvalidArgumentException
+     */
+    private function getChunk(int $limit, int $offset, array $allowedContentTypes, array $contentIds): ContentList
+    {
+        $filter = $this->getFilter($allowedContentTypes, $contentIds);
+        $filter
+            ->withLimit($limit)
+            ->withOffset($offset)
+        ;
+
+        return $this->contentService->find($filter);
+    }
+
+    private function getFilter(array $allowedContentTypes, array $contentIds = []): Filter
+    {
+        $filter = new Filter();
+        $filter->withCriterion(new Query\Criterion\ContentTypeIdentifier($allowedContentTypes));
+
+        if (count($contentIds) > 0) {
+            $filter->andWithCriterion(new Query\Criterion\ContentId($contentIds));
+        }
+
+        return $filter;
+    }
+
+    private function processChunk(ContentList $contentList, ProgressBar $progressBar): void
+    {
+        foreach ($contentList->getIterator() as $content) {
+            try {
+                $this->indexContentWithLocations($content);
+                $progressBar->advance();
+            } catch (IndexPageUnavailableException $exception) {
+                $this->style->error($exception->getMessage());
+            }
+        }
+    }
+
+    private function indexContentWithLocations(Content $content): void
+    {
+        $this->searchHandler->indexContent(
+            $this->persistenceHandler->contentHandler()->load($content->id, $content->versionInfo->versionNo),
+        );
+
+        $locations = $this->persistenceHandler->locationHandler()->loadLocationsByContent($content->id);
+        foreach ($locations as $location) {
+            $this->searchHandler->indexLocation($location);
+        }
+    }
+}

bundle/DependencyInjection/Configuration.php

@@ -8,6 +8,9 @@
 use Symfony\Component\Config\Definition\Builder\TreeBuilder;
 use Symfony\Component\Config\Definition\ConfigurationInterface;
 
+use function array_keys;
+use function is_string;
+
 class Configuration implements ConfigurationInterface
 {
     protected string $rootNodeName;
@@ -25,6 +28,7 @@ public function getConfigTreeBuilder(): TreeBuilder
         $this->addIndexableFieldTypeSection($rootNode);
         $this->addSearchResultExtractorSection($rootNode);
         $this->addAsynchronousIndexingSection($rootNode);
+        $this->addPageIndexingSection($rootNode);
 
         return $treeBuilder;
     }
@@ -73,4 +77,94 @@ private function addAsynchronousIndexingSection(ArrayNodeDefinition $nodeDefinit
                 ->end()
             ->end();
     }
+
+    private function addPageIndexingSection(ArrayNodeDefinition $nodeDefinition): void
+    {
+        $keyValidator = static function ($v) {
+            foreach (array_keys($v) as $key) {
+                if (!is_string($key)) {
+                    return true;
+                }
+            }
+
+            return false;
+        };
+        $nodeDefinition
+            ->children()
+                ->arrayNode('page_indexing')
+                ->addDefaultsIfNotSet()
+                ->info('Page indexing configuration')
+                ->children()
+                    ->booleanNode('enabled')
+                        ->info('Use page indexing')
+                        ->defaultFalse()
+                    ->end()
+                    ->arrayNode('sites')
+                        ->useAttributeAsKey('name')
+                        ->normalizeKeys(false)
+                        ->validate()
+                            ->ifTrue($keyValidator)
+                            ->thenInvalid('Site name must be of string type')
+                        ->end()
+                        ->arrayPrototype()
+                            ->children()
+                                ->integerNode('tree_root_location_id')
+                                    ->info('Site root Location ID')
+                                    ->beforeNormalization()->always(static fn ($v) => is_string($v) ? (int) $v : $v)->end()
+                                ->end()
+                                ->arrayNode('languages_siteaccess_map')
+                                    ->info('Language code mapped to page siteaccess')
+                                    ->useAttributeAsKey('name')
+                                    ->normalizeKeys(false)
+                                    ->validate()
+                                        ->ifTrue($keyValidator)
+                                        ->thenInvalid('Language code must be of string type.')
+                                    ->end()
+                                    ->scalarPrototype()
+                                        ->validate()
+                                            ->ifTrue(static fn ($v) => !is_string($v))
+                                            ->thenInvalid('Siteaccess name must be of string type.')
+                                        ->end()
+                                    ->end()
+                                ->end()
+                                ->arrayNode('fields')
+                                    ->info('Mapping of indexed field names to an array of HTML tag selectors')
+                                    ->validate()
+                                        ->ifTrue($keyValidator)
+                                        ->thenInvalid('Indexed field name must be of string type')
+                                    ->end()
+                                    ->arrayPrototype()
+                                        ->useAttributeAsKey('name')
+                                        ->normalizeKeys(false)
+                                        ->scalarPrototype()
+                                            ->validate()
+                                                ->ifTrue(static fn ($v) => !is_string($v))
+                                                ->thenInvalid('HTML selector must be of string type.')
+                                            ->end()
+                                        ->end()
+                                    ->end()
+                                ->end()
+                                ->arrayNode('allowed_content_types')
+                                    ->info('Content types to index')
+                                    ->useAttributeAsKey('name')
+                                    ->normalizeKeys(false)
+                                    ->scalarPrototype()
+                                        ->validate()
+                                            ->ifTrue(static fn ($v) => !is_string($v))
+                                            ->thenInvalid('Content type identifier must be of string type.')
+                                        ->end()
+                                    ->end()
+                                ->end()
+                                ->scalarNode('host')
+                                    ->info('Host to index page from, defined in .env files')
+                                    ->validate()
+                                        ->ifTrue(static fn ($v) => !is_string($v))
+                                        ->thenInvalid('Host must be of string type.')
+                                    ->end()
+                                ->end()
+                            ->end()
+                    ->end()
+                ->end()
+            ->end();
+    }
 }