diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php index 787b5d24665..51f14b392cc 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php @@ -51,9 +51,4 @@ public function contentStream(): string { return $this->tableNamePrefix . '_contentstream'; } - - public function checkpoint(): string - { - return $this->tableNamePrefix . '_checkpoint'; - } } diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index 39186aa616f..4e998fde99a 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -61,10 +61,8 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPublished; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\NodeType\NodeTypeName; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\NodeTags; @@ -75,7 +73,6 @@ use Neos\ContentRepository\Core\SharedModel\Node\NodeName; use Neos\ContentRepository\Core\SharedModel\Node\ReferenceName; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; /** @@ -93,8 +90,6 @@ final class DoctrineDbalContentGraphProjection implements ContentGraphProjection public const RELATION_DEFAULT_OFFSET = 128; - private DbalCheckpointStorage $checkpointStorage; - public function __construct( private readonly Connection $dbal, private readonly ProjectionContentGraph $projectionContentGraph, @@ -102,11 +97,6 @@ public function __construct( private readonly DimensionSpacePointsRepository $dimensionSpacePointsRepository, private readonly ContentGraphReadModelInterface $contentGraphReadModel ) { - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNames->checkpoint(), - self::class - ); } public function setUp(): void @@ -120,18 +110,10 @@ public function setUp(): void throw new \RuntimeException(sprintf('Failed to setup projection %s: %s', self::class, $e->getMessage()), 1716478255, $e); } } - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->dbal->connect(); } catch (\Throwable $e) { @@ -149,17 +131,9 @@ public function status(): ProjectionStatus return ProjectionStatus::ok(); } - public function reset(): void + public function resetState(): void { $this->truncateDatabaseTables(); - - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); - } - - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; } public function getState(): ContentGraphReadModelInterface @@ -167,43 +141,6 @@ public function getState(): ContentGraphReadModelInterface return $this->contentGraphReadModel; } - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - ContentStreamWasClosed::class, - ContentStreamWasCreated::class, - ContentStreamWasForked::class, - ContentStreamWasRemoved::class, - ContentStreamWasReopened::class, - DimensionShineThroughWasAdded::class, - DimensionSpacePointWasMoved::class, - NodeAggregateNameWasChanged::class, - NodeAggregateTypeWasChanged::class, - NodeAggregateWasMoved::class, - NodeAggregateWasRemoved::class, - NodeAggregateWithNodeWasCreated::class, - NodeGeneralizationVariantWasCreated::class, - NodePeerVariantWasCreated::class, - NodePropertiesWereSet::class, - NodeReferencesWereSet::class, - NodeSpecializationVariantWasCreated::class, - RootNodeAggregateDimensionsWereUpdated::class, - RootNodeAggregateWithNodeWasCreated::class, - RootWorkspaceWasCreated::class, - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - WorkspaceBaseWorkspaceWasChanged::class, - WorkspaceRebaseFailed::class, - WorkspaceWasCreated::class, - WorkspaceWasDiscarded::class, - WorkspaceWasPartiallyDiscarded::class, - WorkspaceWasPartiallyPublished::class, - WorkspaceWasPublished::class, - WorkspaceWasRebased::class, - WorkspaceWasRemoved::class, - ]) || $event instanceof EmbedsContentStreamId; - } - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void { match ($event::class) { @@ -238,7 +175,7 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void WorkspaceWasPublished::class => $this->whenWorkspaceWasPublished($event), WorkspaceWasRebased::class => $this->whenWorkspaceWasRebased($event), WorkspaceWasRemoved::class => $this->whenWorkspaceWasRemoved($event), - default => $event instanceof EmbedsContentStreamId || throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; if ( $event instanceof EmbedsContentStreamId diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php index a5c4d6ae25e..0d69c3c6a22 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php @@ -8,7 +8,7 @@ use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\DimensionSpacePointsRepository; use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\NodeFactory; use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\ProjectionContentGraph; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; /** @@ -24,8 +24,7 @@ public function __construct( } public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, - array $options, + SubscriberFactoryDependencies $projectionFactoryDependencies, ): DoctrineDbalContentGraphProjection { $tableNames = ContentGraphTableNames::create( $projectionFactoryDependencies->contentRepositoryId diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php index 8c6ff9118b8..5961442b9e6 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php @@ -15,7 +15,6 @@ namespace Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection; use Doctrine\DBAL\Connection; -use Doctrine\DBAL\Schema\AbstractSchemaManager; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\ContentStreamForking; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeCreation; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeModification; @@ -26,7 +25,6 @@ use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeVariation; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\SubtreeTagging; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\SchemaBuilder\HypergraphSchemaBuilder; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\Feature\ContentStreamForking\Event\ContentStreamWasForked; use Neos\ContentRepository\Core\Feature\NodeCreation\Event\NodeAggregateWithNodeWasCreated; @@ -41,12 +39,10 @@ use Neos\ContentRepository\Core\Feature\RootNodeCreation\Event\RootNodeAggregateWithNodeWasCreated; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; -use Neos\EventStore\Model\Event\SequenceNumber; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -66,7 +62,6 @@ final class HypergraphProjection implements ContentGraphProjectionInterface use NodeTypeChange; use NodeVariation; - private DbalCheckpointStorage $checkpointStorage; private ProjectionHypergraph $projectionHypergraph; public function __construct( @@ -75,11 +70,6 @@ public function __construct( private readonly ContentGraphReadModelInterface $contentGraphReadModel ) { $this->projectionHypergraph = new ProjectionHypergraph($this->dbal, $this->tableNamePrefix); - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNamePrefix . '_checkpoint', - self::class - ); } @@ -97,18 +87,10 @@ public function setUp(): void create index if not exists restriction_affected on ' . $this->tableNamePrefix . '_restrictionhyperrelation using gin (affectednodeaggregateids); '); - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->getDatabaseConnection()->connect(); } catch (\Throwable $e) { @@ -135,12 +117,9 @@ private function determineRequiredSqlStatements(): array return DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $schema); } - public function reset(): void + public function resetState(): void { $this->truncateDatabaseTables(); - - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); } private function truncateDatabaseTables(): void @@ -151,39 +130,6 @@ private function truncateDatabaseTables(): void $this->dbal->executeQuery('TRUNCATE table ' . $this->tableNamePrefix . '_restrictionhyperrelation'); } - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - // ContentStreamForking - ContentStreamWasForked::class, - // NodeCreation - RootNodeAggregateWithNodeWasCreated::class, - NodeAggregateWithNodeWasCreated::class, - // SubtreeTagging - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - // NodeModification - NodePropertiesWereSet::class, - // NodeReferencing - NodeReferencesWereSet::class, - // NodeRemoval - NodeAggregateWasRemoved::class, - // NodeRenaming - NodeAggregateNameWasChanged::class, - // NodeTypeChange - NodeAggregateTypeWasChanged::class, - // NodeVariation - NodeSpecializationVariantWasCreated::class, - NodeGeneralizationVariantWasCreated::class, - NodePeerVariantWasCreated::class, - // TODO: not yet supported: - //ContentStreamWasRemoved::class, - //DimensionSpacePointWasMoved::class, - //DimensionShineThroughWasAdded::class, - //NodeAggregateWasMoved::class, - ]); - } - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void { match ($event::class) { @@ -209,7 +155,7 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void NodeSpecializationVariantWasCreated::class => $this->whenNodeSpecializationVariantWasCreated($event), NodeGeneralizationVariantWasCreated::class => $this->whenNodeGeneralizationVariantWasCreated($event), NodePeerVariantWasCreated::class => $this->whenNodePeerVariantWasCreated($event), - default => throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; } @@ -228,11 +174,6 @@ public function inSimulation(\Closure $fn): mixed } } - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; - } - public function getState(): ContentGraphReadModelInterface { return $this->contentGraphReadModel; diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php b/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php index 3d3c002c094..a70b0d5d148 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php @@ -7,7 +7,7 @@ use Doctrine\DBAL\Connection; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\HypergraphProjection; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Repository\NodeFactory; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; @@ -28,8 +28,7 @@ public static function graphProjectionTableNamePrefix( } public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, - array $options, + SubscriberFactoryDependencies $projectionFactoryDependencies, ): HypergraphProjection { $tableNamePrefix = self::graphProjectionTableNamePrefix( $projectionFactoryDependencies->contentRepositoryId diff --git a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php index 039156ef9da..9f809c0b9ac 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php @@ -17,12 +17,13 @@ use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto\TraceEntries; use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto\TraceEntryType; use Neos\ContentRepository\Core\EventStore\EventInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Flow\Annotations as Flow; /** - * We had some race conditions in projections, where {@see \Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage} was not working properly. + * We had some race conditions in projections * We saw some non-deterministic, random errors when running the tests - unluckily only on Linux, not on OSX: * On OSX, forking a new subprocess in {@see SubprocessProjectionCatchUpTrigger} is *WAY* slower than in Linux; * and thus the race conditions which appears if two projector instances of the same class run concurrently @@ -73,7 +74,7 @@ * * When {@see onBeforeEvent} is called, we know that we are inside applyEvent() in the diagram above, * thus we know the lock *HAS* been acquired. - * When {@see onBeforeBatchCompleted}is called, we know the lock will be released directly afterwards. + * When {@see onAfterCatchUp}is called, we know the lock will be released directly afterwards. * * We track these timings across processes in a single Redis Stream. Because Redis is single-threaded, * we can be sure that we observe the correct, total order of interleavings *across multiple processes* @@ -107,7 +108,7 @@ final class RaceTrackerCatchUpHook implements CatchUpHookInterface protected $configuration; private bool $inCriticalSection = false; - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { RedisInterleavingLogger::connect($this->configuration['redis']['host'], $this->configuration['redis']['port']); } @@ -126,7 +127,7 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event { } - public function onBeforeBatchCompleted(): void + public function onAfterCatchUp(): void { // we only want to track relevant lock release calls (i.e. if we were in the event processing loop before) if ($this->inCriticalSection) { @@ -134,8 +135,4 @@ public function onBeforeBatchCompleted(): void RedisInterleavingLogger::trace(TraceEntryType::LockWillBeReleasedIfItWasAcquiredBefore); } } - - public function onAfterCatchUp(): void - { - } } diff --git a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php index 389d7a324a5..829ed582f96 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php @@ -14,9 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; /** diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php index 9f8d56cda7f..2fb4fe8ac9b 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php @@ -18,11 +18,14 @@ use Behat\Gherkin\Node\TableNode; use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Helpers\GherkinTableNodeBasedContentDimensionSource; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; use Neos\EventStore\EventStoreInterface; +use PHPUnit\Framework\Assert; use Symfony\Component\Yaml\Yaml; /** @@ -178,17 +181,26 @@ protected function setUpContentRepository(ContentRepositoryId $contentRepository * Catch Up process and the testcase reset. */ $contentRepository = $this->createContentRepository($contentRepositoryId); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); if (!in_array($contentRepository->id, self::$alreadySetUpContentRepositories)) { - $contentRepository->setUp(); + $result = $contentRepositoryMaintainer->setUp(); + Assert::assertNull($result); self::$alreadySetUpContentRepositories[] = $contentRepository->id; } + // todo we TRUNCATE here and do not want to use $contentRepositoryMaintainer->prune(); here as it would not reset the autoincrement sequence number making some assertions impossible /** @var EventStoreInterface $eventStore */ $eventStore = (new \ReflectionClass($contentRepository))->getProperty('eventStore')->getValue($contentRepository); /** @var Connection $databaseConnection */ $databaseConnection = (new \ReflectionClass($eventStore))->getProperty('connection')->getValue($eventStore); $eventTableName = sprintf('cr_%s_events', $contentRepositoryId->value); $databaseConnection->executeStatement('TRUNCATE ' . $eventTableName); - $contentRepository->resetProjectionStates(); + + /** @var SubscriptionEngine $subscriptionEngine */ + $subscriptionEngine = (new \ReflectionClass($contentRepositoryMaintainer))->getProperty('subscriptionEngine')->getValue($contentRepositoryMaintainer); + $result = $subscriptionEngine->reset(); + Assert::assertNull($result->errors); + $result = $subscriptionEngine->boot(); + Assert::assertNull($result->errors); return $contentRepository; } diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php new file mode 100644 index 00000000000..7f71796e96d --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -0,0 +1,133 @@ + + * @internal + * @Flow\Proxy(false) + */ +final class DebugEventProjection implements ProjectionInterface +{ + private DebugEventProjectionState $state; + + private \Closure|null $saboteur = null; + + /** + * @var array + */ + private array $additionalColumnsForSchema = []; + + public function __construct( + private string $tableNamePrefix, + private Connection $dbal + ) { + $this->state = new DebugEventProjectionState($this->tableNamePrefix, $this->dbal); + } + + public function setUp(): void + { + foreach ($this->determineRequiredSqlStatements() as $statement) { + $this->dbal->executeStatement($statement); + } + } + + public function status(): ProjectionStatus + { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + if ($requiredSqlStatements !== []) { + return ProjectionStatus::setupRequired(sprintf('Requires %d SQL statements', count($requiredSqlStatements))); + } + return ProjectionStatus::ok(); + } + + /** + * @return array + */ + private function determineRequiredSqlStatements(): array + { + $schemaManager = $this->dbal->createSchemaManager(); + + $table = new Table($this->tableNamePrefix, [ + (new Column('sequencenumber', Type::getType(Types::INTEGER))), + (new Column('stream', Type::getType(Types::STRING))), + (new Column('type', Type::getType(Types::STRING))), + ...$this->additionalColumnsForSchema + ]); + + $table->setPrimaryKey([ + 'sequencenumber' + ]); + + $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [$table]); + $statements = DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $schema); + + return $statements; + } + + public function resetState(): void + { + $this->dbal->executeStatement('TRUNCATE ' . $this->tableNamePrefix); + } + + public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void + { + try { + $this->dbal->insert($this->tableNamePrefix, [ + 'sequencenumber' => $eventEnvelope->sequenceNumber->value, + 'stream' => $eventEnvelope->streamName->value, + 'type' => $eventEnvelope->event->type->value, + ]); + } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $exception) { + throw new \RuntimeException(sprintf('Must not happen! Debug projection detected duplicate event %s of type %s', $eventEnvelope->sequenceNumber->value, $eventEnvelope->event->type->value), 1732360282, $exception); + } + if ($this->saboteur) { + ($this->saboteur)($eventEnvelope); + } + } + + public function getState(): ProjectionStateInterface + { + return $this->state; + } + + public function injectSaboteur(\Closure $saboteur): void + { + $this->saboteur = $saboteur; + } + + public function killSaboteur(): void + { + $this->saboteur = null; + } + + public function schemaNeedsAdditionalColumn(string $name): void + { + $this->additionalColumnsForSchema[$name] = (new Column($name, Type::getType(Types::STRING)))->setNotnull(false); + } + + public function dropTables(): void + { + $this->dbal->executeStatement('DROP TABLE ' . $this->tableNamePrefix); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php new file mode 100644 index 00000000000..02dde323ad5 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php @@ -0,0 +1,33 @@ + + */ + public function findAppliedSequenceNumbers(): iterable + { + return array_map( + fn ($value) => SequenceNumber::fromInteger((int)$value['sequenceNumber']), + $this->dbal->fetchAllAssociative("SELECT sequenceNumber from {$this->tableNamePrefix}") + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml index 2d0fe1e74c0..7f2fdbc0cc4 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml @@ -31,9 +31,47 @@ Neos: factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeAuthProviderFactory clock: factoryObjectName: Neos\ContentRepositoryRegistry\Factory\Clock\SystemClockFactory + subscriptionStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactory propertyConverters: {} contentGraphProjection: factoryObjectName: Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory + catchUpHooks: {} + projections: + 'Neos.Testing:DebugProjection': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory + options: + instanceId: debug + + t_subscription: + eventStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\EventStore\DoctrineEventStoreFactory + nodeTypeManager: + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory + contentDimensionSource: + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory + authProvider: + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeAuthProviderFactory + clock: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\Clock\SystemClockFactory + subscriptionStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactory + propertyConverters: {} + contentGraphProjection: + factoryObjectName: Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory + catchUpHooks: {} + projections: + 'Vendor.Package:FakeProjection': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory + options: + instanceId: default + 'Vendor.Package:SecondFakeProjection': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory + options: + instanceId: second + catchUpHooks: + 'Vendor.Package:FakeCatchupHook': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory Flow: object: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php new file mode 100644 index 00000000000..ca9c4de33eb --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -0,0 +1,184 @@ +resetDatabase( + $this->getObject(Connection::class), + $contentRepositoryId, + keepSchema: true + ); + + $this->fakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); + $this->fakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); + + FakeProjectionFactory::setProjection( + 'default', + $this->fakeProjection + ); + + $this->secondFakeProjection = new DebugEventProjection( + sprintf('cr_%s_debug_projection', $contentRepositoryId->value), + $this->getObject(Connection::class) + ); + + FakeProjectionFactory::setProjection( + 'second', + $this->secondFakeProjection + ); + + $this->catchupHookForFakeProjection = $this->getMockBuilder(CatchUpHookInterface::class)->getMock(); + + FakeCatchUpHookFactory::setCatchupHook( + $this->secondFakeProjection->getState(), + $this->catchupHookForFakeProjection + ); + + FakeNodeTypeManagerFactory::setConfiguration([]); + FakeContentDimensionSourceFactory::setWithoutDimensions(); + + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($contentRepositoryId); + + $this->setupContentRepositoryDependencies($contentRepositoryId); + } + + final protected function setupContentRepositoryDependencies(ContentRepositoryId $contentRepositoryId) + { + $this->contentRepository = $this->getObject(ContentRepositoryRegistry::class)->get( + $contentRepositoryId + ); + + $subscriptionEngineAndEventStoreAccessor = new class implements ContentRepositoryServiceFactoryInterface { + public EventStoreInterface|null $eventStore; + public SubscriptionEngine|null $subscriptionEngine; + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + $this->eventStore = $serviceFactoryDependencies->eventStore; + $this->subscriptionEngine = $serviceFactoryDependencies->subscriptionEngine; + return new class implements ContentRepositoryServiceInterface + { + }; + } + }; + $this->getObject(ContentRepositoryRegistry::class)->buildService($contentRepositoryId, $subscriptionEngineAndEventStoreAccessor); + $this->eventStore = $subscriptionEngineAndEventStoreAccessor->eventStore; + $this->subscriptionEngine = $subscriptionEngineAndEventStoreAccessor->subscriptionEngine; + } + + final protected function resetDatabase(Connection $connection, ContentRepositoryId $contentRepositoryId, bool $keepSchema): void + { + $connection->prepare('SET FOREIGN_KEY_CHECKS = 0;')->executeStatement(); + foreach ($connection->createSchemaManager()->listTableNames() as $tableNames) { + if (!str_starts_with($tableNames, sprintf('cr_%s_', $contentRepositoryId->value))) { + // speedup deletion, only delete current cr + continue; + } + if ($keepSchema) { + // truncate is faster + $sql = 'TRUNCATE TABLE ' . $tableNames; + } else { + $sql = 'DROP TABLE ' . $tableNames; + } + $connection->prepare($sql)->executeStatement(); + } + $connection->prepare('SET FOREIGN_KEY_CHECKS = 1;')->executeStatement(); + } + + final protected function subscriptionStatus(string $subscriptionId): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null + { + return $this->subscriptionEngine->subscriptionStatus(SubscriptionEngineCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); + } + + final protected function commitExampleContentStreamEvent(): void + { + $this->eventStore->commit( + ContentStreamEventStreamName::fromContentStreamId($cs = ContentStreamId::create())->getEventStreamName(), + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => $cs->value])) + ), + ExpectedVersion::NO_STREAM() + ); + } + + final protected function expectOkayStatus($subscriptionId, SubscriptionStatus $status, SequenceNumber $sequenceNumber): void + { + $actual = $this->subscriptionStatus($subscriptionId); + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString($subscriptionId), + subscriptionStatus: $status, + subscriptionPosition: $sequenceNumber, + subscriptionError: null, + setupStatus: ProjectionStatus::ok(), + ), + $actual + ); + } + + /** + * @template T of object + * @param class-string $className + * + * @return T + */ + final protected function getObject(string $className): object + { + return Bootstrap::$staticObjectManager->get($className); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php new file mode 100644 index 00000000000..3396ec98639 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -0,0 +1,201 @@ +eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + // Todo test that onBeforeEvent|onAfterEvent are in the same transaction and that a rollback will also revert their state + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $exception = new \RuntimeException('This catchup hook is kaputt.') + ); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $this->secondFakeProjection->injectSaboteur(fn () => self::fail('Projection apply is not expected to be called!')); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This catchup hook is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // must be still empty because apply was never called + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterEvent_projectionIsRolledBack() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $exception = new \RuntimeException('This catchup hook is kaputt.') + ); + // TODO pass the error subscription status to onAfterCatchUp, so that in case of an error it can be prevented that mails f.x. will be sent? + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This catchup hook is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // should be empty as we need an exact once delivery + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onBeforeCatchUp_abortsCatchup() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::never())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willThrowException( + new \RuntimeException('This catchup hook is kaputt.') + ); + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + + $this->secondFakeProjection->injectSaboteur(fn () => self::fail('Projection apply is not expected to be called!')); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedFailure = null; + try { + $this->subscriptionEngine->catchUpActive(); + } catch (\Throwable $e) { + $expectedFailure = $e; + } + self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); + + self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" failed onBeforeCatchUp: This catchup hook is kaputt.'); + + // still the initial status + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // must be still empty because apply was never called + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterCatchUp_abortsCatchupAndRollBack() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( + new \RuntimeException('This catchup hook is kaputt.') + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedFailure = null; + try { + $this->subscriptionEngine->catchUpActive(); + } catch (\Throwable $e) { + $expectedFailure = $e; + } + self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); + + self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" failed onAfterCatchUp: This catchup hook is kaputt.'); + + // still the initial status + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // must be empty because full rollback + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php new file mode 100644 index 00000000000..56b9d632597 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php @@ -0,0 +1,50 @@ +eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $expectNoHandledEvents = fn () => self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectOneHandledEvent = fn () => self::assertEquals( + [ + SequenceNumber::fromInteger(1) + ], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectNoHandledEvents); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectNoHandledEvents); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectOneHandledEvent); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willReturnCallback($expectOneHandledEvent); + + $expectNoHandledEvents(); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + $expectOneHandledEvent(); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/DebugEventProjectionTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/DebugEventProjectionTest.php new file mode 100644 index 00000000000..e80be3c98ea --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/DebugEventProjectionTest.php @@ -0,0 +1,107 @@ +debugEventProjection = new DebugEventProjection( + 'test_debug_projection', + Bootstrap::$staticObjectManager->get(Connection::class) + ); + + $this->debugEventProjection->setUp(); + } + + public function tearDown(): void + { + $this->debugEventProjection->resetState(); + } + + /** @test */ + public function fakeProjectionRejectsDuplicateEvents() + { + $fakeEventEnvelope = $this->createExampleEventEnvelopeForPosition( + SequenceNumber::fromInteger(1) + ); + + $this->debugEventProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope + ); + + $this->expectExceptionMessage('Must not happen! Debug projection detected duplicate event 1 of type ContentStreamWasCreated'); + + $this->debugEventProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope + ); + } + + /** @test */ + public function fakeProjectionWithSaboteur() + { + $fakeEventEnvelope1 = $this->createExampleEventEnvelopeForPosition( + SequenceNumber::fromInteger(1) + ); + + $fakeEventEnvelope2 = $this->createExampleEventEnvelopeForPosition( + SequenceNumber::fromInteger(2) + ); + + $this->debugEventProjection->injectSaboteur( + fn (EventEnvelope $eventEnvelope) => + $eventEnvelope->sequenceNumber->value === 2 + ? throw new \RuntimeException('sabotage!!!') + : null + ); + + // catchup + $this->debugEventProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope1 + ); + + $this->expectExceptionMessage('sabotage!!!'); + + $this->debugEventProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope2 + ); + } + + private function createExampleEventEnvelopeForPosition(SequenceNumber $sequenceNumber): EventEnvelope + { + $cs = ContentStreamId::create(); + return new EventEnvelope( + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => $cs->value])) + ), + ContentStreamEventStreamName::fromContentStreamId($cs)->getEventStreamName(), + Event\Version::first(), + $sequenceNumber, + new \DateTimeImmutable() + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php new file mode 100644 index 00000000000..53ded4f33c3 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -0,0 +1,322 @@ +eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionEngine->setup(); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // catchup active tries to apply the commited event + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $exception = new \RuntimeException('This projection is kaputt.') + ); + $expectedStatusForFailedProjection = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::fromSubscriptionIdAndException(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); + + self::assertEquals( + $expectedStatusForFailedProjection, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // todo test retry if reimplemented: https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/RetryStrategy/RetryStrategy.php + // // CatchUp 2 with retry + // $result = $this->subscriptionEngine->catchUpActive(); + // self::assertTrue($result->hasFailed()); + // self::assertEquals($result->errors->first()->message, 'Something really wrong.'); + // self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Something really wrong.'); + + // no retry, nothing to do. + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'This projection is kaputt.'); + self::assertEquals( + $expectedStatusForFailedProjection, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + } + + /** @test */ + public function fixFailedProjection() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // catchup active tries to apply the commited event + $this->fakeProjection->expects(self::exactly(2))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( + new WillThrowException($exception = new \RuntimeException('This projection is kaputt.')), + null // okay again + ); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // catchup active does not change anything + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + // boot neither + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + // still the same state + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + $this->fakeProjection->expects(self::once())->method('resetState'); + + $result = $this->subscriptionEngine->reset(); + self::assertNull($result->errors); + + // expect the subscriptionError to be reset to null + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function projectionIsRolledBackAfterError() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('This projection is kaputt.'); + + $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // should be empty as we need an exact once delivery + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // + // fix projection and catchup + // + + $this->secondFakeProjection->killSaboteur(); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + // subscriptionError is reset + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + // catchup after fix + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit two events + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('Event 2 is kaputt.'); + + // fail at the second event + $this->secondFakeProjection->injectSaboteur( + fn (EventEnvelope $eventEnvelope) => + $eventEnvelope->sequenceNumber->value === 2 + ? throw $exception + : null + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // the first successful event is applied and committet: + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // + // fix projection and catchup + // + + $this->secondFakeProjection->killSaboteur(); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + // subscriptionError is reset, but the position is preserved + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // catchup after fix + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $originalException = new \RuntimeException('This projection is kaputt.'), + ); + + $handleException = null; + try { + $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root'), ContentStreamId::fromString('root-cs'))); + } catch (\RuntimeException $exception) { + $handleException = $exception; + } + self::assertNotNull($handleException); + self::assertEquals('Exception in subscriber "Vendor.Package:FakeProjection" while catching up: This projection is kaputt.', $handleException->getMessage()); + self::assertSame($originalException, $handleException->getPrevious()); + + // workspace is created. The fake projection failed on the first event, but other projections succeed: + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root'))); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // to exception thrown here because the failed projection is not retried and now in error state + $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root-two'), ContentStreamId::fromString('root-cs-two'))); + + // workspace two is created. + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); + self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root-two'))); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2), SequenceNumber::fromInteger(3), SequenceNumber::fromInteger(4)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php new file mode 100644 index 00000000000..421d9a4c212 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php @@ -0,0 +1,117 @@ +eventStore->setup(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionEngine->setup(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // subsequent catchup setup'd does not change the position + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // catchup active does apply the commited event + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function filteringCatchUpActive() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event: + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('apply'); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + $result = $this->subscriptionEngine->catchUpActive($filter); + self::assertNull($result->errors); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + } + + /** @test */ + public function catchupWithNoEventsKeepsThePreviousPositionOfTheSubscribers() + { + $this->eventStore->setup(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionEngine->setup(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // catchup active does apply the commited event + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // empty catchup must keep the sequence numbers of the projections okay + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php new file mode 100644 index 00000000000..a89370bd57e --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php @@ -0,0 +1,66 @@ +eventStore->setup(); + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionEngine->setup(); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->subscriptionEngine->boot(); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // catchup is a noop because there are no unhandled events + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + } + + /** @test */ + public function filteringCatchUpBoot() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + + $result = $this->subscriptionEngine->boot($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php new file mode 100644 index 00000000000..dabdcce9cac --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -0,0 +1,168 @@ +getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); + } + + /** @test */ + public function projectionIsDetachedOnCatchupActive() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + $this->subscriptionEngine->setup(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $mockSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + unset($mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:FakeProjection']); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + self::assertEquals( + DetachedSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + // the state is still active as we do not mutate it during the setup call! + subscriptionStatus: SubscriptionStatus::ACTIVE, + subscriptionPosition: SequenceNumber::none() + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + $this->fakeProjection->expects(self::never())->method('apply'); + // catchup to mark detached subscribers + $result = $this->subscriptionEngine->catchUpActive(); + // todo result should reflect that there was an detachment? Throw error in CR? + self::assertEquals(ProcessedResult::success(1), $result); + + self::assertEquals( + $expectedDetachedState = DetachedSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::none() + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // other projections are not interrupted + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // succeeding catchup's do not handle the detached one: + $this->commitExampleContentStreamEvent(); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // still detached + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + } + + /** @test */ + public function projectionIsDetachedOnSetupAndReattachedIfPossible() + { + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + $this->subscriptionEngine->setup(); + + $this->commitExampleContentStreamEvent(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(1), $result); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // "uninstall" the projection + $originalSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $mockSettings = $originalSettings; + unset($mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:FakeProjection']); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + $this->fakeProjection->expects(self::never())->method('apply'); + // setup to find detached subscribers + $result = $this->subscriptionEngine->setup(); + // todo result should reflect that there was an detachment? + self::assertNull($result->errors); + + $expectedDetachedState = DetachedSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::fromInteger(1) + ); + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // another setup does not reattach, because there is no subscriber + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // "reinstall" the projection + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: null, + setupStatus: ProjectionStatus::ok() // state _IS_ calculate-able at this point, todo better reflect meaning: is detached, but re-attachable! + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // setup does re-attach as the projection is found again + $this->subscriptionEngine->setup(); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(1)); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php new file mode 100644 index 00000000000..0a624d4c08b --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php @@ -0,0 +1,87 @@ +resetDatabase( + $this->getObject(Connection::class), + $this->contentRepository->id, + keepSchema: false + ); + + $crMaintainer = $this->getObject(ContentRepositoryRegistry::class)->buildService($this->contentRepository->id, new ContentRepositoryMaintainerFactory()); + + $status = $crMaintainer->status(); + + self::assertEquals(StatusType::SETUP_REQUIRED, $status->eventStoreStatus->type); + self::assertNull($status->eventStorePosition); + self::assertTrue($status->subscriptionStatus->isEmpty()); + + self::assertNull( + $this->subscriptionStatus('contentGraph') + ); + self::assertNull( + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + self::assertNull( + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // + // setup and fetch status + // + + // only setup content graph so that the other projections are NEW, but still found + $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); + + $actualStatuses = $this->subscriptionEngine->subscriptionStatus(); + + $expected = SubscriptionStatusCollection::fromArray([ + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::ok(), + ), + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('fake needs setup.'), + ), + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements'), + ), + ]); + + self::assertEquals($expected, $actualStatuses); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php new file mode 100644 index 00000000000..86c8ae2e0d5 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -0,0 +1,103 @@ +getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); + } + + /** @test */ + public function newProjectionIsFoundWhenConfigurationIsAdded() + { + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + $this->subscriptionEngine->setup(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + + self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + $newFakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); + $newFakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); + $newFakeProjection->expects(self::exactly(4))->method('status')->willReturnOnConsecutiveCalls( + ProjectionStatus::setupRequired('Set me up'), + ProjectionStatus::setupRequired('Set me up'), + ProjectionStatus::ok(), + ProjectionStatus::ok(), + ); + + FakeProjectionFactory::setProjection( + 'newFake', + $newFakeProjection + ); + + $mockSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:NewFakeProjection'] = [ + 'factoryObjectName' => FakeProjectionFactory::class, + 'options' => [ + 'instanceId' => 'newFake' + ] + ]; + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + $expectedNewState = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:NewFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Set me up') + ); + + // status predicts the NEW state already (without creating it in the db) + self::assertEquals( + $expectedNewState, + $this->subscriptionStatus('Vendor.Package:NewFakeProjection') + ); + + // do something that finds new subscriptions, trigger a setup on a specific projection: + $result = $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); + self::assertNull($result->errors); + + self::assertEquals( + $expectedNewState, + $this->subscriptionStatus('Vendor.Package:NewFakeProjection') + ); + + // setup this projection + $newFakeProjection->expects(self::once())->method('setUp'); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php new file mode 100644 index 00000000000..cbebb621ceb --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php @@ -0,0 +1,52 @@ +fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // commit an event: + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::once())->method('resetState'); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + $result = $this->subscriptionEngine->reset($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php new file mode 100644 index 00000000000..dd021dc8183 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -0,0 +1,221 @@ +eventStore->setup(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionEngine->setup(); + + $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); + $actualStatuses = $this->subscriptionEngine->subscriptionStatus(); + + $expected = SubscriptionStatusCollection::fromArray([ + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::ok(), + ), + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::ok(), + ), + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::ok(), + ), + ]); + + self::assertEquals($expected, $actualStatuses); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function filteringSetup() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + + $result = $this->subscriptionEngine->setup($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::ok() + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + } + + /** @test */ + public function setupIsInvokedForBootingSubscribers() + { + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + // hard reset, so that the tests actually need sql migrations + $this->secondFakeProjection->dropTables(); + + $this->eventStore->setup(); + + // initial setup for FakeProjection + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + // then an update is fetched, the status changes: + + $this->secondFakeProjection->schemaNeedsAdditionalColumn('column_after_update'); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + } + + /** @test */ + public function setupIsInvokedForPreviouslyActiveSubscribers() + { + // Usecase: Setup a content repository and then when the subscribers are active, update to a new patch which requires a setup + + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + // hard reset, so that the tests actually need sql migrations + $this->secondFakeProjection->dropTables(); + + $this->eventStore->setup(); + // setup subscription tables + $result = $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); + self::assertNull($result->errors); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // initial setup for FakeProjection + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // regular work + + $this->commitExampleContentStreamEvent(); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // then an update is fetched, the status changes: + + $this->secondFakeProjection->schemaNeedsAdditionalColumn('column_after_update'); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ACTIVE, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function failingSetupWillMarkProjectionAsErrored() + { + $this->fakeProjection->expects(self::once())->method('setUp')->willThrowException( + $exception = new \RuntimeException('Projection could not be setup') + ); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('Needs setup')); + + $this->eventStore->setup(); + + $result = $this->subscriptionEngine->setup(); + self::assertSame('Projection could not be setup', $result->errors?->first()->message); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::NEW, $exception), + setupStatus: ProjectionStatus::setupRequired('Needs setup'), + ); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php index 569609ee5ce..efa68008845 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php @@ -14,8 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Parallel; -use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Core\Bootstrap; @@ -69,15 +70,11 @@ final protected function setUpContentRepository( ContentRepositoryId $contentRepositoryId ): ContentRepository { $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); - $contentRepository->setUp(); - - $connection = $this->objectManager->get(Connection::class); - + /** @var ContentRepositoryMaintainer $contentRepositoryMaintainer */ + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + $contentRepositoryMaintainer->setUp(); // reset events and projections - $eventTableName = sprintf('cr_%s_events', $contentRepositoryId->value); - $connection->executeStatement('TRUNCATE ' . $eventTableName); - $contentRepository->resetProjectionStates(); - + $contentRepositoryMaintainer->prune(); return $contentRepository; } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php new file mode 100644 index 00000000000..58d0ce6acc9 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php @@ -0,0 +1,230 @@ +log('------ process started ------'); + + $debugProjection = new DebugEventProjection( + 'cr_test_parallel_debug_projection', + $this->objectManager->get(Connection::class) + ); + FakeProjectionFactory::setProjection( + 'debug', + $debugProjection + ); + + FakeContentDimensionSourceFactory::setWithoutDimensions(); + FakeNodeTypeManagerFactory::setConfiguration([ + 'Neos.ContentRepository:Root' => [], + 'Neos.ContentRepository.Testing:Content' => [], + 'Neos.ContentRepository.Testing:Document' => [ + 'properties' => [ + 'title' => [ + 'type' => 'string' + ] + ], + 'childNodes' => [ + 'tethered-a' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-b' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-c' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-d' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-e' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ] + ] + ] + ]); + + $setupLockResource = fopen(self::SETUP_LOCK_PATH, 'w+'); + + $exclusiveNonBlockingLockResult = flock($setupLockResource, LOCK_EX | LOCK_NB); + if ($exclusiveNonBlockingLockResult === false) { + $this->log('waiting for setup'); + if (!flock($setupLockResource, LOCK_SH)) { + throw new \RuntimeException('failed to acquire blocking shared lock'); + } + $this->contentRepository = $this->contentRepositoryRegistry + ->get(ContentRepositoryId::fromString('test_parallel')); + $this->log('wait for setup finished'); + return; + } + + $this->log('setup started'); + $contentRepository = $this->setUpContentRepository(ContentRepositoryId::fromString('test_parallel')); + + $origin = OriginDimensionSpacePoint::createWithoutDimensions(); + $contentRepository->handle(CreateRootWorkspace::create( + WorkspaceName::forLive(), + ContentStreamId::fromString('live-cs-id') + )); + $contentRepository->handle(CreateRootNodeAggregateWithNode::create( + WorkspaceName::forLive(), + NodeAggregateId::fromString('lady-eleonode-rootford'), + NodeTypeName::fromString(NodeTypeName::ROOT_NODE_TYPE_NAME) + )); + $contentRepository->handle(CreateNodeAggregateWithNode::create( + WorkspaceName::forLive(), + NodeAggregateId::fromString('nody-mc-nodeface'), + NodeTypeName::fromString('Neos.ContentRepository.Testing:Document'), + $origin, + NodeAggregateId::fromString('lady-eleonode-rootford'), + initialPropertyValues: PropertyValuesToWrite::fromArray([ + 'title' => 'title-original' + ]) + )); + $contentRepository->handle(CreateWorkspace::create( + WorkspaceName::fromString('user-test'), + WorkspaceName::forLive(), + ContentStreamId::fromString('user-cs-id') + )); + + $this->contentRepository = $contentRepository; + + if (!flock($setupLockResource, LOCK_UN)) { + throw new \RuntimeException('failed to release setup lock'); + } + + $this->log('setup finished'); + } + + /** + * @test + * @group parallel + */ + public function whileANodesArWrittenOnLive(): void + { + $this->log('1. writing started'); + + touch(self::WRITING_IS_RUNNING_FLAG_PATH); + + try { + for ($i = 0; $i <= 100; $i++) { + $this->contentRepository->handle(CreateNodeAggregateWithNode::create( + WorkspaceName::forLive(), + NodeAggregateId::fromString('nody-mc-nodeface-' . $i), + NodeTypeName::fromString('Neos.ContentRepository.Testing:Document'), + OriginDimensionSpacePoint::createWithoutDimensions(), + NodeAggregateId::fromString('lady-eleonode-rootford'), + initialPropertyValues: PropertyValuesToWrite::fromArray([ + 'title' => 'title' + ]) + )); + } + } finally { + unlink(self::WRITING_IS_RUNNING_FLAG_PATH); + } + + $this->log('1. writing finished'); + Assert::assertTrue(true, 'No exception was thrown ;)'); + + $subgraph = $this->contentRepository->getContentGraph(WorkspaceName::forLive())->getSubgraph(DimensionSpacePoint::createWithoutDimensions(), VisibilityConstraints::withoutRestrictions()); + $node = $subgraph->findNodeById(NodeAggregateId::fromString('nody-mc-nodeface-100')); + Assert::assertNotNull($node); + } + + /** + * @test + * @group parallel + */ + public function thenConcurrentPublishLeadsToException(): void + { + if (!is_file(self::WRITING_IS_RUNNING_FLAG_PATH)) { + $this->log('waiting for 2. writing'); + + $this->awaitFile(self::WRITING_IS_RUNNING_FLAG_PATH); + // If write is the process that does the (slowish) setup, and then waits for the rebase to start, + // We give the CR some time to close the content stream + // TODO find another way than to randomly wait!!! + // The problem is, if we dont sleep it happens often that the modification works only then the rebase is startet _really_ + // Doing the modification several times in hope that the second one fails will likely just stop the rebase thread as it cannot close + usleep(10000); + } + + $this->log('2. writing started'); + + for ($i = 0; $i <= 100; $i++) { + $this->contentRepository->handle(CreateNodeAggregateWithNode::create( + WorkspaceName::fromString('user-test'), + NodeAggregateId::fromString('user-nody-mc-nodeface-' . $i), + NodeTypeName::fromString('Neos.ContentRepository.Testing:Document'), + OriginDimensionSpacePoint::createWithoutDimensions(), + NodeAggregateId::fromString('lady-eleonode-rootford'), + initialPropertyValues: PropertyValuesToWrite::fromArray([ + 'title' => 'title' + ]) + )); + } + + $this->log('2. writing finished'); + + Assert::assertTrue(true, 'No exception was thrown ;)'); + + $subgraph = $this->contentRepository->getContentGraph(WorkspaceName::fromString('user-test'))->getSubgraph(DimensionSpacePoint::createWithoutDimensions(), VisibilityConstraints::withoutRestrictions()); + $node = $subgraph->findNodeById(NodeAggregateId::fromString('user-nody-mc-nodeface-100')); + Assert::assertNotNull($node); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php index 1dcc9fcdd03..3c62dcc5149 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php @@ -14,7 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Parallel\WorkspacePublicationDuringWriting; +use Doctrine\DBAL\Connection; use Neos\ContentRepository\BehavioralTests\Tests\Parallel\AbstractParallelTestCase; +use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; @@ -34,6 +36,7 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; +use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; use Neos\EventStore\Exception\ConcurrencyException; use Neos\Flow\ObjectManagement\ObjectManagerInterface; use PHPUnit\Framework\Assert; @@ -51,6 +54,16 @@ public function setUp(): void { parent::setUp(); $this->log('------ process started ------'); + + $debugProjection = new DebugEventProjection( + 'cr_test_parallel_debug_projection', + $this->objectManager->get(Connection::class) + ); + FakeProjectionFactory::setProjection( + 'debug', + $debugProjection + ); + FakeContentDimensionSourceFactory::setWithoutDimensions(); FakeNodeTypeManagerFactory::setConfiguration([ 'Neos.ContentRepository:Root' => [], @@ -155,6 +168,11 @@ public function whileANodesArWrittenOnLive(): void $this->log('writing finished'); Assert::assertTrue(true, 'No exception was thrown ;)'); + + $subgraph = $this->contentRepository->getContentGraph(WorkspaceName::forLive())->getSubgraph(DimensionSpacePoint::createWithoutDimensions(), VisibilityConstraints::withoutRestrictions()); + $node = $subgraph->findNodeById(NodeAggregateId::fromString('nody-mc-nodeface')); + Assert::assertNotNull($node); + Assert::assertSame($node->getProperty('title'), 'changed-title-50'); } /** diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php index f4a37360ed1..566a86c9bc9 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php @@ -14,7 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Parallel\WorkspaceWritingDuringRebase; +use Doctrine\DBAL\Connection; use Neos\ContentRepository\BehavioralTests\Tests\Parallel\AbstractParallelTestCase; +use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; @@ -35,6 +37,7 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; +use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; use Neos\EventStore\Exception\ConcurrencyException; use Neos\Flow\ObjectManagement\ObjectManagerInterface; use PHPUnit\Framework\Assert; @@ -53,6 +56,16 @@ public function setUp(): void { parent::setUp(); $this->log('------ process started ------'); + + $debugProjection = new DebugEventProjection( + 'cr_test_parallel_debug_projection', + $this->objectManager->get(Connection::class) + ); + FakeProjectionFactory::setProjection( + 'debug', + $debugProjection + ); + FakeContentDimensionSourceFactory::setWithoutDimensions(); FakeNodeTypeManagerFactory::setConfiguration([ 'Neos.ContentRepository:Root' => [], diff --git a/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php b/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php index 9becebc5a2d..f004613778f 100644 --- a/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php +++ b/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php @@ -135,11 +135,6 @@ private function handle(RebaseableCommand $rebaseableCommand): void foreach ($eventStream as $eventEnvelope) { $event = $this->eventNormalizer->denormalize($eventEnvelope->event); - - if (!$this->contentRepositoryProjection->canHandle($event)) { - continue; - } - $this->contentRepositoryProjection->apply($event, $eventEnvelope); } } diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 556ba4379be..460cc969898 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -21,27 +21,18 @@ use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\EventStore\InitiatingEventMetadata; use Neos\ContentRepository\Core\Feature\Security\AuthProviderInterface; use Neos\ContentRepository\Core\Feature\Security\Dto\UserId; use Neos\ContentRepository\Core\Feature\Security\Exception\AccessDenied; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\CatchUp; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphInterface; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentSubgraphInterface; -use Neos\ContentRepository\Core\Projection\ProjectionInterface; -use Neos\ContentRepository\Core\Projection\ProjectionsAndCatchUpHooks; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Projection\ProjectionStatuses; -use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStates; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; -use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryStatus; use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStream; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; @@ -49,30 +40,23 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\Workspace; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspaces; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Exception\ConcurrencyException; -use Neos\EventStore\Model\EventEnvelope; -use Neos\EventStore\Model\EventStream\VirtualStreamName; use Psr\Clock\ClockInterface; /** * Main Entry Point to the system. Encapsulates the full event-sourced Content Repository. * * Use this to: - * - set up the necessary database tables and contents via {@see ContentRepository::setUp()} - * - send commands to the system (to mutate state) via {@see ContentRepository::handle()} - * - access projection state (to read state) via {@see ContentRepository::projectionState()} - * - catch up projections via {@see ContentRepository::catchUpProjection()} + * - send commands to the system (to mutate state) via {@see self::handle()} + * - access the content graph read model + * - access 3rd party read models via {@see self::projectionState()} * * @api */ final class ContentRepository { - /** - * @var array, ProjectionStateInterface> - */ - private array $projectionStateCache; - /** * @internal use the {@see ContentRepositoryFactory::getOrBuild()} to instantiate */ @@ -80,9 +64,8 @@ public function __construct( public readonly ContentRepositoryId $id, private readonly CommandBus $commandBus, private readonly EventStoreInterface $eventStore, - private readonly ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks, private readonly EventNormalizer $eventNormalizer, - private readonly EventPersister $eventPersister, + private readonly SubscriptionEngine $subscriptionEngine, private readonly NodeTypeManager $nodeTypeManager, private readonly InterDimensionalVariationGraph $variationGraph, private readonly ContentDimensionSourceInterface $contentDimensionSource, @@ -90,6 +73,7 @@ public function __construct( private readonly ClockInterface $clock, private readonly ContentGraphReadModelInterface $contentGraphReadModel, private readonly CommandHookInterface $commandHook, + private readonly ProjectionStates $projectionStates, ) { } @@ -112,8 +96,9 @@ public function handle(CommandInterface $command): void // simple case if ($toPublish instanceof EventsToPublish) { $eventsToPublish = $this->enrichEventsToPublishWithMetadata($toPublish); - $this->eventPersister->publishWithoutCatchup($eventsToPublish); - $this->catchupProjections(); + $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish->events), $eventsToPublish->expectedVersion); + $catchUpResult = $this->subscriptionEngine->catchUpActive(); + $catchUpResult->throwOnFailure(); return; } @@ -122,7 +107,7 @@ public function handle(CommandInterface $command): void foreach ($toPublish as $yieldedEventsToPublish) { $eventsToPublish = $this->enrichEventsToPublishWithMetadata($yieldedEventsToPublish); try { - $this->eventPersister->publishWithoutCatchup($eventsToPublish); + $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish->events), $eventsToPublish->expectedVersion); } catch (ConcurrencyException $concurrencyException) { // we pass the exception into the generator (->throw), so it could be try-caught and reacted upon: // @@ -134,7 +119,7 @@ public function handle(CommandInterface $command): void // } $yieldedErrorStrategy = $toPublish->throw($concurrencyException); if ($yieldedErrorStrategy instanceof EventsToPublish) { - $this->eventPersister->publishWithoutCatchup($yieldedErrorStrategy); + $this->eventStore->commit($yieldedErrorStrategy->streamName, $this->eventNormalizer->normalizeEvents($yieldedErrorStrategy->events), $yieldedErrorStrategy->expectedVersion); } throw $concurrencyException; } @@ -142,7 +127,8 @@ public function handle(CommandInterface $command): void } finally { // We always NEED to catchup even if there was an unexpected ConcurrencyException to make sure previous commits are handled. // Technically it would be acceptable for the catchup to fail here (due to hook errors) because all the events are already persisted. - $this->catchupProjections(); + $catchUpResult = $this->subscriptionEngine->catchUpActive(); + $catchUpResult->throwOnFailure(); } } @@ -154,119 +140,7 @@ public function handle(CommandInterface $command): void */ public function projectionState(string $projectionStateClassName): ProjectionStateInterface { - if (!isset($this->projectionStateCache)) { - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - if ($projection instanceof ContentGraphProjectionInterface) { - continue; - } - $projectionState = $projection->getState(); - $this->projectionStateCache[$projectionState::class] = $projectionState; - } - } - if (isset($this->projectionStateCache[$projectionStateClassName])) { - /** @var T $projectionState */ - $projectionState = $this->projectionStateCache[$projectionStateClassName]; - return $projectionState; - } - if (in_array(ContentGraphReadModelInterface::class, class_implements($projectionStateClassName), true)) { - throw new \InvalidArgumentException(sprintf('Accessing the internal content repository projection state via %s(%s) is not allowed. Please use the API on the content repository instead.', __FUNCTION__, $projectionStateClassName), 1729338679); - } - - throw new \InvalidArgumentException(sprintf('A projection state of type "%s" is not registered in this content repository instance.', $projectionStateClassName), 1662033650); - } - - /** - * @param class-string> $projectionClassName - */ - public function catchUpProjection(string $projectionClassName, CatchUpOptions $options): void - { - $projection = $this->projectionsAndCatchUpHooks->projections->get($projectionClassName); - - $catchUpHookFactory = $this->projectionsAndCatchUpHooks->getCatchUpHookFactoryForProjection($projection); - $catchUpHook = $catchUpHookFactory?->build(CatchUpHookFactoryDependencies::create( - $this->id, - $projection->getState(), - $this->nodeTypeManager, - $this->contentDimensionSource, - $this->variationGraph - )); - - // TODO allow custom stream name per projection - $streamName = VirtualStreamName::all(); - $eventStream = $this->eventStore->load($streamName); - if ($options->maximumSequenceNumber !== null) { - $eventStream = $eventStream->withMaximumSequenceNumber($options->maximumSequenceNumber); - } - - $eventApplier = function (EventEnvelope $eventEnvelope) use ($projection, $catchUpHook, $options) { - $event = $this->eventNormalizer->denormalize($eventEnvelope->event); - if ($options->progressCallback !== null) { - ($options->progressCallback)($event, $eventEnvelope); - } - if (!$projection->canHandle($event)) { - return; - } - $catchUpHook?->onBeforeEvent($event, $eventEnvelope); - $projection->apply($event, $eventEnvelope); - if ($projection instanceof WithMarkStaleInterface) { - $projection->markStale(); - } - $catchUpHook?->onAfterEvent($event, $eventEnvelope); - }; - - $catchUp = CatchUp::create($eventApplier, $projection->getCheckpointStorage()); - - if ($catchUpHook !== null) { - $catchUpHook->onBeforeCatchUp(); - $catchUp = $catchUp->withOnBeforeBatchCompleted(fn() => $catchUpHook->onBeforeBatchCompleted()); - } - $catchUp->run($eventStream); - $catchUpHook?->onAfterCatchUp(); - } - - public function catchupProjections(): void - { - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - // FIXME optimise by only loading required events once and not per projection - // see https://github.com/neos/neos-development-collection/pull/4988/ - $this->catchUpProjection($projection::class, CatchUpOptions::create()); - } - } - - public function setUp(): void - { - $this->eventStore->setup(); - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - $projection->setUp(); - } - } - - public function status(): ContentRepositoryStatus - { - $projectionStatuses = ProjectionStatuses::createEmpty(); - foreach ($this->projectionsAndCatchUpHooks->projections as $projectionClassName => $projection) { - $projectionStatuses = $projectionStatuses->with($projectionClassName, $projection->status()); - } - return new ContentRepositoryStatus( - $this->eventStore->status(), - $projectionStatuses, - ); - } - - public function resetProjectionStates(): void - { - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - $projection->reset(); - } - } - - /** - * @param class-string> $projectionClassName - */ - public function resetProjectionState(string $projectionClassName): void - { - $projection = $this->projectionsAndCatchUpHooks->projections->get($projectionClassName); - $projection->reset(); + return $this->projectionStates->get($projectionStateClassName); } /** diff --git a/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php b/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php index 52f23d63910..14943d68c97 100644 --- a/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php +++ b/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php @@ -5,6 +5,7 @@ namespace Neos\ContentRepository\Core\EventStore; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\EventStore\Events as DomainEvents; use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Event\ContentStreamWasClosed; use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Event\ContentStreamWasReopened; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; @@ -46,6 +47,7 @@ use Neos\EventStore\Model\Event\EventData; use Neos\EventStore\Model\Event\EventId; use Neos\EventStore\Model\Event\EventType; +use Neos\EventStore\Model\Events; /** * Central authority to convert Content Repository domain events to Event Store EventData and EventType, vice versa. @@ -147,6 +149,11 @@ public function normalize(EventInterface|DecoratedEvent $event): Event ); } + public function normalizeEvents(DomainEvents $events): Events + { + return Events::fromArray($events->map($this->normalize(...))); + } + public function denormalize(Event $event): EventInterface { $eventClassName = $this->getEventClassName($event); diff --git a/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php b/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php deleted file mode 100644 index 1af59ff3ce9..00000000000 --- a/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php +++ /dev/null @@ -1,52 +0,0 @@ -publishWithoutCatchup($eventsToPublish); - $contentRepository->catchUpProjections(); - } - - /** - * TODO Will be refactored via https://github.com/neos/neos-development-collection/pull/5321 - * @throws ConcurrencyException in case the expectedVersion does not match - */ - public function publishWithoutCatchup(EventsToPublish $eventsToPublish): CommitResult - { - $normalizedEvents = Events::fromArray( - $eventsToPublish->events->map($this->eventNormalizer->normalize(...)) - ); - return $this->eventStore->commit( - $eventsToPublish->streamName, - $normalizedEvents, - $eventsToPublish->expectedVersion - ); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index 09c51da4293..ea4593a6a70 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -22,18 +22,28 @@ use Neos\ContentRepository\Core\DimensionSpace\ContentDimensionZookeeper; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\DimensionSpaceCommandHandler; use Neos\ContentRepository\Core\Feature\NodeAggregateCommandHandler; use Neos\ContentRepository\Core\Feature\NodeDuplication\NodeDuplicationCommandHandler; use Neos\ContentRepository\Core\Feature\WorkspaceCommandHandler; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\ProjectionsAndCatchUpHooks; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStates; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; +use Neos\ContentRepository\Core\Subscription\Subscriber\ProjectionSubscriber; +use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepositoryRegistry\Factory\AuthProvider\AuthProviderFactoryInterface; use Neos\EventStore\EventStoreInterface; use Psr\Clock\ClockInterface; +use Psr\Log\LoggerInterface; use Symfony\Component\Serializer\Serializer; /** @@ -43,44 +53,77 @@ */ final class ContentRepositoryFactory { - private ProjectionFactoryDependencies $projectionFactoryDependencies; - private ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks; + private SubscriberFactoryDependencies $subscriberFactoryDependencies; + private SubscriptionEngine $subscriptionEngine; + private ContentGraphProjectionInterface $contentGraphProjection; + private ProjectionStates $additionalProjectionStates; + // guards against recursion and memory overflow + private bool $isBuilding = false; + + // The following properties store "singleton" references of objects for this content repository + private ?ContentRepository $contentRepositoryRuntimeCache = null; + + /** + * @param CatchUpHookFactoryInterface|null $contentGraphCatchUpHookFactory + */ public function __construct( private readonly ContentRepositoryId $contentRepositoryId, - EventStoreInterface $eventStore, + private readonly EventStoreInterface $eventStore, NodeTypeManager $nodeTypeManager, ContentDimensionSourceInterface $contentDimensionSource, Serializer $propertySerializer, - ProjectionsAndCatchUpHooksFactory $projectionsAndCatchUpHooksFactory, private readonly AuthProviderFactoryInterface $authProviderFactory, private readonly ClockInterface $clock, + SubscriptionStoreInterface $subscriptionStore, + ContentGraphProjectionFactoryInterface $contentGraphProjectionFactory, + private readonly CatchUpHookFactoryInterface|null $contentGraphCatchUpHookFactory, private readonly CommandHooksFactory $commandHooksFactory, + private readonly ContentRepositorySubscriberFactories $additionalSubscriberFactories, + LoggerInterface|null $logger = null, ) { $contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); $interDimensionalVariationGraph = new InterDimensionalVariationGraph( $contentDimensionSource, $contentDimensionZookeeper ); - $this->projectionFactoryDependencies = new ProjectionFactoryDependencies( + $eventNormalizer = new EventNormalizer(); + $this->subscriberFactoryDependencies = new SubscriberFactoryDependencies( $contentRepositoryId, - $eventStore, - new EventNormalizer(), + $eventNormalizer, $nodeTypeManager, $contentDimensionSource, $contentDimensionZookeeper, $interDimensionalVariationGraph, new PropertyConverter($propertySerializer), ); - $this->projectionsAndCatchUpHooks = $projectionsAndCatchUpHooksFactory->build($this->projectionFactoryDependencies); + $subscribers = []; + $additionalProjectionStates = []; + foreach ($this->additionalSubscriberFactories as $additionalSubscriberFactory) { + $subscriber = $additionalSubscriberFactory->build($this->subscriberFactoryDependencies); + $additionalProjectionStates[] = $subscriber->projection->getState(); + $subscribers[] = $subscriber; + } + $this->additionalProjectionStates = ProjectionStates::fromArray($additionalProjectionStates); + $this->contentGraphProjection = $contentGraphProjectionFactory->build($this->subscriberFactoryDependencies); + $subscribers[] = $this->buildContentGraphSubscriber(); + $this->subscriptionEngine = new SubscriptionEngine($this->eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, $logger); } - // guards against recursion and memory overflow - private bool $isBuilding = false; - - // The following properties store "singleton" references of objects for this content repository - private ?ContentRepository $contentRepository = null; - private ?EventPersister $eventPersister = null; + private function buildContentGraphSubscriber(): ProjectionSubscriber + { + return new ProjectionSubscriber( + SubscriptionId::fromString('contentGraph'), + $this->contentGraphProjection, + $this->contentGraphCatchUpHookFactory?->build(CatchUpHookFactoryDependencies::create( + $this->contentRepositoryId, + $this->contentGraphProjection->getState(), + $this->subscriberFactoryDependencies->nodeTypeManager, + $this->subscriberFactoryDependencies->contentDimensionSource, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + )), + ); + } /** * Builds and returns the content repository. If it is already built, returns the same instance. @@ -90,75 +133,75 @@ public function __construct( */ public function getOrBuild(): ContentRepository { - if ($this->contentRepository) { - return $this->contentRepository; + if ($this->contentRepositoryRuntimeCache) { + return $this->contentRepositoryRuntimeCache; } if ($this->isBuilding) { throw new \RuntimeException(sprintf('Content repository "%s" was attempted to be build in recursion.', $this->contentRepositoryId->value), 1730552199); } $this->isBuilding = true; - $contentGraphReadModel = $this->projectionsAndCatchUpHooks->contentGraphProjection->getState(); + $contentGraphReadModel = $this->contentGraphProjection->getState(); $commandHandlingDependencies = new CommandHandlingDependencies($contentGraphReadModel); // we dont need full recursion in rebase - e.g apply workspace commands - and thus we can use this set for simulation $commandBusForRebaseableCommands = new CommandBus( $commandHandlingDependencies, new NodeAggregateCommandHandler( - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->contentDimensionZookeeper, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, - $this->projectionFactoryDependencies->propertyConverter, + $this->subscriberFactoryDependencies->nodeTypeManager, + $this->subscriberFactoryDependencies->contentDimensionZookeeper, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + $this->subscriberFactoryDependencies->propertyConverter, ), new DimensionSpaceCommandHandler( - $this->projectionFactoryDependencies->contentDimensionZookeeper, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, + $this->subscriberFactoryDependencies->contentDimensionZookeeper, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, ), new NodeDuplicationCommandHandler( - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->contentDimensionZookeeper, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, + $this->subscriberFactoryDependencies->nodeTypeManager, + $this->subscriberFactoryDependencies->contentDimensionZookeeper, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, ) ); $commandSimulatorFactory = new CommandSimulatorFactory( - $this->projectionsAndCatchUpHooks->contentGraphProjection, - $this->projectionFactoryDependencies->eventNormalizer, + $this->contentGraphProjection, + $this->subscriberFactoryDependencies->eventNormalizer, $commandBusForRebaseableCommands ); $publicCommandBus = $commandBusForRebaseableCommands->withAdditionalHandlers( new WorkspaceCommandHandler( $commandSimulatorFactory, - $this->projectionFactoryDependencies->eventStore, - $this->projectionFactoryDependencies->eventNormalizer, + $this->eventStore, + $this->subscriberFactoryDependencies->eventNormalizer, ) ); $authProvider = $this->authProviderFactory->build($this->contentRepositoryId, $contentGraphReadModel); $commandHooks = $this->commandHooksFactory->build(CommandHooksFactoryDependencies::create( $this->contentRepositoryId, - $this->projectionsAndCatchUpHooks->contentGraphProjection->getState(), - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->contentDimensionSource, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, + $this->contentGraphProjection->getState(), + $this->subscriberFactoryDependencies->nodeTypeManager, + $this->subscriberFactoryDependencies->contentDimensionSource, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, )); - $this->contentRepository = new ContentRepository( + $this->contentRepositoryRuntimeCache = new ContentRepository( $this->contentRepositoryId, $publicCommandBus, - $this->projectionFactoryDependencies->eventStore, - $this->projectionsAndCatchUpHooks, - $this->projectionFactoryDependencies->eventNormalizer, - $this->buildEventPersister(), - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, - $this->projectionFactoryDependencies->contentDimensionSource, + $this->eventStore, + $this->subscriberFactoryDependencies->eventNormalizer, + $this->subscriptionEngine, + $this->subscriberFactoryDependencies->nodeTypeManager, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + $this->subscriberFactoryDependencies->contentDimensionSource, $authProvider, $this->clock, $contentGraphReadModel, $commandHooks, + $this->additionalProjectionStates, ); $this->isBuilding = false; - return $this->contentRepository; + return $this->contentRepositoryRuntimeCache; } /** @@ -177,22 +220,12 @@ public function buildService( ): ContentRepositoryServiceInterface { $serviceFactoryDependencies = ContentRepositoryServiceFactoryDependencies::create( - $this->projectionFactoryDependencies, + $this->subscriberFactoryDependencies, + $this->eventStore, $this->getOrBuild(), - $this->buildEventPersister(), - $this->projectionsAndCatchUpHooks, + $this->contentGraphProjection->getState(), + $this->subscriptionEngine, ); return $serviceFactory->build($serviceFactoryDependencies); } - - private function buildEventPersister(): EventPersister - { - if (!$this->eventPersister) { - $this->eventPersister = new EventPersister( - $this->projectionFactoryDependencies->eventStore, - $this->projectionFactoryDependencies->eventNormalizer, - ); - } - return $this->eventPersister; - } } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php index 08ea272181d..756440449f3 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php @@ -19,11 +19,11 @@ use Neos\ContentRepository\Core\DimensionSpace\ContentDimensionZookeeper; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\ProjectionsAndCatchUpHooks; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; /** @@ -44,9 +44,9 @@ private function __construct( public InterDimensionalVariationGraph $interDimensionalVariationGraph, public PropertyConverter $propertyConverter, public ContentRepository $contentRepository, + public ContentGraphReadModelInterface $contentGraphReadModel, // we don't need CommandBus, because this is included in ContentRepository->handle() - public EventPersister $eventPersister, - public ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks, + public SubscriptionEngine $subscriptionEngine, ) { } @@ -54,14 +54,15 @@ private function __construct( * @internal */ public static function create( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, + EventStoreInterface $eventStore, ContentRepository $contentRepository, - EventPersister $eventPersister, - ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks, + ContentGraphReadModelInterface $contentGraphReadModel, + SubscriptionEngine $subscriptionEngine, ): self { return new self( $projectionFactoryDependencies->contentRepositoryId, - $projectionFactoryDependencies->eventStore, + $eventStore, $projectionFactoryDependencies->eventNormalizer, $projectionFactoryDependencies->nodeTypeManager, $projectionFactoryDependencies->contentDimensionSource, @@ -69,8 +70,8 @@ public static function create( $projectionFactoryDependencies->interDimensionalVariationGraph, $projectionFactoryDependencies->propertyConverter, $contentRepository, - $eventPersister, - $projectionsAndCatchUpHooks, + $contentGraphReadModel, + $subscriptionEngine, ); } } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php new file mode 100644 index 00000000000..399573c23a8 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php @@ -0,0 +1,46 @@ + + * @internal only API for custom content repository integrations + */ +final class ContentRepositorySubscriberFactories implements \IteratorAggregate +{ + /** + * @var array + */ + private array $subscriberFactories; + + private function __construct(ProjectionSubscriberFactory ...$subscriberFactories) + { + $this->subscriberFactories = $subscriberFactories; + } + + /** + * @param array $subscriberFactories + * @return self + */ + public static function fromArray(array $subscriberFactories): self + { + return new self(...$subscriberFactories); + } + + public static function none(): self + { + return new self(); + } + + public function isEmpty(): bool + { + return $this->subscriberFactories === []; + } + + public function getIterator(): \Traversable + { + yield from $this->subscriberFactories; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php new file mode 100644 index 00000000000..203936237e3 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php @@ -0,0 +1,60 @@ +> $projectionFactory + * @param CatchUpHookFactoryInterface|null $catchUpHookFactory + * @param array $projectionFactoryOptions + */ + public function __construct( + private SubscriptionId $subscriptionId, + private ProjectionFactoryInterface $projectionFactory, + private ?CatchUpHookFactoryInterface $catchUpHookFactory, + private array $projectionFactoryOptions, + ) { + } + + public function build(SubscriberFactoryDependencies $dependencies): ProjectionSubscriber + { + $projection = $this->projectionFactory->build($dependencies, $this->projectionFactoryOptions); + $catchUpHook = $this->catchUpHookFactory?->build(CatchUpHookFactoryDependencies::create( + $dependencies->contentRepositoryId, + $projection->getState(), + $dependencies->nodeTypeManager, + $dependencies->contentDimensionSource, + $dependencies->interDimensionalVariationGraph, + )); + + return new ProjectionSubscriber( + $this->subscriptionId, + $projection, + $catchUpHook, + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionsAndCatchUpHooksFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionsAndCatchUpHooksFactory.php deleted file mode 100644 index f48ed1345c0..00000000000 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionsAndCatchUpHooksFactory.php +++ /dev/null @@ -1,92 +0,0 @@ ->, options: array, catchUpHooksFactories: array>}> - */ - private array $factories = []; - - /** - * @param ProjectionFactoryInterface> $factory - * @param array $options - * @return void - * @api - */ - public function registerFactory(ProjectionFactoryInterface $factory, array $options): void - { - $this->factories[get_class($factory)] = [ - 'factory' => $factory, - 'options' => $options, - 'catchUpHooksFactories' => [] - ]; - } - - /** - * @param ProjectionFactoryInterface> $factory - * @param CatchUpHookFactoryInterface $catchUpHookFactory - * @return void - * @api - */ - public function registerCatchUpHookFactory(ProjectionFactoryInterface $factory, CatchUpHookFactoryInterface $catchUpHookFactory): void - { - $this->factories[get_class($factory)]['catchUpHooksFactories'][] = $catchUpHookFactory; - } - - /** - * @internal this method is only called by the {@see ContentRepositoryFactory}, and not by anybody in userland - */ - public function build(ProjectionFactoryDependencies $projectionFactoryDependencies): ProjectionsAndCatchUpHooks - { - $contentGraphProjection = null; - $projectionsArray = []; - $catchUpHookFactoriesByProjectionClassName = []; - foreach ($this->factories as $factoryDefinition) { - $factory = $factoryDefinition['factory']; - $options = $factoryDefinition['options']; - assert($factory instanceof ProjectionFactoryInterface); - - $catchUpHookFactories = CatchUpHookFactories::create(); - foreach ($factoryDefinition['catchUpHooksFactories'] as $catchUpHookFactory) { - assert($catchUpHookFactory instanceof CatchUpHookFactoryInterface); - $catchUpHookFactories = $catchUpHookFactories->with($catchUpHookFactory); - } - - $projection = $factory->build( - $projectionFactoryDependencies, - $options, - ); - $catchUpHookFactoriesByProjectionClassName[$projection::class] = $catchUpHookFactories; - if ($projection instanceof ContentGraphProjectionInterface) { - if ($contentGraphProjection !== null) { - throw new \RuntimeException(sprintf('Content repository requires exactly one %s to be registered.', ContentGraphProjectionInterface::class)); - } - $contentGraphProjection = $projection; - } else { - $projectionsArray[] = $projection; - } - } - - if ($contentGraphProjection === null) { - throw new \RuntimeException(sprintf('Content repository requires the %s to be registered.', ContentGraphProjectionInterface::class)); - } - - return new ProjectionsAndCatchUpHooks($contentGraphProjection, Projections::fromArray($projectionsArray), $catchUpHookFactoriesByProjectionClassName); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php similarity index 93% rename from Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php rename to Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php index 9bb2f0cc31f..8a6af1f94a4 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php @@ -26,11 +26,10 @@ /** * @api because it is used inside the ProjectionsFactory */ -final readonly class ProjectionFactoryDependencies +final readonly class SubscriberFactoryDependencies { public function __construct( public ContentRepositoryId $contentRepositoryId, - public EventStoreInterface $eventStore, public EventNormalizer $eventNormalizer, public NodeTypeManager $nodeTypeManager, public ContentDimensionSourceInterface $contentDimensionSource, diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php deleted file mode 100644 index 8551d5febb8..00000000000 --- a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php +++ /dev/null @@ -1,164 +0,0 @@ -connection->getDatabasePlatform(); - if (!($platform instanceof MySQLPlatform || $platform instanceof PostgreSqlPlatform)) { - throw new \InvalidArgumentException(sprintf('The %s only supports the platforms %s and %s currently. Given: %s', $this::class, MySQLPlatform::class, PostgreSQLPlatform::class, get_debug_type($platform)), 1660556004); - } - if (strlen($this->subscriberId) > 255) { - throw new \InvalidArgumentException('The subscriberId must not exceed 255 characters', 1705673456); - } - $this->platform = $platform; - } - - public function setUp(): void - { - foreach ($this->determineRequiredSqlStatements() as $statement) { - $this->connection->executeStatement($statement); - } - try { - $this->connection->insert($this->tableName, ['subscriberid' => $this->subscriberId, 'appliedsequencenumber' => 0]); - } catch (UniqueConstraintViolationException $e) { - // table and row already exists, ignore - } - } - - public function status(): CheckpointStorageStatus - { - try { - $this->connection->connect(); - } catch (\Throwable $e) { - return CheckpointStorageStatus::error(sprintf('Failed to connect to database for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); - } - try { - $requiredSqlStatements = $this->determineRequiredSqlStatements(); - } catch (\Throwable $e) { - return CheckpointStorageStatus::error(sprintf('Failed to compare database schema for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); - } - if ($requiredSqlStatements !== []) { - return CheckpointStorageStatus::setupRequired(sprintf('The following SQL statement%s required for subscriber "%s": %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', $this->subscriberId, implode(chr(10), $requiredSqlStatements))); - } - try { - $appliedSequenceNumber = $this->connection->fetchOne('SELECT appliedsequencenumber FROM ' . $this->tableName . ' WHERE subscriberid = :subscriberId', ['subscriberId' => $this->subscriberId]); - } catch (\Throwable $e) { - return CheckpointStorageStatus::error(sprintf('Failed to determine initial applied sequence number for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); - } - if ($appliedSequenceNumber === false) { - return CheckpointStorageStatus::setupRequired(sprintf('Initial initial applied sequence number not set for subscriber "%s"', $this->subscriberId)); - } - return CheckpointStorageStatus::ok(); - } - - public function acquireLock(): SequenceNumber - { - if ($this->connection->isTransactionActive()) { - throw new \RuntimeException(sprintf('Failed to acquire checkpoint lock for subscriber "%s" because a transaction is active already', $this->subscriberId), 1652268416); - } - $this->connection->beginTransaction(); - try { - $highestAppliedSequenceNumber = $this->connection->fetchOne('SELECT appliedsequencenumber FROM ' . $this->connection->quoteIdentifier($this->tableName) . ' WHERE subscriberid = :subscriberId ' . $this->platform->getForUpdateSQL() . ' NOWAIT', [ - 'subscriberId' => $this->subscriberId - ]); - } catch (DBALException $exception) { - $this->connection->rollBack(); - if ($exception instanceof LockWaitTimeoutException || ($exception instanceof DBALDriverException && ($exception->getCode() === 3572 || $exception->getCode() === 7))) { - throw new \RuntimeException(sprintf('Failed to acquire checkpoint lock for subscriber "%s" because it is acquired already', $this->subscriberId), 1652279016); - } - throw new \RuntimeException($exception->getMessage(), 1544207778, $exception); - } - if (!is_numeric($highestAppliedSequenceNumber)) { - $this->connection->rollBack(); - throw new \RuntimeException(sprintf('Failed to fetch highest applied sequence number for subscriber "%s". Please run %s::setUp()', $this->subscriberId, $this::class), 1652279139); - } - $this->lockedSequenceNumber = SequenceNumber::fromInteger((int)$highestAppliedSequenceNumber); - return $this->lockedSequenceNumber; - } - - public function updateAndReleaseLock(SequenceNumber $sequenceNumber): void - { - if ($this->lockedSequenceNumber === null) { - throw new \RuntimeException(sprintf('Failed to update and commit checkpoint for subscriber "%s" because the lock has not been acquired successfully before', $this->subscriberId), 1660556344); - } - if (!$this->connection->isTransactionActive()) { - throw new \RuntimeException(sprintf('Failed to update and commit checkpoint for subscriber "%s" because no transaction is active', $this->subscriberId), 1652279314); - } - if ($this->connection->isRollbackOnly()) { - // TODO as described in https://github.com/neos/neos-development-collection/issues/4970 we are in a bad state and cannot commit after a nested transaction was rolled back. - throw new \RuntimeException(sprintf('Failed to update and commit checkpoint for subscriber "%s" because the transaction has been marked for rollback only. See https://github.com/neos/neos-development-collection/issues/4970', $this->subscriberId), 1711964313); - } - try { - if (!$this->lockedSequenceNumber->equals($sequenceNumber)) { - $this->connection->update($this->tableName, ['appliedsequencenumber' => $sequenceNumber->value], ['subscriberid' => $this->subscriberId]); - } - $this->connection->commit(); - } catch (DBALException $exception) { - $this->connection->rollBack(); - throw new \RuntimeException(sprintf('Failed to update and commit highest applied sequence number for subscriber "%s". Please run %s::setUp()', $this->subscriberId, $this::class), 1652279375, $exception); - } finally { - $this->lockedSequenceNumber = null; - } - } - - public function getHighestAppliedSequenceNumber(): SequenceNumber - { - $highestAppliedSequenceNumber = $this->connection->fetchOne('SELECT appliedsequencenumber FROM ' . $this->connection->quoteIdentifier($this->tableName) . ' WHERE subscriberid = :subscriberId ', [ - 'subscriberId' => $this->subscriberId - ]); - if (!is_numeric($highestAppliedSequenceNumber)) { - throw new \RuntimeException(sprintf('Failed to fetch highest applied sequence number for subscriber "%s". Please run %s::setUp()', $this->subscriberId, $this::class), 1652279427); - } - return SequenceNumber::fromInteger((int)$highestAppliedSequenceNumber); - } - - // -------------- - - /** - * @return array - */ - private function determineRequiredSqlStatements(): array - { - $schemaManager = $this->connection->createSchemaManager(); - $tableSchema = new Table( - $this->tableName, - [ - (new Column('subscriberid', Type::getType(Types::STRING)))->setLength(255), - (new Column('appliedsequencenumber', Type::getType(Types::INTEGER))) - ] - ); - $tableSchema->setPrimaryKey(['subscriberid']); - $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [$tableSchema]); - return DbalSchemaDiff::determineRequiredSqlStatements($this->connection, $schema); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUp.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUp.php deleted file mode 100644 index 35cd26467a9..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUp.php +++ /dev/null @@ -1,134 +0,0 @@ -batchSize < 1) { - throw new \InvalidArgumentException(sprintf('batch size must be a positive integer, given: %d', $this->batchSize), 1705672467); - } - } - - /** - * @param \Closure(EventEnvelope): void $eventHandler The callback that is invoked for every {@see EventEnvelope} that is processed - * @param CheckpointStorageInterface $checkpointStorage The checkpoint storage that saves the last processed {@see SequenceNumber} - */ - public static function create(\Closure $eventHandler, CheckpointStorageInterface $checkpointStorage): self - { - return new self($eventHandler, $checkpointStorage, 1, null); - } - - /** - * After how many events should the (database) transaction be committed? - * - * @param int $batchSize Number of events to process before the checkpoint is written - */ - public function withBatchSize(int $batchSize): self - { - if ($batchSize === $this->batchSize) { - return $this; - } - return new self($this->eventHandler, $this->checkpointStorage, $batchSize, $this->onBeforeBatchCompletedHook); - } - - /** - * This hook is called directly before the sequence number is persisted back in CheckpointStorage. - * Use this to trigger any operation which need to happen BEFORE the sequence number update is made - * visible to the outside. - * - * Overrides all previously registered onBeforeBatchCompleted hooks. - * - * @param \Closure(): void $callback the hook being called before the batch is completed - */ - public function withOnBeforeBatchCompleted(\Closure $callback): self - { - return new self($this->eventHandler, $this->checkpointStorage, $this->batchSize, $callback); - } - - /** - * Iterate over the $eventStream, invoke the specified event handler closure for every {@see EventEnvelope} and update - * the last processed sequence number in the {@see CheckpointStorageInterface} - * - * @param EventStreamInterface $eventStream The event stream to process - * @return SequenceNumber The last processed {@see SequenceNumber} - * @throws \Throwable Exceptions that are thrown during callback handling are re-thrown - */ - public function run(EventStreamInterface $eventStream): SequenceNumber - { - $highestAppliedSequenceNumber = $this->checkpointStorage->acquireLock(); - $iteration = 0; - try { - foreach ($eventStream->withMinimumSequenceNumber($highestAppliedSequenceNumber->next()) as $eventEnvelope) { - if ($eventEnvelope->sequenceNumber->value <= $highestAppliedSequenceNumber->value) { - continue; - } - try { - ($this->eventHandler)($eventEnvelope); - } catch (\Exception $e) { - throw new \RuntimeException(sprintf('Exception while catching up to sequence number %d: %s', $eventEnvelope->sequenceNumber->value, $e->getMessage()), 1710707311, $e); - } - $iteration++; - if ($this->batchSize === 1 || $iteration % $this->batchSize === 0) { - if ($this->onBeforeBatchCompletedHook) { - ($this->onBeforeBatchCompletedHook)(); - } - $this->checkpointStorage->updateAndReleaseLock($eventEnvelope->sequenceNumber); - $highestAppliedSequenceNumber = $this->checkpointStorage->acquireLock(); - } else { - $highestAppliedSequenceNumber = $eventEnvelope->sequenceNumber; - } - } - } finally { - try { - if ($this->onBeforeBatchCompletedHook) { - ($this->onBeforeBatchCompletedHook)(); - } - } finally { - $this->checkpointStorage->updateAndReleaseLock($highestAppliedSequenceNumber); - } - } - return $highestAppliedSequenceNumber; - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactories.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php similarity index 88% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactories.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php index efa364124ba..22f2621e2a2 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactories.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php @@ -2,11 +2,13 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; + +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; /** * @implements CatchUpHookFactoryInterface - * @internal + * @api */ final class CatchUpHookFactories implements CatchUpHookFactoryInterface { @@ -30,7 +32,6 @@ public static function create(): self /** * @param CatchUpHookFactoryInterface $catchUpHookFactory - * @return self */ public function with(CatchUpHookFactoryInterface $catchUpHookFactory): self { @@ -50,6 +51,11 @@ private function has(string $catchUpHookFactoryClassName): bool return array_key_exists($catchUpHookFactoryClassName, $this->catchUpHookFactories); } + public function isEmpty(): bool + { + return $this->catchUpHookFactories === []; + } + public function build(CatchUpHookFactoryDependencies $dependencies): CatchUpHookInterface { $catchUpHooks = array_map(static fn(CatchUpHookFactoryInterface $catchUpHookFactory) => $catchUpHookFactory->build($dependencies), $this->catchUpHookFactories); diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php similarity index 94% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryDependencies.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php index 0a98f7e11c3..2b891f34336 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php @@ -12,11 +12,12 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; use Neos\ContentRepository\Core\Dimension\ContentDimensionSourceInterface; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; /** diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryInterface.php similarity index 83% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryInterface.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryInterface.php index 82fc7fea7b4..86e06d571cc 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryInterface.php @@ -2,7 +2,9 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; + +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; /** * @template T of ProjectionStateInterface diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php similarity index 52% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookInterface.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php index fbdfed6e8d5..6dd4301f2a7 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php @@ -2,9 +2,11 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; use Neos\ContentRepository\Core\EventStore\EventInterface; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -18,12 +20,10 @@ interface CatchUpHookInterface { /** - * This hook is called at the beginning of {@see ProjectionInterface::catchUpProjection()}; - * BEFORE the Database Lock is acquired (by {@see CheckpointStorageInterface::acquireLock()}). - * - * @return void + * This hook is called at the beginning of a catch-up run; + * AFTER the Database Lock is acquired ({@see SubscriptionEngine::catchUpActive()}). */ - public function onBeforeCatchUp(): void; + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; /** * This hook is called for every event during the catchup process, **before** the projection @@ -38,22 +38,8 @@ public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $even public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void; /** - * This hook is called directly before the database lock is RELEASED - * in {@see CheckpointStorageInterface::updateAndReleaseLock()}. - * - * It can happen that this method is called multiple times, even without - * having seen Events in the meantime. - * - * If there exist more events which need to be processed, the database lock - * is directly acquired again after it is released. - */ - public function onBeforeBatchCompleted(): void; - - /** - * This hook is called at the END of {@see ProjectionInterface::catchUpProjection()}, directly - * before exiting the method. - * - * At this point, the Database Lock has already been released. + * This hook is called at the END of a catch-up run + * BEFORE the Database Lock is released ({@see SubscriptionEngine::catchUpActive()}). */ public function onAfterCatchUp(): void; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/DelegatingCatchUpHook.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php similarity index 76% rename from Neos.ContentRepository.Core/Classes/Projection/DelegatingCatchUpHook.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php index 3edc73b751d..12a2e73c9b8 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/DelegatingCatchUpHook.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php @@ -2,9 +2,10 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; use Neos\ContentRepository\Core\EventStore\EventInterface; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -13,7 +14,7 @@ * * @internal */ -class DelegatingCatchUpHook implements CatchUpHookInterface +final class DelegatingCatchUpHook implements CatchUpHookInterface { /** * @var CatchUpHookInterface[] @@ -26,10 +27,10 @@ public function __construct( $this->catchUpHooks = $catchUpHooks; } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onBeforeCatchUp(); + $catchUpHook->onBeforeCatchUp($subscriptionStatus); } } @@ -47,13 +48,6 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event } } - public function onBeforeBatchCompleted(): void - { - foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onBeforeBatchCompleted(); - } - } - public function onAfterCatchUp(): void { foreach ($this->catchUpHooks as $catchUpHook) { diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpOptions.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpOptions.php deleted file mode 100644 index 8df856faccc..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpOptions.php +++ /dev/null @@ -1,62 +0,0 @@ -maximumSequenceNumber, - $progressCallback ?? $this->progressCallback, - ); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php deleted file mode 100644 index 43dc37f8ab7..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php +++ /dev/null @@ -1,61 +0,0 @@ -type, $details); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php b/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php deleted file mode 100644 index 3e138d4348d..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php +++ /dev/null @@ -1,12 +0,0 @@ - * @api for creating a custom content repository graph projection implementation, **not for users of the CR** */ -interface ContentGraphProjectionFactoryInterface extends ProjectionFactoryInterface +interface ContentGraphProjectionFactoryInterface { - /** - * @param array $options - */ public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, - array $options, + SubscriberFactoryDependencies $projectionFactoryDependencies, ): ContentGraphProjectionInterface; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php index 6c0de0992d5..d2f03269b1d 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php @@ -4,7 +4,7 @@ namespace Neos\ContentRepository\Core\Projection; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; /** * @template-covariant T of ProjectionInterface @@ -17,7 +17,7 @@ interface ProjectionFactoryInterface * @return T */ public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, array $options, ): ProjectionInterface; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php index 22c65ced9f7..aff57389895 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php @@ -21,21 +21,17 @@ interface ProjectionInterface { /** - * Set up the projection state (create databases, call {@see CheckpointStorageInterface::setUp()}). + * Set up the projection state (create/update required database tables, ...). */ public function setUp(): void; /** - * Determines the status of the projection (not to confuse with {@see getState()}) + * Determines the setup status of the projection. E.g. are the database tables created or any columns missing. */ public function status(): ProjectionStatus; - public function canHandle(EventInterface $event): bool; - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void; - public function getCheckpointStorage(): CheckpointStorageInterface; - /** * NOTE: The ProjectionStateInterface returned must be ALWAYS THE SAME INSTANCE. * @@ -46,5 +42,5 @@ public function getCheckpointStorage(): CheckpointStorageInterface; */ public function getState(): ProjectionStateInterface; - public function reset(): void; + public function resetState(): void; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php new file mode 100644 index 00000000000..81b6d82a0eb --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php @@ -0,0 +1,70 @@ +, ProjectionStateInterface> $statesByClassName + */ + private function __construct( + private array $statesByClassName, + ) { + } + + public static function createEmpty(): self + { + return new self([]); + } + + /** + * @param array $states + */ + public static function fromArray(array $states): self + { + $statesByClassName = []; + foreach ($states as $state) { + if (!$state instanceof ProjectionStateInterface) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', ProjectionStateInterface::class, get_debug_type($state)), 1729687661); + } + if ($state instanceof ContentGraphReadModelInterface) { + throw new \InvalidArgumentException(sprintf('The content graph state (%s) must not be part of the additional projection states', ContentGraphReadModelInterface::class), 1732390657); + } + if (array_key_exists($state::class, $statesByClassName)) { + throw new \InvalidArgumentException(sprintf('An instance of %s is already part of the set', $state::class), 1729687716); + } + $statesByClassName[$state::class] = $state; + } + return new self($statesByClassName); + } + + /** + * Retrieve a single state (aka read model) by its fully qualified PHP class name + * + * @template T of ProjectionStateInterface + * @param class-string $className + * @return T + * @throws \InvalidArgumentException if the specified state class is not registered + */ + public function get(string $className): ProjectionStateInterface + { + if ($className === ContentGraphReadModelInterface::class) { + throw new \InvalidArgumentException(sprintf('Accessing the content repository projection state (%s) via is not allowed. Please use the API on the content repository instead.', ContentGraphReadModelInterface::class), 1732390824); + } + if (!array_key_exists($className, $this->statesByClassName)) { + throw new \InvalidArgumentException(sprintf('A projection state of type "%s" is not registered in this content repository.', $className), 1662033650); + } + /** @var T $state */ + $state = $this->statesByClassName[$className]; + return $state; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php index 517c53d7a7c..17ddc848176 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php @@ -5,6 +5,10 @@ namespace Neos\ContentRepository\Core\Projection; /** + * The setup status of a projection. + * + * E.g. are the database tables created or any columns missing. + * * @api */ final readonly class ProjectionStatus @@ -35,20 +39,4 @@ public static function setupRequired(string $details): self { return new self(ProjectionStatusType::SETUP_REQUIRED, $details); } - - /** - * @param non-empty-string $details - */ - public static function replayRequired(string $details): self - { - return new self(ProjectionStatusType::REPLAY_REQUIRED, $details); - } - - /** - * @param non-empty-string $details - */ - public function withDetails(string $details): self - { - return new self($this->type, $details); - } } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatusType.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatusType.php index 48681927742..285ad8e86d4 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatusType.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatusType.php @@ -9,8 +9,17 @@ */ enum ProjectionStatusType { + /** + * No actions needed + */ case OK; - case ERROR; + /** + * The projection needs to be setup to adjust its schema + * {@see \Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer::setUp()} + */ case SETUP_REQUIRED; - case REPLAY_REQUIRED; + /** + * An error occurred while determining the status (e.g. connection is closed) + */ + case ERROR; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php deleted file mode 100644 index cc146b1a76e..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php +++ /dev/null @@ -1,41 +0,0 @@ - - */ -final readonly class ProjectionStatuses implements \IteratorAggregate -{ - /** - * @param array>, ProjectionStatus> $statuses - */ - private function __construct( - public array $statuses, - ) { - } - - public static function createEmpty(): self - { - return new self([]); - } - - /** - * @param class-string> $projectionClassName - */ - public function with(string $projectionClassName, ProjectionStatus $projectionStatus): self - { - $statuses = $this->statuses; - $statuses[$projectionClassName] = $projectionStatus; - return new self($statuses); - } - - - public function getIterator(): \Traversable - { - yield from $this->statuses; - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/Projections.php b/Neos.ContentRepository.Core/Classes/Projection/Projections.php index e8fd3995a32..a66d0ee9e3b 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/Projections.php +++ b/Neos.ContentRepository.Core/Classes/Projection/Projections.php @@ -4,6 +4,8 @@ namespace Neos\ContentRepository\Core\Projection; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; + /** * An immutable set of Content Repository projections ({@see ProjectionInterface} * @@ -13,7 +15,7 @@ final class Projections implements \IteratorAggregate, \Countable { /** - * @var array>, ProjectionInterface> + * @var array> */ private array $projections; @@ -32,7 +34,7 @@ public static function empty(): self } /** - * @param array> $projections + * @param array> $projections * @return self */ public static function fromArray(array $projections): self @@ -54,45 +56,28 @@ public static function fromArray(array $projections): self } /** - * @template T of ProjectionInterface - * @param class-string $projectionClassName - * @return T + * @return ProjectionInterface */ - public function get(string $projectionClassName): ProjectionInterface + public function get(SubscriptionId $id): ProjectionInterface { - $projection = $this->projections[$projectionClassName] ?? null; - if (!$projection instanceof $projectionClassName) { - throw new \InvalidArgumentException( - sprintf( - 'a projection of type "%s" is not registered in this content repository instance.', - $projectionClassName - ), - 1650120813 - ); + if (!$this->has($id)) { + throw new \InvalidArgumentException(sprintf('a projection with id "%s" is not registered in this content repository instance.', $id->value), 1650120813); } - return $projection; + return $this->projections[$id->value]; } - public function has(string $projectionClassName): bool + public function has(SubscriptionId $id): bool { - return array_key_exists($projectionClassName, $this->projections); + return array_key_exists($id->value, $this->projections); } /** * @param ProjectionInterface $projection * @return self */ - public function with(ProjectionInterface $projection): self - { - return self::fromArray([...$this->projections, $projection]); - } - - /** - * @return list>> - */ - public function getClassNames(): array + public function with(SubscriptionId $id, ProjectionInterface $projection): self { - return array_keys($this->projections); + return self::fromArray([...$this->projections, ...[$id->value => $projection]]); } public function getIterator(): \Traversable diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionsAndCatchUpHooks.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionsAndCatchUpHooks.php deleted file mode 100644 index d7b674babdb..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionsAndCatchUpHooks.php +++ /dev/null @@ -1,35 +0,0 @@ ->, CatchUpHookFactories> $catchUpHookFactoriesByProjectionClassName - */ - public function __construct( - public ContentGraphProjectionInterface $contentGraphProjection, - Projections $additionalProjections, - private array $catchUpHookFactoriesByProjectionClassName, - ) { - $this->projections = $additionalProjections->with($this->contentGraphProjection); - } - - /** - * @param ProjectionInterface $projection - * @return ?CatchUpHookFactoryInterface - */ - public function getCatchUpHookFactoryForProjection(ProjectionInterface $projection): ?CatchUpHookFactoryInterface - { - return $this->catchUpHookFactoriesByProjectionClassName[$projection::class] ?? null; - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php b/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php index baf93b2d1b5..30034de7358 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php @@ -4,8 +4,7 @@ namespace Neos\ContentRepository\Core\Projection; -use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\EventStore\EventPersister; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; /** * Additional marker interface to add to a {@see ProjectionInterface}. @@ -19,7 +18,7 @@ interface WithMarkStaleInterface { /** * Triggered during catching up after applying events - * {@see ContentRepository::catchUpProjection()} + * {@see SubscriptionEngine::catchUpActive()} * * Can be f.e. used to flush caches inside the Projection State. * diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php new file mode 100644 index 00000000000..a8e725dc3b5 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -0,0 +1,249 @@ +eventStore->setup(); + $eventStoreIsEmpty = iterator_count($this->eventStore->load(VirtualStreamName::all())->limit(1)) === 0; + $setupResult = $this->subscriptionEngine->setup(); + if ($setupResult->errors !== null) { + return self::createErrorForReason('setup', $setupResult->errors); + } + if ($eventStoreIsEmpty) { + // note: possibly introduce $skipBooting flag instead + // see https://github.com/patchlevel/event-sourcing/blob/b8591c56b21b049f46bead8e7ab424fd2afe9917/src/Subscription/Engine/DefaultSubscriptionEngine.php#L42 + $bootResult = $this->subscriptionEngine->boot(); + if ($bootResult->errors !== null) { + return self::createErrorForReason('initial catchup', $bootResult->errors); + } + } + return null; + } + + public function status(): ContentRepositoryStatus + { + try { + $lastEventEnvelope = current(iterator_to_array($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1))) ?: null; + $sequenceNumber = $lastEventEnvelope?->sequenceNumber ?? SequenceNumber::none(); + } catch (DBALException) { + $sequenceNumber = null; + } + + return ContentRepositoryStatus::create( + $this->eventStore->status(), + $sequenceNumber, + $this->subscriptionEngine->subscriptionStatus() + ); + } + + public function replaySubscription(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null + { + $subscriptionStatus = $this->subscriptionEngine->subscriptionStatus(SubscriptionEngineCriteria::create([$subscriptionId]))->first(); + if ($subscriptionStatus === null) { + return new Error(sprintf('Subscription "%s" is not registered.', $subscriptionId->value)); + } + if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::NEW) { + return new Error(sprintf('Subscription "%s" is not setup and cannot be replayed.', $subscriptionId->value)); + } + $resetResult = $this->subscriptionEngine->reset(SubscriptionEngineCriteria::create([$subscriptionId])); + if ($resetResult->errors !== null) { + return self::createErrorForReason('reset', $resetResult->errors); + } + $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), $progressCallback); + if ($bootResult->errors !== null) { + return self::createErrorForReason('catchup', $bootResult->errors); + } + return null; + } + + public function replayAllSubscriptions(\Closure|null $progressCallback = null): Error|null + { + $resetResult = $this->subscriptionEngine->reset(); + if ($resetResult->errors !== null) { + return self::createErrorForReason('reset', $resetResult->errors); + } + $bootResult = $this->subscriptionEngine->boot(progressCallback: $progressCallback); + if ($bootResult->errors !== null) { + return self::createErrorForReason('catchup', $bootResult->errors); + } + return null; + } + + /** + * Reactivate a subscription + * + * The explicit catchup is only needed for subscriptions in the error or detached status with an advanced position. + * Running a full replay would work but might be overkill, instead this reactivation will just attempt + * catchup the subscription back to active from its current position. + */ + public function reactivateSubscription(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null + { + $subscriptionStatus = $this->subscriptionEngine->subscriptionStatus(SubscriptionEngineCriteria::create([$subscriptionId]))->first(); + if ($subscriptionStatus === null) { + return new Error(sprintf('Subscription "%s" is not registered.', $subscriptionId->value)); + } + if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::NEW) { + return new Error(sprintf('Subscription "%s" is not setup and cannot be reactivated.', $subscriptionId->value)); + } + + // todo implement https://github.com/patchlevel/event-sourcing/blob/b8591c56b21b049f46bead8e7ab424fd2afe9917/src/Subscription/Engine/DefaultSubscriptionEngine.php#L624 + return null; + } + + /** + * WARNING: Removes all events from the content repository and resets the subscriptions + * This operation cannot be undone. + */ + public function prune(): Error|null + { + // prune all streams: + foreach ($this->findAllContentStreamStreamNames() as $contentStreamStreamName) { + $this->eventStore->deleteStream($contentStreamStreamName); + } + foreach ($this->findAllWorkspaceStreamNames() as $workspaceStreamName) { + $this->eventStore->deleteStream($workspaceStreamName); + } + $resetResult = $this->subscriptionEngine->reset(); + if ($resetResult->errors !== null) { + return self::createErrorForReason('reset', $resetResult->errors); + } + // note: possibly introduce $skipBooting flag like for setup + $bootResult = $this->subscriptionEngine->boot(); + if ($bootResult->errors !== null) { + return self::createErrorForReason('booting', $bootResult->errors); + } + return null; + } + + private static function createErrorForReason(string $method, Errors $errors): Error + { + $message = []; + $message[] = sprintf('%s produced the following error%s', $method, $errors->count() === 1 ? '' : 's'); + foreach ($errors as $error) { + $message[] = sprintf(' Subscription "%s": %s', $error->subscriptionId->value, $error->message); + } + return new Error(join("\n", $message)); + } + + /** + * @return list + */ + private function findAllContentStreamStreamNames(): array + { + $events = $this->eventStore->load( + VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), + EventStreamFilter::create( + EventTypes::create( + // we are only interested in the creation events to limit the amount of events to fetch + EventType::fromString('ContentStreamWasCreated'), + EventType::fromString('ContentStreamWasForked') + ) + ) + ); + $allStreamNames = []; + foreach ($events as $eventEnvelope) { + $allStreamNames[] = $eventEnvelope->streamName; + } + return array_unique($allStreamNames, SORT_REGULAR); + } + + /** + * @return list + */ + private function findAllWorkspaceStreamNames(): array + { + $events = $this->eventStore->load( + VirtualStreamName::forCategory(WorkspaceEventStreamName::EVENT_STREAM_NAME_PREFIX), + EventStreamFilter::create( + EventTypes::create( + // we are only interested in the creation events to limit the amount of events to fetch + EventType::fromString('RootWorkspaceWasCreated'), + EventType::fromString('WorkspaceWasCreated') + ) + ) + ); + $allStreamNames = []; + foreach ($events as $eventEnvelope) { + $allStreamNames[] = $eventEnvelope->streamName; + } + return array_unique($allStreamNames, SORT_REGULAR); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php new file mode 100644 index 00000000000..234b01fcb1d --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php @@ -0,0 +1,24 @@ + + * @api + */ +class ContentRepositoryMaintainerFactory implements ContentRepositoryServiceFactoryInterface +{ + public function build( + ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies + ): ContentRepositoryMaintainer { + return new ContentRepositoryMaintainer( + $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->subscriptionEngine + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 5afbcddc590..5f1991c0a77 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -4,7 +4,6 @@ namespace Neos\ContentRepository\Core\Service; -use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; @@ -23,10 +22,10 @@ use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamForPruning; use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\EventType; use Neos\EventStore\Model\Event\EventTypes; -use Neos\EventStore\Model\Event\StreamName; use Neos\EventStore\Model\EventStream\EventStreamFilter; use Neos\EventStore\Model\EventStream\ExpectedVersion; use Neos\EventStore\Model\EventStream\VirtualStreamName; @@ -39,9 +38,9 @@ class ContentStreamPruner implements ContentRepositoryServiceInterface { public function __construct( - private readonly ContentRepository $contentRepository, private readonly EventStoreInterface $eventStore, - private readonly EventNormalizer $eventNormalizer + private readonly EventNormalizer $eventNormalizer, + private readonly SubscriptionEngine $subscriptionEngine, ) { } @@ -161,10 +160,9 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta } if ($danglingContentStreamsPresent) { - try { - $this->contentRepository->catchUpProjections(); - } catch (\Throwable $e) { - $outputFn(sprintf('Could not catchup after removing unused content streams: %s. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.', $e->getMessage())); + $result = $this->subscriptionEngine->catchUpActive(); + if ($result->hasFailed()) { + $outputFn('Catchup after removing unused content streams led to errors. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.'); } } else { $outputFn('Okay. No pruneable streams in the event stream'); @@ -200,15 +198,6 @@ public function pruneRemovedFromEventStream(\Closure $outputFn): void } } - public function pruneAllWorkspacesAndContentStreamsFromEventStream(): void - { - foreach ($this->findAllContentStreamStreamNames() as $contentStreamStreamName) { - $this->eventStore->deleteStream($contentStreamStreamName); - } - foreach ($this->findAllWorkspaceStreamNames() as $workspaceStreamName) { - $this->eventStore->deleteStream($workspaceStreamName); - } - } /** * Find all removed content streams that are unused in the event stream @@ -411,48 +400,4 @@ private function findAllContentStreams(): array } return $cs; } - - /** - * @return list - */ - private function findAllContentStreamStreamNames(): array - { - $events = $this->eventStore->load( - VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), - EventStreamFilter::create( - EventTypes::create( - // we are only interested in the creation events to limit the amount of events to fetch - EventType::fromString('ContentStreamWasCreated'), - EventType::fromString('ContentStreamWasForked') - ) - ) - ); - $allStreamNames = []; - foreach ($events as $eventEnvelope) { - $allStreamNames[] = $eventEnvelope->streamName; - } - return array_unique($allStreamNames, SORT_REGULAR); - } - - /** - * @return list - */ - private function findAllWorkspaceStreamNames(): array - { - $events = $this->eventStore->load( - VirtualStreamName::forCategory(WorkspaceEventStreamName::EVENT_STREAM_NAME_PREFIX), - EventStreamFilter::create( - EventTypes::create( - // we are only interested in the creation events to limit the amount of events to fetch - EventType::fromString('RootWorkspaceWasCreated'), - EventType::fromString('WorkspaceWasCreated') - ) - ) - ); - $allStreamNames = []; - foreach ($events as $eventEnvelope) { - $allStreamNames[] = $eventEnvelope->streamName; - } - return array_unique($allStreamNames, SORT_REGULAR); - } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php index f9940f8f56a..ecdb4dc2107 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php @@ -16,9 +16,9 @@ class ContentStreamPrunerFactory implements ContentRepositoryServiceFactoryInter public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentStreamPruner { return new ContentStreamPruner( - $serviceFactoryDependencies->contentRepository, $serviceFactoryDependencies->eventStore, - $serviceFactoryDependencies->eventNormalizer + $serviceFactoryDependencies->eventNormalizer, + $serviceFactoryDependencies->subscriptionEngine, ); } } diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php index c7de2e5bb28..db546bf9f9f 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php +++ b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php @@ -14,32 +14,42 @@ namespace Neos\ContentRepository\Core\SharedModel\ContentRepository; -use Neos\ContentRepository\Core\Projection\ProjectionStatuses; -use Neos\ContentRepository\Core\Projection\ProjectionStatusType; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; +use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStore\Status as EventStoreStatus; -use Neos\EventStore\Model\EventStore\StatusType as EventStoreStatusType; /** + * The status information of a content repository. Examined via {@see ContentRepositoryMaintainer::status()} + * * @api */ final readonly class ContentRepositoryStatus { - public function __construct( + /** + * @param EventStoreStatus $eventStoreStatus + * @param SequenceNumber|null $eventStorePosition The position of the event store. NULL if an error occurred in which case a setup must likely be done, see $eventStoreStatus + * @param SubscriptionStatusCollection $subscriptionStatus + */ + private function __construct( public EventStoreStatus $eventStoreStatus, - public ProjectionStatuses $projectionStatuses, + public SequenceNumber|null $eventStorePosition, + public SubscriptionStatusCollection $subscriptionStatus, ) { } - public function isOk(): bool - { - if ($this->eventStoreStatus->type !== EventStoreStatusType::OK) { - return false; - } - foreach ($this->projectionStatuses as $projectionStatus) { - if ($projectionStatus->type !== ProjectionStatusType::OK) { - return false; - } - } - return true; + /** + * @internal + */ + public static function create( + EventStoreStatus $eventStoreStatus, + SequenceNumber|null $eventStorePosition, + SubscriptionStatusCollection $subscriptionStatus, + ): self { + return new self( + $eventStoreStatus, + $eventStorePosition, + $subscriptionStatus + ); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php new file mode 100644 index 00000000000..7ab2ad36ac0 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php @@ -0,0 +1,34 @@ +getMessage(), + $exception, + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php new file mode 100644 index 00000000000..8e8d3b3853f --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -0,0 +1,51 @@ + + * @internal implementation detail of the catchup + */ +final readonly class Errors implements \IteratorAggregate, \Countable +{ + /** + * @var non-empty-array + */ + private array $errors; + + private function __construct( + Error ...$errors + ) { + if ($errors === []) { + throw new \InvalidArgumentException('Errors must not be empty.', 1731612542); + } + $this->errors = array_values($errors); + } + + /** + * @param array $errors + */ + public static function fromArray(array $errors): self + { + return new self(...$errors); + } + + public function getIterator(): \Traversable + { + yield from $this->errors; + } + + public function count(): int + { + return count($this->errors); + } + + public function first(): Error + { + foreach ($this->errors as $error) { + return $error; + } + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php new file mode 100644 index 00000000000..e90e6975fa2 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php @@ -0,0 +1,51 @@ +errors */ + public function hasFailed(): bool + { + return $this->errors !== null; + } + + public function throwOnFailure(): void + { + /** @var Error[] $errors */ + $errors = iterator_to_array($this->errors ?? []); + if ($errors === []) { + return; + } + $firstError = array_shift($errors); + + $additionalFailedSubscribers = array_map(fn (Error $error) => $error->subscriptionId->value, $errors); + + $additionalErrors = $additionalFailedSubscribers === [] ? '' : sprintf(' | And subscribers %s with additional errors.', join(', ', $additionalFailedSubscribers)); + $exceptionMessage = sprintf('Exception in subscriber "%s" while catching up: %s%s', $firstError->subscriptionId->value, $firstError->message, $additionalErrors); + + // todo custom exception! + throw new \RuntimeException($exceptionMessage, 1732132930, $firstError->throwable); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php new file mode 100644 index 00000000000..dab71033f97 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php @@ -0,0 +1,26 @@ +logger?->info('Subscription Engine: Start to setup.'); + + $this->subscriptionStore->setup(); + $this->discoverNewSubscriptions(); + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::fromArray([ + SubscriptionStatus::NEW, + SubscriptionStatus::BOOTING, + SubscriptionStatus::ACTIVE, + SubscriptionStatus::DETACHED, + SubscriptionStatus::ERROR, + ]))); + if ($subscriptions->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions found.'); // todo not happy? Because there must be at least the content graph?!! + return Result::success(); + } + $errors = []; + foreach ($subscriptions as $subscription) { + $error = $this->setupSubscription($subscription); + if ($error !== null) { + $errors[] = $error; + } + } + return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); + } + + public function boot(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult + { + return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::BOOTING, $progressCallback)); + } + + public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult + { + return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::ACTIVE, $progressCallback)); + } + + public function reset(SubscriptionEngineCriteria|null $criteria = null): Result + { + $criteria ??= SubscriptionEngineCriteria::noConstraints(); + + $this->logger?->info('Subscription Engine: Start to reset.'); + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::any())); + if ($subscriptions->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions to reset.'); + return Result::success(); + } + $errors = []; + foreach ($subscriptions as $subscription) { + $error = $this->resetSubscription($subscription); + if ($error !== null) { + $errors[] = $error; + } + } + return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); + } + + public function subscriptionStatus(SubscriptionEngineCriteria|null $criteria = null): SubscriptionStatusCollection + { + $statuses = []; + try { + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::create(ids: $criteria?->ids)); + } catch (TableNotFoundException) { + // the schema is not setup - thus there are no subscribers + return SubscriptionStatusCollection::createEmpty(); + } + foreach ($subscriptions as $subscription) { + if (!$this->subscribers->contain($subscription->id)) { + $statuses[] = DetachedSubscriptionStatus::create( + $subscription->id, + $subscription->status, + $subscription->position + ); + continue; + } + $subscriber = $this->subscribers->get($subscription->id); + $statuses[] = ProjectionSubscriptionStatus::create( + subscriptionId: $subscription->id, + subscriptionStatus: $subscription->status, + subscriptionPosition: $subscription->position, + subscriptionError: $subscription->error, + setupStatus: $subscriber->projection->status(), + ); + } + foreach ($this->subscribers as $subscriber) { + if ($subscriptions->contain($subscriber->id)) { + continue; + } + if ($criteria?->ids?->contain($subscriber->id) === false) { + // this might be a NEW subscription but we dont return it as status is filtered. + continue; + } + // this NEW state is not persisted yet + $statuses[] = ProjectionSubscriptionStatus::create( + subscriptionId: $subscriber->id, + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: $subscriber->projection->status(), + ); + } + return SubscriptionStatusCollection::fromArray($statuses); + } + + private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, SubscriptionId $subscriptionId): Error|null + { + $subscriber = $this->subscribers->get($subscriptionId); + try { + $subscriber->handle($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscriptionId->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); + return Error::fromSubscriptionIdAndException($subscriptionId, $e); + } + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscriptionId->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); + return null; + } + + /** + * Find all subscribers that don't have a corresponding subscription. + * For each match a subscription is added + * + * Note: newly discovered subscriptions are not ACTIVE by default, instead they have to be initialized via {@see self::setup()} explicitly + */ + private function discoverNewSubscriptions(): void + { + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::noConstraints()); + foreach ($this->subscribers as $subscriber) { + if ($subscriptions->contain($subscriber->id)) { + continue; + } + $subscription = new Subscription( + $subscriber->id, + SubscriptionStatus::NEW, + SequenceNumber::fromInteger(0), + null, + null + ); + $this->subscriptionStore->add($subscription); + $this->logger?->info(sprintf('Subscription Engine: New Subscriber "%s" was found and added to the subscription store.', $subscriber->id->value)); + } + } + + /** + * Set up the subscription by retrieving the corresponding subscriber and calling the setUp method on its handler + * If the setup fails, the subscription will be in the {@see SubscriptionStatus::ERROR} state and a corresponding {@see Error} is returned + */ + private function setupSubscription(Subscription $subscription): ?Error + { + if (!$this->subscribers->contain($subscription->id)) { + // mark detached subscriptions as we cannot set up + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::DETACHED, + position: $subscription->position, + subscriptionError: $subscription->error + ); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); + return null; + } + + $subscriber = $this->subscribers->get($subscription->id); + try { + $subscriber->projection->setUp(); + } catch (\Throwable $e) { + // todo wrap in savepoint to ensure error do not mess up the projection? + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the setup method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); + $this->subscriptionStore->update( + $subscription->id, + SubscriptionStatus::ERROR, + $subscription->position, + SubscriptionError::fromPreviousStatusAndException($subscription->status, $e) + ); + return Error::fromSubscriptionIdAndException($subscription->id, $e); + } + + if ($subscription->status === SubscriptionStatus::ACTIVE) { + $this->logger?->debug(sprintf('Subscription Engine: Active subscriber "%s" for "%s" has been re-setup.', $subscriber::class, $subscription->id->value)); + return null; + } else { + $this->subscriptionStore->update( + $subscription->id, + SubscriptionStatus::BOOTING, + $subscription->position, + null + ); + } + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" has been setup, set to %s from previous %s.', $subscriber::class, $subscription->id->value, SubscriptionStatus::BOOTING->name, $subscription->status->name)); + return null; + } + + private function resetSubscription(Subscription $subscription): ?Error + { + $subscriber = $this->subscribers->get($subscription->id); + try { + $subscriber->projection->resetState(); + } catch (\Throwable $e) { + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the resetState method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); + return Error::fromSubscriptionIdAndException($subscription->id, $e); + } + $this->subscriptionStore->update( + $subscription->id, + SubscriptionStatus::BOOTING, + position: SequenceNumber::none(), + subscriptionError: null + ); + $this->logger?->debug(sprintf('Subscription Engine: For Subscriber "%s" for "%s" the resetState method has been executed.', $subscriber::class, $subscription->id->value)); + return null; + } + + private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, SubscriptionStatus $subscriptionStatus, \Closure $progressClosure = null): ProcessedResult + { + $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in state "%s".', $subscriptionStatus->value)); + + $subscriptionEngineCriteria = SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $subscriptionStatus); + return $this->subscriptionStore->transactional( + function () use ($subscriptionEngineCriteria, $subscriptionStatus, $progressClosure) { + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionEngineCriteria); + foreach ($subscriptionsToCatchup as $subscription) { + if (!$this->subscribers->contain($subscription->id)) { + // mark detached subscriptions as we cannot handle them and exclude them from catchup + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::DETACHED, + position: $subscription->position, + subscriptionError: null, + ); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + } + } + + if ($subscriptionsToCatchup->isEmpty()) { + $this->logger?->info(sprintf('Subscription Engine: No subscriptions in state "%s". Finishing catch up', $subscriptionStatus->value)); + return ProcessedResult::success(0); + } + + foreach ($subscriptionsToCatchup as $subscription) { + try { + $this->subscribers->get($subscription->id)->onBeforeCatchUp($subscription->status); + } catch (\Throwable $e) { + // analog to onAfterCatchUp, we tolerate no exceptions here and consider it a critical developer error. + $message = sprintf('Subscriber "%s" failed onBeforeCatchUp: %s', $subscription->id->value, $e->getMessage()); + $this->logger?->critical($message); + throw new CatchUpFailed($message, 1732374000, $e); + } + } + $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); + $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); + + /** @var array $errors */ + $errors = []; + $numberOfProcessedEvents = 0; + /** @var array $highestSequenceNumberForSubscriber */ + $highestSequenceNumberForSubscriber = []; + + $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); + foreach ($eventStream as $eventEnvelope) { + $sequenceNumber = $eventEnvelope->sequenceNumber; + if ($numberOfProcessedEvents > 0) { + $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); + } + if ($progressClosure !== null) { + $progressClosure($eventEnvelope); + } + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + foreach ($subscriptionsToCatchup as $subscription) { + if ($subscription->position->value >= $sequenceNumber->value) { + $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); + continue; + } + $this->subscriptionStore->createSavepoint(); + $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription->id); + if ($error !== null) { + // ERROR Case: + // 1.) roll back the partially applied event on the subscriber + $this->subscriptionStore->rollbackSavepoint(); + // 2.) for the leftover events we are not including this failed subscription for catchup + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + // 3.) update the subscription error state on either its unchanged or new position (if some events worked) + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ERROR, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: SubscriptionError::fromPreviousStatusAndException( + $subscription->status, + $error->throwable + ), + ); + // 4.) invoke onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent we want to "shutdown" this catchup iteration event though we know it failed + // todo put the ERROR $subscriptionStatus into the after hook, so it can properly be reacted upon + try { + $this->subscribers->get($subscription->id)->onAfterCatchUp(); + } catch (\Throwable $e) { + // analog to onBeforeCatchUp, we tolerate no exceptions here and consider it a critical developer error. + $message = sprintf('Subscriber "%s" had an error and also failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); + $this->logger?->critical($message); + throw new CatchUpFailed($message, 1732733740, $e); + } + $errors[] = $error; + continue; + } + // HAPPY Case: + $this->subscriptionStore->releaseSavepoint(); + $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; + } + $numberOfProcessedEvents++; + } + foreach ($subscriptionsToCatchup as $subscription) { + try { + $this->subscribers->get($subscription->id)->onAfterCatchUp(); + } catch (\Throwable $e) { + // analog to onBeforeCatchUp, we tolerate no exceptions here and consider it a critical developer error. + $message = sprintf('Subscriber "%s" failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); + $this->logger?->critical($message); + throw new CatchUpFailed($message, 1732374000, $e); + } + // after catchup mark all subscriptions as active, so they are triggered automatically now. + // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ACTIVE, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: null, + ); + if ($subscription->status !== SubscriptionStatus::ACTIVE) { + $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting.', $subscription->id->value)); + } + } + $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); + return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); + } + ); + } + + /** + * @template T + * @param \Closure(): T $closure + * @return T + */ + private function processExclusively(\Closure $closure): mixed + { + if ($this->processing) { + throw new SubscriptionEngineAlreadyProcessingException('Subscription engine is already processing', 1732714075); + } + $this->processing = true; + try { + return $closure(); + } finally { + $this->processing = false; + } + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php new file mode 100644 index 00000000000..068f5a15a98 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php @@ -0,0 +1,40 @@ +|null $ids + */ + public static function create( + SubscriptionIds|array $ids = null + ): self { + if (is_array($ids)) { + $ids = SubscriptionIds::fromArray($ids); + } + return new self( + $ids + ); + } + + public static function noConstraints(): self + { + return new self( + ids: null + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php new file mode 100644 index 00000000000..2cee008bda8 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php @@ -0,0 +1,14 @@ +|null $ids + * @param SubscriptionStatusFilter|null $status + */ + public static function create( + SubscriptionIds|array $ids = null, + SubscriptionStatusFilter $status = null, + ): self { + if (is_array($ids)) { + $ids = SubscriptionIds::fromArray($ids); + } + return new self( + $ids, + $status ?? SubscriptionStatusFilter::any(), + ); + } + + public static function forEngineCriteriaAndStatus( + SubscriptionEngineCriteria $criteria, + SubscriptionStatusFilter|SubscriptionStatus $status, + ): self { + if ($status instanceof SubscriptionStatus) { + $status = SubscriptionStatusFilter::fromArray([$status]); + } + return new self( + $criteria->ids, + $status, + ); + } + + public static function noConstraints(): self + { + return new self( + ids: null, + status: SubscriptionStatusFilter::any(), + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php new file mode 100644 index 00000000000..85f31072717 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -0,0 +1,44 @@ + $projection + */ + public function __construct( + public readonly SubscriptionId $id, + public readonly ProjectionInterface $projection, + private readonly ?CatchUpHookInterface $catchUpHook + ) { + } + + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void + { + $this->catchUpHook?->onBeforeCatchUp($subscriptionStatus); + } + + public function handle(EventInterface $event, EventEnvelope $eventEnvelope): void + { + $this->catchUpHook?->onBeforeEvent($event, $eventEnvelope); + $this->projection->apply($event, $eventEnvelope); + $this->catchUpHook?->onAfterEvent($event, $eventEnvelope); + } + + public function onAfterCatchUp(): void + { + $this->catchUpHook?->onAfterCatchUp(); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php new file mode 100644 index 00000000000..eba25e39a1a --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php @@ -0,0 +1,87 @@ + + * @internal implementation detail of the catchup + */ +final class Subscribers implements \IteratorAggregate, \Countable, \JsonSerializable +{ + /** + * @param array $subscribersById + */ + private function __construct( + private readonly array $subscribersById + ) { + } + + /** + * @param array $subscribers + */ + public static function fromArray(array $subscribers): self + { + $subscribersById = []; + foreach ($subscribers as $subscriber) { + if (!$subscriber instanceof ProjectionSubscriber) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', ProjectionSubscriber::class, get_debug_type($subscriber)), 1721731490); + } + if (array_key_exists($subscriber->id->value, $subscribersById)) { + throw new \InvalidArgumentException(sprintf('Subscriber with id "%s" is already part of this set', $subscriber->id->value), 1721731494); + } + $subscribersById[$subscriber->id->value] = $subscriber; + } + return new self($subscribersById); + } + + public static function none(): self + { + return self::fromArray([]); + } + + public function with(ProjectionSubscriber $subscriber): self + { + return new self([...$this->subscribersById, $subscriber->id->value => $subscriber]); + } + + public function get(SubscriptionId $id): ProjectionSubscriber + { + if (!$this->contain($id)) { + throw new \InvalidArgumentException(sprintf('Subscriber with the subscription id "%s" not found.', $id->value), 1721731490); + } + return $this->subscribersById[$id->value]; + } + + public function contain(SubscriptionId $id): bool + { + return array_key_exists($id->value, $this->subscribersById); + } + + public function getIterator(): \Traversable + { + yield from array_values($this->subscribersById); + } + + public function count(): int + { + return count($this->subscribersById); + } + + /** + * @return iterable + */ + public function jsonSerialize(): iterable + { + return array_values($this->subscribersById); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php new file mode 100644 index 00000000000..ccfb08f07e5 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -0,0 +1,22 @@ +getMessage(), $previousStatus, $error->getTraceAsString()); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php new file mode 100644 index 00000000000..668c85f0b78 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php @@ -0,0 +1,30 @@ +value === $this->value; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php new file mode 100644 index 00000000000..9e81ae56ef2 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php @@ -0,0 +1,77 @@ + + * @internal implementation detail of the catchup + */ +final class SubscriptionIds implements \IteratorAggregate, \Countable, \JsonSerializable +{ + /** + * @param array $subscriptionIdsById + */ + private function __construct( + private readonly array $subscriptionIdsById + ) { + } + + /** + * @param array $ids + */ + public static function fromArray(array $ids): self + { + $subscriptionIdsById = []; + foreach ($ids as $id) { + if (is_string($id)) { + $id = SubscriptionId::fromString($id); + } + if (!$id instanceof SubscriptionId) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', SubscriptionId::class, get_debug_type($id)), 1731580820); + } + if (array_key_exists($id->value, $subscriptionIdsById)) { + throw new \InvalidArgumentException(sprintf('Subscription id "%s" is already part of this set', $id->value), 1731580838); + } + $subscriptionIdsById[$id->value] = $id; + } + return new self($subscriptionIdsById); + } + + public static function none(): self + { + return self::fromArray([]); + } + + public function getIterator(): \Traversable + { + yield from array_values($this->subscriptionIdsById); + } + + public function count(): int + { + return count($this->subscriptionIdsById); + } + + public function contain(SubscriptionId $id): bool + { + return array_key_exists($id->value, $this->subscriptionIdsById); + } + + /** + * @return list + */ + public function toStringArray(): array + { + return array_values(array_map(static fn (SubscriptionId $id) => $id->value, $this->subscriptionIdsById)); + } + + /** + * @return iterable + */ + public function jsonSerialize(): iterable + { + return array_values($this->subscriptionIdsById); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php new file mode 100644 index 00000000000..487f742fbab --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php @@ -0,0 +1,17 @@ + + */ +final readonly class SubscriptionStatusCollection implements \IteratorAggregate +{ + /** + * @var array $items + */ + private array $items; + + private function __construct( + ProjectionSubscriptionStatus|DetachedSubscriptionStatus ...$items, + ) { + $this->items = $items; + } + + public static function createEmpty(): self + { + return new self(); + } + + /** + * @param array $items + */ + public static function fromArray(array $items): self + { + return new self(...$items); + } + + public function first(): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null + { + foreach ($this->items as $status) { + return $status; + } + return null; + } + + public function getIterator(): \Traversable + { + yield from $this->items; + } + + public function isEmpty(): bool + { + return $this->items === []; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php new file mode 100644 index 00000000000..e531c180329 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php @@ -0,0 +1,64 @@ + + * @internal implementation detail of the catchup + */ +final class SubscriptionStatusFilter implements \IteratorAggregate +{ + /** + * @param array $statusByValue + */ + private function __construct( + private readonly array $statusByValue, + ) { + } + + /** + * @param array $status + */ + public static function fromArray(array $status): self + { + $statusByValue = []; + foreach ($status as $singleStatus) { + if (is_string($singleStatus)) { + $singleStatus = SubscriptionStatus::from($singleStatus); + } + if (!$singleStatus instanceof SubscriptionStatus) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', SubscriptionStatus::class, get_debug_type($singleStatus)), 1731580994); + } + if (array_key_exists($singleStatus->value, $statusByValue)) { + throw new \InvalidArgumentException(sprintf('Status "%s" is already part of this set', $singleStatus->value), 1731581002); + } + $statusByValue[$singleStatus->value] = $singleStatus; + } + return new self($statusByValue); + } + + public static function any(): self + { + return new self([]); + } + + public function getIterator(): \Traversable + { + yield from array_values($this->statusByValue); + } + + public function isEmpty(): bool + { + return $this->statusByValue === []; + } + + /** + * @return list + */ + public function toStringArray(): array + { + return array_values(array_map(static fn (SubscriptionStatus $id) => $id->value, $this->statusByValue)); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php new file mode 100644 index 00000000000..3bba94d0018 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php @@ -0,0 +1,126 @@ + + * @internal implementation detail of the catchup + */ +final class Subscriptions implements \IteratorAggregate, \Countable, \JsonSerializable +{ + /** + * @param array $subscriptionsById + */ + private function __construct( + private readonly array $subscriptionsById + ) { + } + + /** + * @param array $subscriptions + */ + public static function fromArray(array $subscriptions): self + { + $subscriptionsById = []; + foreach ($subscriptions as $subscription) { + if (!$subscription instanceof Subscription) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', Subscription::class, get_debug_type($subscription)), 1729679774); + } + if (array_key_exists($subscription->id->value, $subscriptionsById)) { + throw new \InvalidArgumentException(sprintf('Subscription with id "%s" is contained multiple times in this set', $subscription->id->value), 1731580354); + } + $subscriptionsById[$subscription->id->value] = $subscription; + } + return new self($subscriptionsById); + } + + public static function none(): self + { + return self::fromArray([]); + } + + public function getIterator(): \Traversable + { + yield from $this->subscriptionsById; + } + + public function isEmpty(): bool + { + return $this->subscriptionsById === []; + } + + public function count(): int + { + return count($this->subscriptionsById); + } + + public function contain(SubscriptionId $subscriptionId): bool + { + return array_key_exists($subscriptionId->value, $this->subscriptionsById); + } + + public function get(SubscriptionId $subscriptionId): Subscription + { + if (!$this->contain($subscriptionId)) { + throw new \InvalidArgumentException(sprintf('Subscription with id "%s" not part of this set', $subscriptionId->value), 1723567808); + } + return $this->subscriptionsById[$subscriptionId->value]; + } + + public function without(SubscriptionId $subscriptionId): self + { + $subscriptionsById = $this->subscriptionsById; + unset($subscriptionsById[$subscriptionId->value]); + return new self($subscriptionsById); + } + + /** + * @param \Closure(Subscription): bool $callback + */ + public function filter(\Closure $callback): self + { + return self::fromArray(array_filter($this->subscriptionsById, $callback)); + } + + /** + * @template T + * @param \Closure(Subscription): T $callback + * @return array + */ + public function map(\Closure $callback): array + { + return array_map($callback, $this->subscriptionsById); + } + + public function with(Subscription $subscription): self + { + return new self([...$this->subscriptionsById, $subscription->id->value => $subscription]); + } + + /** + * @return iterable + */ + public function jsonSerialize(): iterable + { + return array_values($this->subscriptionsById); + } + + public function lowestPosition(): SequenceNumber|null + { + if ($this->subscriptionsById === []) { + return null; + } + return SequenceNumber::fromInteger( + min( + array_map( + fn (Subscription $subscription) => $subscription->position->value, + $this->subscriptionsById + ) + ) + ); + } +} diff --git a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php index 753525af34f..bf18eaea3b2 100644 --- a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php +++ b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php @@ -6,7 +6,7 @@ use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; -use Neos\ContentRepository\Core\EventStore\EventPersister; +use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; @@ -14,12 +14,15 @@ use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphInterface; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\StructureAdjustment\Adjustment\DimensionAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\DisallowedChildNodeAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\PropertyAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\StructureAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\TetheredNodeAdjustments; use Neos\ContentRepository\StructureAdjustment\Adjustment\UnknownNodeTypeAdjustment; +use Neos\EventStore\EventStoreInterface; +use Neos\EventStore\Model\Events; class StructureAdjustmentService implements ContentRepositoryServiceInterface { @@ -38,8 +41,10 @@ class StructureAdjustmentService implements ContentRepositoryServiceInterface private readonly ContentGraphInterface $liveContentGraph; public function __construct( - private readonly ContentRepository $contentRepository, - private readonly EventPersister $eventPersister, + ContentRepository $contentRepository, + private readonly EventStoreInterface $eventStore, + private readonly EventNormalizer $eventNormalizer, + private readonly SubscriptionEngine $subscriptionEngine, NodeTypeManager $nodeTypeManager, InterDimensionalVariationGraph $interDimensionalVariationGraph, PropertyConverter $propertyConverter, @@ -98,11 +103,20 @@ public function findAdjustmentsForNodeType(NodeTypeName $nodeTypeName): \Generat public function fixError(StructureAdjustment $adjustment): void { - if ($adjustment->remediation) { - $remediation = $adjustment->remediation; - $eventsToPublish = $remediation(); - assert($eventsToPublish instanceof EventsToPublish); - $this->eventPersister->publishEvents($this->contentRepository, $eventsToPublish); + if (!$adjustment->remediation) { + return; } + $remediation = $adjustment->remediation; + $eventsToPublish = $remediation(); + assert($eventsToPublish instanceof EventsToPublish); + $normalizedEvents = Events::fromArray( + $eventsToPublish->events->map($this->eventNormalizer->normalize(...)) + ); + $this->eventStore->commit( + $eventsToPublish->streamName, + $normalizedEvents, + $eventsToPublish->expectedVersion + ); + $this->subscriptionEngine->catchUpActive(); } } diff --git a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php index b9e75abeaff..534fe279028 100644 --- a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php +++ b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php @@ -16,7 +16,9 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor { return new StructureAdjustmentService( $serviceFactoryDependencies->contentRepository, - $serviceFactoryDependencies->eventPersister, + $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->eventNormalizer, + $serviceFactoryDependencies->subscriptionEngine, $serviceFactoryDependencies->nodeTypeManager, $serviceFactoryDependencies->interDimensionalVariationGraph, $serviceFactoryDependencies->propertyConverter, diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index a6cac6dc8b2..c06fa32af4c 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -22,18 +22,18 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\NodeModification\Dto\PropertyValuesToWrite; use Neos\ContentRepository\Core\NodeType\NodeTypeName; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\FindSubtreeFilter; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\NodeType\NodeTypeCriteria; use Neos\ContentRepository\Core\Projection\ContentGraph\Subtree; use Neos\ContentRepository\Core\Projection\ContentGraph\VisibilityConstraints; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\ContentStreamClosing; -use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeCopying; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeCreation; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeModification; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeMove; @@ -146,7 +146,7 @@ public function iExpectTheGraphProjectionToConsistOfExactlyNodes(int $expectedNu public ContentGraphReadModelInterface|null $instance; public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { - $this->instance = $serviceFactoryDependencies->projectionsAndCatchUpHooks->contentGraphProjection->getState(); + $this->instance = $serviceFactoryDependencies->contentGraphReadModel; return new class implements ContentRepositoryServiceInterface { }; @@ -256,8 +256,9 @@ abstract protected function getContentRepositoryService( */ public function iReplayTheProjection(string $projectionName): void { - $this->currentContentRepository->resetProjectionState($projectionName); - $this->currentContentRepository->catchUpProjection($projectionName, CatchUpOptions::create()); + $contentRepositoryMaintainer = $this->getContentRepositoryService(new ContentRepositoryMaintainerFactory()); + $result = $contentRepositoryMaintainer->replaySubscription(SubscriptionId::fromString($projectionName)); + Assert::assertNull($result); } protected function deserializeProperties(array $properties): PropertyValuesToWrite diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php index 0afbf60c150..ab773a32d30 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php @@ -20,9 +20,9 @@ use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePointSet; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; -use Neos\ContentRepository\Core\EventStore\Events; -use Neos\ContentRepository\Core\EventStore\EventsToPublish; +use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; +use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; +use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\Common\RebasableToOtherWorkspaceInterface; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\Command\AddDimensionShineThrough; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\Command\MoveDimensionSpacePoint; @@ -56,10 +56,12 @@ use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Exception\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Node\ReferenceName; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event; use Neos\EventStore\Model\Event\StreamName; use Neos\EventStore\Model\EventEnvelope; +use Neos\EventStore\Model\Events; use Neos\EventStore\Model\EventStream\ExpectedVersion; use Neos\EventStore\Model\EventStream\VirtualStreamName; use Neos\Utility\Arrays; @@ -287,19 +289,23 @@ protected function publishEvent(string $eventType, StreamName $streamName, array Event\EventData::fromString(json_encode($eventPayload)), Event\EventMetadata::fromArray([]) ); - /** @var EventPersister $eventPersister */ - $eventPersister = (new \ReflectionClass($this->currentContentRepository))->getProperty('eventPersister') - ->getValue($this->currentContentRepository); - /** @var EventNormalizer $eventNormalizer */ - $eventNormalizer = (new \ReflectionClass($eventPersister))->getProperty('eventNormalizer') - ->getValue($eventPersister); - $event = $eventNormalizer->denormalize($artificiallyConstructedEvent); - - $eventPersister->publishEvents($this->currentContentRepository, new EventsToPublish( - $streamName, - Events::with($event), - ExpectedVersion::ANY() - )); + + // HACK can be replaced, once https://github.com/neos/neos-development-collection/pull/5341 is merged + $eventStoreAndSubscriptionEngine = new class implements ContentRepositoryServiceFactoryInterface { + public EventStoreInterface|null $eventStore; + public SubscriptionEngine|null $subscriptionEngine; + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + $this->eventStore = $serviceFactoryDependencies->eventStore; + $this->subscriptionEngine = $serviceFactoryDependencies->subscriptionEngine; + return new class implements ContentRepositoryServiceInterface + { + }; + } + }; + $this->getContentRepositoryService($eventStoreAndSubscriptionEngine); + $eventStoreAndSubscriptionEngine->eventStore->commit($streamName, Events::with($artificiallyConstructedEvent), ExpectedVersion::ANY()); + $eventStoreAndSubscriptionEngine->subscriptionEngine->catchUpActive(); } /** diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php new file mode 100644 index 00000000000..a20650ae93e --- /dev/null +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php @@ -0,0 +1,32 @@ + + * @internal helper to configure custom catchup hook mocks for testing + */ +final class FakeCatchUpHookFactory implements CatchUpHookFactoryInterface +{ + /** + * @var array + */ + private static array $catchupHooks; + + public function build(CatchUpHookFactoryDependencies $dependencies): CatchUpHookInterface + { + return static::$catchupHooks[spl_object_hash($dependencies->projectionState)] ?? throw new \RuntimeException('No catchup hook defined for Fake.'); + } + + public static function setCatchupHook(ProjectionStateInterface $projectionState, CatchUpHookInterface $catchUpHook): void + { + self::$catchupHooks[spl_object_hash($projectionState)] = $catchUpHook; + } +} diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php new file mode 100644 index 00000000000..d0e21e2ec5a --- /dev/null +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php @@ -0,0 +1,37 @@ +>> + * @internal helper to configure custom projection mocks for testing + */ +final class FakeProjectionFactory implements ProjectionFactoryInterface +{ + /** + * @var array> + */ + private static array $projections; + + public function build( + SubscriberFactoryDependencies $projectionFactoryDependencies, + array $options, + ): ProjectionInterface { + return static::$projections[$options['instanceId']] ?? throw new \RuntimeException('No projection defined for Fake.'); + } + + /** + * @param ProjectionInterface $projection + */ + public static function setProjection(string $instanceId, ProjectionInterface $projection): void + { + self::$projections[$instanceId] = $projection; + } +} diff --git a/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php b/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php index c026f755119..e4eeaedd33b 100644 --- a/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php +++ b/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php @@ -13,14 +13,14 @@ * source code. */ -use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\ContentRepositoryRegistry\Exception\ContentRepositoryNotFoundException; -use Neos\EventStore\EventStoreInterface; +use PHPUnit\Framework\Assert; /** * The CR registry subject provider trait for behavioral tests @@ -52,21 +52,18 @@ protected function setUpCRRegistry(): void /** * @Given /^I initialize content repository "([^"]*)"$/ */ - public function iInitializeContentRepository(string $contentRepositoryId): void + public function iInitializeContentRepository(string $rawContentRepositoryId): void { - $contentRepository = $this->getContentRepository(ContentRepositoryId::fromString($contentRepositoryId)); - /** @var EventStoreInterface $eventStore */ - $eventStore = (new \ReflectionClass($contentRepository))->getProperty('eventStore')->getValue($contentRepository); - /** @var Connection $databaseConnection */ - $databaseConnection = (new \ReflectionClass($eventStore))->getProperty('connection')->getValue($eventStore); - $eventTableName = sprintf('cr_%s_events', $contentRepositoryId); - $databaseConnection->executeStatement('TRUNCATE ' . $eventTableName); + $contentRepositoryId = ContentRepositoryId::fromString($rawContentRepositoryId); - if (!in_array($contentRepository->id, self::$alreadySetUpContentRepositories)) { - $contentRepository->setUp(); - self::$alreadySetUpContentRepositories[] = $contentRepository->id; + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + if (!in_array($contentRepositoryId, self::$alreadySetUpContentRepositories)) { + $result = $contentRepositoryMaintainer->setUp(); + Assert::assertNull($result); + self::$alreadySetUpContentRepositories[] = $contentRepositoryId; } - $contentRepository->resetProjectionStates(); + $result = $contentRepositoryMaintainer->prune(); + Assert::assertNull($result); } /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 38149fcbe74..c51957ed96c 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -3,30 +3,39 @@ namespace Neos\ContentRepositoryRegistry\Command; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ProjectionStatusType; -use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; -use Neos\ContentRepository\Core\Service\WorkspaceMaintenanceServiceFactory; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStore\StatusType; +use Neos\Flow\Annotations as Flow; use Neos\Flow\Cli\CommandController; -use Neos\Neos\Domain\Service\WorkspaceService; -use Symfony\Component\Console\Helper\ProgressBar; -use Symfony\Component\Console\Output\ConsoleOutput; use Symfony\Component\Console\Output\Output; +/** + * Set up a content repository + * + * *Initialisation* + * + * The command "./flow cr:setup" sets up the content repository like event store and subscription database tables. + * It is non-destructive. + * + * Note that a reset is not implemented here but for the Neos CMS use-case provided via "./flow site:pruneAll" + * + * *Staus information* + * + * The status of the content repository e.g. if a setup is required or if all subscriptions are active and their position + * can be examined with "./flow cr:status" + * + * See also {@see ContentRepositoryMaintainer} for more information. + */ final class CrCommandController extends CommandController { - - public function __construct( - private readonly ContentRepositoryRegistry $contentRepositoryRegistry, - private readonly ProjectionServiceFactory $projectionServiceFactory, - ) { - parent::__construct(); - } + #[Flow\Inject()] + protected ContentRepositoryRegistry $contentRepositoryRegistry; /** * Sets up and checks required dependencies for a Content Repository instance @@ -38,13 +47,22 @@ public function __construct( * To check if the content repository needs to be setup look into cr:status. * That command will also display information what is about to be migrated. * + * @param bool $quiet If set, no output is generated. This is useful if only the exit code (0 = all OK, 1 = errors or warnings) is of interest * @param string $contentRepository Identifier of the Content Repository to set up */ - public function setupCommand(string $contentRepository = 'default'): void + public function setupCommand(string $contentRepository = 'default', bool $quiet = false): void { + if ($quiet) { + $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); - $this->contentRepositoryRegistry->get($contentRepositoryId)->setUp(); + $result = $contentRepositoryMaintainer->setUp(); + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } $this->outputLine('Content Repository "%s" was set up', [$contentRepositoryId->value]); } @@ -63,35 +81,97 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $status = $this->contentRepositoryRegistry->get($contentRepositoryId)->status(); - - $this->output('Event Store: '); - $this->outputLine(match ($status->eventStoreStatus->type) { + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + $crStatus = $contentRepositoryMaintainer->status(); + $hasErrors = false; + $reactivationRequired = false; + $setupRequired = false; + $bootingRequired = false; + $this->outputLine('Event Store:'); + $this->output(' Setup: '); + $this->outputLine(match ($crStatus->eventStoreStatus->type) { StatusType::OK => 'OK', StatusType::SETUP_REQUIRED => 'Setup required!', StatusType::ERROR => 'ERROR', }); - if ($verbose && $status->eventStoreStatus->details !== '') { - $this->outputFormatted($status->eventStoreStatus->details, [], 2); + if ($crStatus->eventStorePosition) { + $this->outputLine(' Position: %d', [$crStatus->eventStorePosition->value]); + } else { + $this->outputLine(' Position: Loading failed!'); + } + $hasErrors |= $crStatus->eventStoreStatus->type === StatusType::ERROR; + if ($verbose && $crStatus->eventStoreStatus->details !== '') { + $this->outputFormatted($crStatus->eventStoreStatus->details, [], 2); } $this->outputLine(); - foreach ($status->projectionStatuses as $projectionName => $projectionStatus) { - $this->output('Projection "%s": ', [$projectionName]); - $this->outputLine(match ($projectionStatus->type) { - ProjectionStatusType::OK => 'OK', - ProjectionStatusType::SETUP_REQUIRED => 'Setup required!', - ProjectionStatusType::REPLAY_REQUIRED => 'Replay required!', - ProjectionStatusType::ERROR => 'ERROR', - }); - if ($verbose && ($projectionStatus->type !== ProjectionStatusType::OK || $projectionStatus->details)) { - $lines = explode(chr(10), $projectionStatus->details ?: 'No details available.'); - foreach ($lines as $line) { - $this->outputLine(' ' . $line); + $this->outputLine('Subscriptions:'); + if ($crStatus->subscriptionStatus->isEmpty()) { + $this->outputLine('There are no registered subscriptions yet, please run ./flow cr:setup'); + $this->quit(1); + } + foreach ($crStatus->subscriptionStatus as $status) { + if ($status instanceof DetachedSubscriptionStatus) { + $this->outputLine(' %s:', [$status->subscriptionId->value]); + $this->output(' Subscription: '); + $this->output('%s DETACHED', [$status->subscriptionId->value, $status->subscriptionStatus === SubscriptionStatus::DETACHED ? 'is' : 'will be']); + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + } + if ($status instanceof ProjectionSubscriptionStatus) { + $this->outputLine(' %s:', [$status->subscriptionId->value]); + $this->output(' Setup: '); + $this->outputLine(match ($status->setupStatus->type) { + ProjectionStatusType::OK => 'OK', + ProjectionStatusType::SETUP_REQUIRED => 'SETUP REQUIRED', + ProjectionStatusType::ERROR => 'ERROR', + }); + $hasErrors |= $status->setupStatus->type === ProjectionStatusType::ERROR; + $setupRequired |= $status->setupStatus->type === ProjectionStatusType::SETUP_REQUIRED; + if ($verbose && ($status->setupStatus->type !== ProjectionStatusType::OK || $status->setupStatus->details)) { + $lines = explode(chr(10), $status->setupStatus->details ?: 'No details available.'); + foreach ($lines as $line) { + $this->outputLine(' ' . $line); + } + $this->outputLine(); + } + $this->output(' Projection: '); + $this->output(match ($status->subscriptionStatus) { + SubscriptionStatus::NEW => 'NEW', + SubscriptionStatus::BOOTING => 'BOOTING', + SubscriptionStatus::ACTIVE => 'ACTIVE', + SubscriptionStatus::DETACHED => 'DETACHED', + SubscriptionStatus::ERROR => 'ERROR', + }); + if ($crStatus->eventStorePosition?->value > $status->subscriptionPosition->value) { + // projection is behind + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + } else { + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + } + $hasErrors |= $status->subscriptionStatus === SubscriptionStatus::ERROR; + $reactivationRequired |= $status->subscriptionStatus === SubscriptionStatus::ERROR; + $bootingRequired |= $status->subscriptionStatus === SubscriptionStatus::BOOTING; + $reactivationRequired |= $status->subscriptionStatus === SubscriptionStatus::DETACHED; + if ($verbose && $status->subscriptionError !== null) { + $lines = explode(chr(10), $status->subscriptionError->errorMessage ?: 'No details available.'); + foreach ($lines as $line) { + $this->outputLine(' %s', [$line]); + } } - $this->outputLine(); } } - if (!$status->isOk()) { + if ($verbose) { + $this->outputLine(); + if ($setupRequired) { + $this->outputLine('Setup required, please run ./flow cr:setup'); + } + if ($bootingRequired) { + $this->outputLine('Replay needed for BOOTING projections, please run ./flow subscription:replay [subscription-id]'); + } + if ($reactivationRequired) { + $this->outputLine('Reactivation of ERROR or DETACHED projection required, please run ./flow subscription:reactivate [subscription-id]'); + } + } + if ($hasErrors) { $this->quit(1); } } @@ -99,47 +179,35 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo /** * Replays the specified projection of a Content Repository by resetting its state and performing a full catchup. * - * @param string $projection Full Qualified Class Name or alias of the projection to replay (e.g. "contentStream") + * @param string $projection Identifier of the projection to replay * @param string $contentRepository Identifier of the Content Repository instance to operate on * @param bool $force Replay the projection without confirmation. This may take some time! * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) - * @param int $until Until which sequence number should projections be replayed? useful for debugging + * @internal + * @deprecated with Neos 9 Beta 17, please use ./flow subscription:replay instead */ - public function projectionReplayCommand(string $projection, string $contentRepository = 'default', bool $force = false, bool $quiet = false, int $until = 0): void + public function projectionReplayCommand(string $projection, string $contentRepository = 'default', bool $force = false, bool $quiet = false): void { - if ($quiet) { - $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); - } - $progressBar = new ProgressBar($this->output->getOutput()); - $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:16s%/%estimated:-16s% %memory:6s%'); - if (!$force && $quiet) { - $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this projection. This may take some time.'); + $subscriptionId = match($projection) { + 'doctrineDbalContentGraph', + 'Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjection' => 'contentGraph', + 'documentUriPathProjection' => 'Neos.Neos:DocumentUriPathProjection', + 'change' => 'Neos.Neos:PendingChangesProjection', + default => null + }; + if ($subscriptionId === null) { + $this->outputLine('Invalid --projection specified. Please use ./flow subscription:replay [contentGraph|Neos.Neos:DocumentUriPathProjection|...] directly.'); $this->quit(1); } - - if (!$force && !$this->output->askConfirmation(sprintf('> This will replay the projection "%s" in "%s", which may take some time. Are you sure to proceed? (y/n) ', $projection, $contentRepository), false)) { - $this->outputLine('Abort.'); - return; - } - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionServiceFactory); - - $options = CatchUpOptions::create(); - if (!$quiet) { - $this->outputLine('Replaying events for projection "%s" of Content Repository "%s" ...', [$projection, $contentRepositoryId->value]); - $progressBar->start(max($until > 0 ? $until : $projectionService->highestSequenceNumber()->value, 1)); - $options = $options->with(progressCallback: fn () => $progressBar->advance()); - } - if ($until > 0) { - $options = $options->with(maximumSequenceNumber: SequenceNumber::fromInteger($until)); - } - $projectionService->replayProjection($projection, $options); - if (!$quiet) { - $progressBar->finish(); - $this->outputLine(); - $this->outputLine('Done.'); - } + $this->outputLine('Please use ./flow subscription:replay %s instead!', [$subscriptionId]); + $this->forward( + 'replay', + SubscriptionCommandController::class, + array_merge( + ['subscription' => $subscriptionId], + compact('contentRepository', 'force', 'quiet') + ) + ); } /** @@ -148,61 +216,16 @@ public function projectionReplayCommand(string $projection, string $contentRepos * @param string $contentRepository Identifier of the Content Repository instance to operate on * @param bool $force Replay the projection without confirmation. This may take some time! * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) - * @param int $until Until which sequence number should projections be replayed? useful for debugging + * @internal + * @deprecated with Neos 9 Beta 17, please use ./flow subscription:replayall instead */ - public function projectionReplayAllCommand(string $contentRepository = 'default', bool $force = false, bool $quiet = false, int $until = 0): void + public function projectionReplayAllCommand(string $contentRepository = 'default', bool $force = false, bool $quiet = false): void { - if ($quiet) { - $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); - } - $mainSection = ($this->output->getOutput() instanceof ConsoleOutput) ? $this->output->getOutput()->section() : $this->output->getOutput(); - $mainProgressBar = new ProgressBar($mainSection); - $mainProgressBar->setBarCharacter('â–ˆ'); - $mainProgressBar->setEmptyBarCharacter('â–‘'); - $mainProgressBar->setProgressCharacter('â–ˆ'); - $mainProgressBar->setFormat('debug'); - - $subSection = ($this->output->getOutput() instanceof ConsoleOutput) ? $this->output->getOutput()->section() : $this->output->getOutput(); - $progressBar = new ProgressBar($subSection); - $progressBar->setFormat(' %message% - %current%/%max% [%bar%] %percent:3s%% %elapsed:16s%/%estimated:-16s% %memory:6s%'); - if (!$force && $quiet) { - $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this projection. This may take some time.'); - $this->quit(1); - } - - if (!$force && !$this->output->askConfirmation(sprintf('> This will replay all projections in "%s", which may take some time. Are you sure to proceed? (y/n) ', $contentRepository), false)) { - $this->outputLine('Abort.'); - return; - } - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionServiceFactory); - if (!$quiet) { - $this->outputLine('Replaying events for all projections of Content Repository "%s" ...', [$contentRepositoryId->value]); - } - $options = CatchUpOptions::create(); - if (!$quiet) { - $options = $options->with(progressCallback: fn () => $progressBar->advance()); - } - if ($until > 0) { - $options = $options->with(maximumSequenceNumber: SequenceNumber::fromInteger($until)); - } - $highestSequenceNumber = max($until > 0 ? $until : $projectionService->highestSequenceNumber()->value, 1); - $mainProgressBar->start($projectionService->numberOfProjections()); - $mainProgressCallback = null; - if (!$quiet) { - $mainProgressCallback = static function (string $projectionAlias) use ($mainProgressBar, $progressBar, $highestSequenceNumber) { - $mainProgressBar->advance(); - $progressBar->setMessage($projectionAlias); - $progressBar->start($highestSequenceNumber); - $progressBar->setProgress(0); - }; - } - $projectionService->replayAllProjections($options, $mainProgressCallback); - if (!$quiet) { - $mainProgressBar->finish(); - $progressBar->finish(); - $this->outputLine('Done.'); - } + $this->outputLine('Please use ./flow subscription:replayall instead!'); + $this->forward( + 'replayall', + SubscriptionCommandController::class, + compact('contentRepository', 'force', 'quiet') + ); } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/SubscriptionCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/SubscriptionCommandController.php new file mode 100644 index 00000000000..4da391f52c7 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Command/SubscriptionCommandController.php @@ -0,0 +1,180 @@ +output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } + if (!$force && $quiet) { + $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this subscription. This may take some time.'); + $this->quit(1); + } + + if (!$force && !$this->output->askConfirmation(sprintf('> This will replay the subscription "%s" in "%s", which may take some time. Are you sure to proceed? (y/n) ', $subscription, $contentRepository), false)) { + $this->outputLine('Abort.'); + return; + } + + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Replaying events for subscription "%s" of Content Repository "%s" ...', [$subscription, $contentRepositoryId->value]); + // render memory consumption and time remaining + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->replaySubscription(SubscriptionId::fromString($subscription), progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); + $this->outputLine(); + } + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); + } + } + + /** + * Replays all projections of the specified Content Repository by resetting their states and performing a full catchup + * + * @param string $contentRepository Identifier of the Content Repository instance to operate on + * @param bool $force Replay all subscriptions without confirmation. This may take some time! + * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) + */ + public function replayAllCommand(string $contentRepository = 'default', bool $force = false, bool $quiet = false): void + { + if ($quiet) { + $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } + + if (!$force && $quiet) { + $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay all subscriptions. This may take some time.'); + $this->quit(1); + } + + if (!$force && !$this->output->askConfirmation(sprintf('> This will replay all projections in "%s", which may take some time. Are you sure to proceed? (y/n) ', $contentRepository), false)) { + $this->outputLine('Abort.'); + return; + } + + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Replaying events for all projections of Content Repository "%s" ...', [$contentRepositoryId->value]); + // render memory consumption and time remaining + // todo maybe reintroduce pretty output: https://github.com/neos/neos-development-collection/pull/5010 but without using highestSequenceNumber + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->replayAllSubscriptions(progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); + $this->outputLine(); + } + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); + } + } + + /** + * Reactivate a subscription + * + * The explicit catchup is only needed for projections in the error or detached status with an advanced position. + * Running a full replay would work but might be overkill, instead this reactivation will just attempt + * catchup the subscription back to active from its current position. + * + * @param string $subscription Identifier of the subscription to reactivate like it was configured (e.g. "contentGraph", "Vendor.Package:YourProjection") + * @param string $contentRepository Identifier of the Content Repository instance to operate on + * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) + */ + public function reactivateCommand(string $subscription, string $contentRepository = 'default', bool $quiet = false): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Reactivate subscription "%s" of Content Repository "%s" ...', [$subscription, $contentRepositoryId->value]); + // render memory consumption and time remaining + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->reactivateSubscription(SubscriptionId::fromString($subscription), progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); + $this->outputLine(); + } + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); + } + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index 86804d3cd93..4fdec55cd73 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -11,17 +11,21 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryFactory; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\Factory\ProjectionsAndCatchUpHooksFactory; +use Neos\ContentRepository\Core\Factory\ContentRepositorySubscriberFactories; +use Neos\ContentRepository\Core\Factory\ProjectionSubscriberFactory; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactories; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentSubgraphInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Node; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; -use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryIds; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepositoryRegistry\Exception\ContentRepositoryNotFoundException; use Neos\ContentRepositoryRegistry\Exception\InvalidConfigurationException; use Neos\ContentRepositoryRegistry\Factory\AuthProvider\AuthProviderFactoryInterface; @@ -29,6 +33,7 @@ use Neos\ContentRepositoryRegistry\Factory\ContentDimensionSource\ContentDimensionSourceFactoryInterface; use Neos\ContentRepositoryRegistry\Factory\EventStore\EventStoreFactoryInterface; use Neos\ContentRepositoryRegistry\Factory\NodeTypeManager\NodeTypeManagerFactoryInterface; +use Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactoryInterface; use Neos\ContentRepositoryRegistry\SubgraphCachingInMemory\ContentSubgraphWithRuntimeCaches; use Neos\ContentRepositoryRegistry\SubgraphCachingInMemory\SubgraphCachePool; use Neos\EventStore\EventStoreInterface; @@ -37,6 +42,7 @@ use Neos\Utility\Arrays; use Neos\Utility\PositionalArraySorter; use Psr\Clock\ClockInterface; +use Psr\Log\LoggerInterface; use Symfony\Component\Serializer\Normalizer\DenormalizerInterface; use Symfony\Component\Serializer\Normalizer\NormalizerInterface; use Symfony\Component\Serializer\Serializer; @@ -44,7 +50,7 @@ /** * @api */ -#[Flow\Scope("singleton")] +#[Flow\Scope('singleton')] final class ContentRepositoryRegistry { /** @@ -53,13 +59,26 @@ final class ContentRepositoryRegistry private array $factoryInstances = []; /** + * @var array + */ + private array $settings; + + #[Flow\Inject(name: 'Neos.ContentRepositoryRegistry:Logger', lazy: false)] + protected LoggerInterface $logger; + + #[Flow\Inject()] + protected ObjectManagerInterface $objectManager; + + #[Flow\Inject()] + protected SubgraphCachePool $subgraphCachePool; + + /** + * @internal for flow wiring and test cases only * @param array $settings */ - public function __construct( - private readonly array $settings, - private readonly ObjectManagerInterface $objectManager, - private readonly SubgraphCachePool $subgraphCachePool, - ) { + public function injectSettings(array $settings): void + { + $this->settings = $settings; } /** @@ -94,16 +113,6 @@ public function getContentRepositoryIds(): ContentRepositoryIds return ContentRepositoryIds::fromArray($contentRepositoryIds); } - /** - * @internal for test cases only - */ - public function resetFactoryInstance(ContentRepositoryId $contentRepositoryId): void - { - if (array_key_exists($contentRepositoryId->value, $this->factoryInstances)) { - unset($this->factoryInstances[$contentRepositoryId->value]); - } - } - public function subgraphForNode(Node $node): ContentSubgraphInterface { $contentRepository = $this->get($node->contentRepositoryId); @@ -132,6 +141,16 @@ public function buildService(ContentRepositoryId $contentRepositoryId, ContentRe return $this->getFactory($contentRepositoryId)->buildService($contentRepositoryServiceFactory); } + /** + * @internal for test cases only + */ + public function resetFactoryInstance(ContentRepositoryId $contentRepositoryId): void + { + if (array_key_exists($contentRepositoryId->value, $this->factoryInstances)) { + unset($this->factoryInstances[$contentRepositoryId->value]); + } + } + /** * @throws ContentRepositoryNotFoundException | InvalidConfigurationException */ @@ -168,6 +187,8 @@ private function buildFactory(ContentRepositoryId $contentRepositoryId): Content unset($contentRepositorySettings['preset']); } try { + /** @var CatchUpHookFactoryInterface|null $contentGraphCatchUpHookFactory */ + $contentGraphCatchUpHookFactory = $this->buildCatchUpHookFactory($contentRepositoryId, 'contentGraph', $contentRepositorySettings['contentGraphProjection']); $clock = $this->buildClock($contentRepositoryId, $contentRepositorySettings); return new ContentRepositoryFactory( $contentRepositoryId, @@ -175,10 +196,14 @@ private function buildFactory(ContentRepositoryId $contentRepositoryId): Content $this->buildNodeTypeManager($contentRepositoryId, $contentRepositorySettings), $this->buildContentDimensionSource($contentRepositoryId, $contentRepositorySettings), $this->buildPropertySerializer($contentRepositoryId, $contentRepositorySettings), - $this->buildProjectionsFactory($contentRepositoryId, $contentRepositorySettings), $this->buildAuthProviderFactory($contentRepositoryId, $contentRepositorySettings), $clock, + $this->buildSubscriptionStore($contentRepositoryId, $clock, $contentRepositorySettings), + $this->buildContentGraphProjectionFactory($contentRepositoryId, $contentRepositorySettings), + $contentGraphCatchUpHookFactory, $this->buildCommandHooksFactory($contentRepositoryId, $contentRepositorySettings), + $this->buildAdditionalSubscribersFactories($contentRepositoryId, $contentRepositorySettings), + $this->logger, ); } catch (\Exception $exception) { throw InvalidConfigurationException::fromException($contentRepositoryId, $exception); @@ -242,39 +267,44 @@ private function buildPropertySerializer(ContentRepositoryId $contentRepositoryI } /** @param array $contentRepositorySettings */ - private function buildProjectionsFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ProjectionsAndCatchUpHooksFactory + private function buildContentGraphProjectionFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentGraphProjectionFactoryInterface { - $projectionsAndCatchUpHooksFactory = new ProjectionsAndCatchUpHooksFactory(); - - // content graph projection: if (!isset($contentRepositorySettings['contentGraphProjection']['factoryObjectName'])) { throw InvalidConfigurationException::fromMessage('Content repository "%s" does not have the contentGraphProjection.factoryObjectName configured.', $contentRepositoryId->value); } - $projectionFactory = $this->objectManager->get($contentRepositorySettings['contentGraphProjection']['factoryObjectName']); - if (!$projectionFactory instanceof ContentGraphProjectionFactoryInterface) { - throw InvalidConfigurationException::fromMessage('Projection factory object name of contentGraphProjection (content repository "%s") is not an instance of %s but %s.', $contentRepositoryId->value, ContentGraphProjectionFactoryInterface::class, get_debug_type($projectionFactory)); + $contentGraphProjectionFactory = $this->objectManager->get($contentRepositorySettings['contentGraphProjection']['factoryObjectName']); + if (!$contentGraphProjectionFactory instanceof ContentGraphProjectionFactoryInterface) { + throw InvalidConfigurationException::fromMessage('Projection factory object name of contentGraphProjection (content repository "%s") is not an instance of %s but %s.', $contentRepositoryId->value, ContentGraphProjectionFactoryInterface::class, get_debug_type($contentGraphProjectionFactory)); } - $projectionsAndCatchUpHooksFactory->registerFactory($projectionFactory, $contentRepositorySettings['contentGraphProjection']['options'] ?? []); - - $this->registerCatchupHookForProjection($contentRepositorySettings['contentGraphProjection'], $projectionsAndCatchUpHooksFactory, $projectionFactory, 'contentGraphProjection', $contentRepositoryId); + return $contentGraphProjectionFactory; + } - // additional projections: - (is_array($contentRepositorySettings['projections'] ?? [])) || throw InvalidConfigurationException::fromMessage('Content repository "%s" expects projections configured as array.', $contentRepositoryId->value); - foreach ($contentRepositorySettings['projections'] ?? [] as $projectionName => $projectionOptions) { - if ($projectionOptions === null) { + /** + * @param array $projectionOptions + * @return CatchUpHookFactoryInterface|null + */ + private function buildCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, string $projectionName, array $projectionOptions): ?CatchUpHookFactoryInterface + { + if (!isset($projectionOptions['catchUpHooks'])) { + return null; + } + $catchUpHookFactories = CatchUpHookFactories::create(); + foreach ($projectionOptions['catchUpHooks'] as $catchUpHookName => $catchUpHookOptions) { + if ($catchUpHookOptions === null) { + // Allow catch up hooks to be disabled by setting their configuration to `null` continue; } - (is_array($projectionOptions)) || throw InvalidConfigurationException::fromMessage('Projection "%s" (content repository "%s") must be configured as array got %s', $projectionName, $contentRepositoryId->value, get_debug_type($projectionOptions)); - $projectionFactory = isset($projectionOptions['factoryObjectName']) ? $this->objectManager->get($projectionOptions['factoryObjectName']) : null; - if (!$projectionFactory instanceof ProjectionFactoryInterface) { - throw InvalidConfigurationException::fromMessage('Projection factory object name for projection "%s" (content repository "%s") is not an instance of %s but %s.', $projectionName, $contentRepositoryId->value, ProjectionFactoryInterface::class, get_debug_type($projectionFactory)); + $catchUpHookFactory = $this->objectManager->get($catchUpHookOptions['factoryObjectName']); + if (!$catchUpHookFactory instanceof CatchUpHookFactoryInterface) { + throw InvalidConfigurationException::fromMessage('CatchUpHook factory object name for hook "%s" in projection "%s" (content repository "%s") is not an instance of %s but %s', $catchUpHookName, $projectionName, $contentRepositoryId->value, CatchUpHookFactoryInterface::class, get_debug_type($catchUpHookFactory)); } - $projectionsAndCatchUpHooksFactory->registerFactory($projectionFactory, $projectionOptions['options'] ?? []); - - $this->registerCatchupHookForProjection($projectionOptions, $projectionsAndCatchUpHooksFactory, $projectionFactory, $projectionName, $contentRepositoryId); + $catchUpHookFactories = $catchUpHookFactories->with($catchUpHookFactory); + } + if ($catchUpHookFactories->isEmpty()) { + return null; } - return $projectionsAndCatchUpHooksFactory; + return $catchUpHookFactories; } /** @param array $contentRepositorySettings */ @@ -299,21 +329,34 @@ private function buildCommandHooksFactory(ContentRepositoryId $contentRepository return new CommandHooksFactory(...$commandHookFactories); } - /** - * @param ProjectionFactoryInterface> $projectionFactory - */ - private function registerCatchupHookForProjection(mixed $projectionOptions, ProjectionsAndCatchUpHooksFactory $projectionsAndCatchUpHooksFactory, ProjectionFactoryInterface $projectionFactory, string $projectionName, ContentRepositoryId $contentRepositoryId): void + /** @param array $contentRepositorySettings */ + private function buildAdditionalSubscribersFactories(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentRepositorySubscriberFactories { - foreach (($projectionOptions['catchUpHooks'] ?? []) as $catchUpHookOptions) { - if ($catchUpHookOptions === null) { + if (!is_array($contentRepositorySettings['projections'] ?? [])) { + throw InvalidConfigurationException::fromMessage('Content repository "%s" expects projections configured as array.', $contentRepositoryId->value); + } + /** @var array $projectionSubscriberFactories */ + $projectionSubscriberFactories = []; + foreach (($contentRepositorySettings['projections'] ?? []) as $projectionName => $projectionOptions) { + // Allow projections to be disabled by setting their configuration to `null` + if ($projectionOptions === null) { continue; } - $catchUpHookFactory = $this->objectManager->get($catchUpHookOptions['factoryObjectName']); - if (!$catchUpHookFactory instanceof CatchUpHookFactoryInterface) { - throw InvalidConfigurationException::fromMessage('CatchUpHook factory object name for projection "%s" (content repository "%s") is not an instance of %s but %s', $projectionName, $contentRepositoryId->value, CatchUpHookFactoryInterface::class, get_debug_type($catchUpHookFactory)); + if (!is_array($projectionOptions)) { + throw InvalidConfigurationException::fromMessage('Projection "%s" (content repository "%s") must be configured as array got %s', $projectionName, $contentRepositoryId->value, get_debug_type($projectionOptions)); + } + $projectionFactory = isset($projectionOptions['factoryObjectName']) ? $this->objectManager->get($projectionOptions['factoryObjectName']) : null; + if (!$projectionFactory instanceof ProjectionFactoryInterface) { + throw InvalidConfigurationException::fromMessage('Projection factory object name for projection "%s" (content repository "%s") is not an instance of %s but %s.', $projectionName, $contentRepositoryId->value, ProjectionFactoryInterface::class, get_debug_type($projectionFactory)); } - $projectionsAndCatchUpHooksFactory->registerCatchUpHookFactory($projectionFactory, $catchUpHookFactory); + $projectionSubscriberFactories[$projectionName] = new ProjectionSubscriberFactory( + SubscriptionId::fromString($projectionName), + $projectionFactory, + $this->buildCatchUpHookFactory($contentRepositoryId, $projectionName, $projectionOptions), + $projectionOptions['options'] ?? [], + ); } + return ContentRepositorySubscriberFactories::fromArray($projectionSubscriberFactories); } /** @param array $contentRepositorySettings */ @@ -337,4 +380,16 @@ private function buildClock(ContentRepositoryId $contentRepositoryIdentifier, ar } return $clockFactory->build($contentRepositoryIdentifier, $contentRepositorySettings['clock']['options'] ?? []); } + + /** @param array $contentRepositorySettings */ + private function buildSubscriptionStore(ContentRepositoryId $contentRepositoryId, ClockInterface $clock, array $contentRepositorySettings): SubscriptionStoreInterface + { + isset($contentRepositorySettings['subscriptionStore']['factoryObjectName']) || throw InvalidConfigurationException::fromMessage('Content repository "%s" does not have subscriptionStore.factoryObjectName configured.', $contentRepositoryId->value); + $subscriptionStoreFactory = $this->objectManager->get($contentRepositorySettings['subscriptionStore']['factoryObjectName']); + if (!$subscriptionStoreFactory instanceof SubscriptionStoreFactoryInterface) { + throw InvalidConfigurationException::fromMessage('subscriptionStore.factoryObjectName for content repository "%s" is not an instance of %s but %s.', $contentRepositoryId->value, SubscriptionStoreFactoryInterface::class, get_debug_type($subscriptionStoreFactory)); + } + return $subscriptionStoreFactory->build($contentRepositoryId, $clock, $contentRepositorySettings['subscriptionStore']['options'] ?? []); + } } + diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php new file mode 100644 index 00000000000..9c7c9e92005 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -0,0 +1,187 @@ +dbal->createSchemaManager()->createSchemaConfig(); + $schemaConfig->setDefaultTableOptions([ + 'charset' => 'utf8mb4' + ]); + $tableSchema = new Table($this->tableName, [ + (new Column('id', Type::getType(Types::STRING)))->setNotnull(true)->setLength(SubscriptionId::MAX_LENGTH)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', 'ascii_general_ci'), + (new Column('position', Type::getType(Types::INTEGER)))->setNotnull(true), + (new Column('status', Type::getType(Types::STRING)))->setNotnull(true)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', 'ascii_general_ci'), + (new Column('error_message', Type::getType(Types::TEXT)))->setNotnull(false), + (new Column('error_previous_status', Type::getType(Types::STRING)))->setNotnull(false)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', 'ascii_general_ci'), + (new Column('error_trace', Type::getType(Types::TEXT)))->setNotnull(false), + (new Column('last_saved_at', Type::getType(Types::DATETIME_IMMUTABLE)))->setNotnull(true), + ]); + $tableSchema->setPrimaryKey(['id']); + $tableSchema->addIndex(['status']); + $schema = new Schema( + [$tableSchema], + [], + $schemaConfig, + ); + foreach (DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $schema) as $statement) { + $this->dbal->executeStatement($statement); + } + } + + public function findByCriteriaForUpdate(SubscriptionCriteria $criteria): Subscriptions + { + $queryBuilder = $this->dbal->createQueryBuilder() + ->select('*') + ->from($this->tableName) + ->orderBy('id'); + $queryBuilder->forUpdate(); + if ($criteria->ids !== null) { + $queryBuilder->andWhere('id IN (:ids)') + ->setParameter( + 'ids', + $criteria->ids->toStringArray(), + ArrayParameterType::STRING, + ); + } + if (!$criteria->status->isEmpty()) { + $queryBuilder->andWhere('status IN (:status)') + ->setParameter( + 'status', + $criteria->status->toStringArray(), + ArrayParameterType::STRING, + ); + } + $result = $queryBuilder->executeQuery(); + assert($result instanceof Result); + $rows = $result->fetchAllAssociative(); + if ($rows === []) { + return Subscriptions::none(); + } + return Subscriptions::fromArray(array_map(self::fromDatabase(...), $rows)); + } + + public function add(Subscription $subscription): void + { + $row = self::toDatabase($subscription); + $row['id'] = $subscription->id->value; + $row['last_saved_at'] = $this->clock->now()->format('Y-m-d H:i:s'); + $this->dbal->insert( + $this->tableName, + $row, + ); + } + + public function update( + SubscriptionId $subscriptionId, + SubscriptionStatus $status, + SequenceNumber $position, + SubscriptionError|null $subscriptionError, + ): void { + $row = []; + $row['last_saved_at'] = $this->clock->now()->format('Y-m-d H:i:s'); + $row['status'] = $status->name; + $row['position'] = $position->value; + $row['error_message'] = $subscriptionError?->errorMessage; + $row['error_previous_status'] = $subscriptionError?->previousStatus?->name; + $row['error_trace'] = $subscriptionError?->errorTrace; + $this->dbal->update( + $this->tableName, + $row, + [ + 'id' => $subscriptionId->value, + ] + ); + } + + /** + * @return array + */ + private static function toDatabase(Subscription $subscription): array + { + return [ + 'status' => $subscription->status->name, + 'position' => $subscription->position->value, + 'error_message' => $subscription->error?->errorMessage, + 'error_previous_status' => $subscription->error?->previousStatus?->name, + 'error_trace' => $subscription->error?->errorTrace, + ]; + } + + /** + * @param array $row + */ + private static function fromDatabase(array $row): Subscription + { + if (isset($row['error_message'])) { + $subscriptionError = new SubscriptionError($row['error_message'], SubscriptionStatus::from($row['error_previous_status']), $row['error_trace']); + } else { + $subscriptionError = null; + } + $lastSavedAt = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $row['last_saved_at']); + assert($lastSavedAt instanceof DateTimeImmutable); + + return new Subscription( + SubscriptionId::fromString($row['id']), + SubscriptionStatus::from($row['status']), + SequenceNumber::fromInteger($row['position']), + $subscriptionError, + $lastSavedAt, + ); + } + + public function transactional(\Closure $closure): mixed + { + return $this->dbal->transactional($closure); + } + + public function createSavepoint(): void + { + $this->dbal->createSavepoint('SUBSCRIBER'); + } + + public function releaseSavepoint(): void + { + $this->dbal->releaseSavepoint('SUBSCRIBER'); + } + + public function rollbackSavepoint(): void + { + $this->dbal->rollbackSavepoint('SUBSCRIBER'); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactory.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactory.php new file mode 100644 index 00000000000..ef1405837e3 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactory.php @@ -0,0 +1,27 @@ + $options */ + public function build(ContentRepositoryId $contentRepositoryId, ClockInterface $clock, array $options): SubscriptionStoreInterface + { + return new DoctrineSubscriptionStore(sprintf('cr_%s_subscriptions', $contentRepositoryId->value), $this->connection, $clock); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactoryInterface.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactoryInterface.php new file mode 100644 index 00000000000..79e3516887c --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactoryInterface.php @@ -0,0 +1,18 @@ + $options */ + public function build(ContentRepositoryId $contentRepositoryId, ClockInterface $clock, array $options): SubscriptionStoreInterface; +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php deleted file mode 100644 index 69587bb4806..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php +++ /dev/null @@ -1,25 +0,0 @@ -projectionservice->catchupAllProjections(CatchUpOptions::create()); - } -} diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/SubscriptionReplayProcessor.php similarity index 53% rename from Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php rename to Neos.ContentRepositoryRegistry/Classes/Processors/SubscriptionReplayProcessor.php index 7a5a1f9013f..83ce99eaca0 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/SubscriptionReplayProcessor.php @@ -3,22 +3,22 @@ namespace Neos\ContentRepositoryRegistry\Processors; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\ContentRepositoryRegistry\Service\ProjectionService; /** * @internal */ -final class ProjectionResetProcessor implements ProcessorInterface +final readonly class SubscriptionReplayProcessor implements ProcessorInterface { public function __construct( - private readonly ProjectionService $projectionService, + private ContentRepositoryMaintainer $contentRepositoryMaintainer, ) { } public function run(ProcessingContext $context): void { - $this->projectionService->resetAllProjections(); + $this->contentRepositoryMaintainer->replayAllSubscriptions(); } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php deleted file mode 100644 index 06f11984d74..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php +++ /dev/null @@ -1,125 +0,0 @@ -resolveProjectionClassName($projectionAliasOrClassName); - $this->contentRepository->resetProjectionState($projectionClassName); - $this->contentRepository->catchUpProjection($projectionClassName, $options); - } - - public function replayAllProjections(CatchUpOptions $options, ?\Closure $progressCallback = null): void - { - foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { - if ($progressCallback) { - $progressCallback($classNamesAndAlias['alias']); - } - $this->contentRepository->resetProjectionState($classNamesAndAlias['className']); - $this->contentRepository->catchUpProjection($classNamesAndAlias['className'], $options); - } - } - - public function resetAllProjections(): void - { - foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { - $this->contentRepository->resetProjectionState($classNamesAndAlias['className']); - } - } - - public function catchupProjection(string $projectionAliasOrClassName, CatchUpOptions $options): void - { - $projectionClassName = $this->resolveProjectionClassName($projectionAliasOrClassName); - $this->contentRepository->catchUpProjection($projectionClassName, $options); - } - - public function catchupAllProjections(CatchUpOptions $options, ?\Closure $progressCallback = null): void - { - foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { - if ($progressCallback) { - $progressCallback($classNamesAndAlias['alias']); - } - $this->contentRepository->catchUpProjection($classNamesAndAlias['className'], $options); - } - } - - public function highestSequenceNumber(): SequenceNumber - { - foreach ($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1) as $eventEnvelope) { - return $eventEnvelope->sequenceNumber; - } - return SequenceNumber::none(); - } - - public function numberOfProjections(): int - { - return count($this->projections); - } - - /** - * @return class-string> - */ - private function resolveProjectionClassName(string $projectionAliasOrClassName): string - { - $lowerCaseProjectionName = strtolower($projectionAliasOrClassName); - $projectionClassNamesAndAliases = $this->projectionClassNamesAndAliases(); - foreach ($projectionClassNamesAndAliases as $classNamesAndAlias) { - if (strtolower($classNamesAndAlias['className']) === $lowerCaseProjectionName || strtolower($classNamesAndAlias['alias']) === $lowerCaseProjectionName) { - return $classNamesAndAlias['className']; - } - } - throw new \InvalidArgumentException(sprintf( - 'The projection "%s" is not registered for this Content Repository. The following projection aliases (or fully qualified class names) can be used: %s', - $projectionAliasOrClassName, - implode('', array_map(static fn (array $classNamesAndAlias) => sprintf(chr(10) . ' * %s (%s)', $classNamesAndAlias['alias'], $classNamesAndAlias['className']), $projectionClassNamesAndAliases)) - ), 1680519624); - } - - /** - * @return array>, alias: string}> - */ - private function projectionClassNamesAndAliases(): array - { - return array_map( - static fn (string $projectionClassName) => [ - 'className' => $projectionClassName, - 'alias' => self::projectionAlias($projectionClassName), - ], - $this->projections->getClassNames() - ); - } - - private static function projectionAlias(string $className): string - { - $alias = lcfirst(substr(strrchr($className, '\\') ?: '\\' . $className, 1)); - if (str_ends_with($alias, 'Projection')) { - $alias = substr($alias, 0, -10); - } - return $alias; - } -} diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php deleted file mode 100644 index 92114d47f1a..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php +++ /dev/null @@ -1,28 +0,0 @@ - - * @internal - */ -#[Flow\Scope("singleton")] -final class ProjectionServiceFactory implements ContentRepositoryServiceFactoryInterface -{ - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new ProjectionService( - $serviceFactoryDependencies->projectionsAndCatchUpHooks->projections, - $serviceFactoryDependencies->contentRepository, - $serviceFactoryDependencies->eventStore, - ); - } -} diff --git a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php index febdd096d20..3ff7ca35222 100644 --- a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php +++ b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php @@ -4,9 +4,10 @@ namespace Neos\ContentRepositoryRegistry\SubgraphCachingInMemory; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\Flow\Annotations as Flow; use Neos\ContentRepository\Core\EventStore\EventInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\EventStore\Model\EventEnvelope; /** @@ -23,7 +24,7 @@ public function __construct(private readonly SubgraphCachePool $subgraphCachePoo { } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { } @@ -36,10 +37,6 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event $this->subgraphCachePool->reset(); } - public function onBeforeBatchCompleted(): void - { - } - public function onAfterCatchUp(): void { $this->subgraphCachePool->reset(); diff --git a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php index cebe03ebb43..f3017920113 100644 --- a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php +++ b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php @@ -4,9 +4,9 @@ namespace Neos\ContentRepositoryRegistry\SubgraphCachingInMemory; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; /** diff --git a/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml b/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml index df582d41eba..70297ff0415 100644 --- a/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml +++ b/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml @@ -1,9 +1,3 @@ -Neos\ContentRepositoryRegistry\ContentRepositoryRegistry: - arguments: - 1: - setting: Neos.ContentRepositoryRegistry - -# !!! UGLY WORKAROUNDS, because we cannot wire non-Flow class constructor arguments here. # This adds a soft-dependency to the neos/contentgraph-doctrinedbaladapter package Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory: @@ -14,3 +8,12 @@ Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory: value: 'Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory' 2: object: 'Doctrine\DBAL\Connection' + +'Neos.ContentRepositoryRegistry:Logger': + className: Psr\Log\LoggerInterface + scope: singleton + factoryObjectName: Neos\Flow\Log\PsrLoggerFactoryInterface + factoryMethodName: get + arguments: + 1: + value: contentRepositoryLogger diff --git a/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml b/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml index 80574bfb60f..d78307056a8 100644 --- a/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml +++ b/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml @@ -12,6 +12,21 @@ Neos: ignoredClasses: Neos\\ContentRepository\\SharedModel\\NodeType\\NodeTypeManager: true + log: + psr3: + 'Neos\Flow\Log\PsrLoggerFactory': + contentRepositoryLogger: + default: + class: Neos\Flow\Log\Backend\FileBackend + options: + # todo context aware? FLOW_APPLICATION_CONTEXT .. but that contains / + logFileURL: '%FLOW_PATH_DATA%Logs/ContentRepository.log' + createParentDirectories: true + severityThreshold: '%LOG_INFO%' + maximumLogFileSize: 10485760 + logFilesToKeep: 1 + logMessageOrigin: false + ContentRepositoryRegistry: contentRepositories: default: @@ -37,6 +52,9 @@ Neos: clock: factoryObjectName: Neos\ContentRepositoryRegistry\Factory\Clock\SystemClockFactory + subscriptionStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactory + propertyConverters: DateTimeNormalizer: className: Symfony\Component\Serializer\Normalizer\DateTimeNormalizer diff --git a/Neos.ContentRepositoryRegistry/Tests/Functional/ContentRepositoryMaintenanceCommandControllerTest.php b/Neos.ContentRepositoryRegistry/Tests/Functional/ContentRepositoryMaintenanceCommandControllerTest.php new file mode 100644 index 00000000000..cf8de0c0b88 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Tests/Functional/ContentRepositoryMaintenanceCommandControllerTest.php @@ -0,0 +1,142 @@ +crController = $this->getObject(CrCommandController::class); + $this->subscriptionController = $this->getObject(SubscriptionCommandController::class); + + $this->response = new Response(); + $this->bufferedOutput = new BufferedOutput(); + + ObjectAccess::setProperty($this->crController, 'response', $this->response, true); + ObjectAccess::getProperty($this->crController, 'output', true)->setOutput($this->bufferedOutput); + + ObjectAccess::setProperty($this->subscriptionController, 'response', $this->response, true); + ObjectAccess::getProperty($this->subscriptionController, 'output', true)->setOutput($this->bufferedOutput); + } + + /** @test */ + public function setupOnEmptyEventStore(): void + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->crController->setupCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + // projections are marked active because the event store is empty + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + $this->crController->statusCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + } + + /** @test */ + public function setupOnModifiedEventStore(): void + { + $this->eventStore->setup(); + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::once())->method('resetState'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->crController->setupCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->crController->statusCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->subscriptionController->replayCommand(subscription: 'contentGraph', contentRepository: $this->contentRepository->id->value, force: true, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->subscriptionController->replayAllCommand(contentRepository: $this->contentRepository->id->value, force: true, quiet: true); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function projectionInError(): void + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::any())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->crController->setupCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->secondFakeProjection->injectSaboteur(fn () => throw new \RuntimeException('This projection is kaputt.')); + + try { + $this->contentRepository->handle(CreateRootWorkspace::create( + WorkspaceName::forLive(), + ContentStreamId::create() + )); + } catch (\RuntimeException) { + } + + self::assertEquals( + SubscriptionStatus::ERROR, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')?->subscriptionStatus + ); + + try { + $this->crController->statusCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + } catch (StopCommandException) { + } + // exit error code because one projection has a failure + self::assertEquals(1, $this->response->getExitCode()); + self::assertEmpty($this->bufferedOutput->fetch()); + + // repair projection + $this->secondFakeProjection->killSaboteur(); + $this->subscriptionController->reactivateCommand(subscription: 'Vendor.Package:SecondFakeProjection', contentRepository: $this->contentRepository->id->value, quiet: true); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + } +} diff --git a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php index 59e2c733a99..8c2ee16b5d2 100644 --- a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php +++ b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php @@ -7,7 +7,6 @@ use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePointSet; use Neos\ContentRepository\Core\EventStore\EventInterface; -use Neos\ContentRepository\Core\Feature\Common\EmbedsContentStreamId; use Neos\ContentRepository\Core\Feature\Common\EmbedsWorkspaceName; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\Event\DimensionSpacePointWasMoved; use Neos\ContentRepository\Core\Feature\NodeCreation\Event\NodeAggregateWithNodeWasCreated; @@ -19,7 +18,7 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Dto\NodeIdsToPublishOrDiscard; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasDiscarded; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyDiscarded; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\FindDescendantNodesFilter; use Neos\ContentRepository\Core\Projection\ContentGraph\Node; @@ -28,6 +27,7 @@ use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Neos\AssetUsage\Service\AssetUsageIndexingService; @@ -43,7 +43,7 @@ public function __construct( ) { } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { } @@ -88,11 +88,6 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event }; } - - public function onBeforeBatchCompleted(): void - { - } - public function onAfterCatchUp(): void { } diff --git a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php index 4188a10072f..71cf34fe2ea 100644 --- a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php +++ b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php @@ -14,8 +14,8 @@ * source code. */ -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\Neos\AssetUsage\Service\AssetUsageIndexingService; diff --git a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php index 0b94195c9d1..9cb7cf60b24 100644 --- a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php @@ -14,22 +14,25 @@ namespace Neos\Neos\Domain\Pruning; -use Neos\ContentRepository\Core\Service\ContentStreamPruner; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; /** - * Pruning processor that removes all events from the given cr + * Pruning processor that removes all events from the given cr and resets the projections */ final readonly class ContentRepositoryPruningProcessor implements ProcessorInterface { public function __construct( - private ContentStreamPruner $contentStreamPruner, + private ContentRepositoryMaintainer $contentRepositoryMaintainer, ) { } public function run(ProcessingContext $context): void { - $this->contentStreamPruner->pruneAllWorkspacesAndContentStreamsFromEventStream(); + $result = $this->contentRepositoryMaintainer->prune(); + if ($result !== null) { + throw new \RuntimeException($result->getMessage(), 1732461335); + } } } diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 741424a02e2..ebe76134d19 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -17,9 +17,12 @@ use Doctrine\DBAL\Exception as DBALException; use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; -use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Projection\ProjectionStatusType; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Export\Factory\EventStoreImportProcessorFactory; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; @@ -27,8 +30,8 @@ use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; +use Neos\ContentRepositoryRegistry\Processors\SubscriptionReplayProcessor; +use Neos\EventStore\Model\EventStore\StatusType; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; use Neos\Flow\Persistence\PersistenceManagerInterface; @@ -67,8 +70,10 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string } $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + $this->requireDataBaseSchemaToBeSetup(); - $this->requireContentRepositoryToBeSetup($contentRepository); + $this->requireContentRepositoryToBeSetup($contentRepositoryMaintainer, $contentRepositoryId); $filesystem = new Filesystem(new LocalFilesystemAdapter($path)); $context = new ProcessingContext($filesystem, $onMessage); @@ -78,7 +83,9 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->domainRepository, $this->persistenceManager), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - 'Catchup all projections' => new ProjectionCatchupProcessor($this->contentRepositoryRegistry->buildService($contentRepositoryId, new ProjectionServiceFactory())), + // WARNING! We do a replay here even though it will redo the live workspace creation. But otherwise the catchup hooks cannot determine that they need to be skipped as it seems like a regular catchup + // In case we allow to import events into other root workspaces, or don't expect live to be empty (see Import events), this would need to be adjusted, as otherwise existing data will be replayed + 'Replay all subscriptions' => new SubscriptionReplayProcessor($contentRepositoryMaintainer), ]); foreach ($processors as $processorLabel => $processor) { @@ -87,11 +94,18 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string } } - private function requireContentRepositoryToBeSetup(ContentRepository $contentRepository): void + private function requireContentRepositoryToBeSetup(ContentRepositoryMaintainer $contentRepositoryMaintainer, ContentRepositoryId $contentRepositoryId): void { - $status = $contentRepository->status(); - if (!$status->isOk()) { - throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepository->id->value)); + $status = $contentRepositoryMaintainer->status(); + if ($status->eventStoreStatus->type !== StatusType::OK) { + throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepositoryId->value)); + } + foreach ($status->subscriptionStatus as $status) { + if ($status instanceof ProjectionSubscriptionStatus) { + if ($status->setupStatus->type !== ProjectionStatusType::OK) { + throw new \RuntimeException(sprintf('Projection %s in content repository %s is not setup correctly, please run `./flow cr:setup`', $status->subscriptionId->value, $contentRepositoryId->value)); + } + } } } diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index 9a1c7c67537..18ab6f1d78f 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -16,7 +16,7 @@ use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; -use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\ProcessingContext; @@ -24,8 +24,6 @@ use Neos\ContentRepository\Export\Processors; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionResetProcessor; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Pruning\ContentRepositoryPruningProcessor; @@ -66,17 +64,11 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP $this->domainRepository, $this->persistenceManager ), - 'Prune content repository' => new ContentRepositoryPruningProcessor( - $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new ContentStreamPrunerFactory() - ) - ), 'Prune roles and metadata' => new RoleAndMetadataPruningProcessor($contentRepositoryId, $this->workspaceMetadataAndRoleRepository), - 'Reset all projections' => new ProjectionResetProcessor( + 'Prune content repository' => new ContentRepositoryPruningProcessor( $this->contentRepositoryRegistry->buildService( $contentRepositoryId, - new ProjectionServiceFactory() + new ContentRepositoryMaintainerFactory() ) ) ]); diff --git a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php index 1dc70d4fac7..b6bd864021b 100644 --- a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php +++ b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php @@ -10,8 +10,9 @@ use Neos\ContentRepository\Core\Feature\NodeMove\Event\NodeAggregateWasMoved; use Neos\ContentRepository\Core\Feature\NodeRemoval\Event\NodeAggregateWasRemoved; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Flow\Mvc\Routing\RouterCachingService; use Neos\Neos\FrontendRouting\Exception\NodeNotFoundException; @@ -32,7 +33,7 @@ public function __construct( ) { } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { // Nothing to do here } @@ -59,11 +60,6 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event }; } - public function onBeforeBatchCompleted(): void - { - // Nothing to do here - } - public function onAfterCatchUp(): void { // Nothing to do here diff --git a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php index cbdd469d930..b0c4faf2dd6 100644 --- a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php +++ b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php @@ -4,9 +4,9 @@ namespace Neos\Neos\FrontendRouting\CatchUpHook; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\Flow\Mvc\Routing\RouterCachingService; use Neos\Neos\FrontendRouting\Projection\DocumentUriPathFinder; diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php index de859059ae0..016814020bc 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php @@ -24,16 +24,13 @@ use Neos\ContentRepository\Core\Feature\RootNodeCreation\Event\RootNodeAggregateWithNodeWasCreated; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\NodeType\NodeTypeName; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; use Neos\Neos\Domain\Model\SiteNodeName; use Neos\Neos\FrontendRouting\Exception\NodeNotFoundException; @@ -47,7 +44,6 @@ final class DocumentUriPathProjection implements ProjectionInterface, WithMarkSt 'shortcutTarget' => Types::JSON, ]; - private DbalCheckpointStorage $checkpointStorage; private ?DocumentUriPathFinder $stateAccessor = null; /** @@ -60,11 +56,6 @@ public function __construct( private readonly Connection $dbal, private readonly string $tableNamePrefix, ) { - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNamePrefix . '_checkpoint', - self::class - ); } public function setUp(): void @@ -72,18 +63,10 @@ public function setUp(): void foreach ($this->determineRequiredSqlStatements() as $statement) { $this->dbal->executeStatement($statement); } - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->dbal->connect(); } catch (\Throwable $e) { @@ -113,11 +96,9 @@ private function determineRequiredSqlStatements(): array } - public function reset(): void + public function resetState(): void { $this->truncateDatabaseTables(); - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); $this->stateAccessor = null; } @@ -130,27 +111,6 @@ private function truncateDatabaseTables(): void } } - - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - RootNodeAggregateWithNodeWasCreated::class, - RootNodeAggregateDimensionsWereUpdated::class, - NodeAggregateWithNodeWasCreated::class, - NodeAggregateTypeWasChanged::class, - NodePeerVariantWasCreated::class, - NodeGeneralizationVariantWasCreated::class, - NodeSpecializationVariantWasCreated::class, - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - NodeAggregateWasRemoved::class, - NodePropertiesWereSet::class, - NodeAggregateWasMoved::class, - DimensionSpacePointWasMoved::class, - DimensionShineThroughWasAdded::class, - ]); - } - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void { match ($event::class) { @@ -168,15 +128,10 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void NodeAggregateWasMoved::class => $this->whenNodeAggregateWasMoved($event), DimensionSpacePointWasMoved::class => $this->whenDimensionSpacePointWasMoved($event), DimensionShineThroughWasAdded::class => $this->whenDimensionShineThroughWasAdded($event), - default => throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; } - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; - } - public function getState(): DocumentUriPathFinder { if (!$this->stateAccessor) { diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php index 570e0f5485d..c70dfb2dbd3 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php @@ -5,7 +5,7 @@ namespace Neos\Neos\FrontendRouting\Projection; use Doctrine\DBAL\Connection; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; @@ -35,7 +35,7 @@ public static function projectionTableNamePrefix( public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, array $options, ): DocumentUriPathProjection { diff --git a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php index 4acf758411e..6f60eb626d2 100644 --- a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php +++ b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php @@ -34,13 +34,14 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasDiscarded; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyDiscarded; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\NodeAggregate; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateIds; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -135,7 +136,7 @@ public function canHandle(EventInterface $event): bool ]); } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { } @@ -244,10 +245,6 @@ private function scheduleCacheFlushJobForWorkspaceName( ); } - public function onBeforeBatchCompleted(): void - { - } - public function onAfterCatchUp(): void { foreach ($this->flushNodeAggregateRequestsOnAfterCatchUp as $request) { diff --git a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php index a988f1bcac6..35a07b11b70 100644 --- a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php +++ b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php @@ -14,8 +14,8 @@ * source code. */ -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; /** diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php index 4f91f39cea2..ead06f7b588 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php @@ -37,15 +37,12 @@ use Neos\ContentRepository\Core\Feature\NodeVariation\Event\NodeSpecializationVariantWasCreated; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; /** @@ -61,17 +58,10 @@ class ChangeProjection implements ProjectionInterface */ private ?ChangeFinder $changeFinder = null; - private DbalCheckpointStorage $checkpointStorage; - public function __construct( private readonly Connection $dbal, private readonly string $tableNamePrefix, ) { - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNamePrefix . '_checkpoint', - self::class - ); } /** @@ -83,18 +73,10 @@ public function setUp(): void foreach ($this->determineRequiredSqlStatements() as $statement) { $this->dbal->executeStatement($statement); } - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->dbal->connect(); } catch (\Throwable $e) { @@ -146,30 +128,9 @@ private function determineRequiredSqlStatements(): array return $statements; } - public function reset(): void + public function resetState(): void { $this->dbal->exec('TRUNCATE ' . $this->tableNamePrefix); - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); - } - - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - NodeAggregateWasMoved::class, - NodePropertiesWereSet::class, - NodeReferencesWereSet::class, - NodeAggregateWithNodeWasCreated::class, - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - NodeAggregateWasRemoved::class, - DimensionSpacePointWasMoved::class, - NodeGeneralizationVariantWasCreated::class, - NodeSpecializationVariantWasCreated::class, - NodePeerVariantWasCreated::class, - NodeAggregateTypeWasChanged::class, - NodeAggregateNameWasChanged::class, - ]); } public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void @@ -188,15 +149,10 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void NodePeerVariantWasCreated::class => $this->whenNodePeerVariantWasCreated($event), NodeAggregateTypeWasChanged::class => $this->whenNodeAggregateTypeWasChanged($event), NodeAggregateNameWasChanged::class => $this->whenNodeAggregateNameWasChanged($event), - default => throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; } - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; - } - public function getState(): ChangeFinder { if (!$this->changeFinder) { diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php index 8b231897d78..244c1352426 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php @@ -15,7 +15,7 @@ namespace Neos\Neos\PendingChangesProjection; use Doctrine\DBAL\Connection; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; /** @@ -29,7 +29,7 @@ public function __construct( } public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, array $options, ): ChangeProjection { return new ChangeProjection( diff --git a/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php b/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php index 8d22c21adfc..5b0a2d46eb7 100644 --- a/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php +++ b/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php @@ -17,7 +17,7 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\Flow\Mvc\ActionRequest; @@ -67,19 +67,19 @@ private function enableContentRepositorySecurity(): void return; } $contentRepositoryAuthProviderFactory = $this->getObject(ContentRepositoryAuthProviderFactory::class); - $contentGraphProjection = $this->getContentRepositoryService(new class implements ContentRepositoryServiceFactoryInterface { + $contentGraphReadModel = $this->getContentRepositoryService(new class implements ContentRepositoryServiceFactoryInterface { public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { - $contentGraphProjection = $serviceFactoryDependencies->projectionsAndCatchUpHooks->contentGraphProjection; - return new class ($contentGraphProjection) implements ContentRepositoryServiceInterface { + $contentGraphReadModel = $serviceFactoryDependencies->contentGraphReadModel; + return new class ($contentGraphReadModel) implements ContentRepositoryServiceInterface { public function __construct( - public ContentGraphProjectionInterface $contentGraphProjection, + public ContentGraphReadModelInterface $contentGraphReadModel, ) { } }; } - })->contentGraphProjection; - $contentRepositoryAuthProvider = $contentRepositoryAuthProviderFactory->build($this->currentContentRepository->id, $contentGraphProjection->getState()); + })->contentGraphReadModel; + $contentRepositoryAuthProvider = $contentRepositoryAuthProviderFactory->build($this->currentContentRepository->id, $contentGraphReadModel); FakeAuthProvider::replaceAuthProvider($contentRepositoryAuthProvider); $this->crSecurity_contentRepositorySecurityEnabled = true; diff --git a/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature b/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature index 640ac7e62bf..c9fde1e6e6b 100644 --- a/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature +++ b/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature @@ -113,7 +113,7 @@ Feature: Basic routing functionality (match & resolve document nodes in one dime # !!! when caches were still enabled (without calling DocumentUriPathFinder->disableCache()), the replay below will # show really "interesting" (non-correct) results. This was bug #4253. - When I replay the "Neos\Neos\FrontendRouting\Projection\DocumentUriPathProjection" projection + When I replay the "Neos.Neos:DocumentUriPathProjection" projection Then the node "sir-david-nodenborough" in content stream "cs-identifier" and dimension "{}" should resolve to URL "/david-nodenborough-updated-b" And the node "earl-o-documentbourgh" in content stream "cs-identifier" and dimension "{}" should resolve to URL "/david-nodenborough-updated-b/earl-document" diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 12d96178fbb..2477370866c 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -1,9 +1,19 @@ parameters: ignoreErrors: - - message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Projection\\\\Projections\\:\\:getClassNames\" is called\\.$#" + message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Subscription\\\\SubscriptionIds\\:\\:toStringArray\" is called\\.$#" count: 1 - path: Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php + path: Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php + + - + message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Subscription\\\\SubscriptionStatusFilter\\:\\:isEmpty\" is called\\.$#" + count: 1 + path: Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php + + - + message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Subscription\\\\SubscriptionStatusFilter\\:\\:toStringArray\" is called\\.$#" + count: 1 + path: Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php - message: "#^Method Neos\\\\Neos\\\\Controller\\\\Backend\\\\MenuHelper\\:\\:buildModuleList\\(\\) return type has no value type specified in iterable type array\\.$#"