From 3aa93ff64db9a0c884a00b3f60bb2f95950d1b12 Mon Sep 17 00:00:00 2001 From: Markus Friedrich Date: Wed, 31 Aug 2022 14:25:41 +0200 Subject: [PATCH] !!![TASK:BP:11.6] Introduce queue and queue item interfaces As an initial steps to allow custom index queue types interfaces for queue and queue items are introduced. Additionally custom queue classes can be configured via: plugin.tx_solr.index.queue..indexQueue This is a breaking change for all instances that implemented own queues and queue items! Ports: #3764 Resolves: #3763 --- .../Search/AbstractModuleController.php | 5 +- .../Search/IndexQueueModuleController.php | 19 +- Classes/Domain/Index/IndexService.php | 7 +- .../Queue/GarbageRemover/AbstractStrategy.php | 9 +- .../Queue/QueueInitializationService.php | 29 ++- .../Index/Queue/QueueItemRepository.php | 13 +- Classes/IndexQueue/Item.php | 71 +++--- Classes/IndexQueue/ItemInterface.php | 117 +++++++++ .../MountPointAwareItemInterface.php | 81 ++++++ Classes/IndexQueue/Queue.php | 149 ++++++----- ...eueInitializationServiceAwareInterface.php | 36 +++ Classes/IndexQueue/QueueInterface.php | 235 ++++++++++++++++++ .../Configuration/TypoScriptConfiguration.php | 15 ++ Classes/Task/ReIndexTask.php | 8 +- .../Configuration/Reference/TxSolrIndex.rst | 10 + Tests/Integration/IndexQueue/QueueTest.php | 12 +- .../Queue/QueueInitializerServiceTest.php | 6 +- .../TypoScriptConfigurationTest.php | 51 +++- 18 files changed, 746 insertions(+), 127 deletions(-) create mode 100644 Classes/IndexQueue/ItemInterface.php create mode 100644 Classes/IndexQueue/MountPointAwareItemInterface.php create mode 100644 Classes/IndexQueue/QueueInitializationServiceAwareInterface.php create mode 100644 Classes/IndexQueue/QueueInterface.php diff --git a/Classes/Controller/Backend/Search/AbstractModuleController.php b/Classes/Controller/Backend/Search/AbstractModuleController.php index af77a23908..b5541ca4a3 100644 --- a/Classes/Controller/Backend/Search/AbstractModuleController.php +++ b/Classes/Controller/Backend/Search/AbstractModuleController.php @@ -19,6 +19,7 @@ use ApacheSolrForTypo3\Solr\Domain\Site\Site; use ApacheSolrForTypo3\Solr\Domain\Site\SiteRepository; use ApacheSolrForTypo3\Solr\IndexQueue\Queue; +use ApacheSolrForTypo3\Solr\IndexQueue\QueueInterface; use ApacheSolrForTypo3\Solr\System\Mvc\Backend\Service\ModuleDataStorageService; use ApacheSolrForTypo3\Solr\System\Solr\SolrConnection as SolrCoreConnection; use Doctrine\DBAL\Driver\Exception as DBALDriverException; @@ -91,9 +92,9 @@ abstract class AbstractModuleController extends ActionController protected ModuleDataStorageService $moduleDataStorageService; /** - * @var Queue + * @var QueueInterface */ - protected Queue $indexQueue; + protected QueueInterface $indexQueue; /** * @var SiteFinder diff --git a/Classes/Controller/Backend/Search/IndexQueueModuleController.php b/Classes/Controller/Backend/Search/IndexQueueModuleController.php index 4462e62fca..dcd450d889 100644 --- a/Classes/Controller/Backend/Search/IndexQueueModuleController.php +++ b/Classes/Controller/Backend/Search/IndexQueueModuleController.php @@ -17,7 +17,10 @@ use ApacheSolrForTypo3\Solr\Backend\IndexingConfigurationSelectorField; use ApacheSolrForTypo3\Solr\Domain\Index\IndexService; +use ApacheSolrForTypo3\Solr\Domain\Index\Queue\QueueInitializationService; use ApacheSolrForTypo3\Solr\IndexQueue\Queue; +use ApacheSolrForTypo3\Solr\IndexQueue\QueueInterface; +use ApacheSolrForTypo3\Solr\IndexQueue\QueueInitializationServiceAwareInterface; use Psr\Http\Message\ResponseInterface; use TYPO3\CMS\Backend\Form\Exception as BackendFormException; use TYPO3\CMS\Core\Http\RedirectResponse; @@ -34,9 +37,9 @@ class IndexQueueModuleController extends AbstractModuleController { /** - * @var Queue + * @var QueueInterface */ - protected Queue $indexQueue; + protected QueueInterface $indexQueue; /** * Initializes the controller before invoking an action method. @@ -48,9 +51,9 @@ protected function initializeAction() } /** - * @param Queue $indexQueue + * @param QueueInterface $indexQueue */ - public function setIndexQueue(Queue $indexQueue) + public function setIndexQueue(QueueInterface $indexQueue) { $this->indexQueue = $indexQueue; } @@ -121,7 +124,13 @@ public function initializeIndexQueueAction(): ResponseInterface if ((!empty($indexingConfigurationsToInitialize)) && (is_array($indexingConfigurationsToInitialize))) { // initialize selected indexing configuration try { - $initializedIndexingConfigurations = $this->indexQueue->getInitializationService()->initializeBySiteAndIndexConfigurations($this->selectedSite, $indexingConfigurationsToInitialize); + if ($this->indexQueue instanceof QueueInitializationServiceAwareInterface) { + $initializationService = $this->indexQueue->getQueueInitializationService(); + } else { + $initializationService = GeneralUtility::makeInstance(QueueInitializationService::class); + } + + $initializedIndexingConfigurations = $initializationService->initializeBySiteAndIndexConfigurations($this->selectedSite, $indexingConfigurationsToInitialize); } catch (\Throwable $e) { $this->addFlashMessage( sprintf( diff --git a/Classes/Domain/Index/IndexService.php b/Classes/Domain/Index/IndexService.php index b4fc78a340..ea8d8ecec0 100644 --- a/Classes/Domain/Index/IndexService.php +++ b/Classes/Domain/Index/IndexService.php @@ -20,6 +20,7 @@ use ApacheSolrForTypo3\Solr\IndexQueue\Indexer; use ApacheSolrForTypo3\Solr\IndexQueue\Item; use ApacheSolrForTypo3\Solr\IndexQueue\Queue; +use ApacheSolrForTypo3\Solr\IndexQueue\QueueInterface; use ApacheSolrForTypo3\Solr\System\Configuration\TypoScriptConfiguration; use ApacheSolrForTypo3\Solr\System\Logging\SolrLogManager; use ApacheSolrForTypo3\Solr\Task\IndexQueueWorkerTask; @@ -48,9 +49,9 @@ class IndexService protected ?IndexQueueWorkerTask $contextTask = null; /** - * @var Queue + * @var QueueInterface */ - protected Queue $indexQueue; + protected QueueInterface $indexQueue; /** * @var Dispatcher @@ -71,7 +72,7 @@ class IndexService */ public function __construct( Site $site, - Queue $queue = null, + QueueInterface $queue = null, Dispatcher $dispatcher = null, SolrLogManager $solrLogManager = null ) { diff --git a/Classes/Domain/Index/Queue/GarbageRemover/AbstractStrategy.php b/Classes/Domain/Index/Queue/GarbageRemover/AbstractStrategy.php index a11ee0b307..0f3207c9d0 100644 --- a/Classes/Domain/Index/Queue/GarbageRemover/AbstractStrategy.php +++ b/Classes/Domain/Index/Queue/GarbageRemover/AbstractStrategy.php @@ -18,6 +18,7 @@ use ApacheSolrForTypo3\Solr\ConnectionManager; use ApacheSolrForTypo3\Solr\GarbageCollectorPostProcessor; use ApacheSolrForTypo3\Solr\IndexQueue\Queue; +use ApacheSolrForTypo3\Solr\IndexQueue\QueueInterface; use ApacheSolrForTypo3\Solr\System\Logging\SolrLogManager; use ApacheSolrForTypo3\Solr\System\Solr\SolrConnection; use InvalidArgumentException; @@ -30,9 +31,9 @@ abstract class AbstractStrategy { /** - * @var Queue + * @var QueueInterface */ - protected Queue $queue; + protected QueueInterface $queue; /** * @var ConnectionManager @@ -41,11 +42,11 @@ abstract class AbstractStrategy /** * AbstractStrategy constructor. - * @param Queue|null $queue + * @param QueueInterface|null $queue * @param ConnectionManager|null $connectionManager */ public function __construct( - Queue $queue = null, + QueueInterface $queue = null, ConnectionManager $connectionManager = null ) { $this->queue = $queue ?? GeneralUtility::makeInstance(Queue::class); diff --git a/Classes/Domain/Index/Queue/QueueInitializationService.php b/Classes/Domain/Index/Queue/QueueInitializationService.php index 459c381276..52c7bc1cc3 100644 --- a/Classes/Domain/Index/Queue/QueueInitializationService.php +++ b/Classes/Domain/Index/Queue/QueueInitializationService.php @@ -20,12 +20,13 @@ use ApacheSolrForTypo3\Solr\Domain\Site\Site; use ApacheSolrForTypo3\Solr\IndexQueue\InitializationPostProcessor; use ApacheSolrForTypo3\Solr\IndexQueue\Initializer\AbstractInitializer; -use ApacheSolrForTypo3\Solr\IndexQueue\Queue; +use ApacheSolrForTypo3\Solr\IndexQueue\QueueInterface; use Doctrine\DBAL\ConnectionException; use Doctrine\DBAL\Exception as DBALException; use Throwable; use TYPO3\CMS\Core\Utility\GeneralUtility; use UnexpectedValueException; +use ApacheSolrForTypo3\Solr\IndexQueue\QueueInitializationServiceAwareInterface; /** * The queue initialization service is responsible to run the initialization of the index queue for a combination of sites @@ -36,17 +37,14 @@ */ class QueueInitializationService { - /** - * @var Queue - */ - protected Queue $queue; + protected bool $clearQueueOnInitialization = true; /** - * QueueInitializationService constructor. + * @param bool $clearQueueOnInitialization */ - public function __construct(Queue $queue) + public function setClearQueueOnInitialization(bool $clearQueueOnInitialization): void { - $this->queue = $queue; + $this->clearQueueOnInitialization = $clearQueueOnInitialization; } /** @@ -138,10 +136,21 @@ public function initializeBySiteAndIndexConfigurations(Site $site, array $indexi */ protected function applyInitialization(Site $site, string $indexingConfigurationName): bool { + $solrConfiguration = $site->getSolrConfiguration(); + + /** @var QueueInterface $queue */ + $queue = GeneralUtility::makeInstance( + $solrConfiguration->getIndexQueueClassByConfigurationName($indexingConfigurationName) + ); + if ($queue instanceof QueueInitializationServiceAwareInterface) { + $queue->setQueueInitializationService($this); + } + // clear queue - $this->queue->deleteItemsBySite($site, $indexingConfigurationName); + if ($this->clearQueueOnInitialization) { + $queue->deleteItemsBySite($site, $indexingConfigurationName); + } - $solrConfiguration = $site->getSolrConfiguration(); $type = $solrConfiguration->getIndexQueueTypeOrFallbackToConfigurationName($indexingConfigurationName); $initializerClass = $solrConfiguration->getIndexQueueInitializerClassByConfigurationName($indexingConfigurationName); $indexConfiguration = $solrConfiguration->getIndexQueueConfigurationByName($indexingConfigurationName); diff --git a/Classes/Domain/Index/Queue/QueueItemRepository.php b/Classes/Domain/Index/Queue/QueueItemRepository.php index f44f1fdc64..e3aa881468 100644 --- a/Classes/Domain/Index/Queue/QueueItemRepository.php +++ b/Classes/Domain/Index/Queue/QueueItemRepository.php @@ -19,6 +19,7 @@ use ApacheSolrForTypo3\Solr\Domain\Site\Site; use ApacheSolrForTypo3\Solr\IndexQueue\Item; +use ApacheSolrForTypo3\Solr\IndexQueue\ItemInterface; use ApacheSolrForTypo3\Solr\System\Logging\SolrLogManager; use ApacheSolrForTypo3\Solr\System\Records\AbstractRepository; use Doctrine\DBAL\ConnectionException; @@ -150,12 +151,12 @@ public function flushErrorsBySite(Site $site): int /** * Flushes the error for a single item. * - * @param Item $item + * @param ItemInterface $item * @return int affected rows * * @throws DBALException|\Doctrine\DBAL\DBALException */ - public function flushErrorByItem(Item $item): int + public function flushErrorByItem(ItemInterface $item): int { $queryBuilder = $this->getQueryBuilder(); return (int)$this->getPreparedFlushErrorQuery($queryBuilder) @@ -1000,12 +1001,12 @@ public function markItemAsFailed($item, string $errorMessage = ''): int /** * Sets the timestamp of when an item last has been indexed. * - * @param Item $item + * @param ItemInterface $item * @return int affected rows * * @throws DBALException|\Doctrine\DBAL\DBALException */ - public function updateIndexTimeByItem(Item $item): int + public function updateIndexTimeByItem(ItemInterface $item): int { $queryBuilder = $this->getQueryBuilder(); return (int)$queryBuilder @@ -1018,13 +1019,13 @@ public function updateIndexTimeByItem(Item $item): int /** * Sets the change timestamp of an item. * - * @param Item $item + * @param ItemInterface $item * @param int $changedTime * @return int affected rows * * @throws DBALException|\Doctrine\DBAL\DBALException */ - public function updateChangedTimeByItem(Item $item, int $changedTime = 0): int + public function updateChangedTimeByItem(ItemInterface $item, int $changedTime = 0): int { $queryBuilder = $this->getQueryBuilder(); return (int)$queryBuilder diff --git a/Classes/IndexQueue/Item.php b/Classes/IndexQueue/Item.php index 76420642f9..436786a8e4 100644 --- a/Classes/IndexQueue/Item.php +++ b/Classes/IndexQueue/Item.php @@ -35,14 +35,8 @@ * * @author Ingo Renner */ -class Item +class Item implements ItemInterface, MountPointAwareItemInterface { - const STATE_BLOCKED = -1; - - const STATE_PENDING = 0; - - const STATE_INDEXED = 1; - /** * The item's uid in the index queue (tx_solr_indexqueue_item.uid) * @@ -115,6 +109,13 @@ class Item */ protected ?int $recordUid = null; + /** + * The indexing priority + * + * @var int + */ + protected int $indexingPriority = 0; + /** * The record itself * @@ -169,6 +170,7 @@ public function __construct( $this->indexingConfigurationName = $itemMetaData['indexing_configuration'] ?? ''; $this->hasIndexingProperties = (boolean)($itemMetaData['has_indexing_properties'] ?? false); + $this->indexingPriority = (int)($itemMetaData['indexing_priority'] ?? 0); if (!empty($fullRecord)) { $this->record = $fullRecord; @@ -181,7 +183,7 @@ public function __construct( /** * Getter for Index Queue UID * - * @return int + * @return int|null */ public function getIndexQueueUid(): ?int { @@ -201,7 +203,7 @@ public function getRootPageUid(): ?int /** * Returns mount point identifier * - * @return string + * @return string|null */ public function getMountPointIdentifier(): ?string { @@ -233,6 +235,8 @@ public function getHasErrors(): bool } /** + * Items state: pending, indexed, blocked + * * @return int */ public function getState(): int @@ -263,17 +267,17 @@ public function getSite(): ?Site /** * Returns the type/tablename of the queue record. * - * @return mixed|string + * @return string|null */ - public function getType() + public function getType(): ?string { return $this->type; } /** - * @param $type + * @param string $type */ - public function setType($type) + public function setType(string $type): void { $this->type = $type; } @@ -281,9 +285,9 @@ public function setType($type) /** * Returns the name of the index configuration that was used to create this record. * - * @return mixed|string + * @return string */ - public function getIndexingConfigurationName() + public function getIndexingConfigurationName(): string { return $this->indexingConfigurationName; } @@ -291,7 +295,7 @@ public function getIndexingConfigurationName() /** * @param string $indexingConfigurationName */ - public function setIndexingConfigurationName(string $indexingConfigurationName) + public function setIndexingConfigurationName(string $indexingConfigurationName): void { $this->indexingConfigurationName = $indexingConfigurationName; } @@ -299,9 +303,9 @@ public function setIndexingConfigurationName(string $indexingConfigurationName) /** * Returns the timestamp when this queue item was changed. * - * @return int|mixed + * @return int|null */ - public function getChanged() + public function getChanged(): ?int { return $this->changed; } @@ -309,9 +313,9 @@ public function getChanged() /** * Returns the timestamp when this queue item was indexed. * - * @return int|mixed + * @return int|null */ - public function getIndexed() + public function getIndexed(): ?int { return $this->indexed; } @@ -321,7 +325,7 @@ public function getIndexed() * * @param int $changed */ - public function setChanged(int $changed) + public function setChanged(int $changed): void { $this->changed = $changed; } @@ -329,13 +333,14 @@ public function setChanged(int $changed) /** * Returns the uid of related record (item_uid). * - * @return mixed + * @return int The uid of the item record, usually an integer uid, could be a + * different value for non-database-record types. */ public function getRecordUid() { $this->getRecord(); - return $this->record['uid']; + return (int)$this->record['uid']; } /** @@ -365,7 +370,7 @@ public function getRecord(): array * * @param array $record */ - public function setRecord(array $record) + public function setRecord(array $record): void { $this->record = $record; } @@ -386,7 +391,7 @@ public function getRecordPageId(): int * Stores the indexing properties. * @throws DBALException|\Doctrine\DBAL\DBALException */ - public function storeIndexingProperties() + public function storeIndexingProperties(): void { $this->indexQueueIndexingPropertyRepository->removeByRootPidAndIndexQueueUid((int)($this->rootPageUid), (int)($this->indexQueueUid)); @@ -408,7 +413,7 @@ public function hasIndexingProperties(): bool /** * Writes all indexing properties. */ - protected function writeIndexingProperties() + protected function writeIndexingProperties(): void { $properties = []; foreach ($this->indexingProperties as $propertyKey => $propertyValue) { @@ -444,7 +449,7 @@ public function hasIndexingProperty(string $key): bool * @throws DBALDriverException * @throws DBALException */ - public function loadIndexingProperties() + public function loadIndexingProperties(): void { if ($this->indexingPropertiesLoaded) { return; @@ -471,7 +476,7 @@ public function loadIndexingProperties() * @throws DBALDriverException * @throws DBALException */ - public function setIndexingProperty(string $key, $value) + public function setIndexingProperty(string $key, $value): void { // make sure to not interfere with existing indexing properties $this->loadIndexingProperties(); @@ -540,4 +545,14 @@ public function getIndexingPropertyKeys(): array return array_keys($this->indexingProperties); } + + /** + * Returns the index priority. + * + * @return int + */ + public function getIndexPriority(): int + { + return $this->indexingPriority; + } } diff --git a/Classes/IndexQueue/ItemInterface.php b/Classes/IndexQueue/ItemInterface.php new file mode 100644 index 0000000000..3bb37a249a --- /dev/null +++ b/Classes/IndexQueue/ItemInterface.php @@ -0,0 +1,117 @@ + */ -class Queue +class Queue implements QueueInterface, QueueInitializationServiceAwareInterface { /** * @var RootPageResolver @@ -81,26 +81,26 @@ class Queue /** * Queue constructor. - * @param RootPageResolver|null $rootPageResolver - * @param ConfigurationAwareRecordService|null $recordService - * @param QueueItemRepository|null $queueItemRepository - * @param QueueStatisticsRepository|null $queueStatisticsRepository - * @param QueueInitializationService|null $queueInitializationService + * + * @param ?RootPageResolver|null $rootPageResolver + * @param ?ConfigurationAwareRecordService|null $recordService + * @param ?QueueItemRepository|null $queueItemRepository + * @param ?QueueStatisticsRepository|null $queueStatisticsRepository + * @param ?FrontendEnvironment|null $frontendEnvironment */ public function __construct( - RootPageResolver $rootPageResolver = null, - ConfigurationAwareRecordService $recordService = null, - QueueItemRepository $queueItemRepository = null, - QueueStatisticsRepository $queueStatisticsRepository = null, - QueueInitializationService $queueInitializationService = null, - FrontendEnvironment $frontendEnvironment = null + ?RootPageResolver $rootPageResolver = null, + ?ConfigurationAwareRecordService $recordService = null, + ?QueueItemRepository $queueItemRepository = null, + ?QueueStatisticsRepository $queueStatisticsRepository = null, + ?FrontendEnvironment $frontendEnvironment = null ) { $this->logger = GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__); + $this->rootPageResolver = $rootPageResolver ?? GeneralUtility::makeInstance(RootPageResolver::class); $this->recordService = $recordService ?? GeneralUtility::makeInstance(ConfigurationAwareRecordService::class); $this->queueItemRepository = $queueItemRepository ?? GeneralUtility::makeInstance(QueueItemRepository::class); $this->queueStatisticsRepository = $queueStatisticsRepository ?? GeneralUtility::makeInstance(QueueStatisticsRepository::class); - $this->queueInitializationService = $queueInitializationService ?? GeneralUtility::makeInstance(QueueInitializationService::class, /** @scrutinizer ignore-type */ $this); $this->frontendEnvironment = $frontendEnvironment ?? GeneralUtility::makeInstance(FrontendEnvironment::class); } @@ -150,14 +150,46 @@ public function getLastIndexedItemId(int $rootPageId): int return $lastIndexedItemId; } + /** + * @param QueueInitializationService $queueInitializationService + */ + public function setQueueInitializationService(QueueInitializationService $queueInitializationService): void + { + $this->queueInitializationService = $queueInitializationService; + } /** * @return QueueInitializationService */ - public function getInitializationService(): QueueInitializationService + public function getQueueInitializationService(): QueueInitializationService { + if (!isset($this->queueInitializationService)) { + trigger_error( + 'queueInitializationService is no longer initalized automatically, till EXT:solr supports DI' + . ' the QueueInitializationService has to be set manually, fallback will be removed in v13.', + E_USER_DEPRECATED + ); + $this->queueInitializationService = GeneralUtility::makeInstance(QueueInitializationService::class); + } + return $this->queueInitializationService; } + /** + * @return QueueInitializationService + * @deprecated Queue->getInitializationService is deprecated and will be removed in v12. + * Use Queue->getQueueInitializationService instead or create a fresh instance. + */ + public function getInitializationService(): QueueInitializationService + { + trigger_error( + 'Queue->getInitializationService is deprecated and will be removed in v13.' + . ' Use Queue->getQueueInitializationService instead or create a fresh instance.', + E_USER_DEPRECATED + ); + + return $this->getQueueInitializationService(); + } + /** * Marks an item as needing (re)indexing. * @@ -169,12 +201,13 @@ public function getInitializationService(): QueueInitializationService * @param string $itemType The item's type, usually a table name. * @param int|string $itemUid The item's uid, usually an integer uid, could be a different value for non-database-record types. * @param int $forcedChangeTime The change time for the item if set, otherwise value from getItemChangedTime() is used. + * @param array|null $validLanguageUids List of valid language uids, others will be ignored. Depends on your queue implementation, may be irrelevant * @return int Number of updated/created items * @throws DBALDriverException * @throws DBALException|\Doctrine\DBAL\DBALException * @throws Throwable */ - public function updateItem(string $itemType, $itemUid, int $forcedChangeTime = 0): int + public function updateItem(string $itemType, $itemUid, int $forcedChangeTime = 0, ?array $validLanguageUids = null): int { $updateCount = $this->updateOrAddItemForAllRelatedRootPages($itemType, $itemUid, $forcedChangeTime); return $this->postProcessIndexQueueUpdateItem($itemType, $itemUid, $updateCount, $forcedChangeTime); @@ -195,7 +228,7 @@ protected function updateOrAddItemForAllRelatedRootPages(string $itemType, $item { $updateCount = 0; try { - $rootPageIds = $this->rootPageResolver->getResponsibleRootPageIds($itemType, $itemUid); + $rootPageIds = $this->rootPageResolver->getResponsibleRootPageIds($itemType, (int)$itemUid); } catch (InvalidArgumentException $e) { $this->deleteItem($itemType, $itemUid); return 0; @@ -223,8 +256,8 @@ protected function updateOrAddItemForAllRelatedRootPages(string $itemType, $item $itemInQueueForRootPage = $this->containsItemWithRootPageId($itemType, $itemUid, $rootPageId, $indexingConfiguration); if ($itemInQueueForRootPage) { // update changed time if that item is in the queue already - $changedTime = ($forcedChangeTime > 0) ? $forcedChangeTime : $this->getItemChangedTime($itemType, $itemUid); - $updatedRows = $this->queueItemRepository->updateExistingItemByItemTypeAndItemUidAndRootPageId($itemType, $itemUid, $rootPageId, $changedTime, $indexingConfiguration, $indexingPriority); + $changedTime = ($forcedChangeTime > 0) ? $forcedChangeTime : $this->getItemChangedTime($itemType, (int)$itemUid); + $updatedRows = $this->queueItemRepository->updateExistingItemByItemTypeAndItemUidAndRootPageId($itemType, (int)$itemUid, $rootPageId, $changedTime, $indexingConfiguration, $indexingPriority); } else { // add the item since it's not in the queue yet $updatedRows = $this->addNewItem($itemType, $itemUid, $indexingConfiguration, $rootPageId, $indexingPriority); @@ -257,7 +290,7 @@ protected function postProcessIndexQueueUpdateItem( foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessIndexQueueUpdateItem'] as $classReference) { $updateHandler = $this->getHookImplementation($classReference); - $updateCount = $updateHandler->postProcessIndexQueueUpdateItem($itemType, $itemUid, $updateCount, $forcedChangeTime); + $updateCount = $updateHandler->postProcessIndexQueueUpdateItem($itemType, (int)$itemUid, $updateCount, $forcedChangeTime); } return $updateCount; @@ -288,10 +321,10 @@ public function getErrorsBySite(Site $site): array /** * Resets all the errors for all index queue items. * - * @return mixed + * @return int affected rows * @throws DBALException|\Doctrine\DBAL\DBALException */ - public function resetAllErrors() + public function resetAllErrors(): int { return $this->queueItemRepository->flushAllErrors(); } @@ -300,10 +333,10 @@ public function resetAllErrors() * Resets the errors in the index queue for a specific site * * @param Site $site - * @return mixed + * @return int affected rows * @throws DBALException|\Doctrine\DBAL\DBALException */ - public function resetErrorsBySite(Site $site) + public function resetErrorsBySite(Site $site): int { return $this->queueItemRepository->flushErrorsBySite($site); } @@ -311,11 +344,11 @@ public function resetErrorsBySite(Site $site) /** * Resets the error in the index queue for a specific item * - * @param Item $item - * @return mixed + * @param ItemInterface $item + * @return int affected rows * @throws DBALException|\Doctrine\DBAL\DBALException */ - public function resetErrorByItem(Item $item) + public function resetErrorByItem(ItemInterface $item): int { return $this->queueItemRepository->flushErrorByItem($item); } @@ -326,8 +359,7 @@ public function resetErrorByItem(Item $item) * Not meant for public use. * * @param string $itemType The item's type, usually a table name. - * @param int|string $itemUid The item's uid, usually an integer uid, could be a - * different value for non-database-record types. + * @param int|string $itemUid The item's uid, usually an integer uid, could be a different value for non-database-record types. * @param string $indexingConfiguration The item's indexing configuration to use. * Optional, overwrites existing / determined configuration. * @param int $rootPageId @@ -348,31 +380,30 @@ private function addNewItem( $additionalRecordFields = ', doktype, uid'; } - $record = $this->getRecordCached($itemType, $itemUid, $additionalRecordFields); + $record = $this->getRecordCached($itemType, (int)$itemUid, $additionalRecordFields); if (empty($record) || ($itemType === 'pages' && !$this->frontendEnvironment->isAllowedPageType($record, $indexingConfiguration))) { return 0; } - $changedTime = $this->getItemChangedTime($itemType, $itemUid); + $changedTime = $this->getItemChangedTime($itemType, (int)$itemUid); - return $this->queueItemRepository->add($itemType, $itemUid, $rootPageId, $changedTime, $indexingConfiguration, $indexingPriority); + return $this->queueItemRepository->add($itemType, (int)$itemUid, $rootPageId, $changedTime, $indexingConfiguration, $indexingPriority); } /** * Get record to be added in addNewItem * * @param string $itemType The item's type, usually a table name. - * @param int|string $itemUid The item's uid, usually an integer uid, could be a - * different value for non-database-record types. + * @param int $itemUid The item's uid * @param string $additionalRecordFields for sql-query * * @return array|null */ - protected function getRecordCached(string $itemType, $itemUid, string $additionalRecordFields): ?array + protected function getRecordCached(string $itemType, int $itemUid, string $additionalRecordFields): ?array { $cache = GeneralUtility::makeInstance(TwoLevelCache::class, /** @scrutinizer ignore-type */ 'runtime'); - $cacheId = md5('Queue' . ':' . 'getRecordCached' . ':' . $itemType . ':' . $itemUid . ':' . 'pid' . $additionalRecordFields); + $cacheId = md5('Queue' . ':' . 'getRecordCached' . ':' . $itemType . ':' . (string)$itemUid . ':' . 'pid' . $additionalRecordFields); $record = $cache->get($cacheId); if (empty($record)) { @@ -393,13 +424,12 @@ protected function getRecordCached(string $itemType, $itemUid, string $additiona * of an item. * * @param string $itemType The item's table name. - * @param int|string $itemUid The item's uid, usually an integer uid, could be a - * different value for non-database-record types. + * @param int $itemUid The item's uid * @return int Timestamp of the item's changed time or future start time * @throws DBALDriverException * @throws DBALException|\Doctrine\DBAL\DBALException */ - protected function getItemChangedTime(string $itemType, $itemUid): int + protected function getItemChangedTime(string $itemType, int $itemUid): int { $itemTypeHasStartTimeColumn = false; $changedTimeColumns = $GLOBALS['TCA'][$itemType]['ctrl']['tstamp']; @@ -430,7 +460,7 @@ protected function getItemChangedTime(string $itemType, $itemUid): int $pageChangedTime = $this->getPageItemChangedTime($record); } - $localizationsChangedTime = $this->queueItemRepository->getLocalizableItemChangedTime($itemType, (int)$itemUid); + $localizationsChangedTime = $this->queueItemRepository->getLocalizableItemChangedTime($itemType, $itemUid); // if start time exists and start time is higher than last changed timestamp // then set changed to the future start time to make the item @@ -465,7 +495,7 @@ protected function getPageItemChangedTime(array $page): int * * @param string $itemType The item's type, usually a table name. * @param int|string $itemUid The item's uid, usually an integer uid, could be a - * different value for non-database-record types. + * different value for non-database-record types. * @return bool TRUE if the item is found in the queue, FALSE otherwise * @throws DBALDriverException * @throws DBALException|\Doctrine\DBAL\DBALException @@ -480,7 +510,7 @@ public function containsItem(string $itemType, $itemUid): bool * * @param string $itemType The item's type, usually a table name. * @param int|string $itemUid The item's uid, usually an integer uid, could be a - * different value for non-database-record types. + * different value for non-database-record types. * @param int $rootPageId * @param string $indexingConfiguration * @return bool TRUE if the item is found in the queue, FALSE otherwise @@ -498,9 +528,9 @@ public function containsItemWithRootPageId(string $itemType, $itemUid, int $root * * @param string $itemType The item's type, usually a table name. * @param int|string $itemUid The item's uid, usually an integer uid, could be a - * different value for non-database-record types. + * different value for non-database-record types. * @return bool TRUE if the item is found in the queue and marked as - * indexed, FALSE otherwise + * indexed, FALSE otherwise * @throws DBALDriverException * @throws DBALException|\Doctrine\DBAL\DBALException */ @@ -513,12 +543,12 @@ public function containsIndexedItem(string $itemType, $itemUid): bool * Removes an item from the Index Queue. * * @param string $itemType The type of the item to remove, usually a table name. - * @param int|string $itemUid The uid of the item to remove + * @param int|string $itemUid The item's uid, usually an integer uid, could be a different value for non-database-record types. * @throws ConnectionException * @throws DBALException * @throws Throwable */ - public function deleteItem(string $itemType, $itemUid) + public function deleteItem(string $itemType, $itemUid): void { $this->queueItemRepository->deleteItem($itemType, (int)$itemUid); } @@ -531,7 +561,7 @@ public function deleteItem(string $itemType, $itemUid) * @throws DBALException * @throws Throwable */ - public function deleteItemsByType(string $itemType) + public function deleteItemsByType(string $itemType): void { $this->queueItemRepository->deleteItemsByType($itemType); } @@ -547,7 +577,7 @@ public function deleteItemsByType(string $itemType) * @throws \Doctrine\DBAL\DBALException * @throws Throwable */ - public function deleteItemsBySite(Site $site, string $indexingConfigurationName = '') + public function deleteItemsBySite(Site $site, string $indexingConfigurationName = ''): void { $this->queueItemRepository->deleteItemsBySite($site, $indexingConfigurationName); } @@ -555,7 +585,7 @@ public function deleteItemsBySite(Site $site, string $indexingConfigurationName /** * Removes all items from the Index Queue. */ - public function deleteAllItems() + public function deleteAllItems(): void { $this->queueItemRepository->deleteAllItems(); } @@ -577,7 +607,7 @@ public function getItem(int $itemId): ?Item * Gets Index Queue items by type and uid. * * @param string $itemType item type, usually the table name - * @param int|string $itemUid item uid + * @param int|string $itemUid The item's uid, usually an integer uid, could be a different value for non-database-record types. * @return Item[] An array of items matching $itemType and $itemUid * @throws ConnectionException * @throws DBALDriverException @@ -655,35 +685,38 @@ public function getItemsToIndex(Site $site, int $limit = 50): array * Marks an item as failed and causes the indexer to skip the item in the * next run. * - * @param int|Item $item Either the item's Index Queue uid or the complete item + * @param int|ItemInterface $item Either the item's Index Queue uid or the complete item * @param string $errorMessage Error message + * @return int affected rows * @throws DBALException|\Doctrine\DBAL\DBALException */ - public function markItemAsFailed($item, string $errorMessage = '') + public function markItemAsFailed($item, string $errorMessage = ''): int { - $this->queueItemRepository->markItemAsFailed($item, $errorMessage); + return $this->queueItemRepository->markItemAsFailed($item, $errorMessage); } /** * Sets the timestamp of when an item last has been indexed. * - * @param Item $item + * @param ItemInterface $item + * @return int affected rows * @throws DBALException|\Doctrine\DBAL\DBALException */ - public function updateIndexTimeByItem(Item $item) + public function updateIndexTimeByItem(ItemInterface $item): int { - $this->queueItemRepository->updateIndexTimeByItem($item); + return $this->queueItemRepository->updateIndexTimeByItem($item); } /** * Sets the change timestamp of an item. * - * @param Item $item + * @param ItemInterface $item * @param int $forcedChangeTime The change time for the item + * @return int affected rows * @throws DBALException|\Doctrine\DBAL\DBALException */ - public function setForcedChangeTimeByItem(Item $item, int $forcedChangeTime = 0) + public function setForcedChangeTimeByItem(ItemInterface $item, int $forcedChangeTime = 0): int { - $this->queueItemRepository->updateChangedTimeByItem($item, $forcedChangeTime); + return $this->queueItemRepository->updateChangedTimeByItem($item, $forcedChangeTime); } } diff --git a/Classes/IndexQueue/QueueInitializationServiceAwareInterface.php b/Classes/IndexQueue/QueueInitializationServiceAwareInterface.php new file mode 100644 index 0000000000..9b850e82c0 --- /dev/null +++ b/Classes/IndexQueue/QueueInitializationServiceAwareInterface.php @@ -0,0 +1,36 @@ +getValueByPathOrDefaultValue($path, $defaultIfEmpty); } + /** + * This method is used to retrieve the className of a queue initializer for a certain indexing configuration + * + * plugin.tx_solr.index.queue..indexQueue + * + * @param string $configurationName + * @return string + */ + public function getIndexQueueClassByConfigurationName(string $configurationName): string + { + $path = 'plugin.tx_solr.index.queue.' . $configurationName . '.indexQueue'; + return (string)$this->getValueByPathOrDefaultValue($path, Queue::class); + } + /** * Returns the _LOCAL_LANG configuration from the TypoScript. * diff --git a/Classes/Task/ReIndexTask.php b/Classes/Task/ReIndexTask.php index 51c739edd4..fcec5e944b 100644 --- a/Classes/Task/ReIndexTask.php +++ b/Classes/Task/ReIndexTask.php @@ -18,7 +18,7 @@ namespace ApacheSolrForTypo3\Solr\Task; use ApacheSolrForTypo3\Solr\ConnectionManager; -use ApacheSolrForTypo3\Solr\IndexQueue\Queue; +use ApacheSolrForTypo3\Solr\Domain\Index\Queue\QueueInitializationService; use Doctrine\DBAL\ConnectionException as DBALConnectionException; use Doctrine\DBAL\Driver\Exception as DBALDriverException; use Doctrine\DBAL\Exception as DBALException; @@ -59,9 +59,9 @@ public function execute() $cleanUpResult = $this->cleanUpIndex(); // initialize for re-indexing - /* @var Queue $indexQueue */ - $indexQueue = GeneralUtility::makeInstance(Queue::class); - $indexQueueInitializationResults = $indexQueue->getInitializationService() + /* @var QueueInitializationService $indexQueueInitializationService */ + $indexQueueInitializationService = GeneralUtility::makeInstance(QueueInitializationService::class); + $indexQueueInitializationResults = $indexQueueInitializationService ->initializeBySiteAndIndexConfigurations($this->getSite(), $this->indexingConfigurationsToReIndex); return $cleanUpResult && !in_array(false, $indexQueueInitializationResults); diff --git a/Documentation/Configuration/Reference/TxSolrIndex.rst b/Documentation/Configuration/Reference/TxSolrIndex.rst index e6abc49289..7a09481305 100644 --- a/Documentation/Configuration/Reference/TxSolrIndex.rst +++ b/Documentation/Configuration/Reference/TxSolrIndex.rst @@ -224,6 +224,16 @@ Defines the type to index, which is usally the database table. Sometimes you may +queue.[indexConfig].indexQueue +------------------------------ + +:Type: String +:TS Path: plugin.tx_solr.index.queue.[indexConfig].indexQueue +:Since: 11.6 +:Default: + +Class name of custom index queue implementation, falls back to the default index queue (ApacheSolrForTypo3\Solr\IndexQueue\Queue). + queue.[indexConfig].initialization ---------------------------------- diff --git a/Tests/Integration/IndexQueue/QueueTest.php b/Tests/Integration/IndexQueue/QueueTest.php index 6a895e0ca4..7c659c52de 100644 --- a/Tests/Integration/IndexQueue/QueueTest.php +++ b/Tests/Integration/IndexQueue/QueueTest.php @@ -15,6 +15,7 @@ namespace ApacheSolrForTypo3\Solr\Tests\Integration\IndexQueue; +use ApacheSolrForTypo3\Solr\Domain\Index\Queue\QueueInitializationService; use ApacheSolrForTypo3\Solr\Domain\Site\Site; use ApacheSolrForTypo3\Solr\Domain\Site\SiteRepository; use ApacheSolrForTypo3\Solr\IndexQueue\Queue; @@ -43,6 +44,7 @@ protected function setUp(): void parent::setUp(); $this->writeDefaultSolrTestSiteConfiguration(); $this->indexQueue = GeneralUtility::makeInstance(Queue::class); + $this->indexQueue->setQueueInitializationService(GeneralUtility::makeInstance(QueueInitializationService::class)); $this->siteRepository = GeneralUtility::makeInstance(SiteRepository::class); } @@ -78,7 +80,7 @@ public function preFilledQueueContainsRootPageAfterInitialize() // after initialize the prefilled queue item should be lost and the root page should be added again $site = $this->siteRepository->getFirstAvailableSite(); - $this->indexQueue->getInitializationService()->initializeBySiteAndIndexConfiguration($site, 'pages'); + $this->indexQueue->getQueueInitializationService()->initializeBySiteAndIndexConfiguration($site, 'pages'); $this->assertItemsInQueue(1); self::assertTrue($this->indexQueue->containsItem('pages', 1)); @@ -155,10 +157,10 @@ public function mountPagesAreOnlyAddedOnceAfterInitialize() $this->assertEmptyQueue(); $site = $this->siteRepository->getFirstAvailableSite(); - $this->indexQueue->getInitializationService()->initializeBySiteAndIndexConfiguration($site, 'pages'); + $this->indexQueue->getQueueInitializationService()->initializeBySiteAndIndexConfiguration($site, 'pages'); $this->assertItemsInQueue(4); - $this->indexQueue->getInitializationService()->initializeBySiteAndIndexConfiguration($site, 'pages'); + $this->indexQueue->getQueueInitializationService()->initializeBySiteAndIndexConfiguration($site, 'pages'); $this->assertItemsInQueue(4); } @@ -191,7 +193,7 @@ public function canAddCustomPageTypeToTheQueue() }' ); $site = $this->siteRepository->getFirstAvailableSite(); - $this->indexQueue->getInitializationService()->initializeBySiteAndIndexConfiguration($site, 'custom_page_type'); + $this->indexQueue->getQueueInitializationService()->initializeBySiteAndIndexConfiguration($site, 'custom_page_type'); $this->assertItemsInQueue(1); @@ -247,7 +249,7 @@ public function canInitializeMultipleSites() if (is_array($availableSites)) { foreach ($availableSites as $site) { if ($site instanceof Site) { - $this->indexQueue->getInitializationService()->initializeBySiteAndIndexConfiguration($site); + $this->indexQueue->getQueueInitializationService()->initializeBySiteAndIndexConfiguration($site); } } } diff --git a/Tests/Unit/Domain/Index/Queue/QueueInitializerServiceTest.php b/Tests/Unit/Domain/Index/Queue/QueueInitializerServiceTest.php index 2fed50e2cf..99b63eb544 100644 --- a/Tests/Unit/Domain/Index/Queue/QueueInitializerServiceTest.php +++ b/Tests/Unit/Domain/Index/Queue/QueueInitializerServiceTest.php @@ -21,6 +21,7 @@ use ApacheSolrForTypo3\Solr\System\Configuration\TypoScriptConfiguration; use ApacheSolrForTypo3\Solr\Tests\Unit\UnitTest; use PHPUnit\Framework\MockObject\MockObject; +use TYPO3\CMS\Core\Utility\GeneralUtility; /** * @author Timo Hund @@ -33,8 +34,11 @@ class QueueInitializerServiceTest extends UnitTest public function allIndexConfigurationsAreUsedWhenWildcardIsPassed() { $queueMock = $this->getDumbMock(Queue::class); + GeneralUtility::addInstance(Queue::class, $queueMock); + GeneralUtility::addInstance(Queue::class, $queueMock); + /* @var QueueInitializationService|MockObject $service */ - $service = $this->getMockBuilder(QueueInitializationService::class)->onlyMethods(['executeInitializer'])->setConstructorArgs([$queueMock])->getMock(); + $service = $this->getMockBuilder(QueueInitializationService::class)->onlyMethods(['executeInitializer'])->getMock(); $fakeTs = [ 'plugin.' => [ diff --git a/Tests/Unit/System/Configuration/TypoScriptConfigurationTest.php b/Tests/Unit/System/Configuration/TypoScriptConfigurationTest.php index 6b083891b9..58c5869907 100644 --- a/Tests/Unit/System/Configuration/TypoScriptConfigurationTest.php +++ b/Tests/Unit/System/Configuration/TypoScriptConfigurationTest.php @@ -15,6 +15,8 @@ namespace ApacheSolrForTypo3\Solr\Tests\Unit\System\Configuration; +use ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Record; +use ApacheSolrForTypo3\Solr\IndexQueue\Queue; use ApacheSolrForTypo3\Solr\System\Configuration\TypoScriptConfiguration; use ApacheSolrForTypo3\Solr\Tests\Unit\UnitTest; @@ -320,7 +322,6 @@ public function canGetIndexQueueConfigurationNamesByTableName() 'custom_one.' => [ 'type' => 'tx_model_bar', ], - 'custom_two' => 1, 'custom_two.' => [ 'type' => 'tx_model_news', @@ -333,6 +334,54 @@ public function canGetIndexQueueConfigurationNamesByTableName() self::assertEquals(['tx_model_news', 'custom_two'], $configuration->getIndexQueueConfigurationNamesByTableName('tx_model_news')); } + /** + * @test + */ + public function canGetIndexQueueInitializerClassByConfigurationName() + { + $fakeConfigurationArray['plugin.']['tx_solr.'] = [ + 'index.' => [ + 'queue.' => [ + 'tx_model_news' => 1, + 'tx_model_news.' => [ + ], + 'custom_one' => 1, + 'custom_one.' => [ + 'initialization' => 'CustomInitializer', + ], + ], + ], + ]; + + $configuration = new TypoScriptConfiguration($fakeConfigurationArray); + self::assertEquals(Record::class, $configuration->getIndexQueueInitializerClassByConfigurationName('tx_model_news')); + self::assertEquals('CustomInitializer', $configuration->getIndexQueueInitializerClassByConfigurationName('custom_one')); + } + + /** + * @test + */ + public function canGetIndexQueueClassByConfigurationName() + { + $fakeConfigurationArray['plugin.']['tx_solr.'] = [ + 'index.' => [ + 'queue.' => [ + 'tx_model_news' => 1, + 'tx_model_news.' => [ + ], + 'custom_one' => 1, + 'custom_one.' => [ + 'indexQueue' => 'CustomQueue', + ], + ], + ], + ]; + + $configuration = new TypoScriptConfiguration($fakeConfigurationArray); + self::assertEquals(Queue::class, $configuration->getIndexQueueClassByConfigurationName('tx_model_news')); + self::assertEquals('CustomQueue', $configuration->getIndexQueueClassByConfigurationName('custom_one')); + } + /** * @test */