From 6fe7ec396a20a74d6700880d728f006e1b50ac25 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Fri, 31 Oct 2025 15:52:58 -0100 Subject: [PATCH 1/2] files content provider Signed-off-by: Maxence Lange --- lib/AppInfo/Application.php | 3 +- .../Exceptions/NodeNotFoundException.php | 14 ++++ lib/Files/Service/FilesService.php | 66 +++++++++++++++ lib/Files/Service/SharesService.php | 77 ++++++++++++++++++ lib/Provider/FilesContentProvider.php | 81 +++++++++++++++++++ 5 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 lib/Files/Exceptions/NodeNotFoundException.php create mode 100644 lib/Files/Service/FilesService.php create mode 100644 lib/Files/Service/SharesService.php create mode 100644 lib/Provider/FilesContentProvider.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 23792fd6..1591b89d 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -47,7 +47,8 @@ public function register(IRegistrationContext $context): void { $context->registerConfigLexicon(ConfigLexicon::class); $context->registerFullTextSearchService(FullTextSearchService::class); - + $context->registerFullTextSearchContentProvider(FilesContentProvider::class); + $this->registerServices($this->getContainer()); } diff --git a/lib/Files/Exceptions/NodeNotFoundException.php b/lib/Files/Exceptions/NodeNotFoundException.php new file mode 100644 index 00000000..f7428573 --- /dev/null +++ b/lib/Files/Exceptions/NodeNotFoundException.php @@ -0,0 +1,14 @@ +userMountCache->getMountsForFileId($nodeId); + if (empty($mounts ?? [])) { + $this->logger->warning('empty mount for nodeId=' . $nodeId); + return null; + } + + $mount = reset($mounts); + try { + return $this->rootFolder->getUserFolder($mount->getUser()->getUID())->getFirstNodeById($nodeId); + } catch (NotPermittedException|NoUserException $e) { + $this->logger->error('could not find node', ['exception' => $e]); + } + + return null; + } + + public function generateDocument(Node $node): ?Document { + if ($node->getType() !== 'file') { + return null; + } + + $document = new Document(); +// $document->setId((string)$node->getId());` + + $document->setFlags(8); + $document->setContent(base64_encode($node->getContent()), true); + $document->setAccess($this->sharesService->getDocumentAccess($node)); + + return $document; + } + +} + + diff --git a/lib/Files/Service/SharesService.php b/lib/Files/Service/SharesService.php new file mode 100644 index 00000000..e608c34c --- /dev/null +++ b/lib/Files/Service/SharesService.php @@ -0,0 +1,77 @@ +getOwner()?->getUID() ?? ''; + $users = $groups = $circles = $links = []; + foreach($this->getShares($node->getId()) as $share) { + switch ($share['share_type']) { + case IShare::TYPE_USER: + $users[] = $share['share_with']; + break; + case IShare::TYPE_GROUP: + $groups[] = $share['share_with']; + break; + case IShare::TYPE_CIRCLE: + $circles[] = $share['share_with']; + break; + case IShare::TYPE_LINK: + $links[] = $share['token']; + break; + } + } + + return new DocumentAccess( + $owner, + $users, + $groups, + $circles, + $links, + ); + } + + private function getShares(int $nodeId): array { + $qb = $this->connection->getQueryBuilder(); + $qb->select('share_type', 'share_with', 'token') + ->from('share') + ->where( + $qb->expr()->eq('item_source', $qb->createNamedParameter($nodeId, IQueryBuilder::PARAM_INT)), + $qb->expr()->isNotNull('parent') + ); + + $shares = []; + $cursor = $qb->executeQuery(); + while ($row = $cursor->fetch()) { + $shares[] = [ + 'share_type' => $row['share_type'], + 'share_with' => $row['share_with'], + ]; + } + $cursor->closeCursor(); + + return $shares; + } +} + diff --git a/lib/Provider/FilesContentProvider.php b/lib/Provider/FilesContentProvider.php new file mode 100644 index 00000000..6d78e907 --- /dev/null +++ b/lib/Provider/FilesContentProvider.php @@ -0,0 +1,81 @@ +filesService->getNode($nodeId); + if ($node === null) { + return null; + } + + return $this->filesService->generateDocument($node); + } + + // IContentProviderImprovedSearch + public function getSearchTemplate(): ?ISearchTemplate { + return null; + } + + // IContentProviderImprovedSearch + public function improveSearchRequest(ISearchRequest $searchRequest): void { + } + + // IContentProviderImprovedSearch + public function improveSearchResult(ISearchResult $searchResult): void { + } + + // IContentProviderSyncIndex + public function getUnindexedDocuments(IIndexQueryHelper $qh): Generator { + $qh->notNeeded(); + $qb = $this->connection->getQueryBuilder(); + $qb->select('fileid', 'mtime') + ->from('filecache'); + $result = $qb->executeQuery(); + while ($row = $result->fetch()) { + yield new UnindexedDocument($row['fileid'], $row['mtime']); + } + $result->closeCursor(); + } +} + From 0302529f829a61b2a4f2fc7795f1f3a9f9a89086 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Mon, 24 Nov 2025 11:23:35 -0100 Subject: [PATCH 2/2] some fixes Signed-off-by: Maxence Lange --- lib/AppInfo/Application.php | 23 ++++++- lib/Command/Sync.php | 28 ++------ lib/ConfigLexicon.php | 1 + lib/Db/SyncMapper.php | 2 +- lib/Enum/SessionType.php | 4 +- lib/Files/FileEvents.php | 68 +++++++++++++++++++ .../Version33001Date202511271645.php | 2 +- lib/Service/IndexService.php | 57 +++++++++++++++- lib/Service/LockService.php | 5 +- lib/Service/LoggerService.php | 6 +- lib/Service/SyncService.php | 45 ++++++------ 11 files changed, 189 insertions(+), 52 deletions(-) create mode 100644 lib/Files/FileEvents.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 1591b89d..b2105985 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -14,8 +14,15 @@ use NCU\FullTextSearch\IManager; use OC; use OC\FullTextSearch\FullTextSearchManager; +use OCA\Files_FullTextSearch\Listeners\FileChanged; +use OCA\Files_FullTextSearch\Listeners\FileCreated; +use OCA\Files_FullTextSearch\Listeners\FileDeleted; +use OCA\Files_FullTextSearch\Listeners\FileRenamed; +use OCA\Files_FullTextSearch\Listeners\ShareCreated; +use OCA\Files_FullTextSearch\Listeners\ShareDeleted; use OCA\FullTextSearch\Capabilities; use OCA\FullTextSearch\ConfigLexicon; +use OCA\FullTextSearch\Files\FileEvents; use OCA\FullTextSearch\Provider\FilesContentProvider; use OCA\FullTextSearch\Search\UnifiedSearchProvider; use OCA\FullTextSearch\Service\FullTextSearchService; @@ -26,11 +33,17 @@ use OCP\AppFramework\Bootstrap\IBootContext; use OCP\AppFramework\Bootstrap\IBootstrap; use OCP\AppFramework\Bootstrap\IRegistrationContext; +use OCP\Files\Events\Node\NodeCreatedEvent; +use OCP\Files\Events\Node\NodeDeletedEvent; +use OCP\Files\Events\Node\NodeRenamedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; use OCP\FullTextSearch\IFullTextSearchManager; use OCP\IAppConfig; use OCP\INavigationManager; use OCP\IServerContainer; use OCP\IURLGenerator; +use OCP\Share\Events\ShareCreatedEvent; +use OCP\Share\Events\ShareDeletedEvent; use Symfony\Component\Routing\Exception\RouteNotFoundException; use Psr\Container\ContainerInterface; use Throwable; @@ -48,7 +61,15 @@ public function register(IRegistrationContext $context): void { $context->registerFullTextSearchService(FullTextSearchService::class); $context->registerFullTextSearchContentProvider(FilesContentProvider::class); - + + // files life cycle events + $context->registerEventListener(NodeCreatedEvent::class, FileEvents::class); + $context->registerEventListener(NodeWrittenEvent::class, FileEvents::class); + $context->registerEventListener(NodeRenamedEvent::class, FileEvents::class); + $context->registerEventListener(NodeDeletedEvent::class, FileEvents::class); + $context->registerEventListener(ShareCreatedEvent::class, FileEvents::class); + $context->registerEventListener(ShareDeletedEvent::class, FileEvents::class); + $this->registerServices($this->getContainer()); } diff --git a/lib/Command/Sync.php b/lib/Command/Sync.php index d081c401..7d22ce85 100644 --- a/lib/Command/Sync.php +++ b/lib/Command/Sync.php @@ -9,34 +9,16 @@ namespace OCA\FullTextSearch\Command; +use Exception; use OC\Core\Command\Base; -use OCA\FullTextSearch\Exceptions\PlatformTemporaryException; +use OCA\FullTextSearch\Exceptions\LockException; use OCA\FullTextSearch\Service\FullTextSearchService; use OCA\FullTextSearch\Service\LockService; use OCA\FullTextSearch\Service\LoggerService; use OCA\FullTextSearch\Service\SyncService; -use OCA\FullTextSearch\Tools\Traits\TArrayTools; -use Exception; -use OC\Core\Command\InterruptedException; -use OCA\FullTextSearch\ACommandBase; -use OCA\FullTextSearch\Exceptions\TickDoesNotExistException; -use OCA\FullTextSearch\Model\Index as ModelIndex; -use OCA\FullTextSearch\Model\Runner; -use OCA\FullTextSearch\Service\CliService; -use OCA\FullTextSearch\Service\ConfigService; -use OCA\FullTextSearch\Service\IndexService; -use OCA\FullTextSearch\Service\PlatformService; -use OCA\FullTextSearch\Service\ProviderService; -use OCA\FullTextSearch\Service\RunningService; -use OCP\IUserManager; -use OutOfBoundsException; -use Psr\Log\LoggerInterface; -use Symfony\Component\Console\Formatter\OutputFormatterStyle; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; -use Symfony\Component\Console\Terminal; -use Throwable; class Sync extends Base { public function __construct( @@ -52,7 +34,7 @@ protected function configure() { parent::configure(); $this->setName('fulltextsearch:sync') ->setDescription('Index files') - ->addOption('info', '', InputOption::VALUE_NONE, 'display info entries') + ->addOption('info', '', InputOption::VALUE_NONE, 'display info entries') ->addOption('no-output', '', InputOption::VALUE_NONE, 'no output, use nextcloud logs'); } @@ -67,10 +49,12 @@ protected function execute(InputInterface $input, OutputInterface $output) { try { $this->lockService->update(); $this->syncProcess(); + } catch (LockException $e) { + throw $e; } catch (Exception $e) { $this->loggerService->error('Exception while running fulltextsearch:sync', ['exception' => $e]); } - sleep(10); + sleep(15); } return 0; diff --git a/lib/ConfigLexicon.php b/lib/ConfigLexicon.php index ca119f7a..c14eb470 100644 --- a/lib/ConfigLexicon.php +++ b/lib/ConfigLexicon.php @@ -12,6 +12,7 @@ use OCP\Config\Lexicon\ILexicon; use OCP\Config\Lexicon\Strictness; use OCP\Config\ValueType; +use OCP\IAppConfig; /** * Config Lexicon for fulltextsearch. diff --git a/lib/Db/SyncMapper.php b/lib/Db/SyncMapper.php index 4ad25095..7d97be3a 100644 --- a/lib/Db/SyncMapper.php +++ b/lib/Db/SyncMapper.php @@ -29,7 +29,7 @@ public function __construct( /** * @return DocumentSync[] */ - public function getForcedSyncs(int $limit = 100): array { + public function getRequestedSyncs(int $limit = 100): array { $qb = $this->db->getQueryBuilder(); $qb->select('*') ->from($this->getTableName()) diff --git a/lib/Enum/SessionType.php b/lib/Enum/SessionType.php index b43f7ec0..a0fec076 100644 --- a/lib/Enum/SessionType.php +++ b/lib/Enum/SessionType.php @@ -10,8 +10,8 @@ namespace OCA\FullTextSearch\Enum; enum SessionType: string { - case UNKNOWN = ''; - case FORCED = 'forced'; + case CLOSED = ''; + case REQUESTED = 'requested'; case SYNC = 'sync'; case RESYNC = 'sync_recent'; } diff --git a/lib/Files/FileEvents.php b/lib/Files/FileEvents.php new file mode 100644 index 00000000..e370fc57 --- /dev/null +++ b/lib/Files/FileEvents.php @@ -0,0 +1,68 @@ +logger->warning('handle 1'); + try { + $service = $this->fullTextSearchManager->getService(); + } catch (ServiceNotFoundException) { + // no service available + return; + } + $this->logger->warning('handle 2'); + + if ($event instanceof NodeCreatedEvent + || $event instanceof NodeWrittenEvent + || $event instanceof NodeRenamedEvent + || $event instanceof NodeDeletedEvent) { + try { + $node = $event->getNode(); + $service->requestIndex('files', (string)$node->getId()); + } catch (InvalidPathException|NotFoundException) { + // cannot reach document + return; + } + } + + if ($event instanceof ShareCreatedEvent + || $event instanceof ShareDeletedEvent) { + try { + $node = $event->getShare()->getNode(); + $service->requestIndex('files', (string)$node->getId()); + } catch (NotFoundException|InvalidPathException) { + // cannot reach document + return; + } + } + } +} diff --git a/lib/Migration/Version33001Date202511271645.php b/lib/Migration/Version33001Date202511271645.php index 098eb7c5..757146b6 100644 --- a/lib/Migration/Version33001Date202511271645.php +++ b/lib/Migration/Version33001Date202511271645.php @@ -44,7 +44,7 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt ]); $table->addColumn('checksum', Types::STRING, [ 'length' => 16, - 'notnull' => true, + 'default' => '', ]); $table->setPrimaryKey(['provider_id', 'document_id'], 'fts_i_pd'); diff --git a/lib/Service/IndexService.php b/lib/Service/IndexService.php index 119bba9c..1b9d0abc 100644 --- a/lib/Service/IndexService.php +++ b/lib/Service/IndexService.php @@ -10,11 +10,15 @@ use Exception; +use NCU\FullTextSearch\Model\Document; +use OC\FullTextSearch\Model\DocumentAccess; +use OC\FullTextSearch\Model\IndexDocument; use OCA\FullTextSearch\Db\IndexesRequest; use OCA\FullTextSearch\Exceptions\IndexDoesNotExistException; use OCA\FullTextSearch\Exceptions\NotIndexableDocumentException; use OCA\FullTextSearch\Exceptions\PlatformTemporaryException; use OCA\FullTextSearch\Exceptions\ProviderDoesNotExistException; +use OCA\FullTextSearch\Model\DocumentSync; use OCA\FullTextSearch\Model\Index; use OCA\FullTextSearch\Model\IndexOptions; use OCA\FullTextSearch\Model\Runner; @@ -26,7 +30,6 @@ use OCP\FullTextSearch\Model\IRunner; use OCP\FullTextSearch\Service\IIndexService; - /** * Class IndexService * @@ -643,4 +646,56 @@ public function createIndex( return $index; } + + + /** + * convert to API v1 to send a document to the full text search platform for indexing + */ + public function syncDocument(DocumentSync $sync, Document $document): void { + $wrapper = $this->platformService->getPlatform(); + $platform = $wrapper->getPlatform(); + + $platform->initializeIndex(); + + // confirm current document is different from the one indexed on platform + $indexedDocument = $this->convertPreviousIndexDocumentToNewDocument($platform->getDocument($sync->getProviderId(), $sync->getDocumentId())); + if ($document->getChecksum() === $indexedDocument?->getChecksum()) { + return; + } + + try { + $platform->indexDocumentDeprecated($this->convertNewDocumentToPreviousIndexDocument($document)); + } catch (PlatformTemporaryException $e) { + throw $e; +// } catch (Exception $e) { +// throw new IndexDoesNotExistException(); + } + + + } + + /** + * convert Document to deprecated IndexDocument compatible with API v1 + */ + private function convertNewDocumentToPreviousIndexDocument(Document $document): IIndexDocument { + $indexDocument = new IndexDocument($sync->getProviderId(), $sync->getDocumentId()); + + $access = $document->getAccess(); + if ($access !== null) { + $deprecatedAccess = new DocumentAccess($access->getOwnerId()); + $deprecatedAccess->setCircles($access->getCircles()) + ->setGroups($access->getGroups()) + ->setUsers($access->getUsers()) + ->setLinks($access->getLinks()) + ->setViewerId($access->getViewerId()); + $indexDocument->setAccess($deprecatedAccess); + } + + return $indexDocument; + } + + private function convertPreviousIndexDocumentToNewDocument(IIndexDocument $document): Document { + + + } } diff --git a/lib/Service/LockService.php b/lib/Service/LockService.php index cfb5656a..492b7009 100644 --- a/lib/Service/LockService.php +++ b/lib/Service/LockService.php @@ -17,7 +17,7 @@ class LockService { private const LOCK_TIMEOUT = 300; // the final value will be LOCK_TIMEOUT+LOCK_PING_DELAY - private const LOCK_PING_DELAY = 10; + private const LOCK_PING_DELAY = 8; private string $lockId; private int $nextPing = -1; @@ -79,7 +79,7 @@ public function update(): void { // new lock; enforce ping on new lock if ($currentLockId === '') { - throw new LockException('Index not locked'); + throw new LockException('Forced stop'); } // confirm the lock belongs to the current process @@ -94,6 +94,7 @@ public function update(): void { public function unlock(): void { $this->appConfig->deleteKey(Application::APP_ID, ConfigLexicon::LOCK_PING); + $this->appConfig->deleteKey(Application::APP_ID, ConfigLexicon::LOCK_ID); $this->nextPing = -1; } } diff --git a/lib/Service/LoggerService.php b/lib/Service/LoggerService.php index 5a02d894..279e10b8 100644 --- a/lib/Service/LoggerService.php +++ b/lib/Service/LoggerService.php @@ -17,7 +17,7 @@ class LoggerService implements ILoggerService { private ?OutputInterface $output = null; private bool $verbose = false; - private SessionType $currentSessionType = SessionType::UNKNOWN; + private SessionType $currentSessionType = SessionType::CLOSED; public function __construct( private ?LoggerInterface $logger, @@ -56,7 +56,7 @@ public function error(string $entry, array $data = []): void { $this->loggerError->warning('[' . $this->currentSessionType->value . '] ' . $entry, $data); } - public function session(SessionType $sessionType = SessionType::UNKNOWN): void { + public function session(SessionType $sessionType = SessionType::CLOSED): void { $this->currentSessionType = $sessionType; } @@ -66,7 +66,7 @@ private function prepOutput(): string { private function prepEntry(string $entry, string $tag = ''): string { $entry = ($tag !== '') ? '<' . $tag . '>' . $entry . '' : $entry; - $prefix = '(' . str_pad($this->currentSessionType->value . ')', 14, ' ') . ''; + $prefix = '' . str_pad(($this->currentSessionType === SessionType::CLOSED) ? '' : '(' . $this->currentSessionType->value . ')', 14, ' ') . ''; return $prefix . $entry; } } diff --git a/lib/Service/SyncService.php b/lib/Service/SyncService.php index 41dd5375..16c4af4c 100644 --- a/lib/Service/SyncService.php +++ b/lib/Service/SyncService.php @@ -9,12 +9,12 @@ namespace OCA\FullTextSearch\Service; +use NCU\FullTextSearch\IContentProvider; use NCU\FullTextSearch\IManager as IFullTextSearchManager; use OCA\FullTextSearch\ConfigLexicon; use OCA\FullTextSearch\Db\SyncMapper; use OCA\FullTextSearch\Enum\SessionType; use OCA\FullTextSearch\Model\DocumentSync; -use NCU\FullTextSearch\IContentProvider; use OCP\AppFramework\Services\IAppConfig; use OCP\Server; use Psr\Container\ContainerExceptionInterface; @@ -31,6 +31,7 @@ public function __construct( private readonly IFullTextSearchManager $manager, private readonly SyncMapper $mapper, private readonly ProcessService $processService, + private readonly IndexService $indexService, private readonly ProviderService $providerService, private readonly LoggerService $logger, private readonly LoggerInterface $coreLogger, @@ -54,16 +55,20 @@ public function __construct( */ public function smartSync(): bool { // TODO: test link with ES - return ($this->syncForcedIndexes() - || $this->syncContentProviders() - || $this->resyncRecentDocuments() - || $this->syncOlderDocuments(6 * 30 * 24 * 3600) - || $this->syncOlderDocuments(3 * 30 * 24 * 3600) - || $this->syncOlderDocuments(4 * 7 * 24 * 3600) - || $this->syncOlderDocuments(2 * 7 * 24 * 3600) - || $this->syncOlderDocuments(7 * 24 * 3600) - || $this->syncOlderDocuments() - ); + $result = $this->syncRequestedIndexes() + || $this->syncContentProviders() + || $this->resyncRecentDocuments() + || $this->syncOlderDocuments(6 * 30 * 24 * 3600) + || $this->syncOlderDocuments(3 * 30 * 24 * 3600) + || $this->syncOlderDocuments(4 * 7 * 24 * 3600) + || $this->syncOlderDocuments(2 * 7 * 24 * 3600) + || $this->syncOlderDocuments(7 * 24 * 3600) + || $this->syncOlderDocuments(); + + // ending session + $this->logger->session(); + + return $result; } /** @@ -71,9 +76,9 @@ public function smartSync(): bool { * * @return bool FALSE if no out-of-sync documents were found */ - public function syncForcedIndexes(int $limit = 100): bool { - $this->logger->session(SessionType::FORCED); - $syncDocuments = $this->mapper->getForcedSyncs($limit); + public function syncRequestedIndexes(int $limit = 100): bool { + $this->logger->session(SessionType::REQUESTED); + $syncDocuments = $this->mapper->getRequestedSyncs($limit); if (empty($syncDocuments)) { return false; } @@ -88,7 +93,7 @@ public function syncForcedIndexes(int $limit = 100): bool { */ public function syncContentProviders(): bool { $this->logger->session(SessionType::SYNC); - foreach ($this->manager->getContentProviders() as $provider) { + foreach ($this->manager->getContentProviders() as $appId => $provider) { $this->syncContentProvider($provider); } @@ -100,7 +105,7 @@ public function syncContentProvider(string $providerId): void { } private function resyncRecentDocuments(): bool { - $this->logger->session(SessionType::FORCED); + $this->logger->session(SessionType::REQUESTED); return false; } @@ -118,13 +123,13 @@ public function syncDocuments(array $syncDocuments): void { public function syncDocument(DocumentSync $sync): void { $time = time(); $this->logger->action('indexing document ' . $sync->definition()); - $this->indexDocument($sync); + $this->indexDocumentSync($sync); $sync->setIndexed($time); // $this->mapper->insertOrUpdate($sync); } - private function indexDocument(DocumentSync $sync): void { + private function indexDocumentSync(DocumentSync $sync): void { $provider = $this->getContentProvider($sync->getProviderId()); if ($provider === null) { $this->logger->error('provider ' . $sync->getProviderId() . ' not found'); @@ -137,6 +142,7 @@ private function indexDocument(DocumentSync $sync): void { return; } + $document->setDocumentId($sync->getProviderId() . ':' . $sync->getDocumentId()); $checksum = $document->getChecksum(); if (($sync->getChecksum() === $checksum) && ($this->appConfig->getAppValueString(ConfigLexicon::SYNC_REQUIREMENT_LEVEL) >= self::SYNC_LEVEL_CHECKSUM)) { $this->logger->logger('document ' . $sync->definition() . ' seems to be identical to the indexed version'); @@ -146,7 +152,8 @@ private function indexDocument(DocumentSync $sync): void { $sync->setChecksum($checksum); echo json_encode($document->getFlags()) . ' ' . json_encode($document->getContent()) . "\n"; - // get document + $this->indexService->syncDocument($sync, $document); + // send document to ES // confirm it is done sleep(rand(1, 15));