emit the INDEXER_PAGE_ADD event

This allows plugins that add data to the fulltext index to add the same
data to the embeddings. This improves embedding searches with struct
data for example.

Author: splitbrain

Embeddings.php

@@ -2,6 +2,7 @@
 
 namespace dokuwiki\plugin\aichat;
 
+use dokuwiki\Extension\Event;
 use dokuwiki\Extension\PluginInterface;
 use dokuwiki\plugin\aichat\Model\ChatInterface;
 use dokuwiki\plugin\aichat\Model\EmbeddingInterface;
@@ -55,17 +56,18 @@ class Embeddings
      * @param array $config The plugin configuration
      */
     public function __construct(
-        ChatInterface $chatModel,
+        ChatInterface      $chatModel,
         EmbeddingInterface $embedModel,
-        AbstractStorage $storage,
-        $config
-    ) {
+        AbstractStorage    $storage,
+                           $config
+    )
+    {
         $this->chatModel = $chatModel;
         $this->embedModel = $embedModel;
         $this->storage = $storage;
         $this->configChunkSize = $config['chunkSize'];
         $this->configContextChunks = $config['contextChunks'];
-        $this->similarityThreshold = $config['similarityThreshold']/100;
+        $this->similarityThreshold = $config['similarityThreshold'] / 100;
     }
 
     /**
@@ -169,9 +171,10 @@ public function createNewIndex($skipRE = '', $matchRE = '', $clear = false)
      * @param string $page Name of the page to split
      * @param int $firstChunkID The ID of the first chunk of this page
      * @return Chunk[] A list of chunks created for this page
+     * @emits INDEXER_PAGE_ADD support plugins that add additional data to the page
      * @throws \Exception
      */
-    protected function createPageChunks($page, $firstChunkID)
+    public function createPageChunks($page, $firstChunkID)
     {
         $chunkList = [];
 
@@ -184,6 +187,19 @@ protected function createPageChunks($page, $firstChunkID)
             $text = rawWiki($page);
         }
 
+        // allow plugins to modify the text before splitting
+        $eventData = [
+            'page' => $page,
+            'body' => '',
+            'metadata' => ['title' => $page, 'relation_references' => []],
+        ];
+        $event = new Event('INDEXER_PAGE_ADD', $eventData);
+        if ($event->advise_before()) {
+            $text = $eventData['body'] . ' ' . $text;
+        } else {
+            $text = $eventData['body'];
+        }
+
         $parts = $this->splitIntoChunks($text);
         foreach ($parts as $part) {
             if (trim((string)$part) == '') continue; // skip empty chunks
@@ -251,7 +267,7 @@ public function getSimilarChunks($query, $lang = '')
         foreach ($chunks as $chunk) {
             // filter out chunks the user is not allowed to read
             if ($auth && auth_quickaclcheck($chunk->getPage()) < AUTH_READ) continue;
-            if($chunk->getScore() < $this->similarityThreshold) continue;
+            if ($chunk->getScore() < $this->similarityThreshold) continue;
 
             $chunkSize = count($this->getTokenEncoder()->encode($chunk->getText()));
             if ($size + $chunkSize > $this->chatModel->getMaxInputTokenLength()) break; // we have enough
@@ -269,7 +285,7 @@ public function getSimilarChunks($query, $lang = '')
      * @throws \Exception
      * @todo support splitting too long sentences
      */
-    public function splitIntoChunks($text)
+    protected function splitIntoChunks($text)
     {
         $sentenceSplitter = new Sentence();
         $tiktok = $this->getTokenEncoder();
@@ -297,7 +313,8 @@ public function splitIntoChunks($text)
                 $this->rememberSentence($sentence);
             } else {
                 // add current chunk to result
-                $chunks[] = $chunk;
+                $chunk = trim($chunk);
+                if ($chunk !== '') $chunks[] = $chunk;
 
                 // start new chunk with remembered sentences
                 $chunk = implode(' ', $this->sentenceQueue);

Storage/SQLiteStorage.php

@@ -65,6 +65,8 @@ public function startCreation($clear = false)
         if ($clear) {
             /** @noinspection SqlWithoutWhere */
             $this->db->exec('DELETE FROM embeddings');
+            /** @noinspection SqlWithoutWhere */
+            $this->db->exec('DELETE FROM clusters');
         }
     }
 

cli.php

@@ -210,10 +210,9 @@ protected function page($page, $dump = false)
      */
     protected function split($page)
     {
-        $text = rawWiki($page);
-        $chunks = $this->helper->getEmbeddings()->splitIntoChunks($text);
+        $chunks = $this->helper->getEmbeddings()->createPageChunks($page, 0);
         foreach ($chunks as $chunk) {
-            echo $chunk;
+            echo $chunk->getText();
             echo "\n";
             $this->colors->ptln('--------------------------------', Colors::C_LIGHTPURPLE);
         }

cli/simulate.php

@@ -1,8 +1,6 @@
 <?php
 
-use dokuwiki\Extension\CLIPlugin;
 use dokuwiki\plugin\aichat\AbstractCLI;
-use dokuwiki\plugin\aichat\ModelFactory;
 use splitbrain\phpcli\Colors;
 use splitbrain\phpcli\Options;
 
@@ -76,9 +74,15 @@ protected function simulate($questions, $model)
             $this->helper->getEmbeddingModel()->resetUsageStats();
 
             $this->colors->ptln($q, Colors::C_LIGHTPURPLE);
-            $result = $this->helper->askChatQuestion($q, $history);
-            $history[] = [$result['question'], $result['answer']];
-            $this->colors->ptln($result['question'], Colors::C_LIGHTBLUE);
+            try {
+                $result = $this->helper->askChatQuestion($q, $history);
+                $history[] = [$result['question'], $result['answer']];
+                $this->colors->ptln($result['question'], Colors::C_LIGHTBLUE);
+            } catch (Exception $e) {
+                $this->error($e->getMessage());
+                $this->debug($e->getTraceAsString());
+                $result = ['question' => $q, 'answer' => "ERROR\n" . $e->getMessage(), 'sources' => []];
+            }
 
             $record = [
                 'question' => $q,