make threshold configurable

Author: splitbrain

Embeddings.php

@@ -44,6 +44,7 @@ class Embeddings
 
     protected $configChunkSize;
     protected $configContextChunks;
+    protected $similarityThreshold;
 
     /**
      * Embeddings constructor.
@@ -64,6 +65,7 @@ public function __construct(
         $this->storage = $storage;
         $this->configChunkSize = $config['chunkSize'];
         $this->configContextChunks = $config['contextChunks'];
+        $this->similarityThreshold = $config['similarityThreshold']/100;
     }
 
     /**
@@ -249,6 +251,7 @@ public function getSimilarChunks($query, $lang = '')
         foreach ($chunks as $chunk) {
             // filter out chunks the user is not allowed to read
             if ($auth && auth_quickaclcheck($chunk->getPage()) < AUTH_READ) continue;
+            if($chunk->getScore() < $this->similarityThreshold) continue;
 
             $chunkSize = count($this->getTokenEncoder()->encode($chunk->getText()));
             if ($size + $chunkSize > $this->chatModel->getMaxInputTokenLength()) break; // we have enough

Storage/SQLiteStorage.php

@@ -17,8 +17,6 @@
  */
 class SQLiteStorage extends AbstractStorage
 {
-    /** @var float minimum similarity to consider a chunk a match */
-    final public const SIMILARITY_THRESHOLD = 0;
 
     /** @var int Number of documents to randomly sample to create the clusters */
     final public const SAMPLE_SIZE = 2000;
@@ -30,6 +28,9 @@ class SQLiteStorage extends AbstractStorage
 
     protected $useLanguageClusters = false;
 
+    /** @var float minimum similarity to consider a chunk a match */
+    protected $similarityThreshold = 0;
+
     /** @inheritdoc */
     public function __construct(array $config)
     {
@@ -38,6 +39,8 @@ public function __construct(array $config)
 
         $helper = plugin_load('helper', 'aichat');
         $this->useLanguageClusters = $helper->getConf('preferUIlanguage') >= AIChat::LANG_UI_LIMITED;
+
+        $this->similarityThreshold = $config['similarityThreshold']/100;
     }
 
     /** @inheritdoc */
@@ -148,7 +151,7 @@ public function getSimilarChunks($vector, $lang = '', $limit = 4)
                 AND similarity > CAST(? AS FLOAT)
            ORDER BY similarity DESC
               LIMIT ?',
-            [json_encode($vector, JSON_THROW_ON_ERROR), $cluster, self::SIMILARITY_THRESHOLD, $limit]
+            [json_encode($vector, JSON_THROW_ON_ERROR), $cluster, $this->similarityThreshold, $limit]
         );
         $chunks = [];
         foreach ($result as $record) {

conf/default.php

@@ -34,6 +34,7 @@
 $conf['qdrant_collection'] = 'aichat';
 
 $conf['chunkSize'] = 1500;
+$conf['similarityThreshold'] = 75;
 $conf['contextChunks'] = 5;
 $conf['chatHistory'] = 1;
 $conf['rephraseHistory'] = 1;

conf/metadata.php

@@ -40,8 +40,11 @@
 $meta['qdrant_apikey'] = array('password');
 $meta['qdrant_collection'] = array('string');
 
-$meta['chunkSize'] = array('numeric', '_min' => 100);
-$meta['contextChunks'] = array('numeric', '_min' => 1);
+$meta['chunkSize'] = array('numeric', '_min' => 100, '_regexp' => '/^\d+$/');
+$meta['similarityThreshold'] = array('numeric', '_min' => 0, '_max' => 100, '_regexp' => '/^\d+$/');
+$meta['contextChunks'] = array('numeric', '_min' => 1, '_regexp' => '/^\d+$/');
+$meta['chatHistory'] = array('numeric', '_min' => 0, '_regexp' => '/^\d+$/');
+$meta['rephraseHistory'] = array('numeric', '_min' => 0, '_regexp' => '/^\d+$/');
 
 $meta['logging'] = array('onoff');
 $meta['restrict'] = array('string');

lang/en/settings.php

@@ -32,7 +32,8 @@
 $lang['qdrant_collection'] = '📥 <b>Qdrant</b> collection. Will be created.';
 
 $lang['chunkSize'] = 'Maximum number of tokens per chunk.<br>🔄 You need to rebuild the vector storage when changing this setting.';
-$lang['contextChunks'] = 'Number of chunks to send to the AI model for context.';
+$lang['similarityThreshold'] = 'Minimum similarity threshold when selecting sources for a question. 0-100.';
+$lang['contextChunks'] = 'Maximum number of chunks to send to the AI model for context.';
 $lang['chatHistory'] = 'Number of previous chat messages to consider for context in the conversation.';
 $lang['rephraseHistory'] = 'Number of previous chat messages to consider for context when rephrasing a question. Set to 0 to disable rephrasing.';