separate the rephrasing model from the chat model

Rephrasing can be done with faster, simpler models as there is not much
reasoning needed.

Author: splitbrain

Model/AbstractModel.php

@@ -221,6 +221,7 @@ protected function sendAPIRequest($method, $url, $data, $retry = 0)
         if ($this->debug) {
             echo 'Sending ' . $method . ' request to ' . $url . ' with payload:' . "\n";
             print_r($json);
+            echo "\n";
         }
 
         // send request and handle retries
@@ -237,6 +238,7 @@ protected function sendAPIRequest($method, $url, $data, $retry = 0)
         if ($this->debug) {
             echo 'Received response:' . "\n";
             print_r($response);
+            echo "\n";
         }
 
         // decode the response

cli.php

@@ -221,11 +221,15 @@ protected function chat()
     {
         if ($this->loglevel['debug']['enabled']) {
             $this->helper->getChatModel()->setDebug(true);
+            $this->helper->getRephraseModel()->setDebug(true);
+            $this->helper->getEmbedModel()->setDebug(true);
         }
 
         $history = [];
         while ($q = $this->readLine('Your Question')) {
             $this->helper->getChatModel()->resetUsageStats();
+            $this->helper->getRephraseModel()->resetUsageStats();
+            $this->helper->getEmbedModel()->resetUsageStats();
             $result = $this->helper->askChatQuestion($q, $history);
             $this->colors->ptln("Interpretation: {$result['question']}", Colors::C_LIGHTPURPLE);
             $history[] = [$result['question'], $result['answer']];
@@ -277,7 +281,7 @@ protected function models()
                     $name,
                     sprintf(" In: %7d\nOut: %7d", $info['inputTokens'], $info['outputTokens']),
                     sprintf(" In: %.2f\nOut: %.2f", $info['inputTokenPrice'], $info['outputTokenPrice']),
-                    $info['description']."\n"
+                    $info['description'] . "\n"
                 ],
                 [
                     $info['confok'] ? Colors::C_LIGHTGREEN : Colors::C_LIGHTRED,
@@ -300,7 +304,7 @@ protected function models()
                     sprintf("%7d", $info['inputTokens']),
                     sprintf("%.2f", $info['inputTokenPrice']),
                     $info['dimensions'],
-                    $info['description']."\n"
+                    $info['description'] . "\n"
                 ],
                 [
                     $info['confok'] ? Colors::C_LIGHTGREEN : Colors::C_LIGHTRED,
@@ -322,6 +326,8 @@ protected function ask($query)
     {
         if ($this->loglevel['debug']['enabled']) {
             $this->helper->getChatModel()->setDebug(true);
+            $this->helper->getRephraseModel()->setDebug(true);
+            $this->helper->getEmbedModel()->setDebug(true);
         }
 
         $result = $this->helper->askQuestion($query);
@@ -433,9 +439,18 @@ protected function printSources($sources)
      */
     protected function printUsage()
     {
+        $chat = $this->helper->getChatModel()->getUsageStats();
+        $rephrase = $this->helper->getRephraseModel()->getUsageStats();
+        $embed = $this->helper->getEmbedModel()->getUsageStats();
+
         $this->info(
-            'Made {requests} requests in {time}s to Model. Used {tokens} tokens for about ${cost}.',
-            $this->helper->getChatModel()->getUsageStats()
+            'Made {requests} requests in {time}s to models. Used {tokens} tokens for about ${cost}.',
+            [
+                'requests' => $chat['requests'] + $rephrase['requests'] + $embed['requests'],
+                'time' => $chat['time'] + $rephrase['time'] + $embed['time'],
+                'tokens' => $chat['tokens'] + $chat['tokens'] + $embed['tokens'],
+                'cost' => $chat['cost'] + $chat['cost'] + $embed['cost'],
+            ]
         );
     }
 

conf/default.php

@@ -8,6 +8,7 @@
 
 
 $conf['chatmodel'] = 'OpenAI gpt-3.5-turbo';
+$conf['rephrasemodel'] = 'OpenAI gpt-3.5-turbo';
 $conf['embedmodel'] = 'OpenAI text-embedding-ada-002';
 $conf['storage'] = 'SQLite';
 
@@ -35,6 +36,8 @@
 
 $conf['chunkSize'] = 1500;
 $conf['contextChunks'] = 5;
+$conf['chatHistory'] = 1;
+$conf['rephraseHistory'] = 1;
 
 $conf['logging'] = 0;
 $conf['restrict'] = '';

conf/metadata.php

@@ -7,6 +7,7 @@
  */
 
 $meta['chatmodel'] = array(\dokuwiki\plugin\aichat\ModelSetting::class, 'type' => 'chat');
+$meta['rephrasemodel'] = array(\dokuwiki\plugin\aichat\ModelSetting::class, 'type' => 'chat');
 $meta['embedmodel'] = array(\dokuwiki\plugin\aichat\ModelSetting::class, 'type' => 'embedding');
 $meta['storage'] = array('multichoice',
     '_choices' => array(

helper.php

@@ -7,7 +7,6 @@
 use dokuwiki\plugin\aichat\Embeddings;
 use dokuwiki\plugin\aichat\Model\ChatInterface;
 use dokuwiki\plugin\aichat\Model\EmbeddingInterface;
-use dokuwiki\plugin\aichat\Model\OpenAI\Embedding3Small;
 use dokuwiki\plugin\aichat\Storage\AbstractStorage;
 
 /**
@@ -22,6 +21,8 @@ class helper_plugin_aichat extends Plugin
     protected $logger;
     /** @var ChatInterface */
     protected $chatModel;
+    /** @var ChatInterface */
+    protected $rephraseModel;
     /** @var EmbeddingInterface */
     protected $embedModel;
     /** @var Embeddings */
@@ -32,6 +33,7 @@ class helper_plugin_aichat extends Plugin
     /** @var array where to store meta data on the last run */
     protected $runDataFile;
 
+
     /**
      * Constructor. Initializes vendor autoloader
      */
@@ -94,6 +96,26 @@ public function getChatModel()
         return $this->chatModel;
     }
 
+    /**
+     * @return ChatInterface
+     */
+    public function getRephraseModel()
+    {
+        if ($this->rephraseModel instanceof ChatInterface) {
+            return $this->rephraseModel;
+        }
+
+        [$namespace, $name] = sexplode(' ', $this->getConf('rephrasemodel'), 2);
+        $class = '\\dokuwiki\\plugin\\aichat\\Model\\' . $namespace . '\\ChatModel';
+
+        if (!class_exists($class)) {
+            throw new \RuntimeException('No ChatModel found for ' . $namespace);
+        }
+
+        $this->rephraseModel = new $class($name, $this->conf);
+        return $this->rephraseModel;
+    }
+
     /**
      * Access the Embedding Model
      *
@@ -172,7 +194,7 @@ public function getStorage()
      */
     public function askChatQuestion($question, $history = [])
     {
-        if ($history) {
+        if ($history && $this->getConf('rephraseHistory') > 0) {
             $standaloneQuestion = $this->rephraseChatQuestion($question, $history);
         } else {
             $standaloneQuestion = $question;
@@ -204,7 +226,9 @@ public function askQuestion($question, $history = [])
             $history = [];
         }
 
-        $messages = $this->prepareMessages($prompt, $question, $history);
+        $messages = $this->prepareMessages(
+            $this->getChatModel(), $prompt, $question, $history, $this->getConf('chatHistory')
+        );
         $answer = $this->getChatModel()->getAnswer($messages);
 
         return [
@@ -225,29 +249,35 @@ public function askQuestion($question, $history = [])
     public function rephraseChatQuestion($question, $history)
     {
         $prompt = $this->getPrompt('rephrase');
-        $messages = $this->prepareMessages($prompt, $question, $history);
-        return $this->getChatModel()->getAnswer($messages);
+        $messages = $this->prepareMessages(
+            $this->getRephraseModel(), $prompt, $question, $history, $this->getConf('rephraseHistory')
+        );
+        return $this->getRephraseModel()->getAnswer($messages);
     }
 
     /**
      * Prepare the messages for the AI
      *
+     * @param ChatInterface $model The used model
      * @param string $prompt The fully prepared system prompt
      * @param string $question The user question
      * @param array[] $history The chat history [[user, ai], [user, ai], ...]
+     * @param int $historySize The maximum number of messages to use from the history
      * @return array An OpenAI compatible array of messages
      */
-    protected function prepareMessages($prompt, $question, $history)
+    protected function prepareMessages(
+        ChatInterface $model, string $prompt, string $question, array $history, int $historySize
+    ): array
     {
         // calculate the space for context
-        $remainingContext = $this->getChatModel()->getMaxInputTokenLength();
+        $remainingContext = $model->getMaxInputTokenLength();
         $remainingContext -= $this->countTokens($prompt);
         $remainingContext -= $this->countTokens($question);
         $safetyMargin = $remainingContext * 0.05; // 5% safety margin
         $remainingContext -= $safetyMargin;
         // FIXME we may want to also have an upper limit for the history and not always use the full context
 
-        $messages = $this->historyMessages($history, $remainingContext);
+        $messages = $this->historyMessages($history, $remainingContext, $historySize);
         $messages[] = [
             'role' => 'system',
             'content' => $prompt
@@ -265,15 +295,17 @@ protected function prepareMessages($prompt, $question, $history)
      * Only as many messages are used as fit into the token limit
      *
      * @param array[] $history The chat history [[user, ai], [user, ai], ...]
-     * @param int $tokenLimit
+     * @param int $tokenLimit The maximum number of tokens to use
+     * @param int $sizeLimit The maximum number of messages to use
      * @return array
      */
-    protected function historyMessages($history, $tokenLimit)
+    protected function historyMessages(array $history, int $tokenLimit, int $sizeLimit): array
     {
         $remainingContext = $tokenLimit;
 
         $messages = [];
         $history = array_reverse($history);
+        $history = array_slice($history, 0, $sizeLimit);
         foreach ($history as $row) {
             $length = $this->countTokens($row[0] . $row[1]);
             if ($length > $remainingContext) {

lang/en/prompt_rephrase.txt

@@ -1 +1,3 @@
 Given the previous conversation, rephrase the user's follow-up question to be a standalone question. {{LANGUAGE}}
+
+Only reply with the rephrased question, do not answer it.

lang/en/settings.php

@@ -7,6 +7,7 @@
 
 
 $lang['chatmodel'] = 'The 🧠 model to use for chat completion. Configure required credentials below.';
+$lang['rephrasemodel'] = 'The 🧠 model to use for rephrasing questions. Configure required credentials below.';
 $lang['embedmodel'] = 'The 🧠 model to use for text embedding. Configure required credentials below.<br>🔄 You need to rebuild the vector storage when changing this setting.';
 $lang['storage'] = 'Which 📥 vector storage to use. Configure required credentials below.<br>🔄 You need to rebuild the vector storage when changing this setting.';
 
@@ -32,6 +33,8 @@
 
 $lang['chunkSize'] = 'Maximum number of tokens per chunk.<br>🔄 You need to rebuild the vector storage when changing this setting.';
 $lang['contextChunks'] = 'Number of chunks to send to the AI model for context.';
+$lang['chatHistory'] = 'Number of previous chat messages to consider for context in the conversation.';
+$lang['rephraseHistory'] = 'Number of previous chat messages to consider for context when rephrasing a question. Set to 0 to disable rephrasing.';
 
 $lang['logging'] = 'Log all questions and answers. Use the <a href="?do=admin&page=logviewer&facility=aichat">Log Viewer</a> to access.';
 $lang['restrict'] = 'Restrict access to these users and groups (comma separated). Leave empty to allow all users.';