various refactoring and introduction of a simulate command

splitbrain · splitbrain · commit c2b7a1f7fd0f · 2024-03-21T12:50:49.000+01:00
The new command makes it easier to run the same chat questions against
multiple models and compare the results in a spreadsheet
diff --git a/Chunk.php b/Chunk.php
@@ -32,7 +32,11 @@ public function __construct(
 
     public function __toString(): string
     {
-        return $this->page . '#' . $this->id;
+        $string = $this->page . '#' . $this->id;
+        if ($this->score) {
+            $string .= ' (' . $this->score . ')';
+        }
+        return $string;
     }
 
     /**
diff --git a/Embeddings.php b/Embeddings.php
@@ -39,6 +39,9 @@ class Embeddings
     /** @var array remember sentences when chunking */
     private $sentenceQueue = [];
 
+    /** @var int the time spent for the last similar chunk retrieval */
+    public $timeSpent = 0;
+
     protected $configChunkSize;
     protected $configContextChunks;
 
@@ -234,10 +237,11 @@ public function getSimilarChunks($query, $lang = '')
 
         $time = microtime(true);
         $chunks = $this->storage->getSimilarChunks($vector, $lang, $fetch);
+        $this->timeSpent = microtime(true) - $time;
         if ($this->logger instanceof CLI) {
             $this->logger->info(
                 'Fetched {count} similar chunks from store in {time} seconds',
-                ['count' => count($chunks), 'time' => round(microtime(true) - $time, 2)]
+                ['count' => count($chunks), 'time' => round($this->timeSpent, 2)]
             );
         }
 
diff --git a/Model/AbstractModel.php b/Model/AbstractModel.php
@@ -114,7 +114,7 @@ public function getUsageStats()
 
         return [
             'tokens' => $this->inputTokensUsed + $this->outputTokensUsed,
-            'cost' => round($cost / 1_000_000, 4),
+            'cost' => sprintf("%.6f", $cost / 1_000_000),
             'time' => round($this->timeUsed, 2),
             'requests' => $this->requestsMade,
         ];
diff --git a/Model/Mistral/AbstractMistralModel.php b/Model/Mistral/AbstractMistralModel.php
@@ -43,8 +43,8 @@ protected function parseAPIResponse($response)
             $this->outputTokensUsed += $response['usage']['completion_tokens'] ?? 0;
         }
 
-        if (isset($response['error'])) {
-            throw new \Exception('Mistral API error: ' . $response['error']['message']);
+        if (isset($response['object']) && $response['object'] === 'error') {
+            throw new \Exception('Mistral API error: ' . $response['message']);
         }
 
         return $response;
diff --git a/Model/Mistral/ChatModel.php b/Model/Mistral/ChatModel.php
@@ -9,8 +9,25 @@ class ChatModel extends AbstractMistralModel implements ChatInterface
     /** @inheritdoc */
     public function getAnswer(array $messages): string
     {
+        // Mistral allows only for a system message at the beginning of the chat
+        // https://discord.com/channels/1144547040454508606/1220314306844037150
+        $system = '';
+        $chat = [];
+        foreach ($messages as $message) {
+            if ($message['role'] === 'system') {
+                $system .= $message['content'] . "\n";
+            } else {
+                $chat[] = $message;
+            }
+        }
+        $system = trim($system);
+        if ($system) {
+            array_unshift($chat, ['role' => 'system', 'content' => $system]);
+        }
+
+
         $data = [
-            'messages' => $messages,
+            'messages' => $chat,
             'model' => $this->getModelName(),
             'max_tokens' => null,
             'stream' => false,
diff --git a/Model/VoyageAI/EmbeddingModel.php b/Model/VoyageAI/EmbeddingModel.php
@@ -49,8 +49,7 @@ protected function request($endpoint, $data)
     protected function parseAPIResponse($response)
     {
         if (isset($response['usage'])) {
-            $this->inputTokensUsed += $response['usage']['prompt_tokens'];
-            $this->outputTokensUsed += $response['usage']['completion_tokens'] ?? 0;
+            $this->inputTokensUsed += $response['usage']['total_tokens'];
         }
 
         if (isset($response['error'])) {
diff --git a/ModelFactory.php b/ModelFactory.php
@@ -0,0 +1,159 @@
+<?php
+
+namespace dokuwiki\plugin\aichat;
+
+use dokuwiki\plugin\aichat\Model\ChatInterface;
+use dokuwiki\plugin\aichat\Model\EmbeddingInterface;
+
+class ModelFactory
+{
+    /** @var array The plugin configuration */
+    protected array $config;
+
+    public $chatModel;
+    public $rephraseModel;
+    public $embeddingModel;
+
+    protected $debug = false;
+
+    /**
+     * @param array $config The plugin configuration
+     */
+    public function __construct(array $config)
+    {
+        $this->config = $config;
+    }
+
+    /**
+     * Set the debug flag for all models
+     *
+     * @param bool $debug
+     */
+    public function setDebug(bool $debug=true)
+    {
+        $this->debug = $debug;
+        $this->getChatModel()->setDebug($debug);
+        $this->getRephraseModel()->setDebug($debug);
+        $this->getEmbeddingModel()->setDebug($debug);
+    }
+
+    /**
+     * Access a cached Chat Model
+     *
+     * @return ChatInterface
+     * @throws \Exception
+     */
+    public function getChatModel()
+    {
+        if ($this->chatModel instanceof ChatInterface) {
+            return $this->chatModel;
+        }
+        $this->chatModel = $this->loadModel('chat', $this->config['chatmodel']);
+        return $this->chatModel;
+    }
+
+    /**
+     * Access a cached Rephrase Model
+     *
+     * @return ChatInterface
+     * @throws \Exception
+     */
+    public function getRephraseModel()
+    {
+        if ($this->rephraseModel instanceof ChatInterface) {
+            return $this->rephraseModel;
+        }
+        $this->rephraseModel = $this->loadModel('chat', $this->config['chatmodel']);
+        return $this->rephraseModel;
+    }
+
+    /**
+     * Access a cached Embedding Model
+     *
+     * @return EmbeddingInterface
+     */
+    public function getEmbeddingModel()
+    {
+        if ($this->embeddingModel instanceof EmbeddingInterface) {
+            return $this->embeddingModel;
+        }
+        $this->embeddingModel = $this->loadModel('embedding', $this->config['embedmodel']);
+        return $this->embeddingModel;
+    }
+
+    /**
+     * Get all known models
+     *
+     * A (new) instance is returned for each model that is available through the current configuration.
+     *
+     * @param bool $availableOnly Only return models that are available
+     * @param string $typeOnly Only return models of this type ('chat' or 'embedding')
+     * @return array
+     */
+    public function getModels($availableOnly = false, $typeOnly = '')
+    {
+        $result = [
+            'chat' => [],
+            'embedding' => [],
+        ];
+
+        $jsons = glob(__DIR__ . '/Model/*/models.json');
+        foreach ($jsons as $json) {
+            $models = json_decode(file_get_contents($json), true);
+            foreach ($models as $type => $model) {
+                $namespace = basename(dirname($json));
+                foreach ($model as $name => $info) {
+                    try {
+                        $info['instance'] = $this->loadModel($type, "$namespace $name");
+                        $info['instance']->setDebug($this->debug);
+                    } catch (\Exception $e) {
+                        if ($availableOnly) continue;
+                        $info['instance'] = false;
+                    }
+
+                    $result[$type]["$namespace $name"] = $info;
+                }
+            }
+        }
+
+        return $typeOnly ? $result[$typeOnly] : $result;
+    }
+
+
+    /**
+     * Initialize a model by config name
+     *
+     * @param string $type 'chat' or 'embedding'
+     * @param string $name The full model name including provider
+     * @return ChatInterface|EmbeddingInterface
+     * @throws \Exception
+     */
+    public function loadModel(string $type, string $name)
+    {
+        $type = ucfirst(strtolower($type));
+        $prefix = '\\dokuwiki\\plugin\\aichat\\Model\\';
+        $cname = $type . 'Model';
+        $interface = $prefix . $type . 'Interface';
+
+
+        [$namespace, $model] = sexplode(' ', $name, 2, '');
+        $class = $prefix . $namespace . '\\' . $cname;
+
+        if (!class_exists($class)) {
+            throw new \Exception("No $cname found for $namespace");
+        }
+
+        try {
+            $instance = new $class($model, $this->config);
+        } catch (\Exception $e) {
+            throw new \Exception("Failed to initialize $cname for $namespace: " . $e->getMessage(), 0, $e);
+        }
+
+        if (!($instance instanceof $interface)) {
+            throw new \Exception("$cname for $namespace does not implement $interface");
+        }
+
+        return $instance;
+    }
+
+}
diff --git a/cli.php b/cli.php
@@ -2,6 +2,7 @@
 
 use dokuwiki\Extension\CLIPlugin;
 use dokuwiki\plugin\aichat\Chunk;
+use dokuwiki\plugin\aichat\ModelFactory;
 use dokuwiki\Search\Indexer;
 use splitbrain\phpcli\Colors;
 use splitbrain\phpcli\Options;
@@ -18,11 +19,13 @@ class cli_plugin_aichat extends CLIPlugin
     /** @var helper_plugin_aichat */
     protected $helper;
 
+    /** @inheritdoc */
     public function __construct($autocatch = true)
     {
         parent::__construct($autocatch);
         $this->helper = plugin_load('helper', 'aichat');
         $this->helper->setLogger($this);
+        $this->loadConfig();
     }
 
     /** @inheritDoc */
@@ -77,7 +80,10 @@ protected function setup(Options $options)
     /** @inheritDoc */
     protected function main(Options $options)
     {
-        $this->loadConfig();
+        if ($this->loglevel['debug']['enabled']) {
+            $this->helper->factory->setDebug(true);
+        }
+
         ini_set('memory_limit', -1);
         switch ($options->getCmd()) {
             case 'embed':
@@ -219,52 +225,26 @@ protected function split($page)
      */
     protected function chat()
     {
-        if ($this->loglevel['debug']['enabled']) {
-            $this->helper->getChatModel()->setDebug(true);
-            $this->helper->getRephraseModel()->setDebug(true);
-            $this->helper->getEmbedModel()->setDebug(true);
-        }
-
         $history = [];
         while ($q = $this->readLine('Your Question')) {
             $this->helper->getChatModel()->resetUsageStats();
             $this->helper->getRephraseModel()->resetUsageStats();
-            $this->helper->getEmbedModel()->resetUsageStats();
+            $this->helper->getEmbeddingModel()->resetUsageStats();
             $result = $this->helper->askChatQuestion($q, $history);
             $this->colors->ptln("Interpretation: {$result['question']}", Colors::C_LIGHTPURPLE);
             $history[] = [$result['question'], $result['answer']];
             $this->printAnswer($result);
         }
     }
 
+    /**
+     * Print information about the available models
+     *
+     * @return void
+     */
     protected function models()
     {
-        $result = [
-            'chat' => [],
-            'embedding' => [],
-        ];
-
-
-        $jsons = glob(__DIR__ . '/Model/*/models.json');
-        foreach ($jsons as $json) {
-            $models = json_decode(file_get_contents($json), true);
-            foreach ($models as $type => $model) {
-                $namespace = basename(dirname($json));
-                foreach ($model as $name => $info) {
-
-
-                    $class = '\\dokuwiki\\plugin\\aichat\\Model\\' . $namespace . '\\' . ucfirst($type) . 'Model';
-                    try {
-                        new $class($name, $this->conf);
-                        $info['confok'] = true;
-                    } catch (Exception $e) {
-                        $info['confok'] = false;
-                    }
-
-                    $result[$type]["$namespace $name"] = $info;
-                }
-            }
-        }
+        $result = (new ModelFactory($this->conf))->getModels();
 
         $td = new TableFormatter($this->colors);
         $cols = [30, 20, 20, '*'];
@@ -284,7 +264,7 @@ protected function models()
                     $info['description'] . "\n"
                 ],
                 [
-                    $info['confok'] ? Colors::C_LIGHTGREEN : Colors::C_LIGHTRED,
+                    $info['instance'] ? Colors::C_LIGHTGREEN : Colors::C_LIGHTRED,
                 ]
             );
         }
@@ -307,7 +287,7 @@ protected function models()
                     $info['description'] . "\n"
                 ],
                 [
-                    $info['confok'] ? Colors::C_LIGHTGREEN : Colors::C_LIGHTRED,
+                    $info['instance'] ? Colors::C_LIGHTGREEN : Colors::C_LIGHTRED,
                 ]
             );
         }
@@ -324,12 +304,6 @@ protected function models()
      */
     protected function ask($query)
     {
-        if ($this->loglevel['debug']['enabled']) {
-            $this->helper->getChatModel()->setDebug(true);
-            $this->helper->getRephraseModel()->setDebug(true);
-            $this->helper->getEmbedModel()->setDebug(true);
-        }
-
         $result = $this->helper->askQuestion($query);
         $this->printAnswer($result);
     }
@@ -441,7 +415,7 @@ protected function printUsage()
     {
         $chat = $this->helper->getChatModel()->getUsageStats();
         $rephrase = $this->helper->getRephraseModel()->getUsageStats();
-        $embed = $this->helper->getEmbedModel()->getUsageStats();
+        $embed = $this->helper->getEmbeddingModel()->getUsageStats();
 
         $this->info(
             'Made {requests} requests in {time}s to models. Used {tokens} tokens for about ${cost}.',
diff --git a/cli/simulate.php b/cli/simulate.php
diff --git a/helper.php b/helper.php

Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,11 @@ public function __construct(`
`32`	`32`
`33`	`33`	`public function __toString(): string`
`34`	`34`	`{`
`35`		`- return $this->page . '#' . $this->id;`
	`35`	`+ $string = $this->page . '#' . $this->id;`
	`36`	`+ if ($this->score) {`
	`37`	`+ $string .= ' (' . $this->score . ')';`
	`38`	`+ }`
	`39`	`+ return $string;`
`36`	`40`	`}`
`37`	`41`
`38`	`42`	`/**`
Original file line number	Diff line number	Diff line change
`@@ -43,8 +43,8 @@ protected function parseAPIResponse($response)`
`43`	`43`	`$this->outputTokensUsed += $response['usage']['completion_tokens'] ?? 0;`
`44`	`44`	`}`
`45`	`45`
`46`		`- if (isset($response['error'])) {`
`47`		`- throw new \Exception('Mistral API error: ' . $response['error']['message']);`
	`46`	`+ if (isset($response['object']) && $response['object'] === 'error') {`
	`47`	`+ throw new \Exception('Mistral API error: ' . $response['message']);`
`48`	`48`	`}`
`49`	`49`
`50`	`50`	`return $response;`
Original file line number	Diff line number	Diff line change
`@@ -49,8 +49,7 @@ protected function request($endpoint, $data)`
`49`	`49`	`protected function parseAPIResponse($response)`
`50`	`50`	`{`
`51`	`51`	`if (isset($response['usage'])) {`
`52`		`- $this->inputTokensUsed += $response['usage']['prompt_tokens'];`
`53`		`- $this->outputTokensUsed += $response['usage']['completion_tokens'] ?? 0;`
	`52`	`+ $this->inputTokensUsed += $response['usage']['total_tokens'];`
`54`	`53`	`}`
`55`	`54`
`56`	`55`	`if (isset($response['error'])) {`