From ab03878e6f05f97731d4c8d532c427e3ff31882e Mon Sep 17 00:00:00 2001 From: Anil Vishnoi Date: Tue, 4 Feb 2025 19:44:34 -0800 Subject: [PATCH] Make fine-tune and chat eval feature Podman ready Signed-off-by: Anil Vishnoi --- api-server/handlers.go | 4 +- deploy/podman/native/instructlab-ui.yaml | 5 + deploy/podman/native/secret.yaml.example | 2 + installers/podman/ilab-ui-native-installer.sh | 9 +- src/app/api/envConfig/route.ts | 4 +- src/app/api/fine-tune/git/branches/route.ts | 28 +- src/app/api/github/knowledge-files/route.ts | 2 +- .../api/native/git/knowledge-files/route.ts | 7 +- src/app/api/playground/chat/route.ts | 2 +- .../DocumentInformation.tsx | 3 + .../Experimental/ChatEval/ChatEval.tsx | 287 +++++++++--------- 11 files changed, 201 insertions(+), 152 deletions(-) diff --git a/api-server/handlers.go b/api-server/handlers.go index e5728d21..8c8fa2bc 100644 --- a/api-server/handlers.go +++ b/api-server/handlers.go @@ -807,8 +807,8 @@ func (srv *ILabServer) serveModelHandler(modelPath, port string, w http.Response } cmdArgs := []string{ - "serve", "model", - "--model", modelPath, + "serve", + "--model-path", modelPath, "--host", "0.0.0.0", "--port", port, } diff --git a/deploy/podman/native/instructlab-ui.yaml b/deploy/podman/native/instructlab-ui.yaml index 2aa4daf5..10e2e0b0 100644 --- a/deploy/podman/native/instructlab-ui.yaml +++ b/deploy/podman/native/instructlab-ui.yaml @@ -120,6 +120,11 @@ spec: secretKeyRef: name: ui-env key: IL_ENABLE_DEV_MODE + - name: NEXT_PUBLIC_API_SERVER + valueFrom: + secretKeyRef: + name: ui-env + key: NEXT_PUBLIC_API_SERVER ports: - containerPort: 3000 hostPort: 3000 diff --git a/deploy/podman/native/secret.yaml.example b/deploy/podman/native/secret.yaml.example index cdcd49b5..afabcfeb 100644 --- a/deploy/podman/native/secret.yaml.example +++ b/deploy/podman/native/secret.yaml.example @@ -8,6 +8,8 @@ data: NEXT_PUBLIC_TAXONOMY_ROOT_DIR: NEXTAUTH_URL: NEXTAUTH_SECRET: + NEXT_PUBLIC_API_SERVER: + kind: Secret metadata: creationTimestamp: null diff --git a/installers/podman/ilab-ui-native-installer.sh b/installers/podman/ilab-ui-native-installer.sh index 4fe70301..51bbaac2 100755 --- a/installers/podman/ilab-ui-native-installer.sh +++ b/installers/podman/ilab-ui-native-installer.sh @@ -17,6 +17,7 @@ declare AUTH_SECRET="your_super_secret_random_string" declare DEV_MODE="false" declare EXPERIMENTAL_FEATURES="" declare PYENV_DIR="" +declare API_SERVER_URL="" BINARY_INSTALL_DIR=./ IS_ILAB_INSTALLED="true" @@ -350,6 +351,8 @@ if [[ "$COMMAND" == "install" ]]; then EXPERIMENTAL_FEATURES_B64=$(echo -n "false" | base64) fi + API_SERVER_URL_B64=$(echo -n "$API_SERVER_URL" | base64) + # Download secret.yaml file echo -e "${green}Downloading the secret.yaml sample file...${reset}\n" curl -o secret.yaml https://raw.githubusercontent.com/instructlab/ui/main/deploy/podman/native/secret.yaml.example @@ -370,6 +373,7 @@ if [[ "$COMMAND" == "install" ]]; then sed -i "" "s||$AUTH_SECRET_B64|g" secret.yaml sed -i "" "s||$DEV_MODE_B64|g" secret.yaml sed -i "" "s||$EXPERIMENTAL_FEATURES_B64|g" secret.yaml + sed -i "" "s||$API_SERVER_URL_B64|g" secret.yaml else sed -i "s||$UI_DEPLOYMENT_B64|g" secret.yaml sed -i "s||$USERNAME_B64|g" secret.yaml @@ -379,6 +383,7 @@ if [[ "$COMMAND" == "install" ]]; then sed -i "s||$AUTH_SECRET_B64|g" secret.yaml sed -i "s||$DEV_MODE_B64|g" secret.yaml sed -i "s||$EXPERIMENTAL_FEATURES_B64|g" secret.yaml + sed -i "s||$API_SERVER_URL_B64|g" secret.yaml fi if [[ "$IS_ILAB_INSTALLED" == "true" ]]; then @@ -420,7 +425,7 @@ if [[ "$COMMAND" == "install" ]]; then # Check if VARIANT_ID is "rhel_ai" if [ "$VARIANT_ID" == "rhel_ai" ]; then echo -e "${green}Starting API server on OS: RHEL AI running on arch $ARCH ${reset}\n" - nohup ./ilab-apiserver --taxonomy-path "$SELECTED_TAXONOMY_DIR" --rhelai "$CUDA_FLAG" >$ILAB_APISERVER_LOG_FILE 2>&1 & + nohup ./ilab-apiserver --taxonomy-path "$SELECTED_TAXONOMY_DIR" --rhelai --vllm "$CUDA_FLAG" >$ILAB_APISERVER_LOG_FILE 2>&1 & else echo -e "${green}Starting API server on OS: $OS running on arch $ARCH ${reset}\n" nohup ./ilab-apiserver --base-dir "$DISCOVERED_PYENV_DIR" --taxonomy-path "$SELECTED_TAXONOMY_DIR" "$CUDA_FLAG" >$ILAB_APISERVER_LOG_FILE 2>&1 & @@ -479,7 +484,7 @@ elif [[ "$COMMAND" == "uninstall" ]]; then read -r -p "Are you sure you want to uninstall the InstructLab UI stack? (yes/no): " CONFIRM if [[ "$CONFIRM" != "yes" && "$CONFIRM" != "y" ]]; then - echo -e "${green}Uninstallation aborted.${reset}\n" + echo -e "${red}Uninstallation aborted.${reset}\n" exit 0 fi diff --git a/src/app/api/envConfig/route.ts b/src/app/api/envConfig/route.ts index 7a94930b..101e88a0 100644 --- a/src/app/api/envConfig/route.ts +++ b/src/app/api/envConfig/route.ts @@ -18,7 +18,9 @@ export async function GET() { ENABLE_DEV_MODE: process.env.IL_ENABLE_DEV_MODE || 'false', EXPERIMENTAL_FEATURES: process.env.NEXT_PUBLIC_EXPERIMENTAL_FEATURES || '', TAXONOMY_ROOT_DIR: process.env.NEXT_PUBLIC_TAXONOMY_ROOT_DIR || '', - TAXONOMY_KNOWLEDGE_DOCUMENT_REPO: process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'github.com/instructlab-public/taxonomy-knowledge-docs' + TAXONOMY_KNOWLEDGE_DOCUMENT_REPO: + process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'https://github.com/instructlab-public/taxonomy-knowledge-docs', + API_SERVER: process.env.NEXT_PUBLIC_API_SERVER }; return NextResponse.json(envConfig); diff --git a/src/app/api/fine-tune/git/branches/route.ts b/src/app/api/fine-tune/git/branches/route.ts index ccb1902a..c4472832 100644 --- a/src/app/api/fine-tune/git/branches/route.ts +++ b/src/app/api/fine-tune/git/branches/route.ts @@ -5,9 +5,10 @@ import fs from 'fs'; import path from 'path'; const REMOTE_TAXONOMY_ROOT_DIR = process.env.NEXT_PUBLIC_TAXONOMY_ROOT_DIR || ''; +const REMOTE_TAXONOMY_REPO_CONTAINER_MOUNT_DIR = '/tmp/.instructlab-ui'; export async function GET() { - const REPO_DIR = path.join(REMOTE_TAXONOMY_ROOT_DIR, '/taxonomy'); + const REPO_DIR = findTaxonomyRepoPath(); try { console.log(`Checking local taxonomy directory for branches: ${REPO_DIR}`); @@ -63,3 +64,28 @@ export async function GET() { return NextResponse.json({ error: 'Failed to list branches from local taxonomy (fine-tune)' }, { status: 500 }); } } + +function findTaxonomyRepoPath(): string { + let remoteTaxonomyRepoDirFinal: string = ''; + + const remoteTaxonomyRepoContainerMountDir = path.join(REMOTE_TAXONOMY_REPO_CONTAINER_MOUNT_DIR, '/taxonomy'); + const remoteTaxonomyRepoDir = path.join(REMOTE_TAXONOMY_ROOT_DIR, '/taxonomy'); + + // Check if there is taxonomy repository mounted in the container + if (fs.existsSync(remoteTaxonomyRepoContainerMountDir) && fs.readdirSync(remoteTaxonomyRepoContainerMountDir).length !== 0) { + remoteTaxonomyRepoDirFinal = remoteTaxonomyRepoContainerMountDir; + console.log('Remote taxonomy repository ', remoteTaxonomyRepoDir, ' is mounted at:', remoteTaxonomyRepoDirFinal); + } else { + // If remote taxonomy is not mounted, it means it's local deployment and we can directly use the paths + if (fs.existsSync(remoteTaxonomyRepoDir) && fs.readdirSync(remoteTaxonomyRepoDir).length !== 0) { + remoteTaxonomyRepoDirFinal = remoteTaxonomyRepoDir; + } + } + if (remoteTaxonomyRepoDirFinal === '') { + console.warn('Remote taxonomy repository path does not exist.'); + return remoteTaxonomyRepoDirFinal; + } + + console.log('Remote taxonomy repository path:', remoteTaxonomyRepoDirFinal); + return remoteTaxonomyRepoDirFinal; +} diff --git a/src/app/api/github/knowledge-files/route.ts b/src/app/api/github/knowledge-files/route.ts index dfe5abb5..136a23f6 100644 --- a/src/app/api/github/knowledge-files/route.ts +++ b/src/app/api/github/knowledge-files/route.ts @@ -3,7 +3,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { getToken } from 'next-auth/jwt'; const GITHUB_API_URL = 'https://api.github.com'; -const TAXONOMY_DOCUMENTS_REPO = process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'github.com/instructlab-public/taxonomy-knowledge-docs'; +const TAXONOMY_DOCUMENTS_REPO = process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'https://github.com/instructlab-public/taxonomy-knowledge-docs'; const BASE_BRANCH = 'main'; // Interface for the response diff --git a/src/app/api/native/git/knowledge-files/route.ts b/src/app/api/native/git/knowledge-files/route.ts index 93fd8748..55d1626c 100644 --- a/src/app/api/native/git/knowledge-files/route.ts +++ b/src/app/api/native/git/knowledge-files/route.ts @@ -10,7 +10,8 @@ import http from 'isomorphic-git/http/node'; // Constants for repository paths const TAXONOMY_DOCS_ROOT_DIR = process.env.NEXT_PUBLIC_TAXONOMY_ROOT_DIR || ''; const TAXONOMY_DOCS_CONTAINER_MOUNT_DIR = '/tmp/.instructlab-ui'; -const TAXONOMY_KNOWLEDGE_DOCS_REPO_URL = process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'github.com/instructlab-public/taxonomy-knowledge-docs'; +const TAXONOMY_KNOWLEDGE_DOCS_REPO_URL = + process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'https://github.com/instructlab-public/taxonomy-knowledge-docs'; const BASE_BRANCH = 'main'; // Interface for the response @@ -223,10 +224,10 @@ async function cloneTaxonomyDocsRepo() { const taxonomyDocsDirectoryPath = path.join(remoteTaxonomyRepoDirFinal, '/taxonomy-knowledge-docs'); if (fs.existsSync(taxonomyDocsDirectoryPath)) { - console.log(`Using existing taxonomy knowledge docs repository at ${remoteTaxonomyRepoDir}/taxonomy-knowledge-docs.`); + console.log(`Using existing taxonomy knowledge docs repository at ${TAXONOMY_DOCS_ROOT_DIR}/taxonomy-knowledge-docs.`); return taxonomyDocsDirectoryPath; } else { - console.log(`Taxonomy knowledge docs repository not found at ${taxonomyDocsDirectoryPath}. Cloning...`); + console.log(`Taxonomy knowledge docs repository not found at ${TAXONOMY_DOCS_ROOT_DIR}/taxonomy-knowledge-docs. Cloning...`); } try { diff --git a/src/app/api/playground/chat/route.ts b/src/app/api/playground/chat/route.ts index 956dc7b5..beb9f65e 100644 --- a/src/app/api/playground/chat/route.ts +++ b/src/app/api/playground/chat/route.ts @@ -22,7 +22,7 @@ export async function POST(req: NextRequest) { const requestData = { model: modelName, - messages, + messages: messages, stream: true }; diff --git a/src/components/Contribute/Knowledge/Native/DocumentInformation/DocumentInformation.tsx b/src/components/Contribute/Knowledge/Native/DocumentInformation/DocumentInformation.tsx index 25036e92..3bf1c090 100644 --- a/src/components/Contribute/Knowledge/Native/DocumentInformation/DocumentInformation.tsx +++ b/src/components/Contribute/Knowledge/Native/DocumentInformation/DocumentInformation.tsx @@ -141,6 +141,9 @@ const DocumentInformation: React.FC = ({ if (response.status === 201) { const result = await response.json(); console.log('Files uploaded result:', result); + setKnowledgeDocumentRepositoryUrl(result.repoUrl); + setKnowledgeDocumentCommit(result.commitSha); + setDocumentName(result.documentNames.join(', ')); // Populate the patterns field const alertInfo: AlertInfo = { type: 'success', diff --git a/src/components/Experimental/ChatEval/ChatEval.tsx b/src/components/Experimental/ChatEval/ChatEval.tsx index 1473385d..017ee1e9 100644 --- a/src/components/Experimental/ChatEval/ChatEval.tsx +++ b/src/components/Experimental/ChatEval/ChatEval.tsx @@ -42,11 +42,9 @@ import logo from '../../../../public/bot-icon-chat-32x32.svg'; import userLogo from '../../../../public/default-avatar.svg'; import ModelStatusIndicator from '@/components/Experimental/ModelServeStatus/ModelServeStatus'; -// TODO: get nextjs app router server side render working with the patternfly chatbot component. -const MODEL_SERVER_URL = process.env.NEXT_PUBLIC_MODEL_SERVER_URL; - const ChatModelEval: React.FC = () => { const [isUnifiedInput, setIsUnifiedInput] = useState(false); + const [modelServerURL, setModelServerURL] = useState(''); // States for unified input const [questionUnified, setQuestionUnified] = useState(''); @@ -54,14 +52,17 @@ const ChatModelEval: React.FC = () => { // States for left chat const [questionLeft, setQuestionLeft] = useState(''); const [messagesLeft, setMessagesLeft] = useState([]); + const [currentMessageLeft, setCurrentMessageLeft] = React.useState([]); const [selectedModelLeft, setSelectedModelLeft] = useState(null); const [alertMessageLeft, setAlertMessageLeft] = useState<{ title: string; message: string; variant: 'danger' | 'info' } | undefined>(undefined); // States for right chat const [questionRight, setQuestionRight] = useState(''); const [messagesRight, setMessagesRight] = useState([]); + const [currentMessageRight, setCurrentMessageRight] = React.useState([]); const [selectedModelRight, setSelectedModelRight] = useState(null); const [alertMessageRight, setAlertMessageRight] = useState<{ title: string; message: string; variant: 'danger' | 'info' } | undefined>(undefined); + const [freeGpus, setFreeGpus] = useState(0); const [totalGpus, setTotalGpus] = useState(0); @@ -107,15 +108,19 @@ const ChatModelEval: React.FC = () => { }; fetchGpus(); - const intervalId = setInterval(fetchGpus, 20000); + const intervalId = setInterval(fetchGpus, 5000); return () => clearInterval(intervalId); }, []); // Fetch models on component mount useEffect(() => { const fetchDefaultModels = async () => { - // const response = await fetch('/api/envConfig'); - // const envConfig = await response.json(); + const response = await fetch('/api/envConfig'); + const envConfig = await response.json(); + + const modelServerURL = envConfig.API_SERVER.replace(/:\d+/, ''); + console.log('Model server url is set to :', modelServerURL); + setModelServerURL(modelServerURL); const storedEndpoints = localStorage.getItem('endpoints'); const cust: Model[] = storedEndpoints @@ -135,14 +140,14 @@ const ChatModelEval: React.FC = () => { /** * Helper function to map internal model identifiers to chat model names. - * "granite-base-served" => "pre-train" - * "granite-latest-served" => "post-train" + * "granite-base" => "pre-train" + * "granite-latest" => "post-train" * Custom models retain their original modelName. */ const mapModelName = (modelName: string): string => { - if (modelName === 'granite-base-served') { + if (modelName === 'granite-base') { return 'pre-train'; - } else if (modelName === 'granite-latest-served') { + } else if (modelName === 'granite-latest') { return 'post-train'; } return modelName; @@ -197,8 +202,8 @@ const ChatModelEval: React.FC = () => { if (endpoint.includes('serve-base')) { const servedModel: Model = { name: 'Granite base model (Serving)', - apiURL: `${MODEL_SERVER_URL}:8000`, // endpoint for base model - modelName: 'granite-base-served' + apiURL: `${modelServerURL}:8000`, // endpoint for base model + modelName: 'granite-base' }; if (side === 'left') { @@ -209,8 +214,8 @@ const ChatModelEval: React.FC = () => { } else if (endpoint.includes('serve-latest')) { const servedModel: Model = { name: 'Granite fine tune checkpoint (Serving)', - apiURL: `${MODEL_SERVER_URL}:8001`, // endpoint for latest checkpoint - modelName: 'granite-latest-served' + apiURL: `${modelServerURL}:8001`, // endpoint for latest checkpoint + modelName: 'granite-latest' }; if (side === 'left') { @@ -268,19 +273,20 @@ const ChatModelEval: React.FC = () => { body: JSON.stringify({ model_name: 'pre-train' }) }); if (!resp.ok) { - console.error('Failed to unload pre-train:', resp.status, resp.statusText); + console.error('Failed to unload granite base model:', resp.status, resp.statusText); return; } const data = await resp.json(); - console.log('Unload left success:', data); + console.log('Successfully unloaded model served in left:', data); // Optionally clear out the selected model and job logs on the left setSelectedModelLeft(null); setModelJobIdLeft(undefined); setMessagesLeft([]); + setCurrentMessageLeft([]); setAlertMessageLeft({ title: 'Model Unloaded', - message: 'Successfully unloaded the pre-train model.', + message: 'Successfully unloaded the granite-base model.', variant: 'info' }); setTimeout(() => { @@ -299,19 +305,20 @@ const ChatModelEval: React.FC = () => { body: JSON.stringify({ model_name: 'post-train' }) }); if (!resp.ok) { - console.error('Failed to unload post-train:', resp.status, resp.statusText); + console.error('Failed to unload granite latest model:', resp.status, resp.statusText); return; } const data = await resp.json(); - console.log('Unload right success:', data); + console.log('Successfully unloaded model served in right:', data); // Optionally clear out the selected model and job logs on the right setSelectedModelRight(null); setModelJobIdRight(undefined); setMessagesRight([]); + setCurrentMessageRight([]); setAlertMessageRight({ title: 'Model Unloaded', - message: 'Successfully unloaded the post-train model.', + message: 'Successfully unloaded the granite latest model.', variant: 'info' }); setTimeout(() => { @@ -325,19 +332,23 @@ const ChatModelEval: React.FC = () => { // =============== Cleanup (Left / Right) =============== const handleCleanupLeft = () => { setMessagesLeft([]); + setCurrentMessageLeft([]); setAlertMessageLeft(undefined); }; const handleCleanupRight = () => { setMessagesRight([]); + setCurrentMessageRight([]); setAlertMessageRight(undefined); }; const handleSend = async (side: 'left' | 'right', message: string) => { - const trimmedMessage = message.trim(); + const question = message.trim(); + const userMsgId = `${Date.now()}_user_${side}`; + const botMsgId = `${Date.now()}_bot_${side}`; // Prevent sending empty messages - if (!trimmedMessage) { + if (!question) { console.warn(`Attempted to send an empty message on the ${side} side.`); return; } @@ -363,12 +374,47 @@ const ChatModelEval: React.FC = () => { return; } + if (side === 'left') { + if (currentMessageLeft.length > 0) { + const botMessage: MessageProps = { + avatar: logo.src, + id: botMsgId, + name: 'Bot', + role: 'bot', + content: currentMessageLeft.join(''), + timestamp: new Date().toLocaleTimeString(), + isLoading: false, + actions: { + copy: { onClick: () => navigator.clipboard.writeText(currentMessageLeft.join('') || '') } + } + }; + setCurrentMessageLeft([]); + setMessagesLeft((msgs) => [...msgs, botMessage]); + } + } else { + if (currentMessageRight.length > 0) { + const botMessage: MessageProps = { + avatar: logo.src, + id: botMsgId, + name: 'Bot', + role: 'bot', + content: currentMessageRight.join(''), + timestamp: new Date().toLocaleTimeString(), + isLoading: false, + actions: { + copy: { onClick: () => navigator.clipboard.writeText(currentMessageRight.join('') || '') } + } + }; + setCurrentMessageRight([]); + setMessagesRight((msgs) => [...msgs, botMessage]); + } + } + // Add the user's message to the chat - const userMsgId = `${Date.now()}_user_${side}`; const userMessage: MessageProps = { id: userMsgId, role: 'user', - content: trimmedMessage, + content: question, name: 'User', avatar: userLogo.src, timestamp: new Date().toLocaleTimeString() @@ -382,28 +428,10 @@ const ChatModelEval: React.FC = () => { setQuestionRight(''); } - // Add a loading message from the bot - const botMsgId = `${Date.now()}_bot_${side}`; - const botMessage: MessageProps = { - id: botMsgId, - role: 'bot', - content: '', - name: 'Bot', - avatar: logo.src, - timestamp: new Date().toLocaleTimeString(), - isLoading: true - }; - - if (side === 'left') { - setMessagesLeft((msgs) => [...msgs, botMessage]); - } else { - setMessagesRight((msgs) => [...msgs, botMessage]); - } - // Prepare the payload for the backend const messagesPayload = [ { role: 'system', content: systemRole }, - { role: 'user', content: trimmedMessage } + { role: 'user', content: question } ]; const chatModelName = mapModelName(selectedModel.modelName); @@ -415,17 +443,18 @@ const ChatModelEval: React.FC = () => { }; console.log('Sending message to chat endpoint:', selectedModel.apiURL); - try { - // Send the message to the backend - const response = await fetch(`${selectedModel.apiURL}/v1/chat/completions`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Accept: 'text/event-stream' - }, - body: JSON.stringify(requestData) - }); + // Default endpoints (server-side fetch) + const response = await fetch( + `/api/playground/chat?apiURL=${encodeURIComponent(selectedModel.apiURL)}&modelName=${encodeURIComponent(requestData.model)}`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ question, systemRole }) + } + ); if (!response.ok) { const errorText = await response.text(); @@ -441,62 +470,35 @@ const ChatModelEval: React.FC = () => { timestamp: new Date().toLocaleTimeString(), isLoading: false }; - if (side === 'left') { - setMessagesLeft((msgs) => msgs.map((msg) => (msg.id === botMsgId ? errorMessage : msg))); + setMessagesLeft((msgs) => [...msgs, errorMessage]); } else { - setMessagesRight((msgs) => msgs.map((msg) => (msg.id === botMsgId ? errorMessage : msg))); + setMessagesRight((msgs) => [...msgs, errorMessage]); } return; } - - if (!response.body) { - console.error(`No response body received from ${side} side.`); - return; - } - - const reader = response.body.getReader(); - const decoder = new TextDecoder('utf-8'); - let doneReading = false; - let botContent = ''; - - while (!doneReading) { - const { value, done: isDone } = await reader.read(); - doneReading = isDone; - if (value) { - const chunk = decoder.decode(value, { stream: true }); - const lines = chunk.split('\n').filter((line) => line.trim() !== ''); - for (const line of lines) { - if (line.startsWith('data: ')) { - const data = line.replace('data: ', ''); - if (data === '[DONE]') { - doneReading = true; - break; - } - try { - const parsed = JSON.parse(data); - const delta = parsed.choices[0].delta?.content; - if (delta) { - botContent += delta; - if (side === 'left') { - setMessagesLeft((msgs) => msgs.map((msg) => (msg.id === botMsgId ? { ...msg, content: botContent } : msg))); - } else { - setMessagesRight((msgs) => msgs.map((msg) => (msg.id === botMsgId ? { ...msg, content: botContent } : msg))); - } - } - } catch (e) { - console.error('Error parsing JSON:', e); - } + if (response.body) { + const reader = response.body.getReader(); + const textDecoder = new TextDecoder('utf-8'); + let botMessage = ''; + + (async () => { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + const chunk = textDecoder.decode(value, { stream: true }); + botMessage = chunk; + + if (side === 'left') { + setCurrentMessageLeft((prevMsg) => [...prevMsg, botMessage]); + } else { + setCurrentMessageRight((prevMsg) => [...prevMsg, botMessage]); } } - } - } - - // Finalize the bot message by removing the loading state - if (side === 'left') { - setMessagesLeft((msgs) => msgs.map((msg) => (msg.id === botMsgId ? { ...msg, isLoading: false } : msg))); + })(); } else { - setMessagesRight((msgs) => msgs.map((msg) => (msg.id === botMsgId ? { ...msg, isLoading: false } : msg))); + console.error(`No response body received from ${side} side.`); + return; } } catch (error) { console.error(`Error fetching chat response on the ${side} side:`, error); @@ -513,9 +515,9 @@ const ChatModelEval: React.FC = () => { }; if (side === 'left') { - setMessagesLeft((msgs) => msgs.map((msg) => (msg.id === botMsgId ? errorMessage : msg))); + setMessagesLeft((msgs) => [...msgs, errorMessage]); } else { - setMessagesRight((msgs) => msgs.map((msg) => (msg.id === botMsgId ? errorMessage : msg))); + setMessagesRight((msgs) => [...msgs, errorMessage]); } } }; @@ -528,31 +530,6 @@ const ChatModelEval: React.FC = () => { handleSend('right', message); }; - // Add a copy action to bot messages when they are fully loaded - const transformedMessagesLeft = messagesLeft.map((m) => { - if (m.role === 'bot' && m.content && !m.isLoading) { - return { - ...m, - actions: { - copy: { onClick: () => navigator.clipboard.writeText(m.content || '') } - } - }; - } - return m; - }); - - const transformedMessagesRight = messagesRight.map((m) => { - if (m.role === 'bot' && m.content && !m.isLoading) { - return { - ...m, - actions: { - copy: { onClick: () => navigator.clipboard.writeText(m.content || '') } - } - }; - } - return m; - }); - const handleToggleLogsLeft = async (jobId: string, isExpanding: boolean) => { setExpandedJobsLeft((prev) => ({ ...prev, [jobId]: isExpanding })); if (isExpanding && !jobLogsLeft[jobId]) { @@ -687,6 +664,7 @@ const ChatModelEval: React.FC = () => { {/* Unload button (pre-train) */}