diff --git a/api-server/rhelai-install/rhelai-install.sh b/api-server/rhelai-install/rhelai-install.sh index 5bedeb60..162260f6 100755 --- a/api-server/rhelai-install/rhelai-install.sh +++ b/api-server/rhelai-install/rhelai-install.sh @@ -22,11 +22,11 @@ if [ -d "/tmp/api-server" ]; then fi mkdir -p /tmp/api-server -cd /tmp/ui/api-server -wget https://instructlab-ui.s3.us-east-1.amazonaws.com/apiserver/apiserver-linux-amd64.tar.gz -tar -xzf apiserver-linux-amd64.tar.gz -mv apiserver-linux-amd64/ilab-apiserver /usr/local/sbin -rm -rf apiserver-linux-amd64 apiserver-linux-amd64.tar.gz +cd /tmp/api-server +curl -sLO https://instructlab-ui.s3.us-east-1.amazonaws.com/apiserver/apiserver-linux-amd64.tar.gz +tar -xzf apiserver-linux-amd64.tar.gz +mv apiserver-linux-amd64/ilab-apiserver $HOME/.local/bin +rm -rf apiserver-linux-amd64 apiserver-linux-amd64.tar.gz /tmp/api-server CUDA_FLAG="" diff --git a/src/app/api/playground/chat/route.ts b/src/app/api/playground/chat/route.ts index beb9f65e..84f4b66d 100644 --- a/src/app/api/playground/chat/route.ts +++ b/src/app/api/playground/chat/route.ts @@ -10,6 +10,7 @@ export async function POST(req: NextRequest) { const { question, systemRole } = await req.json(); const apiURL = req.nextUrl.searchParams.get('apiURL'); const modelName = req.nextUrl.searchParams.get('modelName'); + const apiKey = req.nextUrl.searchParams.get('apiKey') if (!apiURL || !modelName) { return new NextResponse('Missing API URL or Model Name', { status: 400 }); @@ -26,16 +27,34 @@ export async function POST(req: NextRequest) { stream: true }; - const agent = new https.Agent({ - rejectUnauthorized: false - }); + let agent: https.Agent + if (apiKey && apiKey != "" ) { + agent = new https.Agent({ + rejectUnauthorized: true + }); + } else { + agent = new https.Agent({ + rejectUnauthorized: false + }); + } + + var headers: HeadersInit + if (apiKey && apiKey != "" ) { + headers = { + 'Content-Type': 'application/json', + 'Accept': 'text/event-stream', + 'Authorization': `Bearer: ${apiKey}` + } + } else { + headers = { + 'Content-Type': 'application/json', + 'Accept': 'text/event-stream' + } + } const chatResponse = await fetch(`${apiURL}/v1/chat/completions`, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - accept: 'application/json' - }, + headers: headers, body: JSON.stringify(requestData), agent: apiURL.startsWith('https') ? agent : undefined }); diff --git a/src/app/playground/endpoints/endpointPage.css b/src/app/playground/endpoints/endpointPage.css new file mode 100644 index 00000000..444cb2da --- /dev/null +++ b/src/app/playground/endpoints/endpointPage.css @@ -0,0 +1,5 @@ +.disabled-endpoint { + opacity: 0.4; /* Faded look */ + background-color: #f5f5f5; /* Light gray background (optional) */ + color: #6a6e73; /* PatternFly disabled text color */ + } diff --git a/src/app/playground/endpoints/page.tsx b/src/app/playground/endpoints/page.tsx index 2f5e2e16..946ec337 100644 --- a/src/app/playground/endpoints/page.tsx +++ b/src/app/playground/endpoints/page.tsx @@ -1,9 +1,9 @@ 'use client'; -import React, { useState, useEffect } from 'react'; +import React, { ReactNode, useState, useEffect } from 'react'; import { v4 as uuidv4 } from 'uuid'; import { AppLayout } from '@/components/AppLayout'; -import { Endpoint } from '@/types'; +import { Endpoint, EndpointRequiredFields, ModelEndpointStatus } from '@/types'; import { Breadcrumb, BreadcrumbItem, @@ -15,11 +15,15 @@ import { DataListItem, DataListItemCells, DataListItemRow, + Dropdown, + DropdownList, + DropdownItem, Flex, FlexItem, Form, FormGroup, InputGroup, + MenuToggle, Modal, ModalBody, ModalFooter, @@ -27,10 +31,53 @@ import { ModalVariant, PageBreadcrumb, PageSection, + Popover, TextInput, - Title + Title, + ValidatedOptions } from '@patternfly/react-core'; -import { EyeSlashIcon, EyeIcon } from '@patternfly/react-icons'; +import { BanIcon, CheckCircleIcon, EyeSlashIcon, EllipsisVIcon , EyeIcon, QuestionCircleIcon } from '@patternfly/react-icons'; + +import './endpointPage.css'; +import { status } from 'isomorphic-git'; + +const availableModelEndpointStatusIcon: ReactNode = + +const unavailableModelEndpointStatusIcon: ReactNode = + +const unknownModelEndpointStatusIcon: ReactNode = + + +async function checkEndpointStatus( + endpointURL: string, + modelName: string, + apiKey: string +): Promise { + console.log("checking the model endpoint") + let headers; + if (apiKey != "") { + headers = { + "Content-Type": "application/json", + "Authorization": `Bearer ${apiKey}` + }; + } else { + headers = { + "Content-Type": "application/json", + }; + } + try { + const response = await fetch(`${endpointURL}/v1/models/${modelName}`, { + headers: headers + }); + if (response.ok) { + return ModelEndpointStatus.available; + } else { + return ModelEndpointStatus.unavailable; + } + } catch (error) { + return ModelEndpointStatus.unknown; + } +} interface ExtendedEndpoint extends Endpoint { isApiKeyVisible?: boolean; @@ -40,9 +87,18 @@ const EndpointsPage: React.FC = () => { const [endpoints, setEndpoints] = useState([]); const [isModalOpen, setIsModalOpen] = useState(false); const [currentEndpoint, setCurrentEndpoint] = useState | null>(null); + const [endpointName, setEndpointName] = useState(''); + const [endpointDescription, setEndpointDescription] = useState(''); const [url, setUrl] = useState(''); const [modelName, setModelName] = useState(''); + const [modelDescription, setModelDescription] = useState(''); const [apiKey, setApiKey] = useState(''); + const [endpointStatus, setEndpointStatus] = useState(ModelEndpointStatus.unknown); + const [endpointEnabled, setEndpointEnabled] = useState(true); + const [endpointOptionsOpen, setEndpointOptionsOpen] = React.useState(false); + const [endpointOptionsID, setEndpointOptionsID] = React.useState(''); + const [deleteEndpointModalOpen, setDeleteEndpointModalOpen] = React.useState(false); + const [deleteEndpointName, setDeleteEndpointName] = useState(''); useEffect(() => { const storedEndpoints = localStorage.getItem('endpoints'); @@ -51,6 +107,51 @@ const EndpointsPage: React.FC = () => { } }, []); + useEffect(() => { + async function updateEndpointStatuses() { + const updatedEndpoints = await Promise.all( + endpoints.map(async (endpoint) => { + const status = await checkEndpointStatus( + endpoint.url, + endpoint.modelName, + endpoint.apiKey + ); + + return { + ...endpoint, + status, + }; + }) + ); + + setEndpoints(updatedEndpoints); + } + + const interval = setInterval(() => { + console.log("Running update endpoints") + updateEndpointStatuses(); + }, 10 * 60 * 1000); // run every 10 minutes in miliseconds + + return () => clearInterval(interval); // cleanup on unmount + }, [endpoints]); + + const toggleEndpointEnabled = (endpointId: string, endpointEnabled: boolean) => { + var newEndpointEnabledStatus: boolean + var updatedEndpoints: ExtendedEndpoint[] + endpointEnabled ? newEndpointEnabledStatus = false : newEndpointEnabledStatus = true + updatedEndpoints = endpoints.map(endpoint => { + if (endpoint.id === endpointId) { + return { + ...endpoint, + enabled: newEndpointEnabledStatus, + }; + } + return endpoint; + }); + setEndpoints(updatedEndpoints) + localStorage.setItem('endpoints', JSON.stringify(updatedEndpoints)); + } + const handleModalToggle = () => { setIsModalOpen(!isModalOpen); }; @@ -65,50 +166,106 @@ const EndpointsPage: React.FC = () => { return inputUrl; }; - const handleSaveEndpoint = () => { + const validateEndpointData = (endpoint: ExtendedEndpoint): boolean => { + let returnValue = true + EndpointRequiredFields.forEach((requiredField) => { + if (requiredField != "enabled" && requiredField != "status") { + if (endpoint[requiredField]?.trim().length == 0) { + returnValue = false + } + } + }) + return returnValue + } + + async function handleSaveEndpoint () { const updatedUrl = removeTrailingSlash(url); + const status = await checkEndpointStatus(updatedUrl, modelName, apiKey) if (currentEndpoint) { const updatedEndpoint: ExtendedEndpoint = { id: currentEndpoint.id || uuidv4(), + name: endpointName, + description: endpointDescription, url: updatedUrl, modelName: modelName, + modelDescription: modelDescription, apiKey: apiKey, - isApiKeyVisible: false + isApiKeyVisible: false, + status: status, + enabled: true }; - - const updatedEndpoints = currentEndpoint.id + if (validateEndpointData(updatedEndpoint) == true) { + const updatedEndpoints = currentEndpoint.id ? endpoints.map((ep) => (ep.id === currentEndpoint.id ? updatedEndpoint : ep)) : [...endpoints, updatedEndpoint]; - setEndpoints(updatedEndpoints); - localStorage.setItem('endpoints', JSON.stringify(updatedEndpoints)); - setCurrentEndpoint(null); - setUrl(''); - setModelName(''); - setApiKey(''); - handleModalToggle(); + setEndpoints(updatedEndpoints); + localStorage.setItem('endpoints', JSON.stringify(updatedEndpoints)); + setCurrentEndpoint(null); + setEndpointName(''); + setEndpointDescription(''); + setUrl(''); + setModelName(''); + setModelDescription(''); + setApiKey(''); + setEndpointStatus(ModelEndpointStatus.unknown) + setEndpointEnabled(true) + handleModalToggle(); + } else { + alert("error: please make sure all the required fields are set!") + } } }; - const handleDeleteEndpoint = (id: string) => { - const updatedEndpoints = endpoints.filter((ep) => ep.id !== id); - setEndpoints(updatedEndpoints); - localStorage.setItem('endpoints', JSON.stringify(updatedEndpoints)); + const handleDeleteEndpoint = (id: string, endpointName: string) => { + if (deleteEndpointName && deleteEndpointName == endpointName) { + const updatedEndpoints = endpoints.filter((ep) => ep.id !== id); + setEndpoints(updatedEndpoints); + localStorage.setItem('endpoints', JSON.stringify(updatedEndpoints)); + setDeleteEndpointModalOpen(false) + setDeleteEndpointName('') + } else { + alert("error: endpoint name did not match!") + } }; - const handleEditEndpoint = (endpoint: ExtendedEndpoint) => { + async function handleEditEndpoint (endpoint: ExtendedEndpoint) { + const updatedUrl = removeTrailingSlash(endpoint.url); + const status = await checkEndpointStatus(updatedUrl, endpoint.modelName, endpoint.apiKey) setCurrentEndpoint(endpoint); - setUrl(endpoint.url); + setEndpointName(endpoint.name) + setEndpointDescription(endpoint.description || '') + setUrl(updatedUrl); setModelName(endpoint.modelName); + setModelDescription(endpoint.modelDescription || ''); setApiKey(endpoint.apiKey); + setEndpointStatus(status) handleModalToggle(); }; const handleAddEndpoint = () => { - setCurrentEndpoint({ id: '', url: '', modelName: '', apiKey: '', isApiKeyVisible: false }); + setCurrentEndpoint( + { + id: '', + name: '', + description: '', + url: '', + modelName: '', + modelDescription: '', + apiKey: '', + isApiKeyVisible: false, + status: ModelEndpointStatus.unknown, + enabled: true + } + ); + setEndpointName(''); + setEndpointDescription(''); setUrl(''); setModelName(''); + setModelDescription(''); setApiKey(''); + setEndpointStatus(ModelEndpointStatus.unknown) + setEndpointEnabled(true) handleModalToggle(); }; @@ -136,7 +293,6 @@ const EndpointsPage: React.FC = () => { Custom Model Endpoints - @@ -152,25 +308,64 @@ const EndpointsPage: React.FC = () => { - - + + Add Custom Endpoint + + + + Endpoint Name + , + + Endpoint Status + , + + URL + , + + Model Name + , + + API Key + + ]} + /> + + {endpoints.map((endpoint) => ( - - + + - URL: {endpoint.url} + + {endpoint.name} + + {endpoint.description} , - - Model Name: {endpoint.modelName} + {ModelEndpointStatus[endpoint.status]} {(() => { + switch (endpoint.status) { + case ModelEndpointStatus.available: + return availableModelEndpointStatusIcon; + case ModelEndpointStatus.unavailable: + return unavailableModelEndpointStatusIcon; + case ModelEndpointStatus.unknown: + return unknownModelEndpointStatusIcon; + default: + return unknownModelEndpointStatusIcon; + } + })()} , + {endpoint.url} , + + {endpoint.modelName} + + {endpoint.modelDescription} , - - API Key: {renderApiKey(endpoint.apiKey, endpoint.isApiKeyVisible || false)} + + {renderApiKey(endpoint.apiKey, endpoint.isApiKeyVisible || false)} toggleApiKeyVisibility(endpoint.id)}> {endpoint.isApiKeyVisible ? : } @@ -178,12 +373,79 @@ const EndpointsPage: React.FC = () => { ]} /> - handleEditEndpoint(endpoint)}> - Edit - - handleDeleteEndpoint(endpoint.id)}> - Delete - + {endpoint.enabled == true ? ( + { + toggleEndpointEnabled(endpoint.id, endpoint.enabled) + }}> + disable + + ): ( + { + toggleEndpointEnabled(endpoint.id, endpoint.enabled) + }}> + enable + + )} + setEndpointOptionsOpen(false)} + toggle={(toggleRef) => ( + { + setEndpointOptionsOpen(!endpointOptionsOpen); + setEndpointOptionsID(endpoint.id); + }} + isExpanded={endpointOptionsOpen && endpointOptionsID === endpoint.id} + > + + + )} + isOpen={endpointOptionsOpen && endpointOptionsID === endpoint.id} + popperProps={{ position: 'right' }} + ouiaId="ModelEndpointDropdown" + > + + handleEditEndpoint(endpoint)}>Edit Endpoint + setDeleteEndpointModalOpen(true)}>Delete Endpoint + + + {deleteEndpointModalOpen ? ( + setDeleteEndpointModalOpen(false)} + aria-labelledby="confirm-delete-custom-model-endpoint" + aria-describedby="show-yaml-body-variant" + > + + + The {endpoint.name} custom model endpoint will be deleted. + + Type {endpoint.name} to confirm. + setDeleteEndpointName(value)} + /> + + + {handleDeleteEndpoint(endpoint.id, endpoint.name)}}> + Delete + + , + {setDeleteEndpointName(''); setDeleteEndpointModalOpen(false)}}> + Cancel + + + + ) : null} @@ -201,9 +463,57 @@ const EndpointsPage: React.FC = () => { > + + + Add a custom model endpoint for chat the interface. Use it to compare or interact with remote hosted models. + + - - setUrl(value)} placeholder="Enter URL" /> + + setEndpointName(value)} + placeholder="Enter name" + /> + + + setEndpointDescription(value)} + placeholder="Enter description" + /> + + + + + + + } + > + setUrl(value)} + placeholder="Enter URL" + /> { placeholder="Enter Model Name" /> - + + setModelDescription(value)} + placeholder="Enter description" + /> + + + + + + + }> = ({ const toggle = (toggleRef: React.Ref) => ( - {model ? model.name : 'Select a model'} + {model && model.enabled ? model.name : 'Select a model'} ); const dropdownItems = React.useMemo( () => availableModels.map((model, index) => ( - + {model.name} )), @@ -227,6 +228,20 @@ const ChatBotComponent: React.FunctionComponent = ({ > {dropdownItems} + Can't select your model?} + bodyContent={ + If you model is not selectable, that means you have disabled the custom model endpoint. + + To change this please see the Custom Model Endpoints page. + + } + > + + + + {showCompare ? ( diff --git a/src/components/Chat/ModelsContext.tsx b/src/components/Chat/ModelsContext.tsx index 57c2c6f1..63f2faa0 100644 --- a/src/components/Chat/ModelsContext.tsx +++ b/src/components/Chat/ModelsContext.tsx @@ -29,18 +29,19 @@ const ModelsContextProvider: React.FC = ({ children }) => { const envConfig = await response.json(); const defaultModels: Model[] = [ - { isDefault: true, name: 'Granite-7b', apiURL: envConfig.GRANITE_API, modelName: envConfig.GRANITE_MODEL_NAME }, - { isDefault: true, name: 'Merlinite-7b', apiURL: envConfig.MERLINITE_API, modelName: envConfig.MERLINITE_MODEL_NAME } + { isDefault: true, name: 'Granite-7b', apiURL: envConfig.GRANITE_API, modelName: envConfig.GRANITE_MODEL_NAME, enabled: true }, + { isDefault: true, name: 'Merlinite-7b', apiURL: envConfig.MERLINITE_API, modelName: envConfig.MERLINITE_MODEL_NAME, enabled: true } ]; const storedEndpoints = localStorage.getItem('endpoints'); - const customModels = storedEndpoints ? JSON.parse(storedEndpoints).map((endpoint: Endpoint) => ({ isDefault: false, name: endpoint.modelName, apiURL: `${endpoint.url}`, - modelName: endpoint.modelName + modelName: endpoint.modelName, + enabled: endpoint.enabled, + apiKey: endpoint.apiKey })) : []; diff --git a/src/components/Chat/modelService.ts b/src/components/Chat/modelService.ts index 068d4d6c..a1240bf2 100644 --- a/src/components/Chat/modelService.ts +++ b/src/components/Chat/modelService.ts @@ -26,12 +26,22 @@ export const customModelFetcher = async ( // Client-side fetch if the selected model is a custom endpoint try { + var headers: HeadersInit + if (selectedModel.apiKey && selectedModel.apiKey != "" ) { + headers = { + 'Content-Type': 'application/json', + 'Accept': 'text/event-stream', + 'Authorization': `Bearer: ${selectedModel.apiKey}` + } + } else { + headers = { + 'Content-Type': 'application/json', + 'Accept': 'text/event-stream' + } + } const response = await fetch(`${selectedModel.apiURL}/v1/chat/completions`, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - Accept: 'text/event-stream' - }, + headers: headers, body: JSON.stringify(requestData), signal: newController.signal }); diff --git a/src/types/index.ts b/src/types/index.ts index 2afaefa9..2fc7e5f0 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,17 +1,34 @@ +import { ReactNode } from 'react'; import { ValidatedOptions } from '@patternfly/react-core'; + +export enum ModelEndpointStatus { + available, + unavailable, + unknown +} + export interface Endpoint { id: string; + name: string; + description?: string; url: string; - apiKey: string; modelName: string; + modelDescription?: string; + apiKey: string; + status: ModelEndpointStatus; + enabled: boolean; } +export const EndpointRequiredFields: (keyof Endpoint)[] = ["name", "url", "modelName"]; + export interface Model { isDefault?: boolean; name: string; apiURL: string; modelName: string; + enabled: boolean; + apiKey?: string; } export interface Label {
{endpoint.name}
{endpoint.description}
{endpoint.modelName}
{endpoint.modelDescription}
The {endpoint.name} custom model endpoint will be deleted.
Type {endpoint.name} to confirm.
+ Add a custom model endpoint for chat the interface. Use it to compare or interact with remote hosted models. +