From 46470bea23c06a073d45fce2d6fa11bd8f6e33ff Mon Sep 17 00:00:00 2001 From: "Ben Houston (via MyCoder)" Date: Fri, 14 Mar 2025 11:31:45 +0000 Subject: [PATCH 1/3] docs: add documentation for new providers and update baseUrl references --- README.md | 4 +- packages/docs/docs/providers/index.mdx | 4 + packages/docs/docs/providers/local-openai.md | 123 +++++++++++++++++++ packages/docs/docs/providers/ollama.md | 2 +- packages/docs/docs/providers/xai.md | 80 ++++++++++++ 5 files changed, 210 insertions(+), 3 deletions(-) create mode 100644 packages/docs/docs/providers/local-openai.md create mode 100644 packages/docs/docs/providers/xai.md diff --git a/README.md b/README.md index 123233c..ff84d25 100644 --- a/README.md +++ b/README.md @@ -83,8 +83,8 @@ export default { profile: false, tokenCache: true, - // Ollama configuration (if using local models) - ollamaBaseUrl: 'http://localhost:11434', + // Base URL configuration (for providers that need it) + baseUrl: 'http://localhost:11434', // Example for Ollama }; ``` diff --git a/packages/docs/docs/providers/index.mdx b/packages/docs/docs/providers/index.mdx index 2675d8c..cde7c87 100644 --- a/packages/docs/docs/providers/index.mdx +++ b/packages/docs/docs/providers/index.mdx @@ -13,6 +13,8 @@ MyCoder currently supports the following LLM providers: - [**Anthropic**](./anthropic.md) - Claude models from Anthropic - [**OpenAI**](./openai.md) - GPT models from OpenAI - [**Ollama**](./ollama.md) - Self-hosted open-source models via Ollama +- [**Local OpenAI Compatible**](./local-openai.md) - GPUStack and other OpenAI-compatible servers +- [**xAI**](./xai.md) - Grok models from xAI ## Configuring Providers @@ -52,3 +54,5 @@ For detailed instructions on setting up each provider, see the provider-specific - [Anthropic Configuration](./anthropic.md) - [OpenAI Configuration](./openai.md) - [Ollama Configuration](./ollama.md) +- [Local OpenAI Compatible Configuration](./local-openai.md) +- [xAI Configuration](./xai.md) diff --git a/packages/docs/docs/providers/local-openai.md b/packages/docs/docs/providers/local-openai.md new file mode 100644 index 0000000..caf2cfa --- /dev/null +++ b/packages/docs/docs/providers/local-openai.md @@ -0,0 +1,123 @@ +--- +sidebar_position: 5 +--- + +# Local OpenAI Compatible Servers + +MyCoder supports connecting to local or self-hosted OpenAI-compatible API servers, including solutions like [GPUStack](https://gpustack.ai/), [LM Studio](https://lmstudio.ai/), [Ollama OpenAI compatibility mode](https://github.com/ollama/ollama/blob/main/docs/openai.md), and [LocalAI](https://localai.io/). + +## Setup + +To use a local OpenAI-compatible server with MyCoder: + +1. Install and set up your preferred OpenAI-compatible server +2. Start the server according to its documentation +3. Configure MyCoder to connect to your local server + +### Configuration + +Configure MyCoder to use your local OpenAI-compatible server in your `mycoder.config.js` file: + +```javascript +export default { + // Provider selection - use gpustack for any OpenAI-compatible server + provider: 'gpustack', + model: 'llama3.2', // Use the model name available on your server + + // The base URL for your local server + baseUrl: 'http://localhost:80', // Default for GPUStack, adjust as needed + + // Other MyCoder settings + maxTokens: 4096, + temperature: 0.7, + // ... +}; +``` + +## GPUStack + +[GPUStack](https://gpustack.ai/) is a solution for running AI models on your own hardware. It provides an OpenAI-compatible API server that works seamlessly with MyCoder. + +### Setting up GPUStack + +1. Install GPUStack following the instructions on their website +2. Start the GPUStack server +3. Configure MyCoder to use the `gpustack` provider + +```javascript +export default { + provider: 'gpustack', + model: 'llama3.2', // Choose a model available on your GPUStack instance + baseUrl: 'http://localhost:80', // Default GPUStack URL +}; +``` + +## Other OpenAI-Compatible Servers + +You can use MyCoder with any OpenAI-compatible server by setting the appropriate `baseUrl`: + +### LM Studio + +```javascript +export default { + provider: 'gpustack', + model: 'llama3', // Use the model name as configured in LM Studio + baseUrl: 'http://localhost:1234', // Default LM Studio server URL +}; +``` + +### LocalAI + +```javascript +export default { + provider: 'gpustack', + model: 'gpt-3.5-turbo', // Use the model name as configured in LocalAI + baseUrl: 'http://localhost:8080', // Default LocalAI server URL +}; +``` + +### Ollama (OpenAI Compatibility Mode) + +```javascript +export default { + provider: 'gpustack', + model: 'llama3', // Use the model name as configured in Ollama + baseUrl: 'http://localhost:11434/v1', // Ollama OpenAI compatibility endpoint +}; +``` + +## Hardware Requirements + +Running LLMs locally requires significant hardware resources: + +- Minimum 16GB RAM (32GB+ recommended) +- GPU with at least 8GB VRAM for optimal performance +- SSD storage for model files (models can be 5-20GB each) + +## Best Practices + +- Ensure your local server and the selected model support tool calling/function calling +- Use models optimized for coding tasks when available +- Monitor your system resources when running large models locally +- Consider using a dedicated machine for hosting your local server + +## Troubleshooting + +If you encounter issues with local OpenAI-compatible servers: + +- Verify the server is running and accessible at the configured base URL +- Check that the model name exactly matches what's available on your server +- Ensure the model supports tool/function calling (required for MyCoder) +- Check server logs for specific error messages +- Test the server with a simple curl command to verify API compatibility: + +```bash +curl http://localhost:80/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "llama3.2", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + +For more information, refer to the documentation for your specific OpenAI-compatible server. \ No newline at end of file diff --git a/packages/docs/docs/providers/ollama.md b/packages/docs/docs/providers/ollama.md index 84c31b7..1425890 100644 --- a/packages/docs/docs/providers/ollama.md +++ b/packages/docs/docs/providers/ollama.md @@ -62,7 +62,7 @@ export default { model: 'medragondot/Sky-T1-32B-Preview:latest', // Optional: Custom base URL (defaults to http://localhost:11434) - // ollamaBaseUrl: 'http://localhost:11434', + // baseUrl: 'http://localhost:11434', // Other MyCoder settings maxTokens: 4096, diff --git a/packages/docs/docs/providers/xai.md b/packages/docs/docs/providers/xai.md new file mode 100644 index 0000000..7d1d605 --- /dev/null +++ b/packages/docs/docs/providers/xai.md @@ -0,0 +1,80 @@ +--- +sidebar_position: 6 +--- + +# xAI (Grok) + +[xAI](https://x.ai/) is the company behind Grok, a powerful large language model designed to be helpful, harmless, and honest. Grok models offer strong reasoning capabilities and support for tool calling. + +## Setup + +To use Grok models with MyCoder, you need an xAI API key: + +1. Create an account at [xAI](https://x.ai/) +2. Navigate to the API Keys section and create a new API key +3. Set the API key as an environment variable or in your configuration file + +### Environment Variables + +You can set the xAI API key as an environment variable: + +```bash +export XAI_API_KEY=your_api_key_here +``` + +### Configuration + +Configure MyCoder to use xAI's Grok in your `mycoder.config.js` file: + +```javascript +export default { + // Provider selection + provider: 'xai', + model: 'grok-2-latest', + + // Optional: Set API key directly (environment variable is preferred) + // xaiApiKey: 'your_api_key_here', + + // Other MyCoder settings + maxTokens: 4096, + temperature: 0.7, + // ... +}; +``` + +## Supported Models + +xAI offers several Grok models with different capabilities: + +- `grok-2-latest` (recommended) - The latest Grok-2 model with strong reasoning and tool-calling capabilities +- `grok-1` - The original Grok model + +## Best Practices + +- Grok models excel at coding tasks and technical problem-solving +- They have strong tool-calling capabilities, making them suitable for MyCoder workflows +- For complex programming tasks, use Grok-2 models for best results +- Provide clear, specific instructions for optimal results + +## Custom Base URL + +If you need to use a different base URL for the xAI API (for example, if you're using a proxy or if xAI changes their API endpoint), you can specify it in your configuration: + +```javascript +export default { + provider: 'xai', + model: 'grok-2-latest', + baseUrl: 'https://api.x.ai/v1', // Default xAI API URL +}; +``` + +## Troubleshooting + +If you encounter issues with xAI's Grok: + +- Verify your API key is correct and has sufficient quota +- Check that you're using a supported model name +- For tool-calling issues, ensure your functions are properly formatted +- Monitor your token usage to avoid unexpected costs + +For more information, visit the [xAI Documentation](https://x.ai/docs). \ No newline at end of file From 9359f62e5b2f66c0db76bf9bb00161eb6964a888 Mon Sep 17 00:00:00 2001 From: Ben Houston Date: Fri, 14 Mar 2025 09:56:47 -0400 Subject: [PATCH 2/3] fix: perfect gpustack compatibility, fix openai edge case --- mycoder.config.js | 3 + .../src/core/llm/__tests__/openai.test.ts | 2 +- packages/agent/src/core/llm/provider.ts | 6 - .../agent/src/core/llm/providers/openai.ts | 2 +- packages/cli/src/commands/$default.ts | 15 ++- packages/docs/docs/providers/anthropic.md | 3 - packages/docs/docs/providers/index.mdx | 4 +- packages/docs/docs/providers/local-openai.md | 123 ------------------ packages/docs/docs/providers/openai.md | 22 +++- packages/docs/docs/providers/xai.md | 5 +- 10 files changed, 36 insertions(+), 149 deletions(-) delete mode 100644 packages/docs/docs/providers/local-openai.md diff --git a/mycoder.config.js b/mycoder.config.js index 01663a5..e8a6e82 100644 --- a/mycoder.config.js +++ b/mycoder.config.js @@ -18,6 +18,9 @@ export default { //model: 'llama3.2:3b', //provider: 'xai', //model: 'grok-2-latest', + //provider: 'openai', + //model: 'qwen2.5-coder:14b', + //baseUrl: 'http://192.168.2.66:80/v1-openai', maxTokens: 4096, temperature: 0.7, diff --git a/packages/agent/src/core/llm/__tests__/openai.test.ts b/packages/agent/src/core/llm/__tests__/openai.test.ts index 0f144b9..93048f6 100644 --- a/packages/agent/src/core/llm/__tests__/openai.test.ts +++ b/packages/agent/src/core/llm/__tests__/openai.test.ts @@ -177,7 +177,7 @@ describe('OpenAIProvider', () => { 'role' in toolUseMessage ) { expect(toolUseMessage.role).toBe('assistant'); - expect(toolUseMessage.content).toBe(null); + expect(toolUseMessage.content).toBe(''); // required by gpustack' implementation of openai SDK. if ( 'tool_calls' in toolUseMessage && diff --git a/packages/agent/src/core/llm/provider.ts b/packages/agent/src/core/llm/provider.ts index 1b6e49e..ab01cb3 100644 --- a/packages/agent/src/core/llm/provider.ts +++ b/packages/agent/src/core/llm/provider.ts @@ -57,12 +57,6 @@ export const providerConfig: Record = { model: 'gpt-4o-2024-05-13', factory: (model, options) => new OpenAIProvider(model, options), }, - gpustack: { - docsUrl: 'https://mycoder.ai/docs/provider/local-openai', - model: 'llama3.2', - baseUrl: 'http://localhost:80', - factory: (model, options) => new OpenAIProvider(model, options), - }, ollama: { docsUrl: 'https://mycoder.ai/docs/provider/ollama', model: 'llama3.2', diff --git a/packages/agent/src/core/llm/providers/openai.ts b/packages/agent/src/core/llm/providers/openai.ts index f216672..ee1c235 100644 --- a/packages/agent/src/core/llm/providers/openai.ts +++ b/packages/agent/src/core/llm/providers/openai.ts @@ -154,7 +154,7 @@ export class OpenAIProvider implements LLMProvider { // so we'll include it as a function call in an assistant message return { role: 'assistant', - content: null, + content: '', tool_calls: [ { id: msg.id, diff --git a/packages/cli/src/commands/$default.ts b/packages/cli/src/commands/$default.ts index d6135aa..e1e18d0 100644 --- a/packages/cli/src/commands/$default.ts +++ b/packages/cli/src/commands/$default.ts @@ -114,18 +114,25 @@ export async function executePrompt( throw new Error(`Unknown provider: ${config.provider}`); } - const { keyName } = providerSettings; + // only validate key if baseUrl is not set, otherwise we assume the user is using a local provider let apiKey: string | undefined = undefined; + const { keyName } = providerSettings; if (keyName) { // Then fall back to environment variable + logger.info(`Looking API key in env: ${keyName}`); apiKey = process.env[keyName]; - if (!apiKey) { - logger.error(getProviderApiKeyError(config.provider)); - throw new Error(`${config.provider} API key not found`); + if (!config.baseUrl) { + if (!apiKey) { + logger.error(getProviderApiKeyError(config.provider)); + throw new Error(`${config.provider} API key not found`); + } } } logger.info(`LLM: ${config.provider}/${config.model}`); + if (apiKey) { + logger.info(`Using API key: ${apiKey.slice(0, 4)}...`); + } if (config.baseUrl) { // For Ollama, we check if the base URL is set logger.info(`Using base url: ${config.baseUrl}`); diff --git a/packages/docs/docs/providers/anthropic.md b/packages/docs/docs/providers/anthropic.md index eded1bd..de1b1c7 100644 --- a/packages/docs/docs/providers/anthropic.md +++ b/packages/docs/docs/providers/anthropic.md @@ -32,9 +32,6 @@ export default { provider: 'anthropic', model: 'claude-3-7-sonnet-20250219', - // Optional: Set API key directly (environment variable is preferred) - // anthropicApiKey: 'your_api_key_here', - // Other MyCoder settings maxTokens: 4096, temperature: 0.7, diff --git a/packages/docs/docs/providers/index.mdx b/packages/docs/docs/providers/index.mdx index cde7c87..e0baa3b 100644 --- a/packages/docs/docs/providers/index.mdx +++ b/packages/docs/docs/providers/index.mdx @@ -11,9 +11,8 @@ MyCoder supports multiple Language Model (LLM) providers, giving you flexibility MyCoder currently supports the following LLM providers: - [**Anthropic**](./anthropic.md) - Claude models from Anthropic -- [**OpenAI**](./openai.md) - GPT models from OpenAI +- [**OpenAI**](./openai.md) - GPT models from OpenAI (and OpenAI compatible providers) - [**Ollama**](./ollama.md) - Self-hosted open-source models via Ollama -- [**Local OpenAI Compatible**](./local-openai.md) - GPUStack and other OpenAI-compatible servers - [**xAI**](./xai.md) - Grok models from xAI ## Configuring Providers @@ -54,5 +53,4 @@ For detailed instructions on setting up each provider, see the provider-specific - [Anthropic Configuration](./anthropic.md) - [OpenAI Configuration](./openai.md) - [Ollama Configuration](./ollama.md) -- [Local OpenAI Compatible Configuration](./local-openai.md) - [xAI Configuration](./xai.md) diff --git a/packages/docs/docs/providers/local-openai.md b/packages/docs/docs/providers/local-openai.md deleted file mode 100644 index caf2cfa..0000000 --- a/packages/docs/docs/providers/local-openai.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -sidebar_position: 5 ---- - -# Local OpenAI Compatible Servers - -MyCoder supports connecting to local or self-hosted OpenAI-compatible API servers, including solutions like [GPUStack](https://gpustack.ai/), [LM Studio](https://lmstudio.ai/), [Ollama OpenAI compatibility mode](https://github.com/ollama/ollama/blob/main/docs/openai.md), and [LocalAI](https://localai.io/). - -## Setup - -To use a local OpenAI-compatible server with MyCoder: - -1. Install and set up your preferred OpenAI-compatible server -2. Start the server according to its documentation -3. Configure MyCoder to connect to your local server - -### Configuration - -Configure MyCoder to use your local OpenAI-compatible server in your `mycoder.config.js` file: - -```javascript -export default { - // Provider selection - use gpustack for any OpenAI-compatible server - provider: 'gpustack', - model: 'llama3.2', // Use the model name available on your server - - // The base URL for your local server - baseUrl: 'http://localhost:80', // Default for GPUStack, adjust as needed - - // Other MyCoder settings - maxTokens: 4096, - temperature: 0.7, - // ... -}; -``` - -## GPUStack - -[GPUStack](https://gpustack.ai/) is a solution for running AI models on your own hardware. It provides an OpenAI-compatible API server that works seamlessly with MyCoder. - -### Setting up GPUStack - -1. Install GPUStack following the instructions on their website -2. Start the GPUStack server -3. Configure MyCoder to use the `gpustack` provider - -```javascript -export default { - provider: 'gpustack', - model: 'llama3.2', // Choose a model available on your GPUStack instance - baseUrl: 'http://localhost:80', // Default GPUStack URL -}; -``` - -## Other OpenAI-Compatible Servers - -You can use MyCoder with any OpenAI-compatible server by setting the appropriate `baseUrl`: - -### LM Studio - -```javascript -export default { - provider: 'gpustack', - model: 'llama3', // Use the model name as configured in LM Studio - baseUrl: 'http://localhost:1234', // Default LM Studio server URL -}; -``` - -### LocalAI - -```javascript -export default { - provider: 'gpustack', - model: 'gpt-3.5-turbo', // Use the model name as configured in LocalAI - baseUrl: 'http://localhost:8080', // Default LocalAI server URL -}; -``` - -### Ollama (OpenAI Compatibility Mode) - -```javascript -export default { - provider: 'gpustack', - model: 'llama3', // Use the model name as configured in Ollama - baseUrl: 'http://localhost:11434/v1', // Ollama OpenAI compatibility endpoint -}; -``` - -## Hardware Requirements - -Running LLMs locally requires significant hardware resources: - -- Minimum 16GB RAM (32GB+ recommended) -- GPU with at least 8GB VRAM for optimal performance -- SSD storage for model files (models can be 5-20GB each) - -## Best Practices - -- Ensure your local server and the selected model support tool calling/function calling -- Use models optimized for coding tasks when available -- Monitor your system resources when running large models locally -- Consider using a dedicated machine for hosting your local server - -## Troubleshooting - -If you encounter issues with local OpenAI-compatible servers: - -- Verify the server is running and accessible at the configured base URL -- Check that the model name exactly matches what's available on your server -- Ensure the model supports tool/function calling (required for MyCoder) -- Check server logs for specific error messages -- Test the server with a simple curl command to verify API compatibility: - -```bash -curl http://localhost:80/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "llama3.2", - "messages": [{"role": "user", "content": "Hello!"}] - }' -``` - -For more information, refer to the documentation for your specific OpenAI-compatible server. \ No newline at end of file diff --git a/packages/docs/docs/providers/openai.md b/packages/docs/docs/providers/openai.md index ed9f8e1..c65c61b 100644 --- a/packages/docs/docs/providers/openai.md +++ b/packages/docs/docs/providers/openai.md @@ -38,10 +38,6 @@ export default { provider: 'openai', model: 'gpt-4o', - // Optional: Set API key directly (environment variable is preferred) - // openaiApiKey: 'your_api_key_here', - // openaiOrganization: 'your_organization_id', - // Other MyCoder settings maxTokens: 4096, temperature: 0.7, @@ -60,6 +56,24 @@ MyCoder supports all OpenAI models that have tool/function calling capabilities. You can use any other OpenAI model that supports function calling with MyCoder. The OpenAI provider is not limited to just these listed models. +## Using OpenAI Compatible Providers + +A number of providers offer OpenAI compatible REST API endpoints, such as xAI and [GPUStack](https://gpustack.ai). To point the OpenAI provider to a different provider REST API set the `baseUrl` and also, if applicable, the `OPENAI_API_KEY` to their required key. For example: + +```javascript +export default { + // Provider selection + provider: 'openai', + model: 'qwen2.5', + baseUrl: 'http://localhost/v1-openai', + + // Other MyCoder settings + maxTokens: 4096, + temperature: 0.7, + // ... +}; +``` + ## Best Practices - GPT-4o provides the best balance of performance and cost for most MyCoder tasks diff --git a/packages/docs/docs/providers/xai.md b/packages/docs/docs/providers/xai.md index 7d1d605..0a76308 100644 --- a/packages/docs/docs/providers/xai.md +++ b/packages/docs/docs/providers/xai.md @@ -32,9 +32,6 @@ export default { provider: 'xai', model: 'grok-2-latest', - // Optional: Set API key directly (environment variable is preferred) - // xaiApiKey: 'your_api_key_here', - // Other MyCoder settings maxTokens: 4096, temperature: 0.7, @@ -77,4 +74,4 @@ If you encounter issues with xAI's Grok: - For tool-calling issues, ensure your functions are properly formatted - Monitor your token usage to avoid unexpected costs -For more information, visit the [xAI Documentation](https://x.ai/docs). \ No newline at end of file +For more information, visit the [xAI Documentation](https://x.ai/docs). From c04ee436b02a37d94688803b406cfb0b1e52c281 Mon Sep 17 00:00:00 2001 From: Ben Houston Date: Fri, 14 Mar 2025 10:03:32 -0400 Subject: [PATCH 3/3] fix: disable respawn as it can confuse some LLMs --- packages/agent/src/tools/getTools.ts | 3 +-- packages/agent/src/tools/system/respawn.test.ts | 3 +-- packages/agent/src/tools/system/respawn.ts | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/agent/src/tools/getTools.ts b/packages/agent/src/tools/getTools.ts index 38fb49e..79ee272 100644 --- a/packages/agent/src/tools/getTools.ts +++ b/packages/agent/src/tools/getTools.ts @@ -10,7 +10,6 @@ import { fetchTool } from './io/fetch.js'; import { textEditorTool } from './io/textEditor.js'; import { createMcpTool } from './mcp.js'; import { listBackgroundToolsTool } from './system/listBackgroundTools.js'; -import { respawnTool } from './system/respawn.js'; import { sequenceCompleteTool } from './system/sequenceComplete.js'; import { shellMessageTool } from './system/shellMessage.js'; import { shellStartTool } from './system/shellStart.js'; @@ -39,7 +38,7 @@ export function getTools(options?: GetToolsOptions): Tool[] { shellMessageTool as unknown as Tool, browseStartTool as unknown as Tool, browseMessageTool as unknown as Tool, - respawnTool as unknown as Tool, + //respawnTool as unknown as Tool, this is a confusing tool for now. sleepTool as unknown as Tool, listBackgroundToolsTool as unknown as Tool, ]; diff --git a/packages/agent/src/tools/system/respawn.test.ts b/packages/agent/src/tools/system/respawn.test.ts index b70ea64..2b314b6 100644 --- a/packages/agent/src/tools/system/respawn.test.ts +++ b/packages/agent/src/tools/system/respawn.test.ts @@ -8,9 +8,8 @@ import { respawnTool } from './respawn'; const toolContext: ToolContext = getMockToolContext(); describe('respawnTool', () => { - it('should have correct name and description', () => { + it('should have correct name', () => { expect(respawnTool.name).toBe('respawn'); - expect(respawnTool.description).toContain('Resets the agent context'); }); it('should execute and return confirmation message', async () => { diff --git a/packages/agent/src/tools/system/respawn.ts b/packages/agent/src/tools/system/respawn.ts index 1640764..a740683 100644 --- a/packages/agent/src/tools/system/respawn.ts +++ b/packages/agent/src/tools/system/respawn.ts @@ -20,7 +20,7 @@ const returnSchema = z.object({ export const respawnTool: Tool = { name: 'respawn', description: - 'Resets the agent context to just the system prompt and provided context', + 'Resets the current conversation to just the system prompt and provided input context to this tool.', logPrefix: '🔄', parameters: parameterSchema, returns: returnSchema,