diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml new file mode 100644 index 0000000..effd3d1 --- /dev/null +++ b/.github/workflows/ci-tests.yml @@ -0,0 +1,66 @@ +name: CI Tests + +on: + push: + branches: [ main, master, develop ] + pull_request: + branches: [ main, master, develop ] + +jobs: + test-imports: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Test Import Structure + run: | + python -c "import coderag.config; print('✓ Config import successful')" + python -c "import coderag.embeddings; print('✓ Embeddings import successful')" + python -c "import coderag.index; print('✓ Index import successful')" + python -c "import coderag.search; print('✓ Search import successful')" + python -c "import coderag.monitor; print('✓ Monitor import successful')" + env: + OPENAI_API_KEY: dummy-key-for-testing + + quality-and-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install black flake8 isort mypy pytest + - name: Lint and type-check + run: | + black --check . + isort --check-only . + flake8 . --max-line-length=88 --ignore=E203,W503 + mypy . + - name: Run tests + env: + PYTHONPATH: ${{ github.workspace }} + run: pytest -q diff --git a/.gitignore b/.gitignore index bb814ab..e4e233e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ node_modules/ *.tmp plan.md metadata.npy +test_env/ +*.npy diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f6907cd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +repos: + - repo: https://github.com/psf/black + rev: 23.12.1 + hooks: + - id: black + language_version: python3 + args: ['--line-length=88'] + + - repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + args: ['--max-line-length=88', '--ignore=E203,W503'] + + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + args: ["--profile", "black"] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.8.0 + hooks: + - id: mypy + additional_dependencies: [types-all] + args: [--ignore-missing-imports, --no-strict-optional] + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-merge-conflict + - id: debug-statements \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5d3b824 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,37 @@ +# Repository Guidelines + +## Project Structure & Module Organization +- `coderag/`: Core library (`config.py`, `embeddings.py`, `index.py`, `search.py`, `monitor.py`). +- `app.py`: Streamlit UI. `main.py`: backend/indexer. `prompt_flow.py`: RAG orchestration. +- `scripts/`: Utilities (e.g., `initialize_index.py`, `run_monitor.py`). +- `tests/`: Minimal checks (e.g., `test_faiss.py`). +- `example.env` → copy to `.env` for local secrets; CI lives in `.github/`. + +## Build, Test, and Development Commands +- Create env: `python -m venv venv && source venv/bin/activate`. +- Install deps: `pip install -r requirements.txt`. +- Run backend: `python main.py` (indexes and watches `WATCHED_DIR`). +- Run UI: `streamlit run app.py`. +- Quick test: `python tests/test_faiss.py` (FAISS round‑trip sanity check). +- Quality suite: `pre-commit run --all-files` (black, isort, flake8, mypy, basics). + +## Coding Style & Naming Conventions +- Formatting: Black (88 cols), isort profile "black"; run `black . && isort .`. +- Linting: flake8 with `--ignore=E203,W503` to match Black. +- Typing: mypy (py311 target; ignore missing imports OK). Prefer typed signatures and docstrings. +- Indentation: 4 spaces. Names: `snake_case` for files/functions, `PascalCase` for classes, constants `UPPER_SNAKE`. +- Imports: first‑party module is `coderag` (see `pyproject.toml`). + +## Testing Guidelines +- Place tests in `tests/` as `test_*.py`. Keep unit tests deterministic; mock OpenAI calls where possible. +- Run directly (`python tests/test_faiss.py`) or with pytest if available (`pytest -q`). +- Ensure `.env` or env vars provide `OPENAI_API_KEY` for integration tests; avoid hitting rate limits in CI. + +## Commit & Pull Request Guidelines +- Use Conventional Commits seen in history: `feat:`, `fix:`, `docs:`, `ci:`, `refactor:`, `simplify:`. +- Before pushing: `pre-commit run --all-files` and update docs when behavior changes. +- PRs: clear description, linked issues, steps to validate; include screenshots/GIFs for UI changes; note config changes (`.env`). + +## Security & Configuration Tips +- Never commit secrets. Start with `cp example.env .env`; set `OPENAI_API_KEY`, `WATCHED_DIR`, `FAISS_INDEX_FILE`. +- Avoid logging sensitive data. Regenerate the FAISS index if dimensions or models change (`python scripts/initialize_index.py`). diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..17f4573 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,194 @@ +# 🛠️ Development Guide + +## Setting Up Development Environment + +### 1. Clone and Setup + +```bash +git clone https://github.com/your-username/CodeRAG.git +cd CodeRAG +python -m venv venv +source venv/bin/activate # Windows: venv\Scripts\activate +pip install -r requirements.txt +``` + +### 2. Configure Pre-commit Hooks + +```bash +pip install pre-commit +pre-commit install +``` + +This will run code quality checks on every commit: +- **Black**: Code formatting +- **isort**: Import sorting +- **Flake8**: Linting and style checks +- **MyPy**: Type checking +- **Basic hooks**: Trailing whitespace, file endings, etc. + +### 3. Environment Variables + +Copy `example.env` to `.env` and configure: + +```bash +cp example.env .env +``` + +Required variables: +```env +OPENAI_API_KEY=your_key_here # Required for embeddings and chat +WATCHED_DIR=/path/to/code # Directory to index (default: current dir) +``` + +## Code Quality Standards + +### Type Hints +All functions should have type hints: + +```python +def process_file(filepath: str, content: str) -> Optional[np.ndarray]: + \"\"\"Process a file and return embeddings.\"\"\" + ... +``` + +### Error Handling +Use structured logging and proper exception handling: + +```python +import logging +logger = logging.getLogger(__name__) + +try: + result = risky_operation() +except SpecificError as e: + logger.error(f"Operation failed: {str(e)}") + return None +``` + +### Documentation +Use concise docstrings for public functions: + +```python +def search_code(query: str, k: int = 5) -> List[Dict[str, Any]]: + \"\"\"Search the FAISS index using a text query. + + Args: + query: The search query text + k: Number of results to return + + Returns: + List of search results with metadata + \"\"\" +``` + +## Testing Your Changes + +### Manual Testing +```bash +# Test backend indexing +python main.py + +# Test Streamlit UI (separate terminal) +streamlit run app.py +``` + +### Code Quality Checks +```bash +# Format code +black . +isort . + +# Check linting +flake8 . + +# Type checking +mypy . + +# Run all pre-commit checks +pre-commit run --all-files +``` + +## Adding New Features + +1. **Create feature branch**: `git checkout -b feature/new-feature` +2. **Add logging**: Use the logger for all operations +3. **Add type hints**: Follow existing patterns +4. **Handle errors**: Graceful degradation and user-friendly messages +5. **Update tests**: Add tests for new functionality +6. **Update docs**: Update README if needed + +## Architecture Guidelines + +### Keep It Simple +- Maintain the single-responsibility principle +- Avoid unnecessary abstractions +- Focus on the core RAG functionality + +### Error Handling Strategy +- Log errors with context +- Return None/empty lists for failures +- Show user-friendly messages in UI +- Don't crash the application + +### Performance Considerations +- Limit search results (default: 5) +- Truncate long content for context +- Cache embeddings when possible +- Monitor memory usage with large codebases + +## Debugging Tips + +### Enable Debug Logging +```python +logging.basicConfig(level=logging.DEBUG) +``` + +### Check Index Status +```python +from coderag.index import inspect_metadata +inspect_metadata(5) # Show first 5 entries +``` + +### Test Embeddings +```python +from coderag.embeddings import generate_embeddings +result = generate_embeddings("test code") +print(f"Shape: {result.shape if result is not None else 'None'}") +``` + +## Common Development Issues + +**Import Errors** +- Ensure you're in the virtual environment +- Check PYTHONPATH includes project root +- Verify all dependencies are installed + +**OpenAI API Issues** +- Check API key validity +- Monitor rate limits and usage +- Test with a simple embedding request + +**FAISS Index Corruption** +- Delete existing index files and rebuild +- Check file permissions +- Ensure consistent embedding dimensions + +## Project Structure + +``` +CodeRAG/ +├── coderag/ # Core library +│ ├── __init__.py +│ ├── config.py # Configuration management +│ ├── embeddings.py # OpenAI integration +│ ├── index.py # FAISS operations +│ ├── search.py # Search functionality +│ └── monitor.py # File monitoring +├── scripts/ # Utility scripts +├── tests/ # Test files +├── .github/ # GitHub workflows +├── main.py # Backend service +├── app.py # Streamlit frontend +├── prompt_flow.py # RAG orchestration +└── requirements.txt # Dependencies +``` \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..4722efc --- /dev/null +++ b/README.md @@ -0,0 +1,201 @@ +# 🤖 CodeRAG: AI-Powered Code Retrieval & Assistance + +[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Code Quality](https://github.com/your-username/CodeRAG/workflows/Code%20Quality/badge.svg)](https://github.com/your-username/CodeRAG/actions) + +> **Note**: This POC was innovative for its time, but modern tools like Cursor and Windsurf now apply this principle directly in IDEs. This remains an excellent educational project for understanding RAG implementation. + +## ✨ What is CodeRAG? + +CodeRAG combines **Retrieval-Augmented Generation (RAG)** with AI to provide intelligent coding assistance. Instead of limited context windows, it indexes your entire codebase and provides contextual suggestions based on your complete project. + +### 🎯 Core Idea + +Most coding assistants work with limited scope, but CodeRAG provides the full context of your project by: +- **Real-time indexing** of your entire codebase using FAISS vector search +- **Semantic code search** powered by OpenAI embeddings +- **Contextual AI responses** that understand your project structure + +## 🚀 Quick Start + +### Prerequisites +- Python 3.11+ +- OpenAI API Key ([Get one here](https://platform.openai.com/api-keys)) + +### Installation + +```bash +# Clone the repository +git clone https://github.com/your-username/CodeRAG.git +cd CodeRAG + +# Create virtual environment +python -m venv venv +source venv/bin/activate # On Windows: venv\\Scripts\\activate + +# Install dependencies +pip install -r requirements.txt + +# Configure environment +cp example.env .env +# Edit .env with your OpenAI API key and settings +``` + +### Configuration + +Create a `.env` file with your settings: + +```env +OPENAI_API_KEY=your_openai_api_key_here +OPENAI_EMBEDDING_MODEL=text-embedding-ada-002 +OPENAI_CHAT_MODEL=gpt-4 +WATCHED_DIR=/path/to/your/code/directory +FAISS_INDEX_FILE=./coderag_index.faiss +EMBEDDING_DIM=1536 +``` + +### Running CodeRAG + +```bash +# Start the backend (indexing and monitoring) +python main.py + +# In a separate terminal, start the web interface +streamlit run app.py +``` + +## 📖 How It Works + +```mermaid +graph LR + A[Code Files] --> B[File Monitor] + B --> C[OpenAI Embeddings] + C --> D[FAISS Vector DB] + E[User Query] --> F[Semantic Search] + D --> F + F --> G[Retrieved Context] + G --> H[OpenAI GPT] + H --> I[AI Response] +``` + +1. **Indexing**: CodeRAG monitors your code directory and generates embeddings for Python files +2. **Storage**: Embeddings are stored in a FAISS vector database with metadata +3. **Search**: User queries are embedded and matched against the code database +4. **Generation**: Retrieved code context is sent to GPT models for intelligent responses + +## 🛠️ Architecture + +``` +CodeRAG/ +├── 🧠 coderag/ # Core RAG functionality +│ ├── config.py # Environment configuration +│ ├── embeddings.py # OpenAI embedding generation +│ ├── index.py # FAISS vector operations +│ ├── search.py # Semantic code search +│ └── monitor.py # File system monitoring +├── 🌐 app.py # Streamlit web interface +├── 🔧 main.py # Backend indexing service +├── 🔗 prompt_flow.py # RAG pipeline orchestration +└── 📋 requirements.txt # Dependencies +``` + +### Key Components + +- **🔍 Vector Search**: FAISS-powered similarity search for code retrieval +- **🎯 Smart Embeddings**: OpenAI embeddings capture semantic code meaning +- **📡 Real-time Updates**: Watchdog monitors file changes for live indexing +- **💬 Conversational UI**: Streamlit interface with chat-like experience + +## 🎪 Usage Examples + +### Ask About Your Code +``` +"How does the FAISS indexing work in this codebase?" +"Where is error handling implemented?" +"Show me examples of the embedding generation process" +``` + +### Get Improvements +``` +"How can I optimize the search performance?" +"What are potential security issues in this code?" +"Suggest better error handling for the monitor module" +``` + +### Debug Issues +``` +"Why might the search return no results?" +"How do I troubleshoot OpenAI connection issues?" +"What could cause indexing to fail?" +``` + +## ⚙️ Development + +### Code Quality Tools + +```bash +# Install pre-commit hooks +pip install pre-commit +pre-commit install + +# Run formatting and linting +black . +flake8 . +mypy . +``` + +### Testing + +```bash +# Test FAISS index functionality +python tests/test_faiss.py + +# Test individual components +python scripts/initialize_index.py +python scripts/run_monitor.py +``` + +## 🐛 Troubleshooting + +### Common Issues + +**Search returns no results** +- Check if indexing completed: look for `coderag_index.faiss` file +- Verify OpenAI API key is working +- Ensure your query relates to indexed Python files + +**OpenAI API errors** +- Verify API key in `.env` file +- Check API usage limits and billing +- Ensure model names are correct (gpt-4, text-embedding-ada-002) + +**File monitoring not working** +- Check `WATCHED_DIR` path in `.env` +- Ensure directory contains `.py` files +- Look for error logs in console output + +## 🤝 Contributing + +1. Fork the repository +2. Create a feature branch (`git checkout -b feature/amazing-feature`) +3. Make your changes with proper error handling and type hints +4. Run code quality checks (`pre-commit run --all-files`) +5. Commit your changes (`git commit -m 'Add amazing feature'`) +6. Push to the branch (`git push origin feature/amazing-feature`) +7. Open a Pull Request + +## 📄 License + +This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE-2.0.txt) file for details. + +## 🙏 Acknowledgments + +- [OpenAI](https://openai.com/) for embedding and chat models +- [Facebook AI Similarity Search (FAISS)](https://github.com/facebookresearch/faiss) for vector search +- [Streamlit](https://streamlit.io/) for the web interface +- [Watchdog](https://github.com/gorakhargosh/watchdog) for file monitoring + +--- + +**⭐ If this project helps you, please give it a star!** diff --git a/app.py b/app.py index 74eff29..ee7bec8 100644 --- a/app.py +++ b/app.py @@ -1,39 +1,169 @@ +import logging +from typing import Optional as _Optional + import streamlit as st from openai import OpenAI -from coderag.config import OPENAI_API_KEY, OPENAI_CHAT_MODEL + +from coderag.config import OPENAI_API_KEY from prompt_flow import execute_rag_flow -# Initialize the OpenAI client -client = OpenAI(api_key=OPENAI_API_KEY) +# Configure logging for Streamlit +# Use force=True to ensure Streamlit's default handlers don't suppress ours +logging.basicConfig(level=logging.INFO, force=True) +logger = logging.getLogger(__name__) + +# Initialize the OpenAI client with error handling +client: _Optional[OpenAI] +try: + if OPENAI_API_KEY: + client = OpenAI(api_key=OPENAI_API_KEY) + logger.info("OpenAI client initialized successfully") + else: + client = None + logger.error("OpenAI API key not found") +except Exception as e: + client = None + logger.error(f"Failed to initialize OpenAI client: {e}") -st.title("CodeRAG: Your Coding Assistant") +# Set page config +st.set_page_config( + page_title="CodeRAG: Your Coding Assistant", page_icon="🤖", layout="wide" +) -# Initialize chat history +st.title("🤖 CodeRAG: Your Coding Assistant") +st.markdown("*AI-powered code retrieval and assistance using RAG technology*") + +# Initialize session state if "messages" not in st.session_state: st.session_state.messages = [] +if "conversation_context" not in st.session_state: + st.session_state.conversation_context = [] + +# Sidebar with controls +with st.sidebar: + st.header("Controls") -# Display chat history + if st.button("🗑️ Clear Conversation", type="secondary"): + st.session_state.messages = [] + st.session_state.conversation_context = [] + st.rerun() + + # Status indicators + st.header("Status") + if client: + st.success("✅ OpenAI Connected") + else: + st.error("❌ OpenAI Not Connected") + st.error("Please check your API key in .env file") + + # Conversation stats + if st.session_state.messages: + st.info(f"💬 {len(st.session_state.messages)} messages in conversation") + +# Display chat history with improved formatting for message in st.session_state.messages: with st.chat_message(message["role"]): - st.markdown(message["content"]) + if message["role"] == "assistant" and "error" in message["content"].lower(): + st.error(message["content"]) + else: + st.markdown(message["content"]) + +# Chat input with validation +if not client: + st.warning( + "⚠️ OpenAI client not available. Please configure your API key to use " + "the assistant." + ) + st.stop() + +if prompt := st.chat_input("What is your coding question?", disabled=not client): + # Validate input + if not prompt.strip(): + st.warning("Please enter a valid question.") + st.stop() -# Chat input -if prompt := st.chat_input("What is your coding question?"): + # Add user message st.session_state.messages.append({"role": "user", "content": prompt}) + # Add to conversation context for better continuity + st.session_state.conversation_context.append(f"User: {prompt}") + with st.chat_message("user"): st.markdown(prompt) with st.chat_message("assistant"): message_placeholder = st.empty() - full_response = "" - - try: - response = execute_rag_flow(prompt) - message_placeholder.markdown(response) - full_response = response - except Exception as e: - error_message = f"Error in RAG flow execution: {str(e)}" - st.error(error_message) - full_response = error_message - - st.session_state.messages.append({"role": "assistant", "content": full_response}) \ No newline at end of file + + # Show loading indicator + with st.spinner("🔍 Searching codebase and generating response..."): + try: + # Execute RAG flow with error handling + response = execute_rag_flow(prompt) + + # Check if response indicates an error + if ( + response.startswith("Error:") + or "error occurred" in response.lower() + ): + message_placeholder.error(response) + else: + message_placeholder.markdown(response) + + full_response = response + + except Exception as e: + error_message = f"Unexpected error: {str(e)}" + logger.error(f"Streamlit error: {error_message}") + message_placeholder.error(error_message) + full_response = error_message + + # Add assistant response to session + st.session_state.messages.append( + {"role": "assistant", "content": full_response} + ) + # Add to conversation context + st.session_state.conversation_context.append( + f"Assistant: {full_response[:200]}..." + ) # Truncate for context + + # Keep conversation context manageable (last 10 exchanges) + if len(st.session_state.conversation_context) > 20: + st.session_state.conversation_context = ( + st.session_state.conversation_context[-20:] + ) + +# Footer with helpful information +if not st.session_state.messages: + st.markdown("---") + st.markdown("### 💡 Tips for better results:") + st.markdown( + """ + - Ask specific questions about your code + - Mention file names or functions you're interested in + - Request explanations, improvements, or debugging help + - Ask about code patterns or best practices + """ + ) + + st.markdown("### 🚀 Example queries:") + col1, col2 = st.columns(2) + with col1: + if st.button("📝 Explain the indexing process"): + st.session_state.messages.append( + { + "role": "user", + "content": "Explain how the FAISS indexing works in this codebase", + } + ) + st.rerun() + with col2: + if st.button("🐛 Help debug search issues"): + st.session_state.messages.append( + { + "role": "user", + "content": ( + "How can I debug issues with code search not returning " + "results?" + ), + } + ) + st.rerun() diff --git a/coderag/__init__.py b/coderag/__init__.py index 203562b..143f486 100644 --- a/coderag/__init__.py +++ b/coderag/__init__.py @@ -1 +1 @@ -# __init__.py \ No newline at end of file +# __init__.py diff --git a/coderag/config.py b/coderag/config.py index 46e2e4c..424e7f3 100644 --- a/coderag/config.py +++ b/coderag/config.py @@ -1,4 +1,5 @@ import os + from dotenv import load_dotenv # Load environment variables from the .env file @@ -7,17 +8,21 @@ # === Environment Variables === # OpenAI API key and model settings (loaded from .env) OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") -OPENAI_EMBEDDING_MODEL = os.getenv("OPENAI_EMBEDDING_MODEL", "text-embedding-ada-002") # Default to ada-002 +OPENAI_EMBEDDING_MODEL = os.getenv( + "OPENAI_EMBEDDING_MODEL", "text-embedding-ada-002" +) # Default to ada-002 OPENAI_CHAT_MODEL = os.getenv("OPENAI_CHAT_MODEL", "gpt-4") # Default to GPT-4 # Embedding dimension (from .env or fallback) EMBEDDING_DIM = int(os.getenv("EMBEDDING_DIM", 1536)) # Default to 1536 if not in .env # Project directory (from .env) -WATCHED_DIR = os.getenv("WATCHED_DIR", os.path.join(os.getcwd(), 'CodeRAG')) +WATCHED_DIR = os.getenv("WATCHED_DIR", os.getcwd()) # Path to FAISS index (from .env or fallback) -FAISS_INDEX_FILE = os.getenv("FAISS_INDEX_FILE", os.path.join(WATCHED_DIR, 'coderag_index.faiss')) +FAISS_INDEX_FILE = os.getenv( + "FAISS_INDEX_FILE", os.path.join(WATCHED_DIR, "coderag_index.faiss") +) # === Project-Specific Configuration === # Define the root directory of the project diff --git a/coderag/embeddings.py b/coderag/embeddings.py index b2dd748..3de715e 100644 --- a/coderag/embeddings.py +++ b/coderag/embeddings.py @@ -1,20 +1,84 @@ -from openai import OpenAI +import logging +from typing import List, Optional + import numpy as np +from openai import OpenAI +from tenacity import ( + retry, + stop_after_attempt, + wait_exponential, +) + from coderag.config import OPENAI_API_KEY, OPENAI_EMBEDDING_MODEL -# Initialize the OpenAI client -client = OpenAI(api_key=OPENAI_API_KEY) +logger = logging.getLogger(__name__) + +# Initialize the OpenAI client with error handling +client: Optional[OpenAI] +try: + if not OPENAI_API_KEY: + raise ValueError("OpenAI API key not found in environment variables") + client = OpenAI(api_key=OPENAI_API_KEY) + logger.info(f"OpenAI client initialized with model: {OPENAI_EMBEDDING_MODEL}") +except Exception as e: + logger.error(f"Failed to initialize OpenAI client: {e}") + client = None + + +def _chunk_text(text: str, max_chars: int = 4000) -> List[str]: + """Naive chunking by characters to avoid overly long inputs.""" + text = text.strip() + if len(text) <= max_chars: + return [text] + return [text[i : i + max_chars] for i in range(0, len(text), max_chars)] + + +@retry( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=0.5, max=8), + reraise=True, +) +def _embed_batch(inputs: List[str]) -> np.ndarray: + """Call OpenAI embeddings with basic retry/backoff. Returns shape (n, d).""" + if client is None: + raise RuntimeError("OpenAI client not initialized") + response = client.embeddings.create( + model=OPENAI_EMBEDDING_MODEL, + input=inputs, + timeout=30, + ) + arr = np.array([d.embedding for d in response.data], dtype="float32") + return arr + + +def generate_embeddings(text: str) -> Optional[np.ndarray]: + """Generate embeddings using OpenAI's embedding API. + + Args: + text: The input text to generate embeddings for + + Returns: + numpy array of embeddings or None if generation fails + """ + if not client: + logger.error("OpenAI client not initialized") + return None + + if not text or not text.strip(): + logger.warning("Empty text provided for embedding generation") + return None -def generate_embeddings(text): - """Generate embeddings using the updated OpenAI API.""" try: - response = client.embeddings.create( - model=OPENAI_EMBEDDING_MODEL, - input=[text] # Input should be a list of strings - ) - # Extract the embedding from the response - embeddings = response.data[0].embedding - return np.array(embeddings).astype('float32').reshape(1, -1) + logger.debug(f"Generating embeddings for text of length: {len(text)}") + + chunks = _chunk_text(text, max_chars=4000) + vecs = _embed_batch(chunks) # shape (n, d) + + # Average chunk embeddings for a stable single vector + avg = np.mean(vecs, axis=0, dtype=np.float32).reshape(1, -1) + logger.debug(f"Successfully generated embeddings with shape: {avg.shape}") + return avg + except Exception as e: - print(f"Error generating embeddings with OpenAI: {e}") - return None \ No newline at end of file + logger.error(f"Failed to generate embeddings: {e}") + return None diff --git a/coderag/index.py b/coderag/index.py index 35352d3..d30c5cd 100644 --- a/coderag/index.py +++ b/coderag/index.py @@ -1,62 +1,156 @@ +import logging import os +from typing import Any, Dict, List, Optional + import faiss import numpy as np + from coderag.config import EMBEDDING_DIM, FAISS_INDEX_FILE, WATCHED_DIR -index = faiss.IndexFlatL2(EMBEDDING_DIM) -metadata = [] +logger = logging.getLogger(__name__) -def clear_index(): - """Delete the FAISS index and metadata files if they exist, and reinitialize the index.""" - global index, metadata - - # Delete the FAISS index file - if os.path.exists(FAISS_INDEX_FILE): - os.remove(FAISS_INDEX_FILE) - print(f"Deleted FAISS index file: {FAISS_INDEX_FILE}") - - # Delete the metadata file - metadata_file = "metadata.npy" - if os.path.exists(metadata_file): - os.remove(metadata_file) - print(f"Deleted metadata file: {metadata_file}") - - # Reinitialize the FAISS index and metadata - index = faiss.IndexFlatL2(EMBEDDING_DIM) - metadata = [] - print("FAISS index and metadata cleared and reinitialized.") - -def add_to_index(embeddings, full_content, filename, filepath): +index = faiss.IndexFlatIP(EMBEDDING_DIM) +metadata: List[Dict[str, Any]] = [] + + +def _l2_normalize(mat: np.ndarray) -> np.ndarray: + """Normalize rows to unit length in-place, returns the same array.""" + if mat is None or mat.size == 0: + return mat + faiss.normalize_L2(mat) + return mat + + +def clear_index() -> None: + """Delete the FAISS index and metadata files if they exist, and + reinitialize the index.""" global index, metadata - if embeddings.shape[1] != index.d: - raise ValueError(f"Embedding dimension {embeddings.shape[1]} does not match FAISS index dimension {index.d}") + try: + # Delete the FAISS index file + if os.path.exists(FAISS_INDEX_FILE): + os.remove(FAISS_INDEX_FILE) + logger.info(f"Deleted FAISS index file: {FAISS_INDEX_FILE}") + + # Delete the metadata file + metadata_file = "metadata.npy" + if os.path.exists(metadata_file): + os.remove(metadata_file) + logger.info(f"Deleted metadata file: {metadata_file}") + + # Reinitialize the FAISS index and metadata + index = faiss.IndexFlatIP(EMBEDDING_DIM) + metadata = [] + logger.info("FAISS index and metadata cleared and reinitialized") + + except Exception as e: + logger.error(f"Error clearing index: {str(e)}") + raise + + +def add_to_index( + embeddings: np.ndarray, full_content: str, filename: str, filepath: str +) -> None: + """Add embeddings and metadata to the FAISS index. + + Args: + embeddings: The embedding vectors to add + full_content: The original file content + filename: Name of the file + filepath: Full path to the file + """ + + try: + if embeddings is None or embeddings.size == 0: + logger.warning(f"Empty embeddings provided for {filename}") + return + + if embeddings.shape[1] != index.d: + raise ValueError( + f"Embedding dimension {embeddings.shape[1]} does not match " + f"FAISS index dimension {index.d}" + ) - # Convert absolute filepath to relative path - relative_filepath = os.path.relpath(filepath, WATCHED_DIR) + # Convert absolute filepath to relative path + try: + relative_filepath = os.path.relpath(filepath, WATCHED_DIR) + except ValueError: + logger.warning( + f"Could not create relative path for {filepath}, using " + f"absolute path" + ) + relative_filepath = filepath - index.add(embeddings) - metadata.append({ - "content": full_content, - "filename": filename, - "filepath": relative_filepath # Store relative filepath - }) + # Normalize for cosine similarity (IndexFlatIP) + vecs = embeddings.astype("float32", copy=True) + vecs = _l2_normalize(vecs) + index.add(vecs) + metadata.append( + { + # Store only a snippet to keep metadata small + "content": (full_content[:3000] if full_content else ""), + "filename": filename, + "filepath": relative_filepath, + } + ) -def save_index(): - faiss.write_index(index, FAISS_INDEX_FILE) - with open("metadata.npy", "wb") as f: - np.save(f, metadata) + logger.debug(f"Added {filename} to index (total entries: {index.ntotal})") -def load_index(): + except Exception as e: + logger.error(f"Error adding {filename} to index: {str(e)}") + raise + + +def save_index() -> None: + """Save the FAISS index and metadata to disk.""" + try: + faiss.write_index(index, FAISS_INDEX_FILE) + with open("metadata.npy", "wb") as f: + np.save(f, np.array(metadata, dtype=object)) + logger.debug(f"Index saved with {index.ntotal} entries") + except Exception as e: + logger.error(f"Error saving index: {str(e)}") + raise + + +def load_index() -> Optional[faiss.Index]: + """Load the FAISS index and metadata from disk. + + Returns: + The loaded FAISS index or None if loading fails + """ global index, metadata - index = faiss.read_index(FAISS_INDEX_FILE) - with open("metadata.npy", "rb") as f: - metadata = np.load(f, allow_pickle=True).tolist() - return index -def get_metadata(): + try: + if not os.path.exists(FAISS_INDEX_FILE): + logger.warning(f"FAISS index file not found: {FAISS_INDEX_FILE}") + return None + + if not os.path.exists("metadata.npy"): + logger.warning("Metadata file not found: metadata.npy") + return None + + index = faiss.read_index(FAISS_INDEX_FILE) + with open("metadata.npy", "rb") as f: + metadata = np.load(f, allow_pickle=True).tolist() + + logger.info(f"Loaded index with {index.ntotal} entries") + return index + + except Exception as e: + logger.error(f"Error loading index: {str(e)}") + return None + + +def get_metadata() -> List[Dict[str, Any]]: + """Get the current metadata list. + + Returns: + List of metadata dictionaries + """ return metadata + def retrieve_vectors(n=5): n = min(n, index.ntotal) vectors = np.zeros((n, EMBEDDING_DIM), dtype=np.float32) @@ -64,12 +158,22 @@ def retrieve_vectors(n=5): vectors[i] = index.reconstruct(i) return vectors -def inspect_metadata(n=5): - metadata = get_metadata() - print(f"Inspecting the first {n} metadata entries:") - for i, data in enumerate(metadata[:n]): - print(f"Entry {i}:") - print(f"Filename: {data['filename']}") - print(f"Filepath: {data['filepath']}") - print(f"Content: {data['content'][:100]}...") # Show the first 100 characters - print() + +def inspect_metadata(n: int = 5) -> None: + """Print metadata information for debugging purposes. + + Args: + n: Number of entries to inspect + """ + try: + metadata_list = get_metadata() + logger.info(f"Inspecting the first {n} metadata entries:") + for i, data in enumerate(metadata_list[:n]): + logger.info(f"Entry {i}:") + logger.info(f" Filename: {data['filename']}") + logger.info(f" Filepath: {data['filepath']}") + logger.info( + f" Content: {data['content'][:100]}..." + ) # Show the first 100 characters + except Exception as e: + logger.error(f"Error inspecting metadata: {str(e)}") diff --git a/coderag/monitor.py b/coderag/monitor.py index 61093b5..7484409 100644 --- a/coderag/monitor.py +++ b/coderag/monitor.py @@ -1,44 +1,104 @@ -import time +import logging import os -from watchdog.observers import Observer +import time + from watchdog.events import FileSystemEventHandler -from coderag.index import add_to_index, save_index +from watchdog.observers import Observer + +from coderag.config import IGNORE_PATHS, WATCHED_DIR from coderag.embeddings import generate_embeddings -from coderag.config import WATCHED_DIR, IGNORE_PATHS +from coderag.index import add_to_index, save_index + +logger = logging.getLogger(__name__) + + +def should_ignore_path(path: str) -> bool: + """Check if the given path should be ignored based on the IGNORE_PATHS list. + + Args: + path: File or directory path to check + + Returns: + True if path should be ignored, False otherwise + """ + try: + for ignore_path in IGNORE_PATHS: + if path.startswith(ignore_path): + return True + return False + except Exception as e: + logger.error(f"Error checking ignore path for {path}: {str(e)}") + return True # Err on the side of caution -def should_ignore_path(path): - """Check if the given path should be ignored based on the IGNORE_PATHS list.""" - for ignore_path in IGNORE_PATHS: - if path.startswith(ignore_path): - return True - return False class CodeChangeHandler(FileSystemEventHandler): + """Handle file system events for code changes.""" + def on_modified(self, event): - if event.is_directory or should_ignore_path(event.src_path): - return + """Handle file modification events.""" + try: + if event.is_directory or should_ignore_path(event.src_path): + return + + if event.src_path.endswith(".py"): + logger.info(f"Detected change in file: {event.src_path}") + + # Read file content with error handling + try: + with open(event.src_path, "r", encoding="utf-8") as f: + full_content = f.read() + except (IOError, UnicodeDecodeError) as e: + logger.error(f"Error reading file {event.src_path}: {str(e)}") + return - if event.src_path.endswith(".py"): - print(f"Detected change in file: {event.src_path}") - with open(event.src_path, 'r', encoding='utf-8') as f: - full_content = f.read() - embeddings = generate_embeddings(full_content) - if embeddings is not None and len(embeddings) > 0: - filename = os.path.basename(event.src_path) - add_to_index(embeddings, full_content, filename, event.src_path) - save_index() - print(f"Updated FAISS index for file: {event.src_path}") - -def start_monitoring(): - event_handler = CodeChangeHandler() - observer = Observer() - observer.schedule(event_handler, path=WATCHED_DIR, recursive=True) - observer.start() - print(f"Started monitoring {WATCHED_DIR}...") + # Generate embeddings + embeddings = generate_embeddings(full_content) + if embeddings is not None and embeddings.size > 0: + filename = os.path.basename(event.src_path) + try: + add_to_index(embeddings, full_content, filename, event.src_path) + save_index() + logger.info(f"Updated FAISS index for file: {event.src_path}") + except Exception as e: + logger.error( + f"Error updating index for {event.src_path}: {str(e)}" + ) + else: + logger.warning( + f"Failed to generate embeddings for {event.src_path}" + ) + except Exception as e: + logger.error(f"Unexpected error handling file event: {str(e)}") + + +def start_monitoring() -> None: + """Start monitoring the directory for file changes.""" try: - while True: - time.sleep(1) - except KeyboardInterrupt: - observer.stop() - observer.join() + if not os.path.exists(WATCHED_DIR): + logger.error(f"Watched directory does not exist: {WATCHED_DIR}") + return + + event_handler = CodeChangeHandler() + observer = Observer() + observer.schedule(event_handler, path=WATCHED_DIR, recursive=True) + observer.start() + logger.info(f"Started monitoring {WATCHED_DIR} for changes...") + + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + logger.info("Stopping file monitoring...") + observer.stop() + except Exception as e: + logger.error(f"Error during monitoring: {str(e)}") + observer.stop() + raise + finally: + observer.join() + logger.info("File monitoring stopped") + + except Exception as e: + logger.error(f"Failed to start monitoring: {str(e)}") + raise diff --git a/coderag/search.py b/coderag/search.py index 0477406..ea15a76 100644 --- a/coderag/search.py +++ b/coderag/search.py @@ -1,29 +1,76 @@ -import numpy as np -from coderag.index import load_index, get_metadata +import logging +from typing import Any, Dict, List + +import faiss + from coderag.embeddings import generate_embeddings +from coderag.index import get_metadata, load_index -def search_code(query, k=5): - """Search the FAISS index using a text query.""" - index = load_index() # Load the FAISS index - query_embedding = generate_embeddings(query) # Generate embedding for the query +logger = logging.getLogger(__name__) - if query_embedding is None: - print("Failed to generate query embedding.") - return [] - # Perform the search in FAISS - distances, indices = index.search(query_embedding, k) - - results = [] - for i, idx in enumerate(indices[0]): # Iterate over the search results - if idx < len(get_metadata()): # Ensure the index is within bounds - file_data = get_metadata()[idx] - results.append({ - "filename": file_data["filename"], - "filepath": file_data["filepath"], - "content": file_data["content"], - "distance": distances[0][i] # Access distance using the correct index - }) - else: - print(f"Warning: Index {idx} is out of bounds for metadata with length {len(get_metadata())}") - return results +def search_code(query: str, k: int = 5) -> List[Dict[str, Any]]: + """Search the FAISS index using a text query. + + Args: + query: The search query text + k: Number of results to return (default: 5) + + Returns: + List of search results with filename, filepath, content, and distance + """ + try: + if not query or not query.strip(): + logger.warning("Empty query provided") + return [] + + # Load the FAISS index + index = load_index() + if index is None: + logger.error("Failed to load FAISS index") + return [] + + if index.ntotal == 0: + logger.warning("FAISS index is empty") + return [] + + # Generate embedding for the query + query_embedding = generate_embeddings(query) + if query_embedding is None: + logger.error("Failed to generate query embedding") + return [] + # Normalize for cosine similarity (IndexFlatIP) + faiss.normalize_L2(query_embedding) + + # Perform the search in FAISS + k = min(k, index.ntotal) # Don't search for more items than exist + distances, indices = index.search(query_embedding, k) + + results = [] + metadata = get_metadata() + + for i, idx in enumerate(indices[0]): # Iterate over the search results + if 0 <= idx < len(metadata): # Ensure the index is within bounds + file_data = metadata[idx] + results.append( + { + "filename": file_data["filename"], + "filepath": file_data["filepath"], + "content": file_data["content"], + "distance": float(distances[0][i]), # Convert to Python float + } + ) + else: + logger.warning( + f"Index {idx} is out of bounds for metadata with length " + f"{len(metadata)}" + ) + + logger.debug( + f"Search returned {len(results)} results for query: " f"'{query[:50]}...'" + ) + return results + + except Exception as e: + logger.error(f"Error during code search: {str(e)}") + return [] diff --git a/main.py b/main.py index 0d36325..7061f1c 100644 --- a/main.py +++ b/main.py @@ -1,60 +1,131 @@ -import os import logging -import atexit +import os import warnings -from coderag.index import clear_index, add_to_index, save_index -from coderag.embeddings import generate_embeddings + from coderag.config import WATCHED_DIR -from coderag.monitor import start_monitoring, should_ignore_path +from coderag.embeddings import generate_embeddings +from coderag.index import add_to_index, clear_index, save_index +from coderag.monitor import should_ignore_path, start_monitoring -# Configure logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +# Configure comprehensive logging in the entrypoint only +handlers: list[logging.Handler] = [logging.StreamHandler()] +try: + # Enable file logging only if environment allows it + if os.getenv("CODERAG_ENABLE_FILE_LOGS", "1") == "1": + handlers.append(logging.FileHandler("coderag.log", encoding="utf-8")) +except Exception: + # Ignore file handler failures (e.g., read-only FS) + pass + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + handlers=handlers, + force=True, +) + +logger = logging.getLogger(__name__) # Suppress transformers warnings -warnings.filterwarnings("ignore", category=FutureWarning, module="transformers.tokenization_utils_base") +warnings.filterwarnings( + "ignore", category=FutureWarning, module="transformers.tokenization_utils_base" +) + + +def full_reindex() -> int: + """Perform a full reindex of the entire codebase. + + Returns: + Number of files successfully processed + """ + logger.info("Starting full reindexing of the codebase...") + + if not os.path.exists(WATCHED_DIR): + logger.error(f"Watched directory does not exist: {WATCHED_DIR}") + return 0 -def full_reindex(): - """Perform a full reindex of the entire codebase.""" - logging.info("Starting full reindexing of the codebase...") files_processed = 0 - for root, _, files in os.walk(WATCHED_DIR): - if should_ignore_path(root): # Check if the directory should be ignored - logging.info(f"Ignoring directory: {root}") - continue - - for file in files: - filepath = os.path.join(root, file) - if should_ignore_path(filepath): # Check if the file should be ignored - logging.info(f"Ignoring file: {filepath}") + files_failed = 0 + + try: + for root, _, files in os.walk(WATCHED_DIR): + if should_ignore_path(root): + logger.debug(f"Ignoring directory: {root}") continue - if file.endswith(".py"): - logging.info(f"Processing file: {filepath}") - try: - with open(filepath, 'r', encoding='utf-8') as f: - full_content = f.read() + for file in files: + filepath = os.path.join(root, file) + if should_ignore_path(filepath): + logger.debug(f"Ignoring file: {filepath}") + continue + + if file.endswith(".py"): + logger.debug(f"Processing file: {filepath}") + try: + with open(filepath, "r", encoding="utf-8") as f: + full_content = f.read() + + if not full_content.strip(): + logger.debug(f"Skipping empty file: {filepath}") + continue + + embeddings = generate_embeddings(full_content) + if embeddings is not None: + add_to_index(embeddings, full_content, file, filepath) + files_processed += 1 + else: + logger.warning( + f"Failed to generate embeddings for {filepath}" + ) + files_failed += 1 + + except (IOError, UnicodeDecodeError) as e: + logger.error(f"Error reading file {filepath}: {str(e)}") + files_failed += 1 + except Exception as e: + logger.error( + f"Unexpected error processing file {filepath}: {str(e)}" + ) + files_failed += 1 + + save_index() + logger.info( + f"Full reindexing completed. {files_processed} files processed, " + f"{files_failed} files failed" + ) + return files_processed + + except Exception as e: + logger.error(f"Critical error during reindexing: {str(e)}") + return files_processed + + +def main() -> None: + """Main entry point for the CodeRAG indexing system.""" + try: + logger.info("Starting CodeRAG indexing system") - embeddings = generate_embeddings(full_content) # Generate embeddings - if embeddings is not None: - add_to_index(embeddings, full_content, file, filepath) - else: - logging.warning(f"Failed to generate embeddings for {filepath}") - files_processed += 1 - except Exception as e: - logging.error(f"Error processing file {filepath}: {e}") + # Completely clear the FAISS index and metadata + logger.info("Clearing existing index...") + clear_index() - save_index() - logging.info(f"Full reindexing completed. {files_processed} files processed.") + # Perform a full reindex of the codebase + logger.info("Starting full reindex...") + processed_files = full_reindex() -def main(): - # Completely clear the FAISS index and metadata - clear_index() + if processed_files == 0: + logger.warning("No files were processed during indexing") + else: + logger.info("Indexing complete. Starting file monitoring...") + # Start monitoring the directory for changes + start_monitoring() - # Perform a full reindex of the codebase - full_reindex() + except KeyboardInterrupt: + logger.info("Received interrupt signal, shutting down gracefully") + except Exception as e: + logger.error(f"Critical error in main: {str(e)}") + raise - # Start monitoring the directory for changes - start_monitoring() if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/prompt_flow.py b/prompt_flow.py index 643c3e9..b43d129 100644 --- a/prompt_flow.py +++ b/prompt_flow.py @@ -1,53 +1,123 @@ +import logging +from typing import Optional as _Optional + from openai import OpenAI + from coderag.config import OPENAI_API_KEY, OPENAI_CHAT_MODEL from coderag.search import search_code -client = OpenAI(api_key=OPENAI_API_KEY) +logger = logging.getLogger(__name__) -SYSTEM_PROMPT = """ -You are an expert coding assistant. Your task is to help users with their question. Use the retrieved code context to inform your responses, but feel free to suggest better solutions if appropriate. -""" +# Initialize OpenAI client with error handling +client: _Optional[OpenAI] +try: + if not OPENAI_API_KEY: + raise ValueError("OpenAI API key not found") + client = OpenAI(api_key=OPENAI_API_KEY) + logger.info(f"OpenAI client initialized with chat model: {OPENAI_CHAT_MODEL}") +except Exception as e: + logger.error(f"Failed to initialize OpenAI client: {e}") + client = None -PRE_PROMPT = """ -Based on the user's query and the following code context, provide a helpful response. If improvements can be made, suggest them with explanations. +SYSTEM_PROMPT = ( + "You are an expert coding assistant. Your task is to help users with their " + "question. Use the retrieved code context to inform your responses, but feel " + "free to suggest better solutions if appropriate." +) -User Query: {query} +PRE_PROMPT = ( + "Based on the user's query and the following code context, provide a helpful " + "response. If improvements can be made, suggest them with explanations.\n\n" + "User Query: {query}\n\n" + "Retrieved Code Context:\n{code_context}\n\nYour response:" +) -Retrieved Code Context: -{code_context} -Your response: -""" +def execute_rag_flow(user_query: str) -> str: + """Execute the RAG flow for answering user queries. -def execute_rag_flow(user_query): + Args: + user_query: The user's question or request + + Returns: + AI-generated response based on code context + """ try: + if not client: + logger.error("OpenAI client not initialized") + return ( + "Error: AI service is not available. Please check your " + "OpenAI API key." + ) + + if not user_query or not user_query.strip(): + logger.warning("Empty query received") + return "Please provide a question or request." + + logger.info(f"Processing query: '{user_query[:50]}...'") + # Perform code search search_results = search_code(user_query) - + if not search_results: - return "No relevant code found for your query." - - # Prepare code context - code_context = "\n\n".join([ - f"File: {result['filename']}\n{result['content']}" - for result in search_results[:3] # Limit to top 3 results - ]) - + logger.info("No relevant code found for query") + return ( + "No relevant code found for your query. The codebase might not be " + "indexed yet or your query might be too specific." + ) + + logger.debug(f"Found {len(search_results)} search results") + + # Prepare code context with error handling + try: + code_context = "\n\n".join( + [ + ( + f"File: {result['filename']}\n" + f"Path: {result['filepath']}\n" + # Cosine similarity (IndexFlatIP returns inner product) + f"Similarity: {max(0.0, min(1.0, result['distance'])):.3f}\n" + f"{result['content']}" + ) + for result in search_results[:3] # Limit to top 3 results + ] + ) + except (KeyError, TypeError) as e: + logger.error(f"Error preparing code context: {e}") + return "Error processing search results. Please try again." + # Construct the full prompt full_prompt = PRE_PROMPT.format(query=user_query, code_context=code_context) - - # Generate response using OpenAI - response = client.chat.completions.create( - model=OPENAI_CHAT_MODEL, - messages=[ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": full_prompt} - ], - temperature=0.3, - max_tokens=4000 - ) - - return response.choices[0].message.content.strip() - + + # Generate response using OpenAI with error handling + try: + logger.debug("Sending request to OpenAI") + # Rough heuristic: keep total under ~7000 tokens + est_prompt_tokens = max(1, len(full_prompt) // 4) + max_completion = max(256, min(2000, 7000 - est_prompt_tokens)) + response = client.chat.completions.create( + model=OPENAI_CHAT_MODEL, + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": full_prompt}, + ], + temperature=0.3, + max_tokens=max_completion, + timeout=60, + ) + + if not response.choices or not response.choices[0].message.content: + logger.error("Empty response from OpenAI") + return "Error: Received empty response from AI service." + + result = response.choices[0].message.content.strip() + logger.info("Successfully generated response") + return result + + except Exception as e: + logger.error(f"OpenAI API error: {str(e)}") + return "Error communicating with AI service. Please try again later." + except Exception as e: - return f"Error in RAG flow execution: {e}" \ No newline at end of file + logger.error(f"Unexpected error in RAG flow: {str(e)}") + return "An unexpected error occurred. Please try again." diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a070175 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,31 @@ +[tool.black] +line-length = 88 +target-version = ['py311'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.mypy_cache + | \.tox + | \.venv + | build + | dist + | env +)/ +''' + +[tool.isort] +profile = "black" +multi_line_output = 3 +line_length = 88 +known_first_party = ["coderag"] + +[tool.mypy] +python_version = "3.11" +ignore_missing_imports = true +disallow_untyped_defs = false +warn_unused_ignores = true +warn_redundant_casts = true +check_untyped_defs = true \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c365165..14f0046 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/scripts/initialize_index.py b/scripts/initialize_index.py index 0206a5e..03ee424 100644 --- a/scripts/initialize_index.py +++ b/scripts/initialize_index.py @@ -1,8 +1,10 @@ from coderag.index import save_index + def initialize_index(): save_index() print("FAISS index initialized and saved.") + if __name__ == "__main__": initialize_index() diff --git a/tests/test_faiss.py b/tests/test_faiss.py index f80fc89..ce57bbb 100644 --- a/tests/test_faiss.py +++ b/tests/test_faiss.py @@ -1,28 +1,29 @@ -import faiss -from coderag.index import load_index, retrieve_vectors, inspect_metadata, add_to_index, save_index, clear_index -from coderag.embeddings import generate_embeddings -import os +import numpy as np + +from coderag.config import EMBEDDING_DIM +from coderag.index import ( + add_to_index, + clear_index, + inspect_metadata, + load_index, + retrieve_vectors, + save_index, +) + def test_faiss_index(): # Clear the index before testing clear_index() - # Example text to generate embeddings - example_text = "This is a test document to be indexed." - - # Generate embeddings - embeddings = generate_embeddings(example_text) - if embeddings is None: - print("Embedding generation failed.") - return - - # Add to index - add_to_index(embeddings, example_text, "test_file.py", "test_file.py") + # Create a deterministic dummy embedding (no network needed) + vec = np.ones((1, EMBEDDING_DIM), dtype=np.float32) + # Add to index with small dummy content + add_to_index(vec, "dummy content", "test_file.py", "test_file.py") save_index() # Load the index index = load_index() - + assert index is not None, "Failed to load FAISS index." # Check if index has vectors assert index.ntotal > 0, "FAISS index is empty. No vectors found!" print(f"FAISS index has {index.ntotal} vectors.") @@ -30,9 +31,10 @@ def test_faiss_index(): # Retrieve and inspect vectors vectors = retrieve_vectors(5) print(f"Retrieved {len(vectors)} vectors from the index.") - + # Inspect metadata inspect_metadata(5) + if __name__ == "__main__": test_faiss_index()