Add FileWatcher for project monitoring, expose indexing stats to UI/API, and fix project deletion (#3)

Co-authored-by: Mte90 <403283+Mte90@users.noreply.github.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>

Author: Copilot

.env.example

@@ -30,3 +30,13 @@ CODING_MODEL=gpt-4o-code-preview
 # Host (default 127.0.0.1) and port (default 8000) used when launching uvicorn
 UVICORN_HOST=127.0.0.1
 UVICORN_PORT=8080
+
+# FileWatcher configuration
+# Enable/disable the background file watcher that monitors project changes (default: true)
+FILE_WATCHER_ENABLED=true
+
+# Interval in seconds between directory scans (default: 10, minimum: 5)
+FILE_WATCHER_INTERVAL=10
+
+# Debounce time in seconds before processing detected changes (default: 5, minimum: 1)
+FILE_WATCHER_DEBOUNCE=5

REST_API.md

@@ -101,7 +101,7 @@ Returns list of all registered projects.
 GET /api/projects/{project_id}
 ```
 
-Returns details for a specific project.
+Returns details for a specific project including indexing statistics.
 
 **Response:**
 ```json
@@ -112,10 +112,20 @@ Returns details for a specific project.
   "database_path": "~/.picocode/projects/1234567890abcdef.db",
   "status": "ready",
   "created_at": "2025-11-06T14:30:00",
-  "last_indexed_at": "2025-11-06T15:00:00"
+  "last_indexed_at": "2025-11-06T15:00:00",
+  "indexing_stats": {
+    "file_count": 150,
+    "embedding_count": 450,
+    "is_indexed": true
+  }
 }
 ```
 
+**Indexing Stats Fields:**
+- `file_count`: Number of files indexed in the project
+- `embedding_count`: Number of embeddings (chunks) generated
+- `is_indexed`: Boolean indicating if project has any indexed files
+
 ### Delete Project
 
 ```http

endpoints/project_endpoints.py

@@ -45,6 +45,15 @@ def api_create_project(request: CreateProjectRequest):
     """
     try:
         project = get_or_create_project(request.path, request.name)
+        
+        # Add project to file watcher if available
+        try:
+            from main import _file_watcher
+            if _file_watcher and _file_watcher.is_running():
+                _file_watcher.add_project(project["id"], project["path"])
+        except Exception as e:
+            logger.warning(f"Could not add project to file watcher: {e}")
+        
         return JSONResponse(project)
     except ValueError as e:
         # ValueError is expected for invalid inputs, safe to show message
@@ -86,12 +95,38 @@ def api_get_project(project_id: str):
     
     - **project_id**: Unique project identifier
     
-    Returns project metadata or 404 if not found.
+    Returns project metadata including indexing status and statistics or 404 if not found.
     """
     try:
         project = get_project_by_id(project_id)
         if not project:
             return JSONResponse({"error": "Project not found"}, status_code=404)
+        
+        # Add indexing statistics if project has a database
+        db_path = project.get("database_path")
+        if db_path and os.path.exists(db_path):
+            try:
+                from db.operations import get_project_stats
+                stats = get_project_stats(db_path)
+                project["indexing_stats"] = {
+                    "file_count": stats.get("file_count", 0),
+                    "embedding_count": stats.get("embedding_count", 0),
+                    "is_indexed": stats.get("file_count", 0) > 0
+                }
+            except Exception as e:
+                logger.warning(f"Could not get stats for project {project_id}: {e}")
+                project["indexing_stats"] = {
+                    "file_count": 0,
+                    "embedding_count": 0,
+                    "is_indexed": False
+                }
+        else:
+            project["indexing_stats"] = {
+                "file_count": 0,
+                "embedding_count": 0,
+                "is_indexed": False
+            }
+        
         return JSONResponse(project)
     except Exception as e:
         logger.exception(f"Error getting project: {e}")

endpoints/web_endpoints.py

@@ -33,17 +33,26 @@ def api_health():
     - **status**: "ok" if service is running
     - **version**: API version
     - **features**: List of enabled features
+    - **file_watcher**: Status of the FileWatcher (if enabled)
     
     Use this endpoint for:
     - Load balancer health checks
     - Monitoring systems
     - Service availability verification
     """
-    return JSONResponse({
+    from main import _file_watcher
+    
+    health_data = {
         "status": "ok",
         "version": "0.2.0",
-        "features": ["rag", "per-project-db", "pycharm-api", "incremental-indexing", "rate-limiting", "caching"]
-    })
+        "features": ["rag", "per-project-db", "pycharm-api", "incremental-indexing", "rate-limiting", "caching", "file-watcher"]
+    }
+    
+    # Add file watcher status if available
+    if _file_watcher:
+        health_data["file_watcher"] = _file_watcher.get_status()
+    
+    return JSONResponse(health_data)
 
 
 @router.get("/", response_class=HTMLResponse)

main.py

@@ -8,19 +8,26 @@
 import os
 import uvicorn
 
+from db import operations as db_operations
 from db.operations import get_or_create_project
 from utils.config import CFG
 from utils.logger import get_logger
 from endpoints.project_endpoints import router as project_router
 from endpoints.query_endpoints import router as query_router
 from endpoints.web_endpoints import router as web_router
+from utils.file_watcher import FileWatcher
 
 logger = get_logger(__name__)
 
+# Global FileWatcher instance
+_file_watcher = None
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """Application lifespan handler."""
+    global _file_watcher
+    
     # Project registry is auto-initialized when needed via create_project
 
     # Auto-create default project from configured local_path if it exists
@@ -31,7 +38,42 @@ async def lifespan(app: FastAPI):
         except Exception as e:
             logger.warning(f"Could not create default project: {e}")
 
+    # Start FileWatcher if enabled
+    if CFG.get("file_watcher_enabled", True):
+        try:
+            _file_watcher = FileWatcher(
+                logger=logger,
+                enabled=True,
+                debounce_seconds=CFG.get("file_watcher_debounce", 5),
+                check_interval=CFG.get("file_watcher_interval", 10)
+            )
+            
+            # Add all existing projects to the watcher
+            try:
+                projects = db_operations.list_projects()
+                for project in projects:
+                    if project.get("path") and os.path.exists(project["path"]):
+                        _file_watcher.add_project(project["id"], project["path"])
+            except Exception as e:
+                logger.warning(f"Could not add projects to file watcher: {e}")
+            
+            _file_watcher.start()
+            logger.info("FileWatcher started successfully")
+        except Exception as e:
+            logger.error(f"Failed to start FileWatcher: {e}")
+            _file_watcher = None
+    else:
+        logger.info("FileWatcher is disabled in configuration")
+    
     yield
+    
+    # Stop FileWatcher on shutdown
+    if _file_watcher:
+        try:
+            _file_watcher.stop()
+            logger.info("FileWatcher stopped successfully")
+        except Exception as e:
+            logger.error(f"Error stopping FileWatcher: {e}")
 
 
 app = FastAPI(

templates/index.html

@@ -63,13 +63,16 @@ <h5 class="card-title">Projects</h5>
                     <div class="flex-grow-1">
                       <div class="fw-bold">{{ p.name or p.path.split('/')[-1] }}</div>
                       <small class="text-muted">{{ p.path }}</small><br>
-                      <small class="text-muted">Status: <span class="badge bg-{{ 'success' if p.status=='ready' else 'warning' }}">{{ p.status }}</span></small>
+                      <small class="text-muted">
+                        Status: <span class="badge bg-{{ 'success' if p.status=='ready' else ('warning' if p.status=='indexing' else 'secondary') }}" data-status="{{ p.status }}">{{ p.status }}</span>
+                      </small>
+                      <br>
+                      <small class="text-muted indexing-info" style="display:none;">
+                        Files: <span class="file-count">0</span> | Embeddings: <span class="embedding-count">0</span>
+                      </small>
                     </div>
                     <div>
-                      <form action="/projects/{{ p.id }}" method="post" style="display:inline;" onsubmit="return confirm('Delete this project and its database?');">
-                        <input type="hidden" name="_method" value="DELETE">
-                        <button type="submit" class="btn btn-sm btn-outline-danger">×</button>
-                      </form>
+                      <button type="button" class="btn btn-sm btn-outline-danger delete-project-btn" data-project-id="{{ p.id }}" title="Delete project">×</button>
                     </div>
                   </li>
                 {% endfor %}
@@ -262,28 +265,81 @@ <h5 class="card-title">Chat</h5>
       firstProject.classList.add("active");
     }
 
-    // Poll for project status updates
+    // Poll for project status updates with detailed stats
     setInterval(async () => {
       try {
         const response = await fetch("/projects/status");
         const projects = await response.json();
 
-        // Update project status badges
-        projects.forEach(p => {
+        // Update project status badges and stats
+        for (const p of projects) {
           const item = document.querySelector(`[data-project-id="${p.id}"]`);
           if (item) {
             const badge = item.querySelector('.badge');
             if (badge) {
               badge.className = `badge bg-${p.status === 'ready' ? 'success' : p.status === 'indexing' ? 'warning' : 'secondary'}`;
               badge.textContent = p.status;
             }
+            
+            // Fetch detailed stats for this project
+            try {
+              const detailResponse = await fetch(`/api/projects/${p.id}`);
+              const details = await detailResponse.json();
+              
+              if (details.indexing_stats) {
+                const indexingInfo = item.querySelector('.indexing-info');
+                const fileCount = item.querySelector('.file-count');
+                const embeddingCount = item.querySelector('.embedding-count');
+                
+                if (indexingInfo && fileCount && embeddingCount) {
+                  fileCount.textContent = details.indexing_stats.file_count || 0;
+                  embeddingCount.textContent = details.indexing_stats.embedding_count || 0;
+                  
+                  // Show stats if project has been indexed
+                  if (details.indexing_stats.file_count > 0) {
+                    indexingInfo.style.display = 'block';
+                  } else {
+                    indexingInfo.style.display = 'none';
+                  }
+                }
+              }
+            } catch (detailErr) {
+              // Ignore errors fetching individual project details
+            }
           }
-        });
+        }
       } catch (err) {
         console.error("Error polling status:", err);
       }
     }, 3000);
 
+    // Handle project deletion
+    document.addEventListener('click', async (e) => {
+      if (e.target.classList.contains('delete-project-btn')) {
+        const projectId = e.target.getAttribute('data-project-id');
+        
+        if (!confirm('Delete this project and its database?')) {
+          return;
+        }
+        
+        try {
+          const response = await fetch(`/projects/${projectId}`, {
+            method: 'DELETE'
+          });
+          
+          if (response.ok) {
+            // Reload page to show updated project list
+            window.location.reload();
+          } else {
+            const data = await response.json();
+            alert(`Failed to delete project: ${data.error || 'Unknown error'}`);
+          }
+        } catch (err) {
+          alert(`Error deleting project: ${err.message}`);
+        }
+      }
+    });
+
     // Initial render
     renderChat();
   </script>

utils/config.py

@@ -37,4 +37,9 @@ def _bool_env(name, default):
     # uvicorn host/port (from .env)
     "uvicorn_host": os.getenv("UVICORN_HOST", "127.0.0.1"),
     "uvicorn_port": int(os.getenv("UVICORN_PORT", "8000")),
+
+    # FileWatcher configuration
+    "file_watcher_enabled": _bool_env("FILE_WATCHER_ENABLED", True),
+    "file_watcher_interval": _int_env("FILE_WATCHER_INTERVAL", 10),
+    "file_watcher_debounce": _int_env("FILE_WATCHER_DEBOUNCE", 5),
 }