fix(stuff): copilot bad boy

Author: Mte90

db.py

@@ -3,6 +3,134 @@
 from typing import Any, Dict, List, Optional
 
 from config import CFG  # config (keeps chunk_size etc if needed)
+import atexit
+import logging
+import threading
+import queue
+
+logging.basicConfig(level=logging.INFO)
+_LOG = logging.getLogger(__name__)
+
+# registry of DBWriter instances keyed by database path
+_WRITERS = {}
+_WRITERS_LOCK = threading.Lock()
+
+class _DBTask:
+    def __init__(self, sql, params):
+        self.sql = sql
+        self.params = params
+        self.event = threading.Event()
+        self.rowid = None
+        self.exception = None
+
+class DBWriter:
+    def __init__(self, database_path, timeout_seconds=30):
+        self.database_path = database_path
+        self._q = queue.Queue()
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._worker, daemon=True, name=f"DBWriter-{database_path}")
+        self._timeout_seconds = timeout_seconds
+        self._thread.start()
+
+    def _open_conn(self):
+        conn = sqlite3.connect(self.database_path, timeout=self._timeout_seconds, check_same_thread=False)
+        # Reduce contention and allow concurrent readers during writes
+        conn.execute("PRAGMA journal_mode=WAL;")
+        # Make busy timeout explicit (milliseconds)
+        conn.execute("PRAGMA busy_timeout = 30000;")
+        # Optional: balance durability and performance
+        conn.execute("PRAGMA synchronous = NORMAL;")
+        return conn
+
+    def _worker(self):
+        conn = None
+        try:
+            conn = self._open_conn()
+            cur = conn.cursor()
+            while not self._stop.is_set():
+                try:
+                    task = self._q.get(timeout=0.5)
+                except queue.Empty:
+                    continue
+                if task is None:
+                    # sentinel to stop
+                    break
+                try:
+                    cur.execute(task.sql, task.params)
+                    conn.commit()
+                    task.rowid = cur.lastrowid
+                except Exception as e:
+                    # store exception for the waiting thread to raise or handle
+                    task.exception = e
+                    try:
+                        conn.rollback()
+                    except Exception:
+                        pass
+                    _LOG.exception("Error executing DB task")
+                finally:
+                    task.event.set()
+                    self._q.task_done()
+        except Exception:
+            _LOG.exception("DBWriter thread initialization failed")
+        finally:
+            if conn:
+                try:
+                    conn.close()
+                except Exception:
+                    pass
+
+    def enqueue_and_wait(self, sql, params, wait_timeout=60.0):
+        """
+        Enqueue an SQL write and wait for the background thread to perform it.
+        Returns the lastrowid or raises the exception raised during execution.
+        """
+        task = _DBTask(sql, params)
+        self._q.put(task)
+        completed = task.event.wait(wait_timeout)
+        if not completed:
+            raise TimeoutError(f"Timed out waiting for DB write to {self.database_path}")
+        if task.exception:
+            # re-raise sqlite3.OperationalError or other exceptions
+            raise task.exception
+        return task.rowid
+
+    def enqueue_no_wait(self, sql, params):
+        """
+        Fire-and-forget enqueue (no result returned).
+        """
+        task = _DBTask(sql, params)
+        self._q.put(task)
+        return task
+
+    def stop(self, wait=True):
+        """Stop the worker thread. If wait=True, block until thread joins."""
+        self._stop.set()
+        # enqueue sentinel for immediate exit
+        self._q.put(None)
+        if wait:
+            self._thread.join(timeout=5.0)
+
+def _get_writer(database_path):
+    with _WRITERS_LOCK:
+        w = _WRITERS.get(database_path)
+        if w is None:
+            w = DBWriter(database_path)
+            _WRITERS[database_path] = w
+        return w
+
+def stop_all_writers():
+    """Stop all DBWriter threads (called automatically at process exit)."""
+    with _WRITERS_LOCK:
+        writers = list(_WRITERS.values())
+        _WRITERS.clear()
+    for w in writers:
+        try:
+            w.stop(wait=True)
+        except Exception:
+            _LOG.exception("Error stopping DBWriter")
+
+# ensure cleanup at exit
+atexit.register(stop_all_writers)
 
 # Simple connection helper: we open new connections per operation so the code is robust
 # across threads. We set WAL journal mode for safer concurrency.
@@ -108,21 +236,20 @@ def update_analysis_status(database_path: str, analysis_id: int, status: str) ->
         conn.close()
 
 
-def store_file(database_path: str, analysis_id: int, path: str, content: str, language: str) -> int:
+def store_file(database_path, analysis_id, path, content, language):
     """
-    Insert a file row. Returns the new file id.
+    Insert a file record into the DB using a queued single-writer to avoid
+    sqlite 'database is locked' errors in multithreaded scenarios.
+    Returns lastrowid (same as the previous store_file implementation).
     """
-    conn = _get_connection(database_path)
-    try:
-        cur = conn.cursor()
-        cur.execute(
-            "INSERT INTO files (analysis_id, path, content, language, snippet) VALUES (?, ?, ?, ?, ?)",
-            (analysis_id, path, content, language, (content[:512] if content else "")),
-        )
-        conn.commit()
-        return int(cur.lastrowid)
-    finally:
-        conn.close()
+    snippet = (content[:512] if content else "")
+    sql = "INSERT INTO files (analysis_id, path, content, language, snippet) VALUES (?, ?, ?, ?, ?)"
+    params = (analysis_id, path, content, language, snippet)
+
+    writer = _get_writer(database_path)
+    # We wait for the background writer to complete the insert and then return the row id.
+    # This preserves the synchronous semantics callers expect.
+    return writer.enqueue_and_wait(sql, params, wait_timeout=60.0)
 
 
 def insert_chunk_row_with_null_embedding(database_path: str, file_id: int, path: str, chunk_index: int) -> int: