Merge branch 'main' into pr/85

Author: cyclotruc

src/config.py

@@ -2,6 +2,7 @@
 
 MAX_DISPLAY_SIZE: int = 300_000
 TMP_BASE_PATH: str = "/tmp/gitingest"
+DELETE_REPO_AFTER: int = 60 * 60  # In seconds
 
 EXAMPLE_REPOS: list[dict[str, str]] = [
     {"name": "GitIngest", "url": "https://github.com/cyclotruc/gitingest"},

src/gitingest/parse_query.py

@@ -5,7 +5,7 @@
 import string
 import uuid
 from typing import Any
-from urllib.parse import unquote
+from urllib.parse import unquote, urlparse
 
 from gitingest.exceptions import InvalidPatternError
 from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
@@ -105,6 +105,10 @@ def _parse_url(url: str) -> dict[str, Any]:
     if not url.startswith(("https://", "http://")):
         url = "https://" + url
 
+    # Parse URL and reconstruct it without query parameters and fragments
+    parsed_url = urlparse(url)
+    url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
+
     # Extract domain and path
     url_parts = url.split("/")
     domain = url_parts[2]

src/main.py

@@ -1,6 +1,10 @@
 """ Main module for the FastAPI application. """
 
+import asyncio
 import os
+import shutil
+import time
+from contextlib import asynccontextmanager
 
 from api_analytics.fastapi import Analytics
 from dotenv import load_dotenv
@@ -12,14 +16,86 @@
 from slowapi.errors import RateLimitExceeded
 from starlette.middleware.trustedhost import TrustedHostMiddleware
 
+from config import DELETE_REPO_AFTER, TMP_BASE_PATH
 from routers import download, dynamic, index
 from server_utils import limiter
 
 # Load environment variables from .env file
 load_dotenv()
 
-# Initialize the FastAPI application
-app = FastAPI()
+
+async def remove_old_repositories():
+    """
+    Background task that runs periodically to clean up old repository directories.
+
+    This task:
+    - Scans the TMP_BASE_PATH directory every 60 seconds
+    - Removes directories older than DELETE_REPO_AFTER seconds
+    - Before deletion, logs repository URLs to history.txt if a matching .txt file exists
+    - Handles errors gracefully if deletion fails
+
+    The repository URL is extracted from the first .txt file in each directory,
+    assuming the filename format: "owner-repository.txt"
+    """
+    while True:
+        try:
+            if not os.path.exists(TMP_BASE_PATH):
+                await asyncio.sleep(60)
+                continue
+
+            current_time = time.time()
+
+            for folder in os.listdir(TMP_BASE_PATH):
+                folder_path = os.path.join(TMP_BASE_PATH, folder)
+
+                # Skip if folder is not old enough
+                if current_time - os.path.getctime(folder_path) <= DELETE_REPO_AFTER:
+                    continue
+
+                # Try to log repository URL before deletion
+                try:
+                    txt_files = [f for f in os.listdir(folder_path) if f.endswith(".txt")]
+                    if txt_files:
+                        filename = txt_files[0].replace(".txt", "")
+                        if "-" in filename:
+                            owner, repo = filename.split("-", 1)
+                            repo_url = f"https://github.com/{owner}/{repo}"
+                            with open("history.txt", "a") as history:
+                                history.write(f"{repo_url}\n")
+                except Exception as e:
+                    print(f"Error logging repository URL for {folder_path}: {str(e)}")
+
+                # Delete the folder
+                try:
+                    shutil.rmtree(folder_path)
+                except Exception as e:
+                    print(f"Error deleting {folder_path}: {str(e)}")
+
+        except Exception as e:
+            print(f"Error in remove_old_repositories: {str(e)}")
+
+        await asyncio.sleep(60)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """
+    Lifecycle manager for the FastAPI application.
+    Handles startup and shutdown events.
+    """
+    task = asyncio.create_task(remove_old_repositories())
+
+    yield
+    # Cancel the background task on shutdown
+    task.cancel()
+    try:
+        await task
+    except asyncio.CancelledError:
+        pass
+
+
+# Initialize the FastAPI application with lifespan
+app = FastAPI(lifespan=lifespan)
 app.state.limiter = limiter
 
 

tests/test_parse_query.py

@@ -155,3 +155,11 @@ def test_parse_query_uuid_uniqueness() -> None:
     result1 = parse_query(path, max_file_size=100, from_web=False)
     result2 = parse_query(path, max_file_size=100, from_web=False)
     assert result1["id"] != result2["id"]
+
+
+def test_parse_url_with_query_and_fragment() -> None:
+    url = "https://github.com/user/repo?arg=value#fragment"
+    result = _parse_url(url)
+    assert result["user_name"] == "user"
+    assert result["repo_name"] == "repo"
+    assert result["url"] == "https://github.com/user/repo"  # URL should be cleaned