Package preparation work

cyclotruc · cyclotruc · commit 1cad5b0ddc43 · 2024-12-21T23:49:51.000Z
diff --git a/pytest.ini b/pytest.ini
@@ -4,7 +4,7 @@ testpaths = src/tests
 asyncio_mode = auto
 
 # Coverage configuration
-addopts = --cov=utils --cov=ingest --cov-report=term-missing --cov-report=html
+addopts = --no-cov
 
 python_files = test_*.py
 python_classes = Test*
diff --git a/requirements.txt b/requirements.txt
@@ -5,5 +5,4 @@ slowapi
 tokencost
 pytest
 pytest-asyncio
-pytest-cov
 click>=8.0.0
diff --git a/setup.py b/setup.py
@@ -3,19 +3,39 @@
 setup(
     name="gitingest",
     version="0.1.0",
-    packages=find_packages(),
+    packages=find_packages(where="src"),
+    package_dir={"": "src"},
     include_package_data=True,
     install_requires=[
-        "click",
+        "click>=8.0.0",
+        "tokencost",
     ],
+    extras_require={
+        'dev': [
+            'pytest',
+            'pytest-asyncio',
+        ],
+    },
     entry_points={
         "console_scripts": [
-            "gitingest=src.cli:main",
+            "gitingest=gitingest.cli:main",
         ],
     },
     python_requires=">=3.6",
-    author="Your Name",
+    author="Romain Courtois",
+    author_email="romain@coderamp.io",
     description="A tool to analyze and create text dumps of git repositories",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
-) 
+    url="https://github.com/cyclotruc/gitingest",
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+    ],
+)
diff --git a/src/config.py b/src/config.py
@@ -1,30 +1,5 @@
 MAX_DISPLAY_SIZE = 300000
-MAX_FILE_SIZE = 10000000
 TMP_BASE_PATH = "../tmp"
-CLONE_TIMEOUT = 20
-
-DEFAULT_IGNORE_PATTERNS = [
-    '*.pyc', '*.pyo', '*.pyd', '__pycache__',  # Python
-    'node_modules', 'bower_components',        # JavaScript
-    '.git', '.svn', '.hg', '.gitignore',      # Version control
-    '*.svg', '*.png', '*.jpg', '*.jpeg', '*.gif', # Images
-    'venv', '.venv', 'env',                   # Virtual environments
-    '.idea', '.vscode',                       # IDEs
-    '*.log', '*.bak', '*.swp', '*.tmp',      # Temporary files
-    '.DS_Store',                             # macOS
-    'Thumbs.db',                             # Windows
-    'build', 'dist',                         # Build directories
-    '*.egg-info',                           # Python egg info
-    '*.so', '*.dylib', '*.dll',             # Compiled libraries
-    'package-lock.json', 'yarn.lock',        # Package lock files
-    'pnpm-lock.yaml', 'npm-shrinkwrap.json', # More package lock files
-    'COPYING.*', 'COPYRIGHT',                # More license-related files
-    'AUTHORS', 'AUTHORS.*',                  # Author files
-    'CONTRIBUTORS', 'CONTRIBUTORS.*',        # Contributor files
-    'THANKS', 'THANKS.*',                    # Acknowledgment files
-    'CHANGELOG', 'CHANGELOG.*',              # Change logs
-    'CONTRIBUTING', 'CONTRIBUTING.*'         # Contribution guidelines
-]
 
 EXAMPLE_REPOS = [
     {"name": "Gitingest", "url": "https://github.com/cyclotruc/gitingest"},
diff --git a/src/gitingest/__init__.py b/src/gitingest/__init__.py
@@ -0,0 +1,6 @@
+from .ingest import ingest_from_query
+from .clone import clone_repo
+from .parse_query import parse_query
+
+
+__all__ = ["ingest_from_query", "clone_repo", "parse_query"]
diff --git a/src/gitingest/cli.py b/src/gitingest/cli.py
@@ -1,7 +1,8 @@
-import click
 import os
-from .ingest import analyze_codebase, DEFAULT_IGNORE_PATTERNS, MAX_FILE_SIZE
 import pathlib
+import click
+
+from .ingest import analyze_codebase, DEFAULT_IGNORE_PATTERNS, MAX_FILE_SIZE
 
 def normalize_pattern(pattern: str) -> str:
     pattern = pattern.strip()
diff --git a/src/gitingest/clone.py b/src/gitingest/clone.py
@@ -1,10 +1,11 @@
 import asyncio
 from typing import Tuple
 
-from .decorators import async_timeout
-from config import CLONE_TIMEOUT
+from gitingest.utils import async_timeout
 
-async def check_repo_exists(url: str) -> Tuple[bool, str]:
+CLONE_TIMEOUT = 20
+
+async def check_repo_exists(url: str) -> bool:
     proc = await asyncio.create_subprocess_exec(
         "git",
         "ls-remote",
@@ -13,9 +14,7 @@ async def check_repo_exists(url: str) -> Tuple[bool, str]:
         stderr=asyncio.subprocess.PIPE,
     )
     await proc.communicate()
-    if proc.returncode != 0:
-        return False
-    return True
+    return proc.returncode == 0
 
 @async_timeout(CLONE_TIMEOUT)
 async def clone_repo(query: dict) -> str:
diff --git a/src/gitingest/ingest.py b/src/gitingest/ingest.py
@@ -1,13 +1,13 @@
 import os
 from fnmatch import fnmatch
 from typing import Dict, List, Union
+
 from tokencost import count_string_tokens
 
-# Configuration
-MAX_FILE_SIZE = 10000000  # 10MB
+MAX_FILE_SIZE = 10 * 1024 * 1024  # 10MB
 MAX_DIRECTORY_DEPTH = 20  # Maximum depth of directory traversal
 MAX_FILES = 10000  # Maximum number of files to process
-MAX_TOTAL_SIZE_BYTES = 500 * 1024 * 1024  # Total size limit
+MAX_TOTAL_SIZE_BYTES = 500 * 1024 * 1024  # 500MB
 
 DEFAULT_IGNORE_PATTERNS = [
     # Python
diff --git a/src/gitingest/parse_query.py b/src/gitingest/parse_query.py
@@ -1,10 +1,11 @@
+from typing import List
 import uuid
 import re
 import os 
-from utils.log_convert import logSliderToSize
-from typing import List
 
-from config import TMP_BASE_PATH, DEFAULT_IGNORE_PATTERNS
+from .ingest import DEFAULT_IGNORE_PATTERNS
+
+TMP_BASE_PATH = "../tmp"
 
 def parse_url(url: str) -> dict:
     parsed = {
@@ -71,9 +72,9 @@ def override_ignore_patterns(ignore_patterns: List[str], include_patterns: List[
             ignore_patterns.remove(pattern)
     return ignore_patterns
 
-def parse_query(input_text: str, slider_position: int, pattern_type: str, pattern: str) -> dict:
+def parse_query(input_text: str, max_file_size: int, pattern_type: str, pattern: str) -> dict:
     query = parse_url(input_text)
-    query['max_file_size'] = logSliderToSize(slider_position)
+    query['max_file_size'] = max_file_size
     query['pattern_type'] = pattern_type
     parsed_pattern = parse_patterns(pattern)
     if pattern_type == 'include':
diff --git a/src/gitingest/tests/__init__.py b/src/gitingest/tests/__init__.py
diff --git a/src/gitingest/tests/conftest.py b/src/gitingest/tests/conftest.py
@@ -0,0 +1,9 @@
+import os
+import sys
+
+# Get the absolute path of the project root directory (one level up from tests)
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# Add both the project root and src directory to PYTHONPATH
+sys.path.insert(0, project_root)
+sys.path.insert(0, os.path.join(project_root, 'src')) 
diff --git a/src/gitingest/tests/test_clone.py b/src/gitingest/tests/test_clone.py
@@ -1,5 +1,5 @@
 import pytest
-from utils.clone import clone_repo, check_repo_exists
+from clone import clone_repo, check_repo_exists
 from unittest.mock import patch, AsyncMock
 
 @pytest.mark.asyncio
@@ -11,7 +11,7 @@ async def test_clone_repo_with_commit():
         'local_path': '/tmp/repo'
     }
     
-    with patch('utils.clone.check_repo_exists', return_value=True) as mock_check:
+    with patch('clone.check_repo_exists', return_value=True) as mock_check:
         with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
             mock_process = AsyncMock()
             mock_process.communicate.return_value = (b'output', b'error')
@@ -30,7 +30,7 @@ async def test_clone_repo_without_commit():
         'local_path': '/tmp/repo'
     }
     
-    with patch('utils.clone.check_repo_exists', return_value=True) as mock_check:
+    with patch('clone.check_repo_exists', return_value=True) as mock_check:
         with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
             mock_process = AsyncMock()
             mock_process.communicate.return_value = (b'output', b'error')
@@ -49,7 +49,7 @@ async def test_clone_repo_nonexistent_repository():
         'local_path': '/tmp/repo'
     }
     
-    with patch('utils.clone.check_repo_exists', return_value=False) as mock_check:
+    with patch('gitingest.clone.check_repo_exists', return_value=False) as mock_check:
         with pytest.raises(ValueError, match="Repository not found"):
             await clone_repo(query)
             mock_check.assert_called_once_with(query['url'])
diff --git a/src/gitingest/tests/test_ingest.py b/src/gitingest/tests/test_ingest.py
diff --git a/src/gitingest/tests/test_parse_query.py b/src/gitingest/tests/test_parse_query.py
@@ -1,6 +1,6 @@
 import pytest
-from utils.parse_query import parse_query, parse_url
-from config import DEFAULT_IGNORE_PATTERNS
+from gitingest.parse_query import parse_query, parse_url
+from gitingest.ingest import DEFAULT_IGNORE_PATTERNS
 
 
 def test_parse_url_valid():
@@ -26,7 +26,7 @@ def test_parse_query_basic():
         "https://gitlab.com/user/repo"
     ]
     for url in test_cases:
-        result = parse_query(url, slider_position=50, pattern_type='exclude', pattern='*.txt')
+        result = parse_query(url, max_file_size=50, pattern_type='exclude', pattern='*.txt')
         assert result["user_name"] == "user"
         assert result["repo_name"] == "repo"
         assert result["url"] == url
@@ -35,12 +35,12 @@ def test_parse_query_basic():
 
 def test_parse_query_include_pattern():
     url = "https://github.com/user/repo"
-    result = parse_query(url, slider_position=50, pattern_type='include', pattern='*.py')
+    result = parse_query(url, max_file_size=50, pattern_type='include', pattern='*.py')
     assert result["pattern_type"] == "include"
     assert result["include_patterns"] == ["*.py"]
     assert result["ignore_patterns"] == DEFAULT_IGNORE_PATTERNS
 
 def test_parse_query_invalid_pattern():
     url = "https://github.com/user/repo"
     with pytest.raises(ValueError, match="Pattern.*contains invalid characters"):
-        parse_query(url, slider_position=50, pattern_type='include', pattern='*.py;rm -rf')
+        parse_query(url, max_file_size=50, pattern_type='include', pattern='*.py;rm -rf')
diff --git a/src/gitingest/utils.py b/src/gitingest/utils.py
@@ -1,3 +1,5 @@
+
+## Async Timeout decorator
 import asyncio
 import functools
 from typing import TypeVar, Callable
@@ -17,4 +19,4 @@ async def wrapper(*args, **kwargs) -> T:
             except asyncio.TimeoutError:
                 raise AsyncTimeoutError(f"Operation timed out after {seconds} seconds")
         return wrapper
-    return decorator
+    return decorator
diff --git a/src/main.py b/src/main.py
@@ -1,14 +1,16 @@
+import os
+from dotenv import load_dotenv
+
 from fastapi import FastAPI, Request
 from fastapi.templating import Jinja2Templates
 from fastapi.responses import HTMLResponse, FileResponse, Response
 from fastapi.staticfiles import StaticFiles
-import os
-from dotenv import load_dotenv
-from api_analytics.fastapi import Analytics
 from starlette.middleware.trustedhost import TrustedHostMiddleware
+from api_analytics.fastapi import Analytics
 from slowapi import _rate_limit_exceeded_handler
 from slowapi.errors import RateLimitExceeded
-from utils.limiter import limiter
+
+from server_utils import limiter
 from routers import download, dynamic, index
 
 
diff --git a/src/process_query.py b/src/process_query.py
@@ -1,21 +1,18 @@
-from ingest import ingest_from_query
-from utils.clone import clone_repo
-from utils.parse_query import parse_query
 from typing import List
 from fastapi.templating import Jinja2Templates
 from fastapi import Request
-from config import MAX_DISPLAY_SIZE, EXAMPLE_REPOS
-
 
+from config import MAX_DISPLAY_SIZE, EXAMPLE_REPOS
+from gitingest import ingest_from_query, clone_repo, parse_query
+from server_utils import logSliderToSize
 
 templates = Jinja2Templates(directory="templates")
 
-
 async def process_query(request: Request, input_text: str, slider_position: int, pattern_type: str = "exclude", pattern: str = "", is_index: bool = False) -> str:
-    
     template = "index.jinja" if is_index else "github.jinja"
+    max_file_size = logSliderToSize(slider_position)
     try:
-        query = parse_query(input_text, slider_position, pattern_type, pattern)
+        query = parse_query(input_text, max_file_size, pattern_type, pattern)
         await clone_repo(query)
         summary, tree, content = ingest_from_query(query)
         with open(f"{query['local_path']}.txt", "w") as f:
diff --git a/src/routers/dynamic.py b/src/routers/dynamic.py
@@ -3,7 +3,7 @@
 from fastapi.templating import Jinja2Templates
 
 from process_query import process_query
-from utils.limiter import limiter
+from server_utils import limiter
 
 router = APIRouter()
 templates = Jinja2Templates(directory="templates")
diff --git a/src/routers/index.py b/src/routers/index.py
@@ -2,7 +2,7 @@
 from fastapi.responses import HTMLResponse
 from fastapi.templating import Jinja2Templates
 
-from utils.limiter import limiter
+from server_utils import limiter
 from process_query import process_query
 from config import EXAMPLE_REPOS
 
diff --git a/src/server_utils.py b/src/server_utils.py
@@ -1,5 +1,11 @@
-import math
 
+## Rate Limiter
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+limiter = Limiter(key_func=get_remote_address)
+
+## Logarithmic slider to file size
+import math
 def logSliderToSize(position):
     """Convert slider position to file size in KB"""
     maxp = 500
diff --git a/src/tests/conftest.py b/src/tests/conftest.py
diff --git a/src/utils/__init__.py b/src/utils/__init__.py
diff --git a/src/utils/limiter.py b/src/utils/limiter.py
