From b5ffa5f8aac678af1b45e39064354c0190b80338 Mon Sep 17 00:00:00 2001 From: Filip Christiansen <22807962+filipchristiansen@users.noreply.github.com> Date: Fri, 17 Jan 2025 17:09:58 +0100 Subject: [PATCH 1/6] Refactor CLI to wrap async logic with a sync command function (#136) - Change main() to a synchronous Click command - Introduce _async_main() for async ingest logic - Use asyncio.run(...) to properly await the async function --- src/gitingest/cli.py | 41 ++++++++-- src/main.py | 2 +- tests/test_flow_integration.py | 145 +++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 tests/test_flow_integration.py diff --git a/src/gitingest/cli.py b/src/gitingest/cli.py index ef7761b9..a21a4533 100644 --- a/src/gitingest/cli.py +++ b/src/gitingest/cli.py @@ -2,6 +2,8 @@ # pylint: disable=no-value-for-parameter +import asyncio + import click from config import MAX_FILE_SIZE @@ -9,12 +11,42 @@ @click.command() -@click.argument("source", type=str, required=True) +@click.argument("source", type=str, default=".") @click.option("--output", "-o", default=None, help="Output file path (default: .txt in current directory)") @click.option("--max-size", "-s", default=MAX_FILE_SIZE, help="Maximum file size to process in bytes") @click.option("--exclude-pattern", "-e", multiple=True, help="Patterns to exclude") @click.option("--include-pattern", "-i", multiple=True, help="Patterns to include") -async def main( +def main( + source: str, + output: str | None, + max_size: int, + exclude_pattern: tuple[str, ...], + include_pattern: tuple[str, ...], +): + """ + Main entry point for the CLI. This function is called when the CLI is run as a script. + + It calls the async main function to run the command. + + Parameters + ---------- + source : str + The source directory or repository to analyze. + output : str | None + The path where the output file will be written. If not specified, the output will be written + to a file named `.txt` in the current directory. + max_size : int + The maximum file size to process, in bytes. Files larger than this size will be ignored. + exclude_pattern : tuple[str, ...] + A tuple of patterns to exclude during the analysis. Files matching these patterns will be ignored. + include_pattern : tuple[str, ...] + A tuple of patterns to include during the analysis. Only files matching these patterns will be processed. + """ + # Main entry point for the CLI. This function is called when the CLI is run as a script. + asyncio.run(_async_main(source, output, max_size, exclude_pattern, include_pattern)) + + +async def _async_main( source: str, output: str | None, max_size: int, @@ -24,9 +56,8 @@ async def main( """ Analyze a directory or repository and create a text dump of its contents. - This command analyzes the contents of a specified source directory or repository, - applies custom include and exclude patterns, and generates a text summary of the analysis - which is then written to an output file. + This command analyzes the contents of a specified source directory or repository, applies custom include and + exclude patterns, and generates a text summary of the analysis which is then written to an output file. Parameters ---------- diff --git a/src/main.py b/src/main.py index 556b3e1d..e11cebcc 100644 --- a/src/main.py +++ b/src/main.py @@ -175,7 +175,7 @@ async def rate_limit_exception_handler(request: Request, exc: Exception) -> Resp app.add_middleware(TrustedHostMiddleware, allowed_hosts=allowed_hosts) # Set up template rendering -templates = Jinja2Templates(directory="templates") +templates = Jinja2Templates(directory="src/templates") @app.get("/health") diff --git a/tests/test_flow_integration.py b/tests/test_flow_integration.py new file mode 100644 index 00000000..a8b84f57 --- /dev/null +++ b/tests/test_flow_integration.py @@ -0,0 +1,145 @@ +""" +Integration tests for GitIngest. +These tests cover core functionalities, edge cases, and concurrency handling. +""" + +import shutil +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from main import app + +BASE_DIR = Path(__file__).resolve().parent.parent +TEMPLATE_DIR = BASE_DIR / "src" / "templates" + + +@pytest.fixture(scope="module") +def test_client(): + """Create a test client fixture.""" + with TestClient(app) as client: + client.headers.update({"Host": "localhost"}) + yield client + + +@pytest.fixture(scope="module", autouse=True) +def mock_templates(): + """Mock Jinja2 template rendering to bypass actual file loading.""" + with patch("starlette.templating.Jinja2Templates.TemplateResponse") as mock_template: + mock_template.return_value = "Mocked Template Response" + yield mock_template + + +def cleanup_temp_directories(): + temp_dir = Path("/tmp/gitingest") + if temp_dir.exists(): + try: + shutil.rmtree(temp_dir) + except PermissionError as e: + print(f"Error cleaning up {temp_dir}: {e}") + + +@pytest.fixture(scope="module", autouse=True) +def cleanup(): + """Cleanup temporary directories after tests.""" + yield + cleanup_temp_directories() + + +@pytest.mark.asyncio +async def test_remote_repository_analysis(test_client): # pylint: disable=redefined-outer-name + """Test the complete flow of analyzing a remote repository.""" + form_data = { + "input_text": "https://github.com/octocat/Hello-World", + "max_file_size": "243", + "pattern_type": "exclude", + "pattern": "", + } + + response = test_client.post("/", data=form_data) + assert response.status_code == 200, f"Form submission failed: {response.text}" + assert "Mocked Template Response" in response.text + + +@pytest.mark.asyncio +async def test_invalid_repository_url(test_client): # pylint: disable=redefined-outer-name + """Test handling of an invalid repository URL.""" + form_data = { + "input_text": "https://github.com/nonexistent/repo", + "max_file_size": "243", + "pattern_type": "exclude", + "pattern": "", + } + + response = test_client.post("/", data=form_data) + assert response.status_code == 200, f"Request failed: {response.text}" + assert "Mocked Template Response" in response.text + + +@pytest.mark.asyncio +async def test_large_repository(test_client): # pylint: disable=redefined-outer-name + """Simulate analysis of a large repository with nested folders.""" + form_data = { + "input_text": "https://github.com/large/repo-with-many-files", + "max_file_size": "243", + "pattern_type": "exclude", + "pattern": "", + } + + response = test_client.post("/", data=form_data) + assert response.status_code == 200, f"Request failed: {response.text}" + assert "Mocked Template Response" in response.text + + +@pytest.mark.asyncio +async def test_concurrent_requests(test_client): # pylint: disable=redefined-outer-name + """Test handling of multiple concurrent requests.""" + + def make_request(): + form_data = { + "input_text": "https://github.com/octocat/Hello-World", + "max_file_size": "243", + "pattern_type": "exclude", + "pattern": "", + } + response = test_client.post("/", data=form_data) + assert response.status_code == 200, f"Request failed: {response.text}" + assert "Mocked Template Response" in response.text + + with ThreadPoolExecutor(max_workers=5) as executor: + futures = [executor.submit(make_request) for _ in range(5)] + for future in futures: + future.result() + + +@pytest.mark.asyncio +async def test_large_file_handling(test_client): # pylint: disable=redefined-outer-name + """Test handling of repositories with large files.""" + form_data = { + "input_text": "https://github.com/octocat/Hello-World", + "max_file_size": "1", + "pattern_type": "exclude", + "pattern": "", + } + + response = test_client.post("/", data=form_data) + assert response.status_code == 200, f"Request failed: {response.text}" + assert "Mocked Template Response" in response.text + + +@pytest.mark.asyncio +async def test_repository_with_patterns(test_client): # pylint: disable=redefined-outer-name + """Test repository analysis with include/exclude patterns.""" + form_data = { + "input_text": "https://github.com/octocat/Hello-World", + "max_file_size": "243", + "pattern_type": "include", + "pattern": "*.md", + } + + response = test_client.post("/", data=form_data) + assert response.status_code == 200, f"Request failed: {response.text}" + assert "Mocked Template Response" in response.text From 1846cb79ac88c16450c635987ca38448ba4cabf9 Mon Sep 17 00:00:00 2001 From: RyanL2004 Date: Sat, 18 Jan 2025 02:20:43 +0000 Subject: [PATCH 2/6] fix: Correct static directory path for serving static files --- src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index e11cebcc..0e33794d 100644 --- a/src/main.py +++ b/src/main.py @@ -156,7 +156,7 @@ async def rate_limit_exception_handler(request: Request, exc: Exception) -> Resp app.add_exception_handler(RateLimitExceeded, rate_limit_exception_handler) # Mount static files to serve CSS, JS, and other static assets -app.mount("/static", StaticFiles(directory="static"), name="static") +app.mount("/static", StaticFiles(directory="src/static"), name="static") # Set up API analytics middleware if an API key is provided if app_analytics_key := os.getenv("API_ANALYTICS_KEY"): From c19a379c45d67e7c03f88869ddbe222a2d01f1fe Mon Sep 17 00:00:00 2001 From: RyanL2004 Date: Sat, 18 Jan 2025 18:30:00 +0000 Subject: [PATCH 3/6] fix: Correct static directory path for StaticFiles mount --- src/main.py | 9 +++++-- tests/test_flow_integration.py | 45 ++++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/main.py b/src/main.py index 0e33794d..c7648fd6 100644 --- a/src/main.py +++ b/src/main.py @@ -156,7 +156,12 @@ async def rate_limit_exception_handler(request: Request, exc: Exception) -> Resp app.add_exception_handler(RateLimitExceeded, rate_limit_exception_handler) # Mount static files to serve CSS, JS, and other static assets -app.mount("/static", StaticFiles(directory="src/static"), name="static") +# Mount static files dynamically +static_dir = Path(__file__).parent / "static" +if static_dir.exists(): + app.mount("/static", StaticFiles(directory=static_dir), name="static") +else: + print(f"Warning: Static directory '{static_dir}' does not exist. Skipping static file mount.") # Set up API analytics middleware if an API key is provided if app_analytics_key := os.getenv("API_ANALYTICS_KEY"): @@ -175,7 +180,7 @@ async def rate_limit_exception_handler(request: Request, exc: Exception) -> Resp app.add_middleware(TrustedHostMiddleware, allowed_hosts=allowed_hosts) # Set up template rendering -templates = Jinja2Templates(directory="src/templates") +templates = Jinja2Templates(directory="templates") @app.get("/health") diff --git a/tests/test_flow_integration.py b/tests/test_flow_integration.py index a8b84f57..236988fe 100644 --- a/tests/test_flow_integration.py +++ b/tests/test_flow_integration.py @@ -11,7 +11,7 @@ import pytest from fastapi.testclient import TestClient -from main import app +from src.main import app BASE_DIR = Path(__file__).resolve().parent.parent TEMPLATE_DIR = BASE_DIR / "src" / "templates" @@ -20,9 +20,16 @@ @pytest.fixture(scope="module") def test_client(): """Create a test client fixture.""" - with TestClient(app) as client: - client.headers.update({"Host": "localhost"}) - yield client + with TestClient(app) as client_instance: + client_instance.headers.update({"Host": "localhost"}) + yield client_instance + + +@pytest.fixture(scope="module", autouse=True) +def mock_static_files(): + """Mock the static file mount to avoid directory errors.""" + with patch("src.main.StaticFiles") as mock_static: + yield mock_static @pytest.fixture(scope="module", autouse=True) @@ -50,8 +57,9 @@ def cleanup(): @pytest.mark.asyncio -async def test_remote_repository_analysis(test_client): # pylint: disable=redefined-outer-name +async def test_remote_repository_analysis(request): """Test the complete flow of analyzing a remote repository.""" + client = request.getfixturevalue("test_client") form_data = { "input_text": "https://github.com/octocat/Hello-World", "max_file_size": "243", @@ -59,14 +67,15 @@ async def test_remote_repository_analysis(test_client): # pylint: disable=redef "pattern": "", } - response = test_client.post("/", data=form_data) + response = client.post("/", data=form_data) assert response.status_code == 200, f"Form submission failed: {response.text}" assert "Mocked Template Response" in response.text @pytest.mark.asyncio -async def test_invalid_repository_url(test_client): # pylint: disable=redefined-outer-name +async def test_invalid_repository_url(request): """Test handling of an invalid repository URL.""" + client = request.getfixturevalue("test_client") form_data = { "input_text": "https://github.com/nonexistent/repo", "max_file_size": "243", @@ -74,14 +83,15 @@ async def test_invalid_repository_url(test_client): # pylint: disable=redefined "pattern": "", } - response = test_client.post("/", data=form_data) + response = client.post("/", data=form_data) assert response.status_code == 200, f"Request failed: {response.text}" assert "Mocked Template Response" in response.text @pytest.mark.asyncio -async def test_large_repository(test_client): # pylint: disable=redefined-outer-name +async def test_large_repository(request): """Simulate analysis of a large repository with nested folders.""" + client = request.getfixturevalue("test_client") form_data = { "input_text": "https://github.com/large/repo-with-many-files", "max_file_size": "243", @@ -89,14 +99,15 @@ async def test_large_repository(test_client): # pylint: disable=redefined-outer "pattern": "", } - response = test_client.post("/", data=form_data) + response = client.post("/", data=form_data) assert response.status_code == 200, f"Request failed: {response.text}" assert "Mocked Template Response" in response.text @pytest.mark.asyncio -async def test_concurrent_requests(test_client): # pylint: disable=redefined-outer-name +async def test_concurrent_requests(request): """Test handling of multiple concurrent requests.""" + client = request.getfixturevalue("test_client") def make_request(): form_data = { @@ -105,7 +116,7 @@ def make_request(): "pattern_type": "exclude", "pattern": "", } - response = test_client.post("/", data=form_data) + response = client.post("/", data=form_data) assert response.status_code == 200, f"Request failed: {response.text}" assert "Mocked Template Response" in response.text @@ -116,8 +127,9 @@ def make_request(): @pytest.mark.asyncio -async def test_large_file_handling(test_client): # pylint: disable=redefined-outer-name +async def test_large_file_handling(request): """Test handling of repositories with large files.""" + client = request.getfixturevalue("test_client") form_data = { "input_text": "https://github.com/octocat/Hello-World", "max_file_size": "1", @@ -125,14 +137,15 @@ async def test_large_file_handling(test_client): # pylint: disable=redefined-ou "pattern": "", } - response = test_client.post("/", data=form_data) + response = client.post("/", data=form_data) assert response.status_code == 200, f"Request failed: {response.text}" assert "Mocked Template Response" in response.text @pytest.mark.asyncio -async def test_repository_with_patterns(test_client): # pylint: disable=redefined-outer-name +async def test_repository_with_patterns(request): """Test repository analysis with include/exclude patterns.""" + client = request.getfixturevalue("test_client") form_data = { "input_text": "https://github.com/octocat/Hello-World", "max_file_size": "243", @@ -140,6 +153,6 @@ async def test_repository_with_patterns(test_client): # pylint: disable=redefin "pattern": "*.md", } - response = test_client.post("/", data=form_data) + response = client.post("/", data=form_data) assert response.status_code == 200, f"Request failed: {response.text}" assert "Mocked Template Response" in response.text From 98f59a005c042a860afd78939085cf39bb8e91e5 Mon Sep 17 00:00:00 2001 From: RyanL2004 Date: Wed, 22 Jan 2025 00:05:33 +0000 Subject: [PATCH 4/6] refactor: Skipping static file mount if Static file not found --- src/main.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main.py b/src/main.py index c7648fd6..cb4d2796 100644 --- a/src/main.py +++ b/src/main.py @@ -155,13 +155,11 @@ async def rate_limit_exception_handler(request: Request, exc: Exception) -> Resp # Register the custom exception handler for rate limits app.add_exception_handler(RateLimitExceeded, rate_limit_exception_handler) -# Mount static files to serve CSS, JS, and other static assets -# Mount static files dynamically + +# Mount static files dynamically to serve CSS, JS, and other static assets static_dir = Path(__file__).parent / "static" -if static_dir.exists(): - app.mount("/static", StaticFiles(directory=static_dir), name="static") -else: - print(f"Warning: Static directory '{static_dir}' does not exist. Skipping static file mount.") +app.mount("/static", StaticFiles(directory=static_dir), name="static") + # Set up API analytics middleware if an API key is provided if app_analytics_key := os.getenv("API_ANALYTICS_KEY"): From d94308193c94a8be00374216ee0d8063178e780d Mon Sep 17 00:00:00 2001 From: RyanL2004 Date: Thu, 23 Jan 2025 15:33:06 +0000 Subject: [PATCH 5/6] Update: Branch --- src/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.py b/src/main.py index b00be990..525ac959 100644 --- a/src/main.py +++ b/src/main.py @@ -1,6 +1,7 @@ """ Main module for the FastAPI application. """ import os +from pathlib import Path from api_analytics.fastapi import Analytics from dotenv import load_dotenv From 73a2ba44ba04ff8c9431bf1747b476fc43866f3e Mon Sep 17 00:00:00 2001 From: RyanL2004 Date: Fri, 24 Jan 2025 23:05:47 +0000 Subject: [PATCH 6/6] chore: adapt PR to refactored codebase and resolve conflicts --- src/server/main.py | 2 +- tests/test_flow_integration.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/server/main.py b/src/server/main.py index cac8c8c4..20de4403 100644 --- a/src/server/main.py +++ b/src/server/main.py @@ -27,7 +27,7 @@ # Mount static files dynamically to serve CSS, JS, and other static assets -static_dir = Path(__file__).parent / "static" +static_dir = Path(__file__).parent.parent / "static" app.mount("/static", StaticFiles(directory=static_dir), name="static") diff --git a/tests/test_flow_integration.py b/tests/test_flow_integration.py index 236988fe..8fedeff5 100644 --- a/tests/test_flow_integration.py +++ b/tests/test_flow_integration.py @@ -11,7 +11,7 @@ import pytest from fastapi.testclient import TestClient -from src.main import app +from src.server.main import app BASE_DIR = Path(__file__).resolve().parent.parent TEMPLATE_DIR = BASE_DIR / "src" / "templates" @@ -28,7 +28,8 @@ def test_client(): @pytest.fixture(scope="module", autouse=True) def mock_static_files(): """Mock the static file mount to avoid directory errors.""" - with patch("src.main.StaticFiles") as mock_static: + with patch("src.server.main.StaticFiles") as mock_static: + mock_static.return_value = None # Mocks the StaticFiles response yield mock_static