Merge branch 'main' of https://github.com/RyanL2004/gitingest into flow-integration-tests

Author: RyanL2004

CONTRIBUTING.md

@@ -48,18 +48,18 @@ Thanks for your interest in contributing to Gitingest! 🚀 Gitingest aims to be
    pytest
    ```
 
-8. Run the app locally using Docker to test your changes (optional):
+8. Navigate to src folder
 
    1. Build the Docker image
 
         ``` bash
-        docker build -t gitingest .
+        cd src
         ```
 
-   2. Run the Docker container:
+   2. Run the local web server:
 
       ``` bash
-      docker run -d --name gitingest -p 8000:8000 gitingest
+      uvicorn server.main:app
       ```
 
    3. Open your browser and navigate to `http://localhost:8000` to see the app running.

Dockerfile

@@ -41,4 +41,4 @@ USER appuser
 
 EXPOSE 8000
 
-CMD ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000"]

src/gitingest/cli.py

@@ -6,7 +6,7 @@
 
 import click
 
-from config import MAX_FILE_SIZE, OUTPUT_FILE_PATH
+from gitingest.config import MAX_FILE_SIZE, OUTPUT_FILE_PATH
 from gitingest.repository_ingest import ingest
 
 
@@ -16,12 +16,14 @@
 @click.option("--max-size", "-s", default=MAX_FILE_SIZE, help="Maximum file size to process in bytes")
 @click.option("--exclude-pattern", "-e", multiple=True, help="Patterns to exclude")
 @click.option("--include-pattern", "-i", multiple=True, help="Patterns to include")
+@click.option("--branch", "-b", default=None, help="Branch to clone and ingest")
 def main(
     source: str,
     output: str | None,
     max_size: int,
     exclude_pattern: tuple[str, ...],
     include_pattern: tuple[str, ...],
+    branch: str | None,
 ):
     """
     Main entry point for the CLI. This function is called when the CLI is run as a script.
@@ -41,9 +43,11 @@ def main(
         A tuple of patterns to exclude during the analysis. Files matching these patterns will be ignored.
     include_pattern : tuple[str, ...]
         A tuple of patterns to include during the analysis. Only files matching these patterns will be processed.
+    branch : str | None
+        The branch to clone (optional).
     """
     # Main entry point for the CLI. This function is called when the CLI is run as a script.
-    asyncio.run(_async_main(source, output, max_size, exclude_pattern, include_pattern))
+    asyncio.run(_async_main(source, output, max_size, exclude_pattern, include_pattern, branch))
 
 
 async def _async_main(
@@ -52,6 +56,7 @@ async def _async_main(
     max_size: int,
     exclude_pattern: tuple[str, ...],
     include_pattern: tuple[str, ...],
+    branch: str | None,
 ) -> None:
     """
     Analyze a directory or repository and create a text dump of its contents.
@@ -72,6 +77,8 @@ async def _async_main(
         A tuple of patterns to exclude during the analysis. Files matching these patterns will be ignored.
     include_pattern : tuple[str, ...]
         A tuple of patterns to include during the analysis. Only files matching these patterns will be processed.
+    branch : str | None
+        The branch to clone (optional).
 
     Raises
     ------
@@ -85,7 +92,7 @@ async def _async_main(
 
         if not output:
             output = OUTPUT_FILE_PATH
-        summary, _, _ = await ingest(source, max_size, include_patterns, exclude_patterns, output=output)
+        summary, _, _ = await ingest(source, max_size, include_patterns, exclude_patterns, branch, output=output)
 
         click.echo(f"Analysis complete! Output written to: {output}")
         click.echo("\nSummary:")

src/gitingest/config.py

@@ -0,0 +1,11 @@
+""" Configuration file for the project. """
+
+from pathlib import Path
+
+MAX_FILE_SIZE = 10 * 1024 * 1024  # 10 MB
+MAX_DIRECTORY_DEPTH = 20  # Maximum depth of directory traversal
+MAX_FILES = 10_000  # Maximum number of files to process
+MAX_TOTAL_SIZE_BYTES = 500 * 1024 * 1024  # 500 MB
+
+OUTPUT_FILE_PATH = "digest.txt"
+TMP_BASE_PATH = Path("/tmp/gitingest")

src/gitingest/query_ingestion.py

@@ -6,7 +6,7 @@
 
 import tiktoken
 
-from config import MAX_DIRECTORY_DEPTH, MAX_FILES, MAX_TOTAL_SIZE_BYTES
+from gitingest.config import MAX_DIRECTORY_DEPTH, MAX_FILES, MAX_TOTAL_SIZE_BYTES
 from gitingest.exceptions import (
     AlreadyVisitedError,
     InvalidNotebookError,

src/gitingest/query_parser.py

@@ -9,7 +9,7 @@
 from pathlib import Path
 from urllib.parse import unquote, urlparse
 
-from config import MAX_FILE_SIZE, TMP_BASE_PATH
+from gitingest.config import MAX_FILE_SIZE, TMP_BASE_PATH
 from gitingest.exceptions import InvalidPatternError
 from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
 from gitingest.repository_clone import _check_repo_exists, fetch_remote_branch_list
@@ -163,7 +163,7 @@ async def _parse_repo_source(source: str) -> ParsedQuery:
 
     _id = str(uuid.uuid4())
     slug = f"{user_name}-{repo_name}"
-    local_path = Path(TMP_BASE_PATH) / _id / slug
+    local_path = TMP_BASE_PATH / _id / slug
     url = f"https://{host}/{user_name}/{repo_name}"
 
     parsed = ParsedQuery(

src/gitingest/repository_ingest.py

@@ -4,7 +4,7 @@
 import inspect
 import shutil
 
-from config import TMP_BASE_PATH
+from gitingest.config import TMP_BASE_PATH
 from gitingest.query_ingestion import run_ingest_query
 from gitingest.query_parser import ParsedQuery, parse_query
 from gitingest.repository_clone import CloneConfig, clone_repo
@@ -15,6 +15,7 @@ async def ingest(
     max_file_size: int = 10 * 1024 * 1024,  # 10 MB
     include_patterns: set[str] | str | None = None,
     exclude_patterns: set[str] | str | None = None,
+    branch: str | None = None,
     output: str | None = None,
 ) -> tuple[str, str, str]:
     """
@@ -35,6 +36,8 @@ async def ingest(
         Pattern or set of patterns specifying which files to include. If `None`, all files are included.
     exclude_patterns : set[str] | str | None, optional
         Pattern or set of patterns specifying which files to exclude. If `None`, no files are excluded.
+    branch : str | None, optional
+        The branch to clone and ingest. If `None`, the default branch is used.
     output : str | None, optional
         File path where the summary and content should be written. If `None`, the results are not written to a file.
 
@@ -61,17 +64,23 @@ async def ingest(
         )
 
         if parsed_query.url:
+            selected_branch = branch if branch else parsed_query.branch  # prioritize branch argument
+            parsed_query.branch = selected_branch
+
             # Extract relevant fields for CloneConfig
             clone_config = CloneConfig(
                 url=parsed_query.url,
                 local_path=str(parsed_query.local_path),
                 commit=parsed_query.commit,
-                branch=parsed_query.branch,
+                branch=selected_branch,
             )
             clone_result = clone_repo(clone_config)
 
             if inspect.iscoroutine(clone_result):
-                asyncio.run(clone_result)
+                if asyncio.get_event_loop().is_running():
+                    await clone_result
+                else:
+                    asyncio.run(clone_result)
             else:
                 raise TypeError("clone_repo did not return a coroutine as expected.")
 

src/routers/__init__.py

@@ -11,10 +11,9 @@
 from slowapi.errors import RateLimitExceeded
 from starlette.middleware.trustedhost import TrustedHostMiddleware
 
-from config import templates
-from routers import download, dynamic, index
-from server_utils import limiter
-from utils import lifespan, rate_limit_exception_handler
+from server.routers import download, dynamic, index
+from server.server_config import templates
+from server.server_utils import lifespan, limiter, rate_limit_exception_handler
 
 # Load environment variables from .env file
 load_dotenv()