Merge branch 'main' into VsCode-Extension

Author: ShreyPurohit

.pre-commit-config.yaml

@@ -39,7 +39,7 @@ repos:
         description: "Automatically convert relative imports to absolute. (Use `args: [--never]` to revert.)"
 
   - repo: https://github.com/psf/black
-    rev: 24.10.0
+    rev: 25.1.0
     hooks:
       - id: black
 
@@ -61,7 +61,7 @@ repos:
         description: "Enforce that python3.6+ type annotations are used instead of type comments."
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
+    rev: 6.0.1
     hooks:
       - id: isort
         description: "Sort imports alphabetically, and automatically separated into sections and by type."
@@ -73,7 +73,7 @@ repos:
       - id: djlint-reformat-jinja
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.43.0
+    rev: v0.44.0
     hooks:
       - id: markdownlint
         description: "Lint markdown files."
@@ -88,23 +88,23 @@ repos:
         files: ^src/
 
   - repo: https://github.com/pycqa/pylint
-    rev: v3.3.3
+    rev: v3.3.6
     hooks:
       - id: pylint
         name: pylint for source
         files: ^src/
         additional_dependencies:
           [
-            chardet,
-            click,
-            fastapi-analytics,
+            click>=8.0.0,
+            "fastapi[standard]>=0.109.1",
+            pydantic,
             pytest-asyncio,
             python-dotenv,
             slowapi,
-            starlette,
+            starlette>=0.40.0,
             tiktoken,
             tomli,
-            uvicorn,
+            uvicorn>=0.11.7,
           ]
       - id: pylint
         name: pylint for tests
@@ -113,17 +113,16 @@ repos:
           - --rcfile=tests/.pylintrc
         additional_dependencies:
           [
-            chardet,
-            click,
-            fastapi-analytics,
-            pytest,
+            click>=8.0.0,
+            "fastapi[standard]>=0.109.1",
+            pydantic,
             pytest-asyncio,
             python-dotenv,
             slowapi,
-            starlette,
-            tomli,
+            starlette>=0.40.0,
             tiktoken,
-            uvicorn,
+            tomli,
+            uvicorn>=0.11.7,
           ]
 
   - repo: meta

pyproject.toml

@@ -6,9 +6,15 @@ readme = {file = "README.md", content-type = "text/markdown" }
 requires-python = ">= 3.8"
 dependencies = [
     "click>=8.0.0",
+    "fastapi[standard]>=0.109.1",  # Vulnerable to https://osv.dev/vulnerability/PYSEC-2024-38
+    "pydantic",
+    "python-dotenv",
+    "slowapi",
+    "starlette>=0.40.0",  # Vulnerable to https://osv.dev/vulnerability/GHSA-f96h-pmfr-66vw
     "tiktoken",
     "tomli",
     "typing_extensions; python_version < '3.10'",
+    "uvicorn>=0.11.7",  # Vulnerable to https://osv.dev/vulnerability/PYSEC-2020-150
 ]
 
 license = {file = "LICENSE"}

requirements.txt

@@ -1,6 +1,6 @@
-chardet
 click>=8.0.0
 fastapi[standard]>=0.109.1  # Vulnerable to https://osv.dev/vulnerability/PYSEC-2024-38
+pydantic
 python-dotenv
 slowapi
 starlette>=0.40.0  # Vulnerable to https://osv.dev/vulnerability/GHSA-f96h-pmfr-66vw

src/gitingest/__init__.py

@@ -1,8 +1,8 @@
-""" Gitingest: A package for ingesting data from Git repositories. """
+"""Gitingest: A package for ingesting data from Git repositories."""
 
-from gitingest.cloning import clone
+from gitingest.cloning import clone_repo
 from gitingest.entrypoint import ingest, ingest_async
 from gitingest.ingestion import ingest_query
 from gitingest.query_parsing import parse_query
 
-__all__ = ["ingest_query", "clone", "parse_query", "ingest", "ingest_async"]
+__all__ = ["ingest_query", "clone_repo", "parse_query", "ingest", "ingest_async"]

src/gitingest/cli.py

@@ -1,4 +1,4 @@
-""" Command-line interface for the Gitingest package. """
+"""Command-line interface for the Gitingest package."""
 
 # pylint: disable=no-value-for-parameter
 

src/gitingest/cloning.py

@@ -1,18 +1,18 @@
-""" This module contains functions for cloning a Git repository to a local path. """
+"""This module contains functions for cloning a Git repository to a local path."""
 
-import asyncio
 import os
 from pathlib import Path
-from typing import List, Optional, Tuple
+from typing import Optional
 
-from gitingest.ingestion_schema import CloneConfig
+from gitingest.schemas import CloneConfig
+from gitingest.utils.git_utils import check_repo_exists, ensure_git_installed, run_command
 from gitingest.utils.timeout_wrapper import async_timeout
 
 TIMEOUT: int = 60
 
 
 @async_timeout(TIMEOUT)
-async def clone(config: CloneConfig) -> None:
+async def clone_repo(config: CloneConfig) -> None:
     """
     Clone a repository to a local path based on the provided configuration.
 
@@ -47,7 +47,7 @@ async def clone(config: CloneConfig) -> None:
         raise OSError(f"Failed to create parent directory {parent_dir}: {exc}") from exc
 
     # Check if the repository exists
-    if not await _check_repo_exists(url):
+    if not await check_repo_exists(url):
         raise ValueError("Repository not found, make sure it is public")
 
     clone_cmd = ["git", "clone", "--single-branch"]
@@ -64,7 +64,8 @@ async def clone(config: CloneConfig) -> None:
     clone_cmd += [url, local_path]
 
     # Clone the repository
-    await _run_command(*clone_cmd)
+    await ensure_git_installed()
+    await run_command(*clone_cmd)
 
     if commit or partial_clone:
         checkout_cmd = ["git", "-C", local_path]
@@ -81,148 +82,4 @@ async def clone(config: CloneConfig) -> None:
             checkout_cmd += ["checkout", commit]
 
         # Check out the specific commit and/or subpath
-        await _run_command(*checkout_cmd)
-
-
-async def _check_repo_exists(url: str) -> bool:
-    """
-    Check if a Git repository exists at the provided URL.
-
-    Parameters
-    ----------
-    url : str
-        The URL of the Git repository to check.
-    Returns
-    -------
-    bool
-        True if the repository exists, False otherwise.
-
-    Raises
-    ------
-    RuntimeError
-        If the curl command returns an unexpected status code.
-    """
-    proc = await asyncio.create_subprocess_exec(
-        "curl",
-        "-I",
-        url,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
-    )
-    stdout, _ = await proc.communicate()
-
-    if proc.returncode != 0:
-        return False
-
-    response = stdout.decode()
-    status_code = _get_status_code(response)
-
-    if status_code in (200, 301):
-        return True
-
-    if status_code in (404, 302):
-        return False
-
-    raise RuntimeError(f"Unexpected status code: {status_code}")
-
-
-async def fetch_remote_branch_list(url: str) -> List[str]:
-    """
-    Fetch the list of branches from a remote Git repository.
-    Parameters
-    ----------
-    url : str
-        The URL of the Git repository to fetch branches from.
-    Returns
-    -------
-    List[str]
-        A list of branch names available in the remote repository.
-    """
-    fetch_branches_command = ["git", "ls-remote", "--heads", url]
-    stdout, _ = await _run_command(*fetch_branches_command)
-    stdout_decoded = stdout.decode()
-
-    return [
-        line.split("refs/heads/", 1)[1]
-        for line in stdout_decoded.splitlines()
-        if line.strip() and "refs/heads/" in line
-    ]
-
-
-async def _run_command(*args: str) -> Tuple[bytes, bytes]:
-    """
-    Execute a command asynchronously and captures its output.
-
-    Parameters
-    ----------
-    *args : str
-        The command and its arguments to execute.
-
-    Returns
-    -------
-    Tuple[bytes, bytes]
-        A tuple containing the stdout and stderr of the command.
-
-    Raises
-    ------
-    RuntimeError
-        If command exits with a non-zero status.
-    """
-    await check_git_installed()
-
-    # Execute the requested command
-    proc = await asyncio.create_subprocess_exec(
-        *args,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
-    )
-    stdout, stderr = await proc.communicate()
-    if proc.returncode != 0:
-        error_message = stderr.decode().strip()
-        raise RuntimeError(f"Command failed: {' '.join(args)}\nError: {error_message}")
-
-    return stdout, stderr
-
-
-async def check_git_installed() -> None:
-    """
-    Check if Git is installed and accessible on the system.
-
-    Raises
-    ------
-    RuntimeError
-        If Git is not installed or if the Git command exits with a non-zero status.
-    """
-    try:
-        proc = await asyncio.create_subprocess_exec(
-            "git",
-            "--version",
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
-        _, stderr = await proc.communicate()
-        if proc.returncode != 0:
-            error_message = stderr.decode().strip() if stderr else "Git command not found"
-            raise RuntimeError(f"Git is not installed or not accessible: {error_message}")
-
-    except FileNotFoundError as exc:
-        raise RuntimeError("Git is not installed. Please install Git before proceeding.") from exc
-
-
-def _get_status_code(response: str) -> int:
-    """
-    Extract the status code from an HTTP response.
-
-    Parameters
-    ----------
-    response : str
-        The HTTP response string.
-
-    Returns
-    -------
-    int
-        The status code of the response
-    """
-    status_line = response.splitlines()[0].strip()
-    status_code = int(status_line.split(" ", 2)[1])
-    return status_code
+        await run_command(*checkout_cmd)

src/gitingest/config.py

@@ -1,4 +1,4 @@
-""" Configuration file for the project. """
+"""Configuration file for the project."""
 
 import tempfile
 from pathlib import Path

src/gitingest/entrypoint.py

@@ -1,11 +1,11 @@
-""" Main entry point for ingesting a source and processing its contents. """
+"""Main entry point for ingesting a source and processing its contents."""
 
 import asyncio
 import inspect
 import shutil
 from typing import Optional, Set, Tuple, Union
 
-from gitingest.cloning import clone
+from gitingest.cloning import clone_repo
 from gitingest.config import TMP_BASE_PATH
 from gitingest.ingestion import ingest_query
 from gitingest.query_parsing import IngestionQuery, parse_query
@@ -53,7 +53,7 @@ async def ingest_async(
     Raises
     ------
     TypeError
-        If `clone` does not return a coroutine, or if the `source` is of an unsupported type.
+        If `clone_repo` does not return a coroutine, or if the `source` is of an unsupported type.
     """
     repo_cloned = False
 
@@ -71,15 +71,15 @@ async def ingest_async(
             query.branch = selected_branch
 
             clone_config = query.extract_clone_config()
-            clone_coroutine = clone(clone_config)
+            clone_coroutine = clone_repo(clone_config)
 
             if inspect.iscoroutine(clone_coroutine):
                 if asyncio.get_event_loop().is_running():
                     await clone_coroutine
                 else:
                     asyncio.run(clone_coroutine)
             else:
-                raise TypeError("clone did not return a coroutine as expected.")
+                raise TypeError("clone_repo did not return a coroutine as expected.")
 
             repo_cloned = True
 

src/gitingest/ingestion.py

@@ -1,13 +1,13 @@
-""" Functions to ingest and analyze a codebase directory or single file. """
+"""Functions to ingest and analyze a codebase directory or single file."""
 
 import warnings
 from pathlib import Path
 from typing import Tuple
 
 from gitingest.config import MAX_DIRECTORY_DEPTH, MAX_FILES, MAX_TOTAL_SIZE_BYTES
-from gitingest.filesystem_schema import FileSystemNode, FileSystemNodeType, FileSystemStats
 from gitingest.output_formatters import format_node
 from gitingest.query_parsing import IngestionQuery
+from gitingest.schemas import FileSystemNode, FileSystemNodeType, FileSystemStats
 from gitingest.utils.ingestion_utils import _should_exclude, _should_include
 from gitingest.utils.path_utils import _is_safe_symlink
 

src/gitingest/output_formatters.py

@@ -4,8 +4,8 @@
 
 import tiktoken
 
-from gitingest.filesystem_schema import FileSystemNode, FileSystemNodeType
 from gitingest.query_parsing import IngestionQuery
+from gitingest.schemas import FileSystemNode, FileSystemNodeType
 
 
 def format_node(node: FileSystemNode, query: IngestionQuery) -> Tuple[str, str, str]: