feat(auth): support private GitHub repos & correct sparse-checkout flow

* CLI: new `--token/-t` flag (fallback to `GITHUB_TOKEN`)
* clone_repo:
  * injects Basic-auth header when a PAT is supplied
  * validates PAT format (`github_pat_*`)
* git_utils:
  * `create_git_auth_header`, `validate_github_token`, `create_git_command`
  * `_check_github_repo_exists` & branch-listing now work with tokens
* os_utils.ensure_directory extracted for reuse
* tests updated to reflect new call signatures

Author: filipchristiansen

src/gitingest/cli.py

@@ -44,13 +44,24 @@
     ),
 )
 @click.option("--branch", "-b", default=None, help="Branch to clone and ingest")
+@click.option(
+    "--token",
+    "-t",
+    envvar="GITHUB_TOKEN",
+    default=None,
+    help=(
+        "GitHub personal access token for accessing private repositories. "
+        "If omitted, the CLI will look for the GITHUB_TOKEN environment variable."
+    ),
+)
 def main(
     source: str,
     output: Optional[str],
     max_size: int,
     exclude_pattern: Tuple[str, ...],
     include_pattern: Tuple[str, ...],
     branch: Optional[str],
+    token: Optional[str],
 ):
     """
     Main entry point for the CLI. This function is called when the CLI is run as a script.
@@ -71,6 +82,9 @@ def main(
         Glob patterns for including files in the output.
     branch : str, optional
         Specific branch to ingest (defaults to the repository's default).
+    token: str, optional
+        GitHub personal-access token (PAT). Needed when *source* refers to a
+        **private** repository. Can also be set via the ``GITHUB_TOKEN`` env var.
     """
 
     asyncio.run(
@@ -81,6 +95,7 @@ def main(
             exclude_pattern=exclude_pattern,
             include_pattern=include_pattern,
             branch=branch,
+            token=token,
         )
     )
 
@@ -92,6 +107,7 @@ async def _async_main(
     exclude_pattern: Tuple[str, ...],
     include_pattern: Tuple[str, ...],
     branch: Optional[str],
+    token: Optional[str],
 ) -> None:
     """
     Analyze a directory or repository and create a text dump of its contents.
@@ -113,6 +129,9 @@ async def _async_main(
         Glob patterns for including files in the output.
     branch : str, optional
         Specific branch to ingest (defaults to the repository's default).
+    token: str, optional
+        GitHub personal-access token (PAT). Needed when *source* refers to a
+        **private** repository. Can also be set via the ``GITHUB_TOKEN`` env var.
 
     Raises
     ------
@@ -135,6 +154,7 @@ async def _async_main(
             exclude_patterns=exclude_patterns,
             branch=branch,
             output=output,
+            token=token,
         )
 
         click.echo(f"Analysis complete! Output written to: {output}")

src/gitingest/cloning.py

@@ -1,17 +1,24 @@
 """This module contains functions for cloning a Git repository to a local path."""
 
-import os
 from pathlib import Path
 from typing import Optional
 
 from gitingest.config import DEFAULT_TIMEOUT
 from gitingest.schemas import CloneConfig
-from gitingest.utils.git_utils import check_repo_exists, ensure_git_installed, run_command
+from gitingest.utils.git_utils import (
+    check_repo_exists,
+    create_git_auth_header,
+    create_git_command,
+    ensure_git_installed,
+    run_command,
+    validate_github_token,
+)
+from gitingest.utils.os_utils import ensure_directory
 from gitingest.utils.timeout_wrapper import async_timeout
 
 
 @async_timeout(DEFAULT_TIMEOUT)
-async def clone_repo(config: CloneConfig) -> None:
+async def clone_repo(config: CloneConfig, token: Optional[str] = None) -> None:
     """
     Clone a repository to a local path based on the provided configuration.
 
@@ -23,11 +30,15 @@ async def clone_repo(config: CloneConfig) -> None:
     ----------
     config : CloneConfig
         The configuration for cloning the repository.
+    token : str, optional
+        GitHub personal-access token (PAT). Needed when *source* refers to a
+        **private** repository. Can also be set via the ``GITHUB_TOKEN`` env var.
+        Must start with 'github_pat_' for GitHub repositories.
 
     Raises
     ------
     ValueError
-        If the repository is not found or if the provided URL is invalid.
+        If the repository is not found, if the provided URL is invalid, or if the token format is invalid.
     """
     # Extract and validate query parameters
     url: str = config.url
@@ -36,14 +47,22 @@ async def clone_repo(config: CloneConfig) -> None:
     branch: Optional[str] = config.branch
     partial_clone: bool = config.subpath != "/"
 
+    # Validate token if provided
+    if token and url.startswith("https://github.com"):
+        validate_github_token(token)
+
     # Create parent directory if it doesn't exist
-    await _ensure_directory(Path(local_path).parent)
+    await ensure_directory(Path(local_path).parent)
 
     # Check if the repository exists
-    if not await check_repo_exists(url):
-        raise ValueError("Repository not found, make sure it is public")
+    if not await check_repo_exists(url, token=token):
+        raise ValueError("Repository not found. Make sure it is public or that you have provided a valid token.")
+
+    clone_cmd = ["git"]
+    if token and url.startswith("https://github.com"):
+        clone_cmd += ["-c", create_git_auth_header(token)]
 
-    clone_cmd = ["git", "clone", "--single-branch"]
+    clone_cmd += ["clone", "--single-branch"]
     # TODO: Re-enable --recurse-submodules when submodule support is needed
 
     if partial_clone:
@@ -67,28 +86,10 @@ async def clone_repo(config: CloneConfig) -> None:
             # When ingesting from a file url (blob/branch/path/file.txt), we need to remove the file name.
             subpath = str(Path(subpath).parent.as_posix())
 
-        await run_command("git", "-C", local_path, "sparse-checkout", "set", subpath)
+        checkout_cmd = create_git_command(["git"], local_path, url, token)
+        await run_command(*checkout_cmd, "sparse-checkout", "set", subpath)
 
     # Checkout the commit if it is provided
     if commit:
-        await run_command("git", "-C", local_path, "checkout", commit)
-
-
-async def _ensure_directory(path: Path) -> None:
-    """
-    Ensure the directory exists, creating it if necessary.
-
-    Parameters
-    ----------
-    path : Path
-        The path to ensure exists
-
-    Raises
-    ------
-    OSError
-        If the directory cannot be created
-    """
-    try:
-        os.makedirs(path, exist_ok=True)
-    except OSError as exc:
-        raise OSError(f"Failed to create directory {path}: {exc}") from exc
+        checkout_cmd = create_git_command(["git"], local_path, url, token)
+        await run_command(*checkout_cmd, "checkout", commit)

src/gitingest/entrypoint.py

@@ -17,6 +17,7 @@ async def ingest_async(
     include_patterns: Optional[Union[str, Set[str]]] = None,
     exclude_patterns: Optional[Union[str, Set[str]]] = None,
     branch: Optional[str] = None,
+    token: Optional[str] = None,
     output: Optional[str] = None,
 ) -> Tuple[str, str, str]:
     """
@@ -39,6 +40,9 @@ async def ingest_async(
         Pattern or set of patterns specifying which files to exclude. If `None`, no files are excluded.
     branch : str, optional
         The branch to clone and ingest. If `None`, the default branch is used.
+    token : str, optional
+        GitHub personal-access token (PAT). Needed when *source* refers to a
+        **private** repository. Can also be set via the ``GITHUB_TOKEN`` env var.
     output : str, optional
         File path where the summary and content should be written. If `None`, the results are not written to a file.
 
@@ -71,7 +75,7 @@ async def ingest_async(
             query.branch = selected_branch
 
             clone_config = query.extract_clone_config()
-            clone_coroutine = clone_repo(clone_config)
+            clone_coroutine = clone_repo(clone_config, token=token)
 
             if inspect.iscoroutine(clone_coroutine):
                 if asyncio.get_event_loop().is_running():
@@ -102,6 +106,7 @@ def ingest(
     include_patterns: Optional[Union[str, Set[str]]] = None,
     exclude_patterns: Optional[Union[str, Set[str]]] = None,
     branch: Optional[str] = None,
+    token: Optional[str] = None,
     output: Optional[str] = None,
 ) -> Tuple[str, str, str]:
     """
@@ -124,6 +129,9 @@ def ingest(
         Pattern or set of patterns specifying which files to exclude. If `None`, no files are excluded.
     branch : str, optional
         The branch to clone and ingest. If `None`, the default branch is used.
+    token : str, optional
+        GitHub personal-access token (PAT). Needed when *source* refers to a
+        **private** repository. Can also be set via the ``GITHUB_TOKEN`` env var.
     output : str, optional
         File path where the summary and content should be written. If `None`, the results are not written to a file.
 
@@ -146,6 +154,7 @@ def ingest(
             include_patterns=include_patterns,
             exclude_patterns=exclude_patterns,
             branch=branch,
+            token=token,
             output=output,
         )
     )

src/gitingest/query_parsing.py

@@ -94,7 +94,7 @@ async def parse_query(
     )
 
 
-async def _parse_remote_repo(source: str) -> IngestionQuery:
+async def _parse_remote_repo(source: str, token: Optional[str] = None) -> IngestionQuery:
     """
     Parse a repository URL into a structured query dictionary.
 
@@ -107,6 +107,9 @@ async def _parse_remote_repo(source: str) -> IngestionQuery:
     ----------
     source : str
         The URL or domain-less slug to parse.
+    token : str, optional
+        GitHub personal-access token (PAT). Needed when *source* refers to a
+        **private** repository. Can also be set via the ``GITHUB_TOKEN`` env var.
 
     Returns
     -------
@@ -128,7 +131,7 @@ async def _parse_remote_repo(source: str) -> IngestionQuery:
             _validate_host(tmp_host)
         else:
             # No scheme, no domain => user typed "user/repo", so we'll guess the domain.
-            host = await try_domains_for_user_and_repo(*_get_user_and_repo_from_path(source))
+            host = await try_domains_for_user_and_repo(*_get_user_and_repo_from_path(source), token=token)
             source = f"{host}/{source}"
 
         source = "https://" + source
@@ -285,7 +288,7 @@ def _parse_local_dir_path(path_str: str) -> IngestionQuery:
     )
 
 
-async def try_domains_for_user_and_repo(user_name: str, repo_name: str) -> str:
+async def try_domains_for_user_and_repo(user_name: str, repo_name: str, token: Optional[str] = None) -> str:
     """
     Attempt to find a valid repository host for the given user_name and repo_name.
 
@@ -295,6 +298,9 @@ async def try_domains_for_user_and_repo(user_name: str, repo_name: str) -> str:
         The username or owner of the repository.
     repo_name : str
         The name of the repository.
+    token : str, optional
+        GitHub personal-access token (PAT). Needed when *source* refers to a
+        **private** repository. Can also be set via the ``GITHUB_TOKEN`` env var.
 
     Returns
     -------
@@ -308,6 +314,6 @@ async def try_domains_for_user_and_repo(user_name: str, repo_name: str) -> str:
     """
     for domain in KNOWN_GIT_HOSTS:
         candidate = f"https://{domain}/{user_name}/{repo_name}"
-        if await check_repo_exists(candidate):
+        if await check_repo_exists(candidate, token=token if domain == "github.com" else None):
             return domain
     raise ValueError(f"Could not find a valid repository host for '{user_name}/{repo_name}'.")