feat(logging): enhance ingestion with detailed contextual logs

Author: MickaelCa

src/gitingest/clone.py

@@ -16,12 +16,16 @@
     resolve_commit,
     run_command,
 )
+from gitingest.utils.logging_config import get_logger
 from gitingest.utils.os_utils import ensure_directory_exists_or_create
 from gitingest.utils.timeout_wrapper import async_timeout
 
 if TYPE_CHECKING:
     from gitingest.schemas import CloneConfig
 
+# Initialize logger for this module
+logger = get_logger(__name__)
+
 
 @async_timeout(DEFAULT_TIMEOUT)
 async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
@@ -49,14 +53,35 @@ async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
     local_path: str = config.local_path
     partial_clone: bool = config.subpath != "/"
 
+    logger.info(
+        "Starting git clone operation",
+        extra={
+            "url": url,
+            "local_path": local_path,
+            "partial_clone": partial_clone,
+            "subpath": config.subpath,
+            "branch": config.branch,
+            "tag": config.tag,
+            "commit": config.commit,
+            "include_submodules": config.include_submodules,
+        },
+    )
+
+    logger.debug("Ensuring git is installed")
     await ensure_git_installed()
+
+    logger.debug("Creating local directory", extra={"parent_path": str(Path(local_path).parent)})
     await ensure_directory_exists_or_create(Path(local_path).parent)
 
+    logger.debug("Checking if repository exists", extra={"url": url})
     if not await check_repo_exists(url, token=token):
+        logger.error("Repository not found", extra={"url": url})
         msg = "Repository not found. Make sure it is public or that you have provided a valid token."
         raise ValueError(msg)
 
+    logger.debug("Resolving commit reference")
     commit = await resolve_commit(config, token=token)
+    logger.debug("Resolved commit", extra={"commit": commit})
 
     clone_cmd = ["git"]
     if token and is_github_host(url):
@@ -69,20 +94,30 @@ async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
     clone_cmd += [url, local_path]
 
     # Clone the repository
+    logger.info("Executing git clone command", extra={"command": " ".join([*clone_cmd[:-1], "<url>", local_path])})
     await run_command(*clone_cmd)
+    logger.info("Git clone completed successfully")
 
     # Checkout the subpath if it is a partial clone
     if partial_clone:
+        logger.info("Setting up partial clone for subpath", extra={"subpath": config.subpath})
         await checkout_partial_clone(config, token=token)
+        logger.debug("Partial clone setup completed")
 
     git = create_git_command(["git"], local_path, url, token)
 
     # Ensure the commit is locally available
+    logger.debug("Fetching specific commit", extra={"commit": commit})
     await run_command(*git, "fetch", "--depth=1", "origin", commit)
 
     # Write the work-tree at that commit
+    logger.info("Checking out commit", extra={"commit": commit})
     await run_command(*git, "checkout", commit)
 
     # Update submodules
     if config.include_submodules:
+        logger.info("Updating submodules")
         await run_command(*git, "submodule", "update", "--init", "--recursive", "--depth=1")
+        logger.debug("Submodules updated successfully")
+
+    logger.info("Git clone operation completed successfully", extra={"local_path": local_path})

src/gitingest/entrypoint.py

@@ -20,6 +20,7 @@
 from gitingest.utils.auth import resolve_token
 from gitingest.utils.compat_func import removesuffix
 from gitingest.utils.ignore_patterns import load_ignore_patterns
+from gitingest.utils.logging_config import get_logger
 from gitingest.utils.pattern_utils import process_patterns
 from gitingest.utils.query_parser_utils import KNOWN_GIT_HOSTS
 
@@ -28,6 +29,9 @@
 
     from gitingest.schemas import IngestionQuery
 
+# Initialize logger for this module
+logger = get_logger(__name__)
+
 
 async def ingest_async(
     source: str,
@@ -83,19 +87,23 @@ async def ingest_async(
         - The content of the files in the repository or directory.
 
     """
+    logger.info("Starting ingestion process", extra={"source": source})
+
     token = resolve_token(token)
 
     source = removesuffix(source.strip(), ".git")
 
     # Determine the parsing method based on the source type
     if urlparse(source).scheme in ("https", "http") or any(h in source for h in KNOWN_GIT_HOSTS):
         # We either have a full URL or a domain-less slug
+        logger.info("Parsing remote repository", extra={"source": source})
         query = await parse_remote_repo(source, token=token)
         query.include_submodules = include_submodules
         _override_branch_and_tag(query, branch=branch, tag=tag)
 
     else:
         # Local path scenario
+        logger.info("Processing local directory", extra={"source": source})
         query = parse_local_dir_path(source)
 
     query.max_file_size = max_file_size
@@ -109,11 +117,35 @@ async def ingest_async(
 
     query.include_submodules = include_submodules
 
+    logger.debug(
+        "Configuration completed",
+        extra={
+            "max_file_size": query.max_file_size,
+            "include_submodules": query.include_submodules,
+            "include_gitignored": include_gitignored,
+            "has_include_patterns": bool(query.include_patterns),
+            "has_exclude_patterns": bool(query.ignore_patterns),
+        },
+    )
+
     async with _clone_repo_if_remote(query, token=token):
+        if query.url:
+            logger.info("Repository cloned, starting file processing")
+        else:
+            logger.info("Starting local directory processing")
+
         if not include_gitignored:
+            logger.debug("Applying gitignore patterns")
             _apply_gitignores(query)
+
+        logger.info("Processing files and generating output")
         summary, tree, content = ingest_query(query)
+
+        if output:
+            logger.debug("Writing output to file", extra={"output_path": output})
         await _write_output(tree, content=content, target=output)
+
+        logger.info("Ingestion completed successfully")
         return summary, tree, content
 
 

src/gitingest/ingestion.py

@@ -41,16 +41,30 @@ def ingest_query(query: IngestionQuery) -> tuple[str, str, str]:
         If the path cannot be found, is not a file, or the file has no content.
 
     """
+    logger.info(
+        "Starting file ingestion",
+        extra={
+            "slug": query.slug,
+            "subpath": query.subpath,
+            "local_path": str(query.local_path),
+            "max_file_size": query.max_file_size,
+        },
+    )
+
     subpath = Path(query.subpath.strip("/")).as_posix()
     path = query.local_path / subpath
 
     if not path.exists():
+        logger.error("Path not found", extra={"path": str(path), "slug": query.slug})
         msg = f"{query.slug} cannot be found"
         raise ValueError(msg)
 
     if (query.type and query.type == "blob") or query.local_path.is_file():
         # TODO: We do this wrong! We should still check the branch and commit!
+        logger.info("Processing single file", extra={"file_path": str(path)})
+
         if not path.is_file():
+            logger.error("Expected file but found non-file", extra={"path": str(path)})
             msg = f"Path {path} is not a file"
             raise ValueError(msg)
 
@@ -66,11 +80,21 @@ def ingest_query(query: IngestionQuery) -> tuple[str, str, str]:
         )
 
         if not file_node.content:
+            logger.error("File has no content", extra={"file_name": file_node.name})
             msg = f"File {file_node.name} has no content"
             raise ValueError(msg)
 
+        logger.info(
+            "Single file processing completed",
+            extra={
+                "file_name": file_node.name,
+                "file_size": file_node.size,
+            },
+        )
         return format_node(file_node, query=query)
 
+    logger.info("Processing directory", extra={"directory_path": str(path)})
+
     root_node = FileSystemNode(
         name=path.name,
         type=FileSystemNodeType.DIRECTORY,
@@ -82,6 +106,17 @@ def ingest_query(query: IngestionQuery) -> tuple[str, str, str]:
 
     _process_node(node=root_node, query=query, stats=stats)
 
+    logger.info(
+        "Directory processing completed",
+        extra={
+            "total_files": root_node.file_count,
+            "total_directories": root_node.dir_count,
+            "total_size_bytes": root_node.size,
+            "stats_total_files": stats.total_files,
+            "stats_total_size": stats.total_size,
+        },
+    )
+
     return format_node(root_node, query=query)