wip before jinja

Author: NicolasIRAGNE

src/gitingest/entrypoint.py

@@ -45,12 +45,11 @@ async def ingest_async(
     include_submodules: bool = False,
     token: str | None = None,
     output: str | None = None,
-) -> tuple[str, str, str]:
+) -> str:
     """Ingest a source and process its contents.
 
     This function analyzes a source (URL or local path), clones the corresponding repository (if applicable),
-    and processes its files according to the specified query parameters. It returns a summary, a tree-like
-    structure of the files, and the content of the files. The results can optionally be written to an output file.
+    and processes its files according to the specified query parameters. It returns a single digest string.
 
     The output is generated lazily using a Context object and its .generate_digest() method.
 
@@ -82,11 +81,8 @@ async def ingest_async(
 
     Returns
     -------
-    tuple[str, str, str]
-        A tuple containing:
-        - A summary string of the analyzed repository or directory.
-        - A tree-like string representation of the file structure.
-        - The content of the files in the repository or directory.
+    str
+        The full digest string.
 
     """
     logger.info("Starting ingestion process", extra={"source": source})
@@ -146,11 +142,10 @@ async def ingest_async(
         if output:
             logger.debug("Writing output to file", extra={"output_path": output})
         context = ingest_query(query)
-        summary, tree, content = context.generate_digest()
-        await _write_output(tree, content=content, target=output)
-
+        digest = context.generate_digest()
+        await _write_output(digest, content=None, target=output)
         logger.info("Ingestion completed successfully")
-        return summary, tree, content
+        return digest
 
 
 def ingest(
@@ -165,12 +160,11 @@ def ingest(
     include_submodules: bool = False,
     token: str | None = None,
     output: str | None = None,
-) -> tuple[str, str, str]:
+) -> str:
     """Provide a synchronous wrapper around ``ingest_async``.
 
     This function analyzes a source (URL or local path), clones the corresponding repository (if applicable),
-    and processes its files according to the specified query parameters. It returns a summary, a tree-like
-    structure of the files, and the content of the files. The results can optionally be written to an output file.
+    and processes its files according to the specified query parameters. It returns a single digest string.
 
     The output is generated lazily using a Context object and its .generate_digest() method.
 
@@ -202,18 +196,14 @@ def ingest(
 
     Returns
     -------
-    tuple[str, str, str]
-        A tuple containing:
-        - A summary string of the analyzed repository or directory.
-        - A tree-like string representation of the file structure.
-        - The content of the files in the repository or directory.
+    str
+        The full digest string.
 
     See Also
     --------
     ``ingest_async`` : The asynchronous version of this function.
 
     """
-    import asyncio
     context = asyncio.run(ingest_async(
         source,
         max_file_size=max_file_size,

src/gitingest/ingestion.py

@@ -6,8 +6,9 @@
 from typing import TYPE_CHECKING
 
 from gitingest.config import MAX_DIRECTORY_DEPTH, MAX_FILES, MAX_TOTAL_SIZE_BYTES
-from gitingest.output_formatter import format_node
-from gitingest.schemas import FileSystemNode, FileSystemNodeType, FileSystemStats, Context
+from gitingest.output_formatter import DefaultFormatter
+from gitingest.schemas import FileSystemNode, FileSystemStats, Context
+from gitingest.schemas.filesystem import FileSystemDirectory, FileSystemFile, FileSystemSymlink
 from gitingest.utils.ingestion_utils import _should_exclude, _should_include
 from gitingest.utils.logging_config import get_logger
 
@@ -69,11 +70,16 @@ def ingest_query(query: IngestionQuery) -> Context:
 
         relative_path = path.relative_to(query.local_path)
 
-        file_node = FileSystemNode(
+        # file_node = FileSystemNode(
+        #     name=path.name,
+        #     type=FileSystemNodeType.FILE,
+        #     size=path.stat().st_size,
+        #     file_count=1,
+        #     path_str=str(relative_path),
+        #     path=path,
+        # )
+        file_node = FileSystemFile(
             name=path.name,
-            type=FileSystemNodeType.FILE,
-            size=path.stat().st_size,
-            file_count=1,
             path_str=str(relative_path),
             path=path,
         )
@@ -90,13 +96,15 @@ def ingest_query(query: IngestionQuery) -> Context:
                 "file_size": file_node.size,
             },
         )
-		return Context([file_node])
 
-    logger.info("Processing directory", extra={"directory_path": str(path)})
-
-    root_node = FileSystemNode(
+    # root_node = FileSystemNode(
+    #     name=path.name,
+    #     type=FileSystemNodeType.DIRECTORY,
+    #     path_str=str(path.relative_to(query.local_path)),
+    #     path=path,
+    # )
+    root_node = FileSystemDirectory(
         name=path.name,
-        type=FileSystemNodeType.DIRECTORY,
         path_str=str(path.relative_to(query.local_path)),
         path=path,
     )
@@ -116,7 +124,7 @@ def ingest_query(query: IngestionQuery) -> Context:
         },
     )
 
-    return Context([root_node])
+    return Context([root_node], DefaultFormatter(), query)
 
 
 def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystemStats) -> None:
@@ -160,9 +168,8 @@ def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystem
                 continue
             _process_file(path=sub_path, parent_node=node, stats=stats, local_path=query.local_path)
         elif sub_path.is_dir():
-            child_directory_node = FileSystemNode(
+            child_directory_node = FileSystemDirectory(
                 name=sub_path.name,
-                type=FileSystemNodeType.DIRECTORY,
                 path_str=str(sub_path.relative_to(query.local_path)),
                 path=sub_path,
                 depth=node.depth + 1,
@@ -200,9 +207,8 @@ def _process_symlink(path: Path, parent_node: FileSystemNode, stats: FileSystemS
         The base path of the repository or directory being processed.
 
     """
-    child = FileSystemNode(
+    child = FileSystemSymlink(
         name=path.name,
-        type=FileSystemNodeType.SYMLINK,
         path_str=str(path.relative_to(local_path)),
         path=path,
         depth=parent_node.depth + 1,
@@ -212,7 +218,7 @@ def _process_symlink(path: Path, parent_node: FileSystemNode, stats: FileSystemS
     parent_node.file_count += 1
 
 
-def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStats, local_path: Path) -> None:
+def _process_file(path: Path, parent_node: FileSystemDirectory, stats: FileSystemStats, local_path: Path) -> None:
     """Process a file in the file system.
 
     This function checks the file's size, increments the statistics, and reads its content.
@@ -222,7 +228,7 @@ def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStat
     ----------
     path : Path
         The full path of the file.
-    parent_node : FileSystemNode
+    parent_node : FileSystemDirectory
         The dictionary to accumulate the results.
     stats : FileSystemStats
         Statistics tracking object for the total file count and size.
@@ -257,11 +263,8 @@ def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStat
     stats.total_files += 1
     stats.total_size += file_size
 
-    child = FileSystemNode(
+    child = FileSystemFile(
         name=path.name,
-        type=FileSystemNodeType.FILE,
-        size=file_size,
-        file_count=1,
         path_str=str(path.relative_to(local_path)),
         path=path,
         depth=parent_node.depth + 1,

src/gitingest/output_formatter.py

@@ -8,8 +8,11 @@
 import requests.exceptions
 import tiktoken
 
-from gitingest.schemas import FileSystemNode, FileSystemNodeType
+from gitingest.schemas import FileSystemNode
 from gitingest.utils.compat_func import readlink
+from functools import singledispatchmethod
+from gitingest.schemas import Source, FileSystemFile, FileSystemDirectory, FileSystemSymlink
+from gitingest.schemas.filesystem import SEPARATOR
 from gitingest.utils.logging_config import get_logger
 
 if TYPE_CHECKING:
@@ -24,43 +27,41 @@
 ]
 
 
-def format_node(node: FileSystemNode, query: IngestionQuery) -> tuple[str, str, str]:
-    """Generate a summary, directory structure, and file contents for a given file system node.
-
-    If the node represents a directory, the function will recursively process its contents.
-
-    Parameters
-    ----------
-    node : FileSystemNode
-        The file system node to be summarized.
-    query : IngestionQuery
-        The parsed query object containing information about the repository and query parameters.
-
-    Returns
-    -------
-    tuple[str, str, str]
-        A tuple containing the summary, directory structure, and file contents.
-
-    """
-    is_single_file = node.type == FileSystemNodeType.FILE
-    summary = _create_summary_prefix(query, single_file=is_single_file)
-
-    if node.type == FileSystemNodeType.DIRECTORY:
-        summary += f"Files analyzed: {node.file_count}\n"
-    elif node.type == FileSystemNodeType.FILE:
-        summary += f"File: {node.name}\n"
-        summary += f"Lines: {len(node.content.splitlines()):,}\n"
-
-    tree = "Directory structure:\n" + _create_tree_structure(query, node=node)
-
-    content = _gather_file_contents(node)
-
-    token_estimate = _format_token_count(tree + content)
-    if token_estimate:
-        summary += f"\nEstimated tokens: {token_estimate}"
-
-    return summary, tree, content
-
+class Formatter:
+    @singledispatchmethod
+    def format(self, node: Source, query):
+        return f"{getattr(node, 'content', '')}"
+
+    @format.register
+    def _(self, node: FileSystemFile, query):
+        return (
+            f"{SEPARATOR}\n"
+            f"{node.name}\n"
+            f"{SEPARATOR}\n\n"
+            f"{node.content}"
+        )
+
+    @format.register
+    def _(self, node: FileSystemDirectory, query):
+        formatted = []
+        for child in node.children:
+            formatted.append(self.format(child, query))
+        return "\n".join(formatted)
+
+    @format.register
+    def _(self, node: FileSystemSymlink, query):
+        target = getattr(node, 'target', None)
+        target_str = f" -> {target}" if target else ""
+        return (
+            f"{SEPARATOR}\n"
+            f"{node.name}{target_str}\n"
+            f"{SEPARATOR}\n"
+        )
+
+class DefaultFormatter(Formatter):
+    pass
+
+# Backward compatibility
 
 def _create_summary_prefix(query: IngestionQuery, *, single_file: bool = False) -> str:
     """Create a prefix string for summarizing a repository or local directory.

src/gitingest/schemas/__init__.py

@@ -1,7 +1,7 @@
 """Module containing the schemas for the Gitingest package."""
 
 from gitingest.schemas.cloning import CloneConfig
-from gitingest.schemas.filesystem import Context, FileSystemNode, FileSystemNodeType, FileSystemStats
+from gitingest.schemas.filesystem import FileSystemNode, FileSystemFile, FileSystemDirectory, FileSystemSymlink, FileSystemStats, Context, Source
 from gitingest.schemas.ingestion import IngestionQuery
 
-__all__ = ["CloneConfig", "FileSystemNode", "FileSystemNodeType", "FileSystemStats", "IngestionQuery", "Context"]
+__all__ = ["CloneConfig", "FileSystemNode", "FileSystemFile", "FileSystemDirectory", "FileSystemSymlink", "FileSystemStats", "IngestionQuery", "Context"]