fix(ignore-patterns): resolve .git and nested path exclusions with improved .gitignore parsing

Author: Dai Hung PHAM

src/gitingest/ingest_from_query.py

@@ -19,14 +19,28 @@ def should_include(path: str, base_path: str, include_patterns: List[str]) -> bo
     return include
 
 def should_exclude(path: str, base_path: str, ignore_patterns: List[str]) -> bool:
-    rel_path = path.replace(base_path, "").lstrip(os.sep)
+    """
+    Check if a file or directory should be ignored.
+
+    Args:
+        path (str): Path to check.
+        base_path (str): Root base path.
+        ignore_patterns (List[str]): List of patterns to ignore.
+
+    Returns:
+        bool: True if the path should be ignored.
+    """
+    rel_path = os.path.relpath(path, base_path).replace("\\", "/")
     for pattern in ignore_patterns:
-        if pattern == '':
-            continue
-        if fnmatch(rel_path, pattern):
+        if fnmatch(rel_path, pattern) or fnmatch(os.path.basename(path), pattern):
+            return True
+        # Special case for directories ending with /
+        if os.path.isdir(path) and fnmatch(rel_path + '/', pattern):
             return True
     return False
 
+
+
 def is_safe_symlink(symlink_path: str, base_path: str) -> bool:
     """Check if a symlink points to a location within the base directory."""
     try:
@@ -96,8 +110,10 @@ def scan_directory(path: str, query: dict, seen_paths: set = None, depth: int =
     try:
         for item in os.listdir(path):
             item_path = os.path.join(path, item)
+            print(f"Checking path: {path}")
 
             if should_exclude(item_path, base_path, ignore_patterns):
+                print(f"Checking path: {path}")
                 continue
 
             is_file = os.path.isfile(item_path)

src/gitingest/parse_query.py

@@ -93,40 +93,81 @@ def parse_url(url: str) -> dict:
         parsed["subpath"] = "/" + "/".join(path_parts[4:])
     return parsed
 
+### 📝 **Normalize Pattern**
 def normalize_pattern(pattern: str) -> str:
+    """
+    Normalize a pattern by stripping and formatting.
+
+    Args:
+        pattern (str): The ignore pattern.
+
+    Returns:
+        str: Normalized pattern.
+    """
     pattern = pattern.strip()
     pattern = pattern.lstrip(os.sep)
     if pattern.endswith(os.sep):
         pattern += "*"
     return pattern
 
+### 📝 **Parse Patterns**
 def parse_patterns(pattern: Union[List[str], str]) -> List[str]:
+    """
+    Parse and validate patterns.
+
+    Args:
+        pattern (Union[List[str], str]): Patterns to parse.
+
+    Returns:
+        List[str]: Parsed patterns.
+    """
     if isinstance(pattern, list):
         pattern = ",".join(pattern)
 
     for p in pattern.split(","):
         if not all(c.isalnum() or c in "-_./+*" for c in p.strip()):
-            raise ValueError(f"Pattern '{p}' contains invalid characters. Only alphanumeric characters, dash (-), underscore (_), dot (.), forward slash (/), plus (+), and asterisk (*) are allowed.")
-    patterns = [normalize_pattern(p) for p in pattern.split(",")]
-    return patterns
+            raise ValueError(
+                f"Pattern '{p}' contains invalid characters. Only alphanumeric characters, dash (-), underscore (_), dot (.), forward slash (/), plus (+), and asterisk (*) are allowed."
+            )
+    return [normalize_pattern(p) for p in pattern.split(",")]
 
+### 📝 **Override Ignore Patterns**
 def override_ignore_patterns(ignore_patterns: List[str], include_patterns: List[str]) -> List[str]:
+    """
+    Remove include patterns from ignore patterns.
+
+    Args:
+        ignore_patterns (List[str]): Ignore patterns.
+        include_patterns (List[str]): Include patterns.
+
+    Returns:
+        List[str]: Updated ignore patterns.
+    """
     for pattern in include_patterns:
         if pattern in ignore_patterns:
             ignore_patterns.remove(pattern)
     return ignore_patterns
 
 
+### 📝 **Parse Path**
 def parse_path(path: str) -> dict:
-    
-    query = {
+    """
+    Parse a local file path.
+
+    Args:
+        path (str): File path.
+
+    Returns:
+        dict: Parsed path details.
+    """
+    return {
         "local_path": os.path.abspath(path),
         "slug": os.path.basename(os.path.dirname(path)) + "/" + os.path.basename(path),
         "subpath": "/",
         "id": str(uuid.uuid4()),
         "url": None,
     }
-    return query
+
 
 def parse_query(source: str, max_file_size: int, from_web: bool, include_patterns: Union[List[str], str] = None, ignore_patterns: Union[List[str], str] = None) -> dict:
     if from_web:
@@ -154,3 +195,89 @@ def parse_query(source: str, max_file_size: int, from_web: bool, include_pattern
 
     return query
 
+### 📝 **Parse .gitignore**
+def parse_gitignore(gitignore_path: str) -> List[str]:
+    """
+    Parse .gitignore and return ignore patterns.
+
+    Args:
+        gitignore_path (str): Path to the .gitignore file.
+
+    Returns:
+        List[str]: List of ignore patterns.
+    """
+    ignore_patterns = []
+    if os.path.exists(gitignore_path):
+        with open(gitignore_path, 'r', encoding='utf-8') as file:
+            for line in file:
+                line = line.strip()
+                if line and not line.startswith('#'):
+                    # Ensure directory patterns end with '/'
+                    if os.path.isdir(os.path.join(os.path.dirname(gitignore_path), line)):
+                        line = line.rstrip('/') + '/'
+                    ignore_patterns.append(line)
+    return ignore_patterns
+
+
+### 📝 **Parse Query**
+def parse_query(source: str, max_file_size: int, from_web: bool,
+                include_patterns: Union[List[str], str] = None,
+                ignore_patterns: Union[List[str], str] = None) -> dict:
+    """
+    Parse the query and apply ignore patterns.
+
+    Args:
+        source (str): Source path or URL.
+        max_file_size (int): Maximum file size.
+        from_web (bool): Web source or local.
+        include_patterns (Union[List[str], str]): Include patterns.
+        ignore_patterns (Union[List[str], str]): Ignore patterns.
+
+    Returns:
+        dict: Query object with patterns.
+    """
+    if from_web:
+        query = parse_url(source)
+    else:
+        query = parse_path(source)
+    
+    query['max_file_size'] = max_file_size
+
+    # Start with default ignore patterns
+    final_ignore_patterns = DEFAULT_IGNORE_PATTERNS.copy()
+
+    # Load from .gitignore
+    gitignore_path = os.path.join(query['local_path'], '.gitignore')
+    print(f"find .gitignore on project --> {gitignore_path}")
+
+    if os.path.exists(gitignore_path):
+        gitignore_patterns = parse_gitignore(gitignore_path)
+        final_ignore_patterns.extend(gitignore_patterns)
+        print(f"\n🛡️  Patterns from: {gitignore_path}")
+        for pattern in gitignore_patterns:
+            print(f"  - {pattern}")
+    # Add user-defined ignore patterns
+    if ignore_patterns:
+        final_ignore_patterns.extend(parse_patterns(ignore_patterns))
+    
+    # Handle include patterns
+    if include_patterns:
+        include_patterns = parse_patterns(include_patterns)
+        final_ignore_patterns = override_ignore_patterns(final_ignore_patterns, include_patterns)
+    
+    query['ignore_patterns'] = final_ignore_patterns
+    query['include_patterns'] = include_patterns
+    # 🖨️ Print patterns to the console
+    print("\n🛡️  Applied Ignore Patterns:")
+    for pattern in final_ignore_patterns:
+        print(f"  - {pattern}")
+    
+    if include_patterns:
+        print("\n✅ Included Patterns:")
+        for pattern in include_patterns:
+            print(f"  - {pattern}")
+    else:
+        print("\n✅ Included Patterns: None")
+
+    return query
+    return query