Merge branch 'main' into refactor/gitingest-structure

Author: filipchristiansen

src/gitingest/cli.py

@@ -3,6 +3,7 @@
 
 import click
 
+from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
 from gitingest.ingest import ingest
 from gitingest.ingest_from_query import MAX_FILE_SIZE
 

src/gitingest/clone.py

@@ -116,9 +116,6 @@ async def clone_repo(config: CloneConfig) -> Tuple[bytes, bytes]:
     if not local_path:
         raise ValueError("The 'local_path' parameter is required.")
 
-    # if commit and branch:
-    #     raise ValueError("Provide either 'commit' or 'branch', not both.")
-
     # Check if the repository exists
     if not await check_repo_exists(url):
         raise ValueError("Repository not found, make sure it is public")

src/gitingest/ingest.py

@@ -25,6 +25,7 @@ def ingest(
             ignore_patterns=exclude_patterns,
         )
         if query['url']:
+
             # Extract relevant fields for CloneConfig
             clone_config = CloneConfig(
                 url=f"https://github.com/{query['slug']}.git",
@@ -33,6 +34,7 @@ def ingest(
                 branch=query.get('branch'),
             )
             clone_result = clone_repo(clone_config)
+
             if inspect.iscoroutine(clone_result):
                 asyncio.run(clone_result)
             else:

src/gitingest/parse_query.py

@@ -10,7 +10,22 @@
 
 
 def parse_url(url: str) -> Dict[str, Any]:
+    parsed = {
+        "user_name": None,
+        "repo_name": None,
+        "type": None,
+        "branch": None,
+        "commit": None,
+        "subpath": "/",
+        "local_path": None,
+        "url": None,
+        "slug": None,
+        "id": None,
+    }
+
     url = url.split(" ")[0]
+    url = unquote(url)  # Decode URL-encoded characters
+    
     if not url.startswith('https://'):
         url = 'https://' + url
 
@@ -22,33 +37,37 @@ def parse_url(url: str) -> Dict[str, Any]:
     if len(path_parts) < 2:
         raise ValueError("Invalid repository URL. Please provide a valid Git repository URL.")
 
-    user_name = path_parts[0]
-    repo_name = path_parts[1]
-    slug = f"{user_name}-{repo_name}"
-    _id = str(uuid.uuid4())
+    parsed["user_name"] = path_parts[0]
+    parsed["repo_name"] = path_parts[1]
 
-    parsed = {
-        "url": f"https://{domain}/{user_name}/{repo_name}",
-        "local_path": f"{TMP_BASE_PATH}/{_id}/{slug}",
-        "commit": None,
-        "branch": None,
-        "user_name": user_name,
-        "repo_name": repo_name,
-        "type": None,
-        "subpath": "/",
-        "slug": slug,
-        "id": _id,
-    }
+    # Keep original URL format but with decoded components
+    parsed["url"] = f"https://{domain}/{parsed['user_name']}/{parsed['repo_name']}"
+    parsed['slug'] = f"{parsed['user_name']}-{parsed['repo_name']}"
+    parsed["id"] = str(uuid.uuid4())
+    parsed["local_path"] = f"{TMP_BASE_PATH}/{parsed['id']}/{parsed['slug']}"
 
     if len(path_parts) > 3:
-        parsed["type"] = path_parts[2]
-        branch = path_parts[3]
 
-        parsed["branch"] = branch
-        if len(branch) == 40 and all(c in HEX_DIGITS for c in branch):
-            parsed["commit"] = branch
+        parsed["type"] = path_parts[2]  # Usually 'tree' or 'blob'
+        
+        # Find the commit hash or reconstruct the branch name
+        remaining_parts = path_parts[3:]
+        if remaining_parts[0] and len(remaining_parts[0]) == 40 and all(c in HEX_DIGITS for c in remaining_parts[0]):
+            parsed["commit"] = remaining_parts[0]
+            parsed["subpath"] = "/" + "/".join(remaining_parts[1:]) if len(remaining_parts) > 1 else "/"
+        else:
+            # Handle branch names with slashes and special characters
+            for i, part in enumerate(remaining_parts):
+                if part in ('tree', 'blob'):
+                    # Found another type indicator, everything before this was the branch name
+                    parsed["branch"] = "/".join(remaining_parts[:i])
+                    parsed["subpath"] = "/" + "/".join(remaining_parts[i+2:]) if len(remaining_parts) > i+2 else "/"
+                    break
+            else:
+                # No additional type indicator found, assume everything is part of the branch name
+                parsed["branch"] = "/".join(remaining_parts)
+                parsed["subpath"] = "/"
 
-        parsed["subpath"] += "/".join(path_parts[4:])
 
     return parsed
 
@@ -111,6 +130,7 @@ def parse_query(
     include_patterns: Optional[Union[List[str], str]] = None,
     ignore_patterns: Optional[Union[List[str], str]] = None,
 ) -> Dict[str, Any]:
+
     """
     Parses the input source to construct a query dictionary with specified parameters.
 
@@ -159,5 +179,4 @@ def parse_query(
             'include_patterns': parsed_include,
         }
     )
-
     return query

src/gitingest/tests/test_clone.py

@@ -19,7 +19,6 @@ async def test_clone_repo_with_commit() -> None:
             mock_process = AsyncMock()
             mock_process.communicate.return_value = (b'output', b'error')
             mock_exec.return_value = mock_process
-
             await clone_repo(clone_config)
             mock_check.assert_called_once_with(clone_config.url)
             assert mock_exec.call_count == 2  # Clone and checkout calls
@@ -43,9 +42,11 @@ async def test_clone_repo_without_commit() -> None:
 @pytest.mark.asyncio
 async def test_clone_repo_nonexistent_repository() -> None:
     clone_config = CloneConfig(
-        url='https://github.com/user/nonexistent-repo', local_path='/tmp/repo', commit=None, branch='main'
+        url='https://github.com/user/nonexistent-repo',
+        local_path='/tmp/repo',
+        commit=None,
+        branch='main',
     )
-
     with patch('gitingest.clone.check_repo_exists', return_value=False) as mock_check:
         with pytest.raises(ValueError, match="Repository not found"):
             await clone_repo(clone_config)