refactor: convert key functions and tests to asynchronous

- Made `parse_query` in query_processor.py asynchronous
- Made `main` in cli.py asynchronous
- Made `ingest` in repository_ingest.py asynchronous
- Updated test functions in test_query_parser.py to support async

Author: filipchristiansen

src/gitingest/cli.py

@@ -14,7 +14,7 @@
 @click.option("--max-size", "-s", default=MAX_FILE_SIZE, help="Maximum file size to process in bytes")
 @click.option("--exclude-pattern", "-e", multiple=True, help="Patterns to exclude")
 @click.option("--include-pattern", "-i", multiple=True, help="Patterns to include")
-def main(
+async def main(
     source: str,
     output: str | None,
     max_size: int,
@@ -54,7 +54,7 @@ def main(
 
         if not output:
             output = "digest.txt"
-        summary, _, _ = ingest(source, max_size, include_patterns, exclude_patterns, output=output)
+        summary, _, _ = await ingest(source, max_size, include_patterns, exclude_patterns, output=output)
 
         click.echo(f"Analysis complete! Output written to: {output}")
         click.echo("\nSummary:")

src/gitingest/query_parser.py

@@ -15,7 +15,7 @@
 HEX_DIGITS = set(string.hexdigits)
 
 
-def parse_query(
+async def parse_query(
     source: str,
     max_file_size: int,
     from_web: bool,

src/gitingest/repository_ingest.py

@@ -10,7 +10,7 @@
 from gitingest.repository_clone import CloneConfig, clone_repo
 
 
-def ingest(
+async def ingest(
     source: str,
     max_file_size: int = 10 * 1024 * 1024,  # 10 MB
     include_patterns: list[str] | str | None = None,
@@ -52,7 +52,7 @@ def ingest(
         If `clone_repo` does not return a coroutine, or if the `source` is of an unsupported type.
     """
     try:
-        query = parse_query(
+        query = await parse_query(
             source=source,
             max_file_size=max_file_size,
             from_web=False,

src/query_processor.py

@@ -77,7 +77,7 @@ async def process_query(
     }
 
     try:
-        query = parse_query(
+        query = await parse_query(
             source=input_text,
             max_file_size=max_file_size,
             from_web=True,

tests/test_query_parser.py

@@ -8,9 +8,9 @@
 from gitingest.query_parser import _parse_patterns, _parse_url, parse_query
 
 
-def test_parse_url_valid_https() -> None:
+async def test_parse_url_valid_https() -> None:
     """
-    Test `_parse_url` with valid HTTPS URLs from supported platforms (GitHub, GitLab, Bitbucket).
+    Test `_parse_url` with valid HTTPS URLs from supported platforms (GitHub, GitLab, Bitbucket, Gitea).
     Verifies that user and repository names are correctly extracted.
     """
     test_cases = [
@@ -19,13 +19,13 @@ def test_parse_url_valid_https() -> None:
         "https://bitbucket.org/user/repo",
     ]
     for url in test_cases:
-        result = _parse_url(url)
+        result = await _parse_url(url)
         assert result["user_name"] == "user"
         assert result["repo_name"] == "repo"
         assert result["url"] == url
 
 
-def test_parse_url_valid_http() -> None:
+async def test_parse_url_valid_http() -> None:
     """
     Test `_parse_url` with valid HTTP URLs from supported platforms.
     Verifies that user and repository names, as well as the slug, are correctly extracted.
@@ -36,88 +36,88 @@ def test_parse_url_valid_http() -> None:
         "http://bitbucket.org/user/repo",
     ]
     for url in test_cases:
-        result = _parse_url(url)
+        result = await _parse_url(url)
         assert result["user_name"] == "user"
         assert result["repo_name"] == "repo"
         assert result["slug"] == "user-repo"
 
 
-def test_parse_url_invalid() -> None:
+async def test_parse_url_invalid() -> None:
     """
     Test `_parse_url` with an invalid URL that does not include a repository structure.
     Verifies that a ValueError is raised with an appropriate error message.
     """
-    url = "https://only-domain.com"
+    url = "https://github.com"
     with pytest.raises(ValueError, match="Invalid repository URL"):
-        _parse_url(url)
+        await _parse_url(url)
 
 
-def test_parse_query_basic() -> None:
+async def test_parse_query_basic() -> None:
     """
     Test `parse_query` with basic inputs including valid repository URLs.
     Verifies that user and repository names, URL, and ignore patterns are correctly parsed.
     """
     test_cases = ["https://github.com/user/repo", "https://gitlab.com/user/repo"]
     for url in test_cases:
-        result = parse_query(url, max_file_size=50, from_web=True, ignore_patterns="*.txt")
+        result = await parse_query(url, max_file_size=50, from_web=True, ignore_patterns="*.txt")
         assert result["user_name"] == "user"
         assert result["repo_name"] == "repo"
         assert result["url"] == url
         assert "*.txt" in result["ignore_patterns"]
 
 
-def test_parse_query_mixed_case() -> None:
+async def test_parse_query_mixed_case() -> None:
     """
     Test `parse_query` with mixed case URLs.
     """
     url = "Https://GitHub.COM/UsEr/rEpO"
-    result = parse_query(url, max_file_size=50, from_web=True)
+    result = await parse_query(url, max_file_size=50, from_web=True)
     assert result["user_name"] == "user"
     assert result["repo_name"] == "repo"
 
 
-def test_parse_query_include_pattern() -> None:
+async def test_parse_query_include_pattern() -> None:
     """
     Test `parse_query` with an include pattern.
     Verifies that the include pattern is set correctly and default ignore patterns are applied.
     """
     url = "https://github.com/user/repo"
-    result = parse_query(url, max_file_size=50, from_web=True, include_patterns="*.py")
+    result = await parse_query(url, max_file_size=50, from_web=True, include_patterns="*.py")
     assert result["include_patterns"] == ["*.py"]
     assert set(result["ignore_patterns"]) == set(DEFAULT_IGNORE_PATTERNS)
 
 
-def test_parse_query_invalid_pattern() -> None:
+async def test_parse_query_invalid_pattern() -> None:
     """
     Test `parse_query` with an invalid pattern containing special characters.
     Verifies that a ValueError is raised with an appropriate error message.
     """
     url = "https://github.com/user/repo"
     with pytest.raises(ValueError, match="Pattern.*contains invalid characters"):
-        parse_query(url, max_file_size=50, from_web=True, include_patterns="*.py;rm -rf")
+        await parse_query(url, max_file_size=50, from_web=True, include_patterns="*.py;rm -rf")
 
 
-def test_parse_url_with_subpaths() -> None:
+async def test_parse_url_with_subpaths() -> None:
     """
     Test `_parse_url` with a URL containing a branch and subpath.
     Verifies that user name, repository name, branch, and subpath are correctly extracted.
     """
     url = "https://github.com/user/repo/tree/main/subdir/file"
-    result = _parse_url(url)
+    result = await _parse_url(url)
     assert result["user_name"] == "user"
     assert result["repo_name"] == "repo"
     assert result["branch"] == "main"
     assert result["subpath"] == "/subdir/file"
 
 
-def test_parse_url_invalid_repo_structure() -> None:
+async def test_parse_url_invalid_repo_structure() -> None:
     """
     Test `_parse_url` with an invalid repository structure in the URL.
     Verifies that a ValueError is raised with an appropriate error message.
     """
     url = "https://github.com/user"
     with pytest.raises(ValueError, match="Invalid repository URL"):
-        _parse_url(url)
+        await _parse_url(url)
 
 
 def test_parse_patterns_valid() -> None:
@@ -140,35 +140,35 @@ def test_parse_patterns_invalid_characters() -> None:
         _parse_patterns(patterns)
 
 
-def test_parse_query_with_large_file_size() -> None:
+async def test_parse_query_with_large_file_size() -> None:
     """
     Test `parse_query` with a very large file size limit.
     Verifies that the file size limit and default ignore patterns are set correctly.
     """
     url = "https://github.com/user/repo"
-    result = parse_query(url, max_file_size=10**9, from_web=True)
+    result = await parse_query(url, max_file_size=10**9, from_web=True)
     assert result["max_file_size"] == 10**9
     assert result["ignore_patterns"] == DEFAULT_IGNORE_PATTERNS
 
 
-def test_parse_query_empty_patterns() -> None:
+async def test_parse_query_empty_patterns() -> None:
     """
     Test `parse_query` with empty include and ignore patterns.
     Verifies that the include patterns are set to None and default ignore patterns are applied.
     """
     url = "https://github.com/user/repo"
-    result = parse_query(url, max_file_size=50, from_web=True, include_patterns="", ignore_patterns="")
+    result = await parse_query(url, max_file_size=50, from_web=True, include_patterns="", ignore_patterns="")
     assert result["include_patterns"] is None
     assert result["ignore_patterns"] == DEFAULT_IGNORE_PATTERNS
 
 
-def test_parse_query_include_and_ignore_overlap() -> None:
+async def test_parse_query_include_and_ignore_overlap() -> None:
     """
     Test `parse_query` with overlapping include and ignore patterns.
     Verifies that overlapping patterns are removed from the ignore patterns.
     """
     url = "https://github.com/user/repo"
-    result = parse_query(
+    result = await parse_query(
         url,
         max_file_size=50,
         from_web=True,
@@ -180,50 +180,50 @@ def test_parse_query_include_and_ignore_overlap() -> None:
     assert "*.txt" in result["ignore_patterns"]
 
 
-def test_parse_query_local_path() -> None:
+async def test_parse_query_local_path() -> None:
     """
     Test `parse_query` with a local file path.
     Verifies that the local path is set, a unique ID is generated, and the slug is correctly created.
     """
     path = "/home/user/project"
-    result = parse_query(path, max_file_size=100, from_web=False)
+    result = await parse_query(path, max_file_size=100, from_web=False)
     tail = Path("home/user/project")
     assert result["local_path"].parts[-len(tail.parts) :] == tail.parts
     assert result["id"] is not None
     assert result["slug"] == "user/project"
 
 
-def test_parse_query_relative_path() -> None:
+async def test_parse_query_relative_path() -> None:
     """
     Test `parse_query` with a relative file path.
     Verifies that the local path and slug are correctly resolved.
     """
     path = "./project"
-    result = parse_query(path, max_file_size=100, from_web=False)
+    result = await parse_query(path, max_file_size=100, from_web=False)
     tail = Path("project")
     assert result["local_path"].parts[-len(tail.parts) :] == tail.parts
     assert result["slug"].endswith("project")
 
 
-def test_parse_query_empty_source() -> None:
+async def test_parse_query_empty_source() -> None:
     """
     Test `parse_query` with an empty source input.
     Verifies that a ValueError is raised with an appropriate error message.
     """
     with pytest.raises(ValueError, match="Invalid repository URL"):
-        parse_query("", max_file_size=100, from_web=True)
+        await parse_query("", max_file_size=100, from_web=True)
 
 
-def test_parse_url_branch_and_commit_distinction() -> None:
+async def test_parse_url_branch_and_commit_distinction() -> None:
     """
     Test `_parse_url` with URLs containing either a branch name or a commit hash.
     Verifies that the branch and commit are correctly distinguished.
     """
     url_branch = "https://github.com/user/repo/tree/main"
     url_commit = "https://github.com/user/repo/tree/abcd1234abcd1234abcd1234abcd1234abcd1234"
 
-    result_branch = _parse_url(url_branch)
-    result_commit = _parse_url(url_commit)
+    result_branch = await _parse_url(url_branch)
+    result_commit = await _parse_url(url_commit)
 
     assert result_branch["branch"] == "main"
     assert result_branch["commit"] is None
@@ -232,23 +232,23 @@ def test_parse_url_branch_and_commit_distinction() -> None:
     assert result_commit["commit"] == "abcd1234abcd1234abcd1234abcd1234abcd1234"
 
 
-def test_parse_query_uuid_uniqueness() -> None:
+async def test_parse_query_uuid_uniqueness() -> None:
     """
     Test `parse_query` to ensure that each call generates a unique UUID for the query result.
     """
     path = "/home/user/project"
-    result1 = parse_query(path, max_file_size=100, from_web=False)
-    result2 = parse_query(path, max_file_size=100, from_web=False)
+    result1 = await parse_query(path, max_file_size=100, from_web=False)
+    result2 = await parse_query(path, max_file_size=100, from_web=False)
     assert result1["id"] != result2["id"]
 
 
-def test_parse_url_with_query_and_fragment() -> None:
+async def test_parse_url_with_query_and_fragment() -> None:
     """
     Test `_parse_url` with a URL containing query parameters and a fragment.
     Verifies that the URL is cleaned and other fields are correctly extracted.
     """
     url = "https://github.com/user/repo?arg=value#fragment"
-    result = _parse_url(url)
+    result = await _parse_url(url)
     assert result["user_name"] == "user"
     assert result["repo_name"] == "repo"
     assert result["url"] == "https://github.com/user/repo"  # URL should be cleaned