Skip to content

Commit 0b20091

Browse files
filipchristiansenfilipchristiansen
authored
Merge branch 'main' into 365_improve_ui_and_backend
2 parents dba583e + 38c2317 commit 0b20091

File tree

15 files changed

+134
-40
lines changed

15 files changed

+134
-40
lines changed

‎README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,9 @@ gitingest https://github.com/username/private-repo --token github_pat_...
122122
# Or set it as an environment variable
123123
export GITHUB_TOKEN=github_pat_...
124124
gitingest https://github.com/username/private-repo
125+
126+
# Include repository submodules
127+
gitingest https://github.com/username/repo-with-submodules --include-submodules
125128
```
126129

127130
By default, files listed in `.gitignore` are skipped. Use `--include-gitignored` if you
@@ -163,6 +166,9 @@ summary, tree, content = ingest("https://github.com/username/private-repo", toke
163166
import os
164167
os.environ["GITHUB_TOKEN"] = "github_pat_..."
165168
summary, tree, content = ingest("https://github.com/username/private-repo")
169+
170+
# Include repository submodules
171+
summary, tree, content = ingest("https://github.com/username/repo-with-submodules", include_submodules=True)
166172
```
167173

168174
By default, this won't write a file but can be enabled with the `output` argument.

‎src/gitingest/cli.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ class _CLIArgs(TypedDict):
2020
include_pattern: tuple[str, ...]
2121
branch: str | None
2222
include_gitignored: bool
23+
include_submodules: bool
2324
token: str | None
2425
output: str | None
2526

@@ -47,6 +48,12 @@ class _CLIArgs(TypedDict):
4748
default=False,
4849
help="Include files matched by .gitignore and .gitingestignore",
4950
)
51+
@click.option(
52+
"--include-submodules",
53+
is_flag=True,
54+
help="Include repository's submodules in the analysis",
55+
default=False,
56+
)
5057
@click.option(
5158
"--token",
5259
"-t",
@@ -94,6 +101,9 @@ def main(**cli_kwargs: Unpack[_CLIArgs]) -> None:
94101
$ gitingest https://github.com/user/private-repo -t ghp_token
95102
$ GITHUB_TOKEN=ghp_token gitingest https://github.com/user/private-repo
96103
104+
Include submodules:
105+
$ gitingest https://github.com/user/repo --include-submodules
106+
97107
"""
98108
asyncio.run(_async_main(**cli_kwargs))
99109

@@ -106,6 +116,7 @@ async def _async_main(
106116
include_pattern: tuple[str, ...] | None = None,
107117
branch: str | None = None,
108118
include_gitignored: bool = False,
119+
include_submodules: bool = False,
109120
token: str | None = None,
110121
output: str | None = None,
111122
) -> None:
@@ -129,6 +140,8 @@ async def _async_main(
129140
Git branch to ingest. If ``None``, the repository's default branch is used.
130141
include_gitignored : bool
131142
If ``True``, also ingest files matched by ``.gitignore`` or ``.gitingestignore`` (default: ``False``).
143+
include_submodules : bool
144+
If ``True``, recursively include all Git submodules within the repository (default: ``False``).
132145
token : str | None
133146
GitHub personal access token (PAT) for accessing private repositories.
134147
Can also be set via the ``GITHUB_TOKEN`` environment variable.
@@ -155,14 +168,15 @@ async def _async_main(
155168
click.echo(f"Analyzing source, output will be written to '{output_target}'...", err=True)
156169

157170
summary, _, _ = await ingest_async(
158-
source=source,
171+
source,
159172
max_file_size=max_size,
160173
include_patterns=include_patterns,
161174
exclude_patterns=exclude_patterns,
162175
branch=branch,
163-
output=output_target,
164176
include_gitignored=include_gitignored,
177+
include_submodules=include_submodules,
165178
token=token,
179+
output=output_target,
166180
)
167181
except Exception as exc:
168182
# Convert any exception into Click.Abort so that exit status is non-zero

‎src/gitingest/clone.py‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,9 @@ async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
6363
clone_cmd += ["-c", create_git_auth_header(token, url=url)]
6464

6565
clone_cmd += ["clone", "--single-branch"]
66-
# TODO: Re-enable --recurse-submodules when submodule support is needed
66+
67+
if config.include_submodules:
68+
clone_cmd += ["--recurse-submodules"]
6769

6870
if partial_clone:
6971
clone_cmd += ["--filter=blob:none", "--sparse"]
@@ -86,15 +88,28 @@ async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
8688

8789
# Checkout the subpath if it is a partial clone
8890
if partial_clone:
89-
subpath = config.subpath.lstrip("/")
90-
if config.blob:
91-
# When ingesting from a file url (blob/branch/path/file.txt), we need to remove the file name.
92-
subpath = str(Path(subpath).parent.as_posix())
93-
94-
checkout_cmd = create_git_command(["git"], local_path, url, token)
95-
await run_command(*checkout_cmd, "sparse-checkout", "set", subpath)
91+
await _checkout_partial_clone(config, token)
9692

9793
# Checkout the commit if it is provided
9894
if commit:
9995
checkout_cmd = create_git_command(["git"], local_path, url, token)
10096
await run_command(*checkout_cmd, "checkout", commit)
97+
98+
99+
async def _checkout_partial_clone(config: CloneConfig, token: str | None) -> None:
100+
"""Configure sparse-checkout for a partially cloned repository.
101+
102+
Parameters
103+
----------
104+
config : CloneConfig
105+
The configuration for cloning the repository, including subpath and blob flag.
106+
token : str | None
107+
GitHub personal access token (PAT) for accessing private repositories.
108+
109+
"""
110+
subpath = config.subpath.lstrip("/")
111+
if config.blob:
112+
# Remove the file name from the subpath when ingesting from a file url (e.g. blob/branch/path/file.txt)
113+
subpath = str(Path(subpath).parent.as_posix())
114+
checkout_cmd = create_git_command(["git"], config.local_path, config.url, token)
115+
await run_command(*checkout_cmd, "sparse-checkout", "set", subpath)

‎src/gitingest/entrypoint.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ async def ingest_async(
2727
branch: str | None = None,
2828
tag: str | None = None,
2929
include_gitignored: bool = False,
30+
include_submodules: bool = False,
3031
token: str | None = None,
3132
output: str | None = None,
3233
) -> tuple[str, str, str]:
@@ -52,6 +53,8 @@ async def ingest_async(
5253
The tag to clone and ingest. If ``None``, no tag is used.
5354
include_gitignored : bool
5455
If ``True``, include files ignored by ``.gitignore`` and ``.gitingestignore`` (default: ``False``).
56+
include_submodules : bool
57+
If ``True``, recursively include all Git submodules within the repository (default: ``False``).
5558
token : str | None
5659
GitHub personal access token (PAT) for accessing private repositories.
5760
Can also be set via the ``GITHUB_TOKEN`` environment variable.
@@ -86,6 +89,8 @@ async def ingest_async(
8689
if query.url:
8790
_override_branch_and_tag(query, branch=branch, tag=tag)
8891

92+
query.include_submodules = include_submodules
93+
8994
async with _clone_repo_if_remote(query, token=token):
9095
summary, tree, content = ingest_query(query)
9196
await _write_output(tree, content=content, target=output)
@@ -101,6 +106,7 @@ def ingest(
101106
branch: str | None = None,
102107
tag: str | None = None,
103108
include_gitignored: bool = False,
109+
include_submodules: bool = False,
104110
token: str | None = None,
105111
output: str | None = None,
106112
) -> tuple[str, str, str]:
@@ -126,6 +132,8 @@ def ingest(
126132
The tag to clone and ingest. If ``None``, no tag is used.
127133
include_gitignored : bool
128134
If ``True``, include files ignored by ``.gitignore`` and ``.gitingestignore`` (default: ``False``).
135+
include_submodules : bool
136+
If ``True``, recursively include all Git submodules within the repository (default: ``False``).
129137
token : str | None
130138
GitHub personal access token (PAT) for accessing private repositories.
131139
Can also be set via the ``GITHUB_TOKEN`` environment variable.
@@ -156,6 +164,7 @@ def ingest(
156164
branch=branch,
157165
tag=tag,
158166
include_gitignored=include_gitignored,
167+
include_submodules=include_submodules,
159168
token=token,
160169
output=output,
161170
),

‎src/gitingest/schemas/ingestion.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212

1313
@dataclass
14-
class CloneConfig:
14+
class CloneConfig: # pylint: disable=too-many-instance-attributes
1515
"""Configuration for cloning a Git repository.
1616
1717
This class holds the necessary parameters for cloning a repository to a local path, including
@@ -33,6 +33,8 @@ class CloneConfig:
3333
The subpath to clone from the repository (default: ``"/"``).
3434
blob: bool
3535
Whether the repository is a blob (default: ``False``).
36+
include_submodules: bool
37+
Whether to clone submodules (default: ``False``).
3638
3739
"""
3840

@@ -43,6 +45,7 @@ class CloneConfig:
4345
tag: str | None = None
4446
subpath: str = "/"
4547
blob: bool = False
48+
include_submodules: bool = False
4649

4750

4851
class IngestionQuery(BaseModel): # pylint: disable=too-many-instance-attributes
@@ -78,6 +81,8 @@ class IngestionQuery(BaseModel): # pylint: disable=too-many-instance-attributes
7881
The patterns to ignore (default: ``set()``).
7982
include_patterns : set[str] | None
8083
The patterns to include.
84+
include_submodules : bool
85+
Whether to include all Git submodules within the repository. (default: ``False``)
8186
8287
"""
8388

@@ -95,6 +100,7 @@ class IngestionQuery(BaseModel): # pylint: disable=too-many-instance-attributes
95100
max_file_size: int = Field(default=MAX_FILE_SIZE)
96101
ignore_patterns: set[str] = set() # TODO: ignore_patterns and include_patterns have the same type
97102
include_patterns: set[str] | None = None
103+
include_submodules: bool = False
98104

99105
def extract_clone_config(self) -> CloneConfig:
100106
"""Extract the relevant fields for the CloneConfig object.
@@ -122,6 +128,7 @@ def extract_clone_config(self) -> CloneConfig:
122128
tag=self.tag,
123129
subpath=self.subpath,
124130
blob=self.type == "blob",
131+
include_submodules=self.include_submodules,
125132
)
126133

127134
def ensure_url(self) -> None:

‎src/server/templates/components/_macros.jinja‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{# Icon link #}
2-
{% macro icon_link(href, icon, label) -%}
2+
{% macro footer_icon_link(href, icon, label) -%}
33
<a href="{{ href }}"
44
target="_blank"
55
rel="noopener noreferrer"

‎src/server/templates/components/footer.jinja‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
1-
{% from 'components/_macros.jinja' import icon_link %}
1+
{% from 'components/_macros.jinja' import footer_icon_link %}
22
<footer class="w-full border-t-[3px] border-gray-900 mt-auto">
33
<div class="max-w-4xl mx-auto px-4 py-4">
44
<div class="grid grid-cols-2 items-center text-gray-900 text-sm">
55
{# Left column — Chrome + PyPI #}
66
<div class="flex items-center space-x-4">
7-
{{ icon_link('https://chromewebstore.google.com/detail/adfjahbijlkjfoicpjkhjicpjpjfaood',
7+
{{ footer_icon_link('https://chromewebstore.google.com/detail/adfjahbijlkjfoicpjkhjicpjpjfaood',
88
'icons/chrome.svg',
99
'Chrome Extension') }}
10-
{{ icon_link('https://pypi.org/project/gitingest',
10+
{{ footer_icon_link('https://pypi.org/project/gitingest',
1111
'icons/python.svg',
1212
'Python Package') }}
1313
</div>
1414
{# Right column - Discord #}
1515
<div class="flex justify-end">
16-
{{ icon_link('https://discord.gg/zerRaGK9EC',
16+
{{ footer_icon_link('https://discord.gg/zerRaGK9EC',
1717
'icons/discord.svg',
1818
'Discord') }}
1919
</div>

‎src/server/templates/components/git_form.jinja‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@
4040
<!-- Pattern type selector -->
4141
<div class="relative flex items-center">
4242
<select id="pattern_type"
43-
onchange="changePattern()"
4443
name="pattern_type"
45-
class="w-21 py-2 pl-2 pr-6 appearance-none bg-[#e6e8eb] focus:outline-none border-r-[3px] border-gray-900 cursor-pointer">
44+
onchange="changePattern()"
45+
class="pattern-select">
4646
<option value="exclude"
4747
{% if pattern_type == 'exclude' or not pattern_type %}selected{% endif %}>
4848
Exclude

‎src/server/templates/components/navbar.jinja‎

Lines changed: 11 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,34 @@
11
<header class="sticky top-0 bg-[#FFFDF8] border-b-[3px] border-gray-900 z-50">
22
<div class="max-w-4xl mx-auto px-4">
33
<div class="flex justify-between items-center h-16">
4-
<!-- Logo -->
4+
{# Logo #}
55
<div class="flex items-center gap-4">
66
<h1 class="text-2xl font-bold tracking-tight">
77
<a href="/" class="hover:opacity-80 transition-opacity">
88
<span class="text-gray-900">Git</span><span class="text-[#FE4A60]">ingest</span>
99
</a>
1010
</h1>
1111
</div>
12-
<!-- Navigation with updated styling -->
12+
{# Navigation with updated styling #}
1313
<nav class="flex items-center space-x-6">
14-
<a href="/llm.txt"
15-
class="text-gray-900 hover:-translate-y-0.5 transition-transform flex items-center">
14+
<a href="/llm.txt" class="link-bounce flex items-center text-gray-900">
1615
<span class="badge-new">NEW</span>
1716
/llm.txt
1817
</a>
18+
{# GitHub link #}
1919
<div class="flex items-center gap-2">
2020
<a href="https://github.com/cyclotruc/gitingest"
2121
target="_blank"
2222
rel="noopener noreferrer"
23-
class="text-gray-900 hover:-translate-y-0.5 transition-transform flex items-center gap-1.5">
24-
<svg class="w-4 h-4"
25-
fill="currentColor"
26-
viewBox="0 0 24 24"
27-
aria-hidden="true">
28-
<path fill-rule="evenodd" d="M12 2C6.477 2 2 6.484 2 12.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0112 6.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.202 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.943.359.309.678.92.678 1.855 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0022 12.017C22 6.484 17.522 2 12 2z" clip-rule="evenodd">
29-
</path>
30-
</svg>
23+
class="link-bounce flex items-center gap-1.5 text-gray-900">
24+
<img src="/static/icons/github.svg" class="w-4 h-4" alt="GitHub logo">
3125
GitHub
3226
</a>
33-
<div class="flex items-center text-sm text-gray-600">
34-
<svg class="w-4 h-4 text-[#ffc480] mr-1"
35-
fill="currentColor"
36-
viewBox="0 0 20 20">
37-
<path d="M9.049 2.927c.3-.921 1.603-.921 1.902 0l1.07 3.292a1 1 0 00.95.69h3.462c.969 0 1.371 1.24.588 1.81l-2.8 2.034a1 1 0 00-.364 1.118l1.07 3.292c.3.921-.755 1.688-1.54 1.118l-2.8-2.034a1 1 0 00-1.175 0l-2.8 2.034c-.784.57-1.838-.197-1.539-1.118l1.07-3.292a1 1 0 00-.364-1.118L2.98 8.72c-.783-.57-.38-1.81.588-1.81h3.461a1 1 0 00.951-.69l1.07-3.292z" />
38-
</svg>
27+
{# Star counter #}
28+
<div class="no-drag flex items-center text-sm text-gray-600">
29+
<img src="/static/svg/github-star.svg"
30+
class="w-4 h-4 mr-1"
31+
alt="GitHub star icon">
3932
<span id="github-stars">0</span>
4033
</div>
4134
</div>

‎src/static/css/tailwind.css‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
@import "tailwindcss";
22

3-
@layer utilities {
3+
@layer components {
44
.badge-new {
55
@apply inline-block -rotate-6 -translate-y-1 mx-1 px-1
66
bg-[#FE4A60] border border-gray-900 text-white
@@ -36,9 +36,24 @@
3636
translate-y-4 sm:translate-y-10 md:translate-y-2 lg:translate-y-4;
3737
}
3838

39+
.pattern-select {
40+
@apply min-w-max appearance-none pr-6 pl-2 py-2
41+
bg-[#e6e8eb] border-r-[3px] border-gray-900
42+
cursor-pointer focus:outline-none;
43+
}
44+
45+
}
46+
47+
@layer utilities {
48+
3949
.no-drag {
4050
@apply pointer-events-none select-none;
4151
-webkit-user-drag: none;
4252
}
4353

54+
.link-bounce {
55+
@apply transition-transform
56+
hover:-translate-y-0.5;
57+
}
58+
4459
}

0 commit comments

Comments
 (0)