Make gitingest.com parsable , notebook now supports output and some metadata

src/gitingest/notebook_utils.py

@@ -6,7 +6,7 @@
 from typing import Any
 
 
-def process_notebook(file: Path) -> str:
+def process_notebook(file: Path , parse_notebook_output: bool = True) -> str:
     """
     Process a Jupyter notebook file and return an executable Python script as a string.
 
@@ -45,6 +45,7 @@ def process_notebook(file: Path) -> str:
         notebook = worksheets[0]
 
     result = []
+    cell_count=0
 
     for cell in notebook["cells"]:
         cell_type = cell.get("cell_type")
@@ -61,6 +62,24 @@ def process_notebook(file: Path) -> str:
         if cell_type in ("markdown", "raw"):
             str_ = f'"""\n{str_}\n"""'
 
+        # Extract Output from cell
+        if parse_notebook_output and (("outputs" in cell) and (cell["outputs"] != [])):
+            sample_output=""
+            for output in cell["outputs"]:
+                if output["output_type"] == "stream" and output["text"] != []:
+                    sample_output += "".join(output["text"]) + "\n"
+                elif (output["output_type"] in ["execute_result","display_data"]) and ("data" in output) and ("text/plain" in output["data"]):
+                    sample_output += "".join(output["data"]["text/plain"]) + "\n"
+                elif (output["output_type"]=="error" and ("evalue" in output) ):
+                    sample_output += f"{output.get("ename","Error")} : " + "".join(output["evalue"]) + "\n"
+            str_ += f'\n# Output:\n"""{sample_output}"""\n'
+
+        # Add Cell Info
+        cell_count+=1 
+        str_ = f"# Cell {cell_count} ; Type : ({cell_type})\n" + str_
+
+
+
         result.append(str_)
 
     return "\n\n".join(result)

src/gitingest/query_ingestion.py

@@ -140,7 +140,7 @@ def _is_text_file(file_path: Path) -> bool:
         return False
 
 
-def _read_file_content(file_path: Path) -> str:
+def _read_file_content(file_path: Path , parse_notebook_output: bool = True) -> str:
     """
     Read the content of a file.
 
@@ -152,6 +152,8 @@ def _read_file_content(file_path: Path) -> str:
     ----------
     file_path : Path
         The path to the file to read.
+    parse_output_notebook : bool
+        Whether to parse the output of the notebook-cells.
 
     Returns
     -------
@@ -160,7 +162,7 @@ def _read_file_content(file_path: Path) -> str:
     """
     try:
         if file_path.suffix == ".ipynb":
-            return process_notebook(file_path)
+            return process_notebook(file_path, parse_notebook_output)
 
         with open(file_path, encoding="utf-8", errors="ignore") as f:
             return f.read()

src/gitingest/query_parser.py

@@ -21,6 +21,7 @@
     "bitbucket.org",
     "gitea.com",
     "codeberg.org",
+    "gitingest.com"
 ]
 
 

tests/query_parser/test_query_parser.py

@@ -17,6 +17,9 @@ async def test_parse_url_valid_https() -> None:
         "https://github.com/user/repo",
         "https://gitlab.com/user/repo",
         "https://bitbucket.org/user/repo",
+        "https://gitea.com/user/repo",
+        "https://codeberg.com/user/repo",
+        "https://gitingest.com/user/repo",
     ]
     for url in test_cases:
         result = await _parse_repo_source(url)
@@ -34,6 +37,8 @@ async def test_parse_url_valid_http() -> None:
         "http://github.com/user/repo",
         "http://gitlab.com/user/repo",
         "http://bitbucket.org/user/repo",
+        "https://gitingest.com/user/repo",
+        "http://gitea.com/user/repo",
     ]
     for url in test_cases:
         result = await _parse_repo_source(url)