chore: Apply pre-commit fixes

Author: sidmohan0

README.md

@@ -29,22 +29,22 @@ pip install datafog
 
 DataFog uses `extras` to manage dependencies for optional features like specific OCR engines or Apache Spark integration. You can install these as needed:
 
-*   **OCR (Tesseract):** For image scanning using Tesseract. Requires Tesseract OCR engine to be installed on your system separately.
-    ```bash
-    pip install "datafog[ocr]"
-    ```
-*   **OCR (Donut):** For image scanning using the Donut document understanding model.
-    ```bash
-    pip install "datafog[donut]"
-    ```
-*   **Spark:** For processing data using PySpark.
-    ```bash
-    pip install "datafog[spark]"
-    ```
-*   **All:** To install all optional features at once.
-    ```bash
-    pip install "datafog[all]"
-    ```
+- **OCR (Tesseract):** For image scanning using Tesseract. Requires Tesseract OCR engine to be installed on your system separately.
+  ```bash
+  pip install "datafog[ocr]"
+  ```
+- **OCR (Donut):** For image scanning using the Donut document understanding model.
+  ```bash
+  pip install "datafog[donut]"
+  ```
+- **Spark:** For processing data using PySpark.
+  ```bash
+  pip install "datafog[spark]"
+  ```
+- **All:** To install all optional features at once.
+  ```bash
+  pip install "datafog[all]"
+  ```
 
 # CLI
 

datafog/processing/image_processing/donut_processor.py

@@ -27,7 +27,8 @@
         "torch is not installed. Please install it to use Donut features: pip install 'datafog[donut]'"
     )
 try:
-    from transformers import DonutProcessor as TransformersDonutProcessor, VisionEncoderDecoderModel
+    from transformers import DonutProcessor as TransformersDonutProcessor
+    from transformers import VisionEncoderDecoderModel
 except ModuleNotFoundError:
     raise ModuleNotFoundError(
         "transformers is not installed. Please install it to use Donut features: pip install 'datafog[donut]'"

datafog/processing/spark_processing/pyspark_udfs.py

@@ -7,67 +7,60 @@
 on text data.
 """
 
-import logging
-import sys
 import importlib
+import logging
 import subprocess
+import sys
+import traceback
+from typing import List
 
-# Attempt imports and provide helpful error messages
 try:
-    from pyspark.sql.functions import udf
-    from pyspark.sql.types import StringType, ArrayType
-except ModuleNotFoundError:
-    raise ModuleNotFoundError(
-        "pyspark is not installed. Please install it to use Spark features: pip install datafog[spark]"
-    )
+    import spacy
+except ImportError:
+    print("Spacy not found. Please install it: pip install spacy")
+    print("and download the model: python -m spacy download en_core_web_lg")
+    spacy = None
+    traceback.print_exc()
+    sys.exit(1)
 
 try:
-    import spacy
-except ModuleNotFoundError:
-    # Spacy is a core dependency, but let's provide a helpful message just in case.
-    raise ModuleNotFoundError(
-        "spacy is not installed. Please ensure datafog is installed correctly: pip install datafog"
+    from pyspark.sql import SparkSession
+    from pyspark.sql.functions import udf
+    from pyspark.sql.types import ArrayType, StringType
+except ImportError:
+    print(
+        "PySpark not found. Please install it with the [spark] extra: pip install 'datafog[spark]'"
     )
 
+    # Set placeholders to allow module import even if pyspark is not installed
+    def placeholder_udf(*args, **kwargs):
+        return None
 
-from typing import List
-
-PII_ANNOTATION_LABELS = ["DATE_TIME", "LOC", "NRP", "ORG", "PER"]
-MAXIMAL_STRING_SIZE = 1000000
-
+    def placeholder_arraytype(x):
+        return None
 
-def pii_annotator(text: str, broadcasted_nlp) -> List[List[str]]:
-    """Extract features using en_core_web_lg model.
+    def placeholder_stringtype():
+        return None
 
-    Returns:
-        list[list[str]]: Values as arrays in order defined in the PII_ANNOTATION_LABELS.
-    """
-    if text:
-        if len(text) > MAXIMAL_STRING_SIZE:
-            # Cut the strings for required sizes
-            text = text[:MAXIMAL_STRING_SIZE]
-        nlp = broadcasted_nlp.value
-        doc = nlp(text)
+    udf = placeholder_udf
+    ArrayType = placeholder_arraytype
+    StringType = placeholder_stringtype
+    SparkSession = None  # Define a placeholder
+    traceback.print_exc()
+    # Do not exit, allow basic import but functions using Spark will fail later if called
 
-        # Pre-create dictionary with labels matching to expected extracted entities
-        classified_entities: dict[str, list[str]] = {
-            _label: [] for _label in PII_ANNOTATION_LABELS
-        }
-        for ent in doc.ents:
-            # Add entities from extracted values
-            classified_entities[ent.label_].append(ent.text)
+from datafog.processing.text_processing.spacy_pii_annotator import pii_annotator
 
-        return [_ent for _ent in classified_entities.values()]
-    else:
-        return [[] for _ in PII_ANNOTATION_LABELS]
+PII_ANNOTATION_LABELS = ["DATE_TIME", "LOC", "NRP", "ORG", "PER"]
+MAXIMAL_STRING_SIZE = 1000000
 
 
 def broadcast_pii_annotator_udf(
     spark_session=None, spacy_model: str = "en_core_web_lg"
 ):
     """Broadcast PII annotator across Spark cluster and create UDF"""
     if not spark_session:
-        spark_session = SparkSession.builder.getOrCreate()
+        spark_session = SparkSession.builder.getOrCreate()  # noqa: F821
     broadcasted_nlp = spark_session.sparkContext.broadcast(spacy.load(spacy_model))
 
     pii_annotation_udf = udf(

datafog/services/spark_service.py

@@ -5,16 +5,16 @@
 JSON reading, and package management.
 """
 
-import sys
 import importlib
-import subprocess
-import logging
 import json
+import logging
+import subprocess
+import sys
 from typing import Any, List, Optional
 
 # Attempt to import pyspark and provide a helpful error message if missing
 try:
-    from pyspark.sql import SparkSession, DataFrame
+    from pyspark.sql import DataFrame, SparkSession
 except ModuleNotFoundError:
     raise ModuleNotFoundError(
         "pyspark is not installed. Please install it to use Spark features: pip install datafog[spark]"

notes/ROADMAP.md

@@ -1,4 +1,3 @@
-
 ---
 
 ### **v4.1.0 — Baseline stability**
@@ -11,68 +10,68 @@
 
 ### **v4.2.0 — Faster spaCy path**
 
-* **MUST** hold the spaCy `nlp` object in a module-level cache (singleton).  
-* **MUST** replace per-doc loops with `nlp.pipe(batch_size=?, n_process=-1)`.  
-* **MUST** run spaCy and Tesseract calls in `asyncio.to_thread()` (or a thread-pool) so the event-loop stays free.  
-* **SHOULD** expose `PIPE_BATCH_SIZE` env var for tuning.
+- **MUST** hold the spaCy `nlp` object in a module-level cache (singleton).
+- **MUST** replace per-doc loops with `nlp.pipe(batch_size=?, n_process=-1)`.
+- **MUST** run spaCy and Tesseract calls in `asyncio.to_thread()` (or a thread-pool) so the event-loop stays free.
+- **SHOULD** expose `PIPE_BATCH_SIZE` env var for tuning.
 
 ---
 
 ### **v4.3.0 — Strong types, predictable output**
 
-* **MUST** make `_process_text` always return `Dict[str, Dict]`.  
-* **MUST** add `mypy --strict` to CI; fix any revealed issues.  
-* **SHOULD** convert `datafog.config` to a Pydantic v2 `BaseSettings`.
+- **MUST** make `_process_text` always return `Dict[str, Dict]`.
+- **MUST** add `mypy --strict` to CI; fix any revealed issues.
+- **SHOULD** convert `datafog.config` to a Pydantic v2 `BaseSettings`.
 
 ---
 
 ### **v4.4.0 — Clean OCR architecture**
 
-* **MUST** split `ImageService` into `TesseractOCR` and `DonutOCR`, each with `extract_text(Image)->str`.  
-* **MUST** let users pick via `ImageService(backend="tesseract"|"donut")` or the `DATAFOG_DEFAULT_OCR` env var.  
-* **SHOULD** add unit tests that stub each backend independently.
+- **MUST** split `ImageService` into `TesseractOCR` and `DonutOCR`, each with `extract_text(Image)->str`.
+- **MUST** let users pick via `ImageService(backend="tesseract"|"donut")` or the `DATAFOG_DEFAULT_OCR` env var.
+- **SHOULD** add unit tests that stub each backend independently.
 
 ---
 
 ### **v4.5.0 — Rust-powered pattern matching (optional wheel)**
 
-* **MUST** create a PyO3 extension `datafog._fastregex` that wraps `aho-corasick` / `regex-automata`.  
-* **MUST** auto-import it when available; fall back to pure-Python silently.  
-* **SHOULD** publish platform wheels under `pip install "datafog[fastregex]"`.
+- **MUST** create a PyO3 extension `datafog._fastregex` that wraps `aho-corasick` / `regex-automata`.
+- **MUST** auto-import it when available; fall back to pure-Python silently.
+- **SHOULD** publish platform wheels under `pip install "datafog[fastregex]"`.
 
 ---
 
 ### **v4.6.0 — Streaming and zero-copy**
 
-* **MUST** add `async def stream_text_pipeline(iterable[str]) -> AsyncIterator[Result]`.  
-* **MUST** scan CSV/JSON via `pyarrow.dataset` to avoid reading the whole file into RAM.  
-* **SHOULD** provide example notebook comparing latency/bandwidth vs. v4.5.
+- **MUST** add `async def stream_text_pipeline(iterable[str]) -> AsyncIterator[Result]`.
+- **MUST** scan CSV/JSON via `pyarrow.dataset` to avoid reading the whole file into RAM.
+- **SHOULD** provide example notebook comparing latency/bandwidth vs. v4.5.
 
 ---
 
 ### **v4.7.0 — GPU / transformer toggle**
 
-* **MUST** accept `DataFog(use_gpu=True)` which loads `en_core_web_trf` in half precision if CUDA is present.  
-* **MUST** fall back gracefully on CPU-only hosts.  
-* **SHOULD** benchmark and log model choice at INFO level.
+- **MUST** accept `DataFog(use_gpu=True)` which loads `en_core_web_trf` in half precision if CUDA is present.
+- **MUST** fall back gracefully on CPU-only hosts.
+- **SHOULD** benchmark and log model choice at INFO level.
 
 ---
 
 ### **v4.8.0 — Fast anonymizer core**
 
-* **MUST** rewrite `Anonymizer.replace_pii/redact_pii/hash_pii` in Cython (single-pass over the string).  
-* **MUST** switch hashing to OpenSSL EVP via `cffi` for SHA-256/SHA3-256.  
-* **SHOULD** guard with `pip install "datafog[fast]"`.
+- **MUST** rewrite `Anonymizer.replace_pii/redact_pii/hash_pii` in Cython (single-pass over the string).
+- **MUST** switch hashing to OpenSSL EVP via `cffi` for SHA-256/SHA3-256.
+- **SHOULD** guard with `pip install "datafog[fast]"`.
 
 ---
 
 ### **v4.9.0 — Edge & CI polish**
 
-* **MUST** compile the annotator and anonymizer to WebAssembly using `maturin`, package as `_datafog_wasm`.  
-* **MUST** auto-load WASM build on `wasmtime` when `import datafog.wasm` succeeds.  
-* **MUST** cache spaCy model artefacts in GitHub Actions with `actions/cache`, keyed by `model-hash`.  
-* **SHOULD** update docs and `README.md` badges for new extras and WASM support.
+- **MUST** compile the annotator and anonymizer to WebAssembly using `maturin`, package as `_datafog_wasm`.
+- **MUST** auto-load WASM build on `wasmtime` when `import datafog.wasm` succeeds.
+- **MUST** cache spaCy model artefacts in GitHub Actions with `actions/cache`, keyed by `model-hash`.
+- **SHOULD** update docs and `README.md` badges for new extras and WASM support.
 
 ---
 
-Use this ladder as-is, bumping **only the minor version** each time, so v4.0.x callers never break.
+Use this ladder as-is, bumping **only the minor version** each time, so v4.0.x callers never break.

notes/v4.1.0-tickets.md

@@ -10,13 +10,15 @@
 Currently, the package version might be duplicated or inconsistently defined. We need to centralize the version definition in `datafog/__about__.py`.
 
 **Tasks:**
+
 1.  Ensure `datafog/__about__.py` exists and contains a `__version__` string variable (e.g., `__version__ = "4.1.0"`).
 2.  Modify `setup.py` to read this `__version__` variable from `datafog/__about__.py`. Common patterns involve reading the file and executing its content in a temporary namespace or using regular expressions.
 3.  Remove any hardcoded `version` assignment within `setup.py` itself.
 4.  Verify that `pip install .` and building distributions (`sdist`, `wheel`) correctly pick up the version from `__about__.py`.
 
 **Acceptance Criteria:**
-- The package version is defined *only* in `datafog/__about__.py`.
+
+- The package version is defined _only_ in `datafog/__about__.py`.
 - `setup.py` successfully reads the version from `__about__.py` during installation and build processes.
 - Running `import datafog; print(datafog.__version__)` (if applicable) shows the correct version.
 
@@ -30,12 +32,14 @@ Currently, the package version might be duplicated or inconsistently defined. We
 The codebase currently uses functions like `ensure_installed()` that attempt to `pip install` missing dependencies at runtime. This practice is unreliable, can hide dependency issues, slow down startup, and interfere with environment management. We must remove this pattern and adopt a "fail fast" approach.
 
 **Tasks:**
+
 1.  Identify all code locations where runtime `pip install` commands are executed (e.g., calls to `ensure_installed`, `subprocess.run(['pip', 'install', ...])`).
 2.  Remove these runtime installation calls entirely.
 3.  Replace them with standard `import` statements. If an `ImportError` occurs, the program should exit gracefully, clearly stating which dependency is missing and how to install it (e.g., "Please install the 'X' package: pip install datafog[feature]").
 4.  Ensure all necessary dependencies are listed correctly in `setup.py`'s `install_requires` or `extras_require`.
 
 **Acceptance Criteria:**
+
 - No code attempts to install packages using `pip` or similar mechanisms during program execution.
 - If an optional dependency (part of an `extra`) is needed but not installed, the program raises an `ImportError` with a helpful message instructing the user how to install the required extra.
 - Core dependencies listed in `install_requires` are assumed to be present; missing core dependencies will naturally cause `ImportError` on startup.
@@ -50,18 +54,20 @@ The codebase currently uses functions like `ensure_installed()` that attempt to
 The project offers optional OCR functionality using Tesseract and/or Donut models, which have their own dependencies. These optional dependencies need to be formally defined using `extras_require` in `setup.py` and documented for users.
 
 **Tasks:**
-1.  Identify all dependencies required *only* for Tesseract functionality.
-2.  Identify all dependencies required *only* for Donut functionality.
+
+1.  Identify all dependencies required _only_ for Tesseract functionality.
+2.  Identify all dependencies required _only_ for Donut functionality.
 3.  Define appropriate extras in the `extras_require` dictionary within `setup.py`. Suggestions:
-    *   `'ocr': ['pytesseract', 'pillow', ...]` (for Tesseract)
-    *   `'donut': ['transformers[torch]', 'sentencepiece', ...]` (for Donut)
-    *   Optionally, a combined extra: `'all_ocr': ['pytesseract', 'pillow', 'transformers[torch]', 'sentencepiece', ...]` or include dependencies in a general `'ocr'` extra if they don't conflict significantly.
+    - `'ocr': ['pytesseract', 'pillow', ...]` (for Tesseract)
+    - `'donut': ['transformers[torch]', 'sentencepiece', ...]` (for Donut)
+    - Optionally, a combined extra: `'all_ocr': ['pytesseract', 'pillow', 'transformers[torch]', 'sentencepiece', ...]` or include dependencies in a general `'ocr'` extra if they don't conflict significantly.
 4.  Update the `README.md` and any installation documentation (e.g., `docs/installation.md`) to explain these extras and how users can install them (e.g., `pip install "datafog[ocr]"` or `pip install "datafog[donut]"`).
 
 **Acceptance Criteria:**
+
 - `setup.py` contains an `extras_require` section defining keys like `ocr` and/or `donut`.
 - Installing the package with these extras (e.g., `pip install .[ocr]`) successfully installs the associated dependencies.
 - Documentation clearly explains the available extras and the installation commands.
-- Core installation (`pip install .`) does *not* install the OCR-specific dependencies.
+- Core installation (`pip install .`) does _not_ install the OCR-specific dependencies.
 
 ---