Merge 2.3.1dev for v2.3.1 release readiness (#89)

Author: mikeqfu

docs/source/requirements.txt

@@ -1,9 +1,10 @@
-furo==2024.8.6
-pandas==2.3.1
-psycopg2==2.9.10
-pyproj==3.7.1
-shapely==2.1.1
+furo==2025.9.25
+pandas==2.3.3
+pandas-stubs==2.3.2.250926
+psycopg2==2.9.11
+pyproj==3.7.2
+shapely==2.1.2
 sphinx-copybutton==0.5.2
-sphinx-new-tab-link==0.8.0
+sphinx-new-tab-link==0.8.1
 sphinx-toggleprompt==0.6.0
-sqlalchemy==2.0.41
+sqlalchemy==2.0.44

pyhelpers/data/.metadata

@@ -5,7 +5,7 @@
     "Author": "Qian Fu",
     "Affiliation": "School of Engineering, University of Birmingham",
     "Email": "q.fu@bham.ac.uk",
-    "Version": "2.3.0",
+    "Version": "2.3.1",
     "License": "MIT",
     "First release": "September 2019"
 }

pyhelpers/ops/downloads.py

@@ -168,7 +168,7 @@ def _download_file_from_url(response, path_to_file, chunk_multiplier=1, desc=Non
 def download_file_from_url(url, path_to_file, if_exists='replace', max_retries=5,
                            requests_session_args=None, verbose=False, print_wrap_limit=None,
                            chunk_multiplier=1, desc=None, bar_format=None, colour=None,
-                           validate=True, **kwargs):
+                           validate=True, stream_download=False, **kwargs):
     # noinspection PyShadowingNames
     """
     Downloads a file from a valid URL.
@@ -212,6 +212,11 @@ def download_file_from_url(url, path_to_file, if_exists='replace', max_retries=5
     :param validate: Whether to validate if the downloaded file size matches the expected content
         length; defaults to ``True``.
     :type validate: bool
+    :param stream_download: When `stream_download=True`, use streaming download
+        (memory-efficient, perferred for large files);
+        When `stream_download=False`, not streaming (simpler/faster for small files);
+        defaults to ``False``.
+    :type stream_download: bool
     :param kwargs: [Optional] Additional parameters passed to the method `tqdm.tqdm()`_.
 
     .. _`tqdm.tqdm()`: https://tqdm.github.io/docs/tqdm/
@@ -272,8 +277,10 @@ def download_file_from_url(url, path_to_file, if_exists='replace', max_retries=5
 
         os.makedirs(os.path.dirname(path_to_file_), exist_ok=True)  # Ensure the directory exists
 
-        # Streaming, so we can iterate over the response
-        with session.get(url=url, stream=True, headers=fake_headers) as response:
+        # If streaming, we can iterate over the response
+        stream_download_ = verbose or stream_download
+
+        with session.get(url=url, stream=stream_download_, headers=fake_headers) as response:
             if response.status_code != 200:
                 print(f"Failed to retrieve file. HTTP Status Code: {response.status_code}.")
                 return None
@@ -292,8 +299,17 @@ def download_file_from_url(url, path_to_file, if_exists='replace', max_retries=5
                 )
 
             else:
-                with open(path_to_file_, mode='wb') as f:  # Open the file in binary write mode
-                    shutil.copyfileobj(fsrc=response.raw, fdst=f)  # type: ignore
+                if stream_download_:
+                    encoding = response.headers.get('Content-Encoding', '')
+                    if 'gzip' in encoding or 'deflate' in encoding:
+                        response.raw.decode_content = True
+
+                    with open(path_to_file_, mode='wb') as f:  # Open the file in binary write mode
+                        shutil.copyfileobj(fsrc=response.raw, fdst=f)  # type: ignore
+
+                else:
+                    with open(path_to_file_, mode='wb') as f:
+                        f.write(response.content)
 
             # Validate download if necessary
             if validate and os.stat(path_to_file).st_size == 0:

pyhelpers/ops/general.py

@@ -345,3 +345,71 @@ def get_ansi_colour_code(colours, show_valid_colours=False):
     """
 
     return _get_ansi_colour_code(colours=colours, show_valid_colours=show_valid_colours)
+
+
+def get_project_structure(start_path, ignore_dirs=None, out_file=None, encoding='utf-8',
+                          print_in_console=True):
+    """
+    Prints and/or writes the directory and file structure of a given project folder
+    starting from ``start_path``.
+
+    The output shows a tree-like hierarchy with branch symbols for better readability.
+
+    :param start_path: Path to the root directory whose structure to visualize;
+        can be absolute or relative to the current working directory.
+    :type start_path: str | pathlib.Path
+    :param ignore_dirs: Optional set of directory names to ignore during traversal;
+        defaults to ``{'__pycache__'}``.
+    :type ignore_dirs: None | typying.Iterable
+    :param out_file: Optional file path to write the structure output. If ``None`` (default),
+        no file will be written.
+        If specified, the structure will be saved to the specified file path.
+    :type out_file: str | None
+    :param encoding: The encoding to use when writing to the output file; defaults to ``'utf-8'``.
+    :type encoding: str
+    :param print_in_console: Whether to print the structure to the console; defaults to ``True``.
+    :type print_in_console: bool
+
+    **Examples**::
+
+        >>> from pyhelpers.ops import get_project_structure
+        >>> get_project_structure(start_path="pyhelpers")
+        >>> get_project_structure("my_project", print_in_console=False, out_file="structure.txt")
+    """
+
+    if ignore_dirs is None:
+        ignore_dirs = {'__pycache__'}
+
+    def _print(text, file_handle=None):
+        if print_in_console:
+            try:
+                print(text)
+            except UnicodeEncodeError:
+                # Fallback for consoles that can't handle Unicode
+                ascii_text = text.replace('├', '+').replace('│', '|').replace('──', '--')
+                print(ascii_text)
+
+        if file_handle is not None:
+            file_handle.write(text + '\n')
+
+    f = None
+    if out_file is not None:
+        f = open(out_file, 'w', encoding=encoding)
+
+    try:
+        for root, dirs, files in os.walk(start_path):
+            dirs[:] = [d for d in dirs if d not in ignore_dirs]
+
+            level = root.replace(start_path, '').count(os.sep)
+            indent = '│   ' * level + '├── '
+            line = f"{indent}{os.path.basename(root)}/"
+            _print(line, f)
+
+            sub_indent = '│   ' * (level + 1)
+            for file in sorted(files):
+                file_line = f"{sub_indent}├── {file}"
+                _print(file_line, f)
+
+    finally:
+        if f is not None:
+            f.close()

pyhelpers/ops/manipulation.py

@@ -812,7 +812,7 @@ def downcast_numeric_columns(*dataframes):
                 df_[int_cols] = df_[int_cols].apply(pd.to_numeric, downcast='integer')
 
             # Process all float columns
-            float_cols = df_.select_dtypes(include=['floating']).columns
+            float_cols = df_.select_dtypes(include=['floating']).columns  # noqa
             if not float_cols.empty:
                 df_[float_cols] = df_[float_cols].apply(pd.to_numeric, downcast='float')
 
@@ -822,14 +822,14 @@ def downcast_numeric_columns(*dataframes):
         elif isinstance(df, pl.DataFrame):  # noqa
             df_ = df.clone()
             # Process integer columns
-            int_cols = [col for col in df.columns if df[col].dtype.is_integer()]
+            int_cols = [col for col in df_.columns if df_[col].dtype.is_integer()]
             if int_cols:
-                df_ = df_.with_columns([pl.col(col).shrink_dtype() for col in int_cols]) # noqa
+                df_ = df_.with_columns([df_[col].shrink_dtype() for col in int_cols]) # noqa
 
             # Process float columns
-            float_cols = [col for col in df.columns if df[col].dtype.is_float()]
+            float_cols = [col for col in df_.columns if df_[col].dtype.is_float()]
             if float_cols:
-                df_ = df_.with_columns([pl.col(col).shrink_dtype() for col in float_cols]) # noqa
+                df_ = df_.with_columns([df_[col].shrink_dtype() for col in float_cols]) # noqa
 
             results.append(df_)
 

requirements.txt

@@ -1,42 +1,43 @@
-build==1.2.2.post1
+build==1.3.0
 folium==0.20.0
 fqdn==1.5.1
-furo==2024.8.6
-gdal==3.11.1
+furo==2025.9.25
+gdal==3.11.4
 isoduration==20.11.0
 jaraco.collections==5.1.0
 jsonpointer==3.0.0
-matplotlib==3.10.3
-networkx==3.5
-nltk==3.9.1
-notebook==7.4.4
+matplotlib==3.10.7
+networkx==3.6
+nltk==3.9.2
+notebook==7.5.0
 odfpy==1.4.1
 openpyxl==3.1.5
-orjson==3.10.18
-pandas==2.3.1
+orjson==3.11.4
+pandas==2.3.3
+pandas-stubs==2.3.2.250926
 pdfkit==1.0.0
 pip-chill==1.0.3
 pkginfo==1.12.1.2
-polars==1.31.0
-psycopg2==2.9.10
-pyarrow==20.0.0
-pyodbc==5.2.0
-pypandoc==1.15
-pyproj==3.7.1
-pytest-cov==6.2.1
-python-rapidjson==1.20
-rapidfuzz==3.13.0
-scikit-learn==1.7.0
-shapely==2.1.1
+polars==1.35.2
+psycopg2==2.9.11
+pyarrow==22.0.0
+pyodbc==5.3.0
+pypandoc==1.16.2
+pyproj==3.7.2
+pytest-cov==7.0.0
+python-rapidjson==1.22
+rapidfuzz==3.14.3
+scikit-learn==1.7.2
+shapely==2.1.2
 sphinx-copybutton==0.5.2
-sphinx-new-tab-link==0.8.0
+sphinx-new-tab-link==0.8.1
 sphinx-toggleprompt==0.6.0
-sqlalchemy==2.0.41
+sqlalchemy==2.0.44
 tinycss2==1.4.0
 tomli==2.0.1
-twine==6.1.0
-ujson==5.10.0
+twine==6.2.0
+ujson==5.11.0
 uri-template==1.3.0
 webcolors==24.11.1
 xlsx2csv==0.8.4
-xlsxwriter==3.2.5
+xlsxwriter==3.2.9

tests/test_ops/test_downloads.py

@@ -19,14 +19,17 @@ def test_is_downloadable():
 
 
 @pytest.mark.parametrize('verbose', [True, False])
-def test_download_file_from_url(capfd, verbose):
+@pytest.mark.parametrize('stream_download', [True, False])
+def test_download_file_from_url(capfd, verbose, stream_download):
     logo_url = 'https://www.python.org/static/community_logos/python-logo-master-v3-TM.png'
 
     path_to_img = "ops-download_file_from_url-demo.png"
-    download_file_from_url(logo_url, path_to_img, verbose=verbose, colour='green')
+    download_file_from_url(
+        logo_url, path_to_img, verbose=verbose, colour='green', stream_download=stream_download)
     assert os.path.isfile(path_to_img)
 
-    download_file_from_url(logo_url, path_to_img, if_exists='pass', verbose=verbose)
+    download_file_from_url(
+        logo_url, path_to_img, if_exists='pass', verbose=verbose, stream_download=stream_download)
     out, _ = capfd.readouterr()
     if verbose:
         assert "Aborting download." in out
@@ -35,7 +38,7 @@ def test_download_file_from_url(capfd, verbose):
 
     path_to_img_ = tempfile.NamedTemporaryFile()
     path_to_img = path_to_img_.name + ".png"
-    download_file_from_url(logo_url, path_to_img, verbose=verbose)
+    download_file_from_url(logo_url, path_to_img, verbose=verbose, stream_download=stream_download)
     assert os.path.isfile(path_to_img)
 
     os.remove(path_to_img_.name)