0.2.0 - add external URL proxy support

Add the external URL proxy support for fetching the cross-origin resources.

Author: cainmagi

Changelog.md

@@ -4,13 +4,14 @@
 
 ## CHANGELOG
 
-### 0.2.0 @ 03/25/2025
+### 0.2.0 @ 03/31/2025
 
 #### :mega: New
 
 1. Add the Dash component project based on TypeScript. The project contains a `PlainDownloader` component.
 2. Add the default downloader component `Downloader`. It is implemented by using `StreamSaver.js`.
 3. Add the cross-origin support when serving the data to clients (e.g. the browser).
+4. Add the external URL proxy support for fetching the cross-origin resources.
 
 #### :floppy_disk: Change
 

dash_file_cache/caches/tempfile.py

@@ -193,6 +193,9 @@ def dump(self, key: str, info: th.CachedFileInfo, data: th.CachedData) -> None:
                 while chunk:
                     fobj.write(chunk)
                     chunk = _data.read(self.__chunk_size)
+        elif data["type"] == "request":
+            with open(path + ".tmp", "w") as fobj:
+                json.dump(data, fobj, ensure_ascii=False)
         else:
             raise TypeError(
                 "cache: The value to be dumped is not recognized: {0}".format(data)
@@ -233,6 +236,10 @@ def _deferred():
             elif file_type == "bytes":
                 fobj = open(path + ".tmp", "rb")
                 return th.CachedBytesIO(type=file_type, data=fobj)
+            elif file_type == "request":
+                with open(path + ".tmp", "r") as fobj:
+                    _data: th.CachedRequest = json.load(fobj)
+                return _data
             else:
                 raise TypeError(
                     "cache: The type {0} of the key to be loaded is not "

dash_file_cache/caches/typehints.py

@@ -19,12 +19,14 @@
 
 import os
 
-from typing import Union, IO, TypeVar
+from typing import Union, Optional, IO, TypeVar
 
 try:
     from typing import Callable
+    from typing import Dict
 except ImportError:
     from collections.abc import Callable
+    from builtins import dict as Dict
 
 from typing_extensions import Literal, TypedDict
 
@@ -49,7 +51,7 @@
 class CachedFileInfo(TypedDict):
     """The metadata of the cached file."""
 
-    type: Literal["path", "str", "bytes"]
+    type: Literal["path", "str", "bytes", "request"]
     """The type os this cached data."""
 
     data_size: int
@@ -90,6 +92,31 @@ class CachedPath(TypedDict):
     """The path to the file on the local disk."""
 
 
+class CachedRequest(TypedDict):
+    """The URL and request configuration specifying a remote file. This configuration
+    is mainly used when a cross-domain URL needs to be accessed or an authentication
+    protected file is to be read.
+
+    In this case, the `CachedRequest` will only store the URL and the request
+    configurations. The request stream will be established and forwarded to the
+    users when this cached data item is to be streamed.
+    """
+
+    type: Literal["request"]
+    """The type os this cached data."""
+
+    url: str
+    """The URL referring to the remote file."""
+
+    headers: Dict[str, str]
+    """A collection of headers to be used when accessing the remote file."""
+
+    file_name_fallback: Optional[str]
+    """A fall-back file name. It is used when the response of the request does not
+    provide a file name. If this valus is not specified, will use the URL to guess
+    the file name."""
+
+
 class CachedStringIO(TypedDict):
     """The data of one cached `StringIO` data."""
 
@@ -112,5 +139,5 @@ class CachedBytesIO(TypedDict):
     file name and the `mime_type`."""
 
 
-CachedData = Union[CachedPath, CachedStringIO, CachedBytesIO]
+CachedData = Union[CachedPath, CachedRequest, CachedStringIO, CachedBytesIO]
 """The typehint of the cached data."""

dash_file_cache/services/__init__.py

@@ -23,11 +23,12 @@
 
 # Import sub-modules.
 from . import utilities
+from . import reqstream
 from . import data
 
 from .data import ServiceData
 
-__all__ = ("utilities", "data", "ServiceData")
+__all__ = ("utilities", "reqstream", "data", "ServiceData")
 
 # Set this local module as the prefered one
 __path__ = extend_path(__path__, __name__)

dash_file_cache/services/data.py

@@ -20,13 +20,14 @@
 import os
 import io
 import uuid
+import urllib.parse
 
 from typing import Union, Optional, IO, AnyStr, Any
 
 try:
-    from typing import Iterator, Callable
+    from typing import Iterator, Callable, Mapping
 except ImportError:
-    from collections.abc import Iterator, Callable
+    from collections.abc import Iterator, Callable, Mapping
 
 from typing_extensions import ClassVar, Literal
 
@@ -40,12 +41,15 @@
     CachedPath,
     CachedStringIO,
     CachedBytesIO,
+    CachedRequest,
 )
 from ..caches.abstract import CacheAbstract
 from ..caches.memory import CacheQueue
 from .utilities import no_cache, get_server
 from ..utilities import StreamFinalizer
 
+from .reqstream import DeferredRequestStream
+
 
 __all__ = ("ServiceData",)
 
@@ -118,6 +122,69 @@ def allowed_cross_origin(self) -> str:
         cross-origin data delivery will not be used."""
         return self.__allowed_cross_origin
 
+    def register_request(
+        self,
+        url: Union[str, urllib.parse.ParseResult],
+        headers: Optional[Mapping[str, str]] = None,
+        file_name_fallback: str = "",
+        download: bool = True,
+    ) -> str:
+        """Register the a remote request to the cache.
+
+        Arguments
+        ---------
+        url: `str | urllib.parse.ParseResult`
+            The string-form or parsed URL of the remote file.
+
+        headers: `Mapping[str, str] | None`
+            The customized headers for this request. If not specified, will not use
+            any headers.
+
+        file_name_fallback: `str`
+            The fall-back file name. If specified, it will be used as the saved file
+            name when the remote service does not provide the file name.
+
+        download: `bool`
+            If specified, will mark the returned address as a downloadable link.
+
+        Returns
+        -------
+        #1: `str`
+            The URL that would be used for accessing this temporarily cached file.
+        """
+        url_parsed = urllib.parse.urlparse(url) if isinstance(url, str) else url
+        file_name = (
+            url_parsed.path.replace("\\", "/").rsplit("/", maxsplit=1)[-1].strip()
+        )
+        if not file_name:
+            file_name = "Unknown"
+
+        info = CachedFileInfo(
+            type="request",
+            data_size=0,
+            file_name=file_name,
+            content_type="",
+            mime_type="application/octet-stream",
+            one_time_service=False,
+        )
+        data = CachedRequest(
+            type="request",
+            url=url_parsed.geturl(),
+            headers=dict(headers) if headers else dict(),
+            file_name_fallback=file_name_fallback,
+        )
+
+        uid = uuid.uuid4().hex
+
+        if isinstance(self.__cache, CacheQueue):
+            cache = self.__cache.mirror
+        else:
+            cache = self.__cache
+        cache.dump(key=uid, info=info, data=data)
+        return "{0}?uid={1}{2}".format(
+            self.__addr, uid, "&download=true" if download else ""
+        )
+
     def register(
         self,
         fobj: Union[str, os.PathLike, io.StringIO, io.BytesIO],
@@ -229,6 +296,11 @@ def loader() -> IO[Any]:
             _data = data()
             if _data["type"] == "path":
                 fobj = open(_data["path"], "rb")
+            elif _data["type"] == "request":
+                raise TypeError(
+                    "service: Should not use the request data in file-based data "
+                    "loader."
+                )
             else:
                 fobj = _data["data"]
             fobj.seek(0, io.SEEK_SET)
@@ -264,7 +336,9 @@ def _stream_add_headers(
         """Private method of `stream()`
 
         Add customized headers to the data service response."""
-        resp.headers["Content-Length"] = str(info["data_size"])
+        data_size = info["data_size"]
+        if isinstance(data_size, str) or (isinstance(data_size, int) and data_size > 0):
+            resp.headers["Content-Length"] = str(data_size)
         if self.__allowed_cross_origin:
             resp.headers["Access-Control-Allow-Origin"] = self.__allowed_cross_origin
             resp.headers["Access-Control-Allow-Credentials"] = "true"
@@ -305,41 +379,53 @@ def stream(self, uid: str, download: bool = False) -> flask.Response:
 
         info, deferred = self.__cache.load(uid)
 
-        if info["data_size"] <= 0:
-            raise FileNotFoundError(
-                "services: The requested file {0} is empty.".format(uid)
-            )
-
         file_type = info["type"]
-        if file_type not in ("path", "str", "bytes"):
+        if file_type not in ("path", "str", "bytes", "request"):
             raise TypeError(
                 "service: Cannot recognize the type of fobj: " "{0}".format(file_type)
             )
 
-        one_time_service = info["one_time_service"]
-        at_closed = self._stream_get_at_closed(cache=self.cache, uid=uid)
+        if file_type != "request" and info["data_size"] <= 0:
+            raise FileNotFoundError(
+                "services: The requested file {0} is empty.".format(uid)
+            )
 
-        def provider(_deferred: Callable[[], IO[AnyStr]]) -> Iterator[AnyStr]:
-            """Streaming data provider."""
+        if info["type"] == "request":
+            val = deferred()
+            if val["type"] != "request":
+                raise TypeError(
+                    "service: The data type ({0}) and the info type ({1}) does not "
+                    "match.".format(info["type"], val["type"])
+                )
+            streamer = DeferredRequestStream(info=info, data=val)
+            stream = streamer.provide(chunk_size=self.__chunk_size)
+            info = streamer.info
+        else:
+            one_time_service = info["one_time_service"]
+            at_closed = self._stream_get_at_closed(cache=self.cache, uid=uid)
+
+            def provider(_deferred: Callable[[], IO[AnyStr]]) -> Iterator[AnyStr]:
+                """Streaming data provider."""
 
-            with StreamFinalizer(
-                _deferred(), callback_on_exit=at_closed if one_time_service else None
-            ) as _fobj:
-                data = _fobj.read(self.__chunk_size)
-                while data:
-                    yield data
+                with StreamFinalizer(
+                    _deferred(),
+                    callback_on_exit=at_closed if one_time_service else None,
+                ) as _fobj:
                     data = _fobj.read(self.__chunk_size)
+                    while data:
+                        yield data
+                        data = _fobj.read(self.__chunk_size)
+
+            stream = provider(self._stream_data_to_loader(deferred))
 
         resp = flask.Response(
-            flask.stream_with_context(provider(self._stream_data_to_loader(deferred))),
+            flask.stream_with_context(stream),
             content_type=(
                 "application/octet-stream" if download else info["content_type"]
             ),
             mimetype=info["mime_type"],
         )
         self._stream_add_headers(resp, info=info, uid=uid, download=download)
-        print(resp.headers)
-
         return resp
 
     def serve(