Release 2.2.0

conftest.py

@@ -9,7 +9,6 @@
 import functools
 import logging
 import os
-import sys
 from copy import deepcopy
 from pathlib import Path
 
@@ -22,7 +21,6 @@
 import pytest
 from packaging.version import Version
 
-from imas.backends.imas_core.imas_interface import has_imas as _has_imas
 from imas.backends.imas_core.imas_interface import ll_interface, lowlevel
 from imas.dd_zip import dd_etree, dd_xml_versions, latest_dd_version
 from imas.ids_defs import (
@@ -39,17 +37,7 @@
 
 os.environ["IMAS_AL_DISABLE_VALIDATE"] = "1"
 
-
-try:
-    import imas  # noqa
-except ImportError:
-
-    class SkipOnIMASAccess:
-        def __getattr__(self, attr):
-            pytest.skip("This test requires the `imas` HLI, which is not available.")
-
-    # Any test that tries to access an attribute from the `imas` package will be skipped
-    sys.modules["imas"] = SkipOnIMASAccess()
+import imas  # noqa
 
 
 def pytest_addoption(parser):
@@ -78,7 +66,6 @@ def pytest_addoption(parser):
     if "not available" in str(iex.message):
         _BACKENDS.pop("mdsplus")
 
-
 try:
     import pytest_xdist
 except ImportError:
@@ -91,28 +78,11 @@ def worker_id():
 @pytest.fixture(params=_BACKENDS)
 def backend(pytestconfig: pytest.Config, request: pytest.FixtureRequest):
     backends_provided = any(map(pytestconfig.getoption, _BACKENDS))
-    if not _has_imas:
-        if backends_provided:
-            raise RuntimeError(
-                "Explicit backends are provided, but IMAS is not available."
-            )
-        pytest.skip("No IMAS available, skip tests using a backend")
     if backends_provided and not pytestconfig.getoption(request.param):
         pytest.skip(f"Tests for {request.param} backend are skipped.")
     return _BACKENDS[request.param]
 
 
-@pytest.fixture()
-def has_imas():
-    return _has_imas
-
-
-@pytest.fixture()
-def requires_imas():
-    if not _has_imas:
-        pytest.skip("No IMAS available")
-
-
 def pytest_generate_tests(metafunc):
     if "ids_name" in metafunc.fixturenames:
         if metafunc.config.getoption("ids"):
@@ -214,7 +184,7 @@ def wrapper(*args, **kwargs):
 
 
 @pytest.fixture
-def log_lowlevel_calls(monkeypatch, requires_imas):
+def log_lowlevel_calls(monkeypatch):
     """Debugging fixture to log calls to the imas lowlevel module."""
     for al_function in dir(lowlevel):
         if al_function.startswith("ual_") or al_function.startswith("al"):

docs/source/api.rst

@@ -7,16 +7,24 @@ This page provides an auto-generated summary of IMAS-Python's API. For more deta
 and examples, refer to the relevant chapters in the main part of the
 documentation.
 
-IMAS-Python IDS manipulation
-----------------------------
+IMAS-Python public API
+----------------------
 
 .. currentmodule:: imas
 
 .. autosummary::
 
+   convert_core_edge_plasma.convert_to_plasma_profiles
+   convert_core_edge_plasma.convert_to_plasma_sources
+   convert_core_edge_plasma.convert_to_plasma_transport
    db_entry.DBEntry
+   ids_convert.convert_ids
+   ids_data_type.IDSDataType
    ids_factory.IDSFactory
-   ids_toplevel.IDSToplevel
+   ids_identifiers.identifiers
+   ids_metadata.IDSMetadata
+   ids_metadata.IDSType
    ids_primitive.IDSPrimitive
-   ids_structure.IDSStructure
    ids_struct_array.IDSStructArray
+   ids_structure.IDSStructure
+   ids_toplevel.IDSToplevel

docs/source/multi-dd.rst

@@ -276,27 +276,110 @@ You need to explicitly convert the data, which you can do as follows:
       entry.put(imas.convert_ids(equilibrium, entry.dd_version))
 
 
+.. _`UDA backend and DD versions`:
 
-.. _`DD background`:
+UDA backend caching and Data Dictionary versions
+------------------------------------------------
 
-Background information
-----------------------
+If you try to load data from a different Data Dictionary version with the UDA backend,
+you may see the following error:
+
+.. code-block:: text
+
+  The Data Dictionary version of the data (3.38.1) is different from the Data
+  Dictionary version of the DBEntry (3.42.0). This is not supported when using the
+  UDA backend.
+
+There are three possible workarounds. The first two require passing an additional option
+in the IMAS UDA URI: please see the `imas-core documentation
+<https://imas-core.readthedocs.io/en/stable/user_guide/backends_guide.html#query-keys-specific-for-the-uda-backend>`__
+for more details on these URI options.
+
+1. Use UDA fetch to bypass the cache problem. You can do this by appending ``&fetch=1``
+   to the URI when you create the :py:class:`~imas.db_entry.DBEntry`.
+
+   Note that this will download the entire IDS files from the remote server, this may
+   not be desired if you only want to read a single time slice.
+2. Disable the UDA cache. You can do this by appending ``&cache_mode=none`` to the URI
+   when you create the :py:class:`~imas.db_entry.DBEntry`.
+
+   Note that this may make the ``get()`` (a lot) slower, since a separate request needs
+   to be sent to the remote UDA server for every data variable. However, this may still
+   be the best performing option if you are only interested in a subset of all the data
+   in an IDS (and use :ref:`lazy loading`).
+3. Explicitly provide the data dictionary version when you create the
+   :py:class:`~imas.db_entry.DBEntry`, setting it to match the Data Dictionary version
+   of the data you want to load. To obtain the version of the data on the remote server
+   from the field `ids_properties.put_version.data_dictionary` via a _lazy_ ``get()``
+   with ``autoconvert=False`` option and using the ``&cache_mode=none`` query in the URI.
+
+   Note that you may need to call ``imas.convert_ids`` to convert the IDS to your
+   desired Data Dictionary version.
+
+All three possible workarounds are shown in the examples below:
+
+.. md-tab-set::
+
+    .. md-tab-item:: Original code
+
+      .. code-block:: python
+
+        import imas
+
+        URI = (
+            "imas://uda.iter.org:56565/uda?backend=hdf5"
+            "&path=/work/imas/shared/imasdb/ITER/3/121013/50"
+        )
+        with imas.DBEntry(URI, "r") as entry:
+            cp = entry.get("core_profiles")
 
-Since IMAS-Python needs to have access to multiple DD versions it was chosen to
-bundle these with the code at build-time, in setup.py. If a git clone of the
-Data Dictionary succeeds, the setup tools automatically download saxon and
-generate ``IDSDef.xml`` for each of the tagged versions in the DD git
-repository. These are then gathered into ``IDSDef.zip``, which is
-distributed inside the IMAS-Python package.
+    .. md-tab-item:: 1. Use UDA fetch
 
-To update the set of data dictionaries new versions can be added to the zipfile.
-A reinstall of the package will ensure that all available versions are included
-in IMAS-Python. Additionally an explicit path to an XML file can be specified, which
-is useful for development.
+      .. code-block:: python
 
-Automated tests have been provided that check the loading of all of the DD
-versions tagged in the data-dictionary git repository.
+        import imas
 
+        URI = (
+            "imas://uda.iter.org:56565/uda?backend=hdf5"
+            "&path=/work/imas/shared/imasdb/ITER/3/121013/50&fetch=1"
+        )
+        with imas.DBEntry(URI, "r") as entry:
+            cp = entry.get("core_profiles")
+
+    .. md-tab-item:: 2. Disable the UDA cache
+
+      .. code-block:: python
+
+        import imas
+
+        URI = (
+            "imas://uda.iter.org:56565/uda?backend=hdf5"
+            "&path=/work/imas/shared/imasdb/ITER/3/121013/50&cache_mode=none"
+        )
+        with imas.DBEntry(URI, "r") as entry:
+            cp = entry.get("core_profiles")
+
+    .. md-tab-item:: 3. Explicitly provide the DD version
+
+      .. code-block:: python
+
+        import imas
+
+        URI = (
+            "imas://uda.iter.org:56565/uda?backend=hdf5"
+            "&path=/work/imas/shared/imasdb/ITER/3/121013/50"
+        )
+        with imas.DBEntry(URI, "r", dd_version="3.38.1") as entry:
+            cp = entry.get("core_profiles")
+
+        # Optional: convert the IDS to your desired DD version
+        cp = imas.convert_ids(cp, "3.42.0")
+
+
+.. _`DD background`:
+
+Background information
+----------------------
 
 Data Dictionary definitions
 '''''''''''''''''''''''''''

imas/__init__.py

@@ -1,30 +1,58 @@
 # This file is part of IMAS-Python.
 # You should have received the IMAS-Python LICENSE file with this project.
 
-# isort: skip_file
-
 from packaging.version import Version as _V
 
-from ._version import version as __version__  # noqa: F401
-from ._version import version_tuple  # noqa: F401
-
 # Import logging _first_
-from . import setup_logging
+# isort: off
+from . import setup_logging  # noqa: F401
+
+# isort: on
 
-# Import main user API objects in the imas module
+# Ensure that `imas.util` is loaded when importing imas
+from . import util  # noqa: F401
+
+# Public API:
+from ._version import version as __version__
+from ._version import version_tuple
+from .convert_core_edge_plasma import (
+    convert_to_plasma_profiles,
+    convert_to_plasma_sources,
+    convert_to_plasma_transport,
+)
 from .db_entry import DBEntry
-from .ids_factory import IDSFactory
 from .ids_convert import convert_ids
+from .ids_data_type import IDSDataType
+from .ids_factory import IDSFactory
 from .ids_identifiers import identifiers
-
-# Load the IMAS-Python IMAS AL/DD core
-from . import (
-    db_entry,
-    dd_zip,
-    util,
-)
+from .ids_metadata import IDSMetadata, IDSType
+from .ids_primitive import IDSPrimitive
+from .ids_struct_array import IDSStructArray
+from .ids_structure import IDSStructure
+from .ids_toplevel import IDSToplevel
 
 PUBLISHED_DOCUMENTATION_ROOT = "https://imas-python.readthedocs.io/en/latest/"
 """URL to the published documentation."""
 OLDEST_SUPPORTED_VERSION = _V("3.22.0")
 """Oldest Data Dictionary version that is supported by IMAS-Python."""
+
+__all__ = [
+    "__version__",
+    "version_tuple",
+    "DBEntry",
+    "IDSDataType",
+    "IDSFactory",
+    "IDSMetadata",
+    "IDSPrimitive",
+    "IDSStructure",
+    "IDSStructArray",
+    "IDSToplevel",
+    "IDSType",
+    "convert_ids",
+    "convert_to_plasma_profiles",
+    "convert_to_plasma_sources",
+    "convert_to_plasma_transport",
+    "identifiers",
+    "PUBLISHED_DOCUMENTATION_ROOT",
+    "OLDEST_SUPPORTED_VERSION",
+]

imas/_to_xarray.py

@@ -27,7 +27,7 @@ def to_xarray(ids: IDSToplevel, *paths: str) -> xarray.Dataset:
     # block checks if the paths are valid, and by using "metadata.path_string" we ensure
     # that / are used as separator.
     try:
-        paths = [ids.metadata[path].path_string for path in paths]
+        paths: list[str] = [ids.metadata[path].path_string for path in paths]
     except KeyError as exc:
         raise ValueError(str(exc)) from None
 

imas/backends/db_entry_impl.py

@@ -78,7 +78,7 @@ def get(
         destination: IDSToplevel,
         lazy: bool,
         nbc_map: Optional[NBCPathMap],
-    ) -> None:
+    ) -> IDSToplevel:
         """Implement DBEntry.get/get_slice/get_sample. Load data from the data source.
 
         Args:

imas/backends/imas_core/al_context.py

@@ -115,7 +115,7 @@ def timerange_action(
         tmin: float,
         tmax: float,
         dtime: Optional[numpy.ndarray],
-        interpolation_method: int,
+        interpolation_method: Optional[int],
     ) -> "ALContext":
         """Begin a new timerange action for use in a ``with`` context."""
         ctx = ll_interface.begin_timerange_action(
@@ -163,7 +163,7 @@ def write_data(self, path: str, timebasepath: str, data: Any) -> None:
         """Call ual_write_data with this context."""
         status = ll_interface.write_data(self.ctx, path, timebasepath, data)
         if status != 0:
-            raise LowlevelError(f"write data at {path!r}: {status=}")
+            raise LowlevelError(f"write data at {path!r}", status)
 
     def list_all_occurrences(self, ids_name: str) -> List[int]:
         """List all occurrences of this IDS."""
@@ -359,7 +359,7 @@ def timerange_action(
         tmin: float,
         tmax: float,
         dtime: Optional[numpy.ndarray],
-        interpolation_method: int,
+        interpolation_method: Optional[int],
     ) -> Iterator["LazyALContext"]:
         """Lazily start a lowlevel timerange action, see
         :meth:`ALContext.timerange_action`.