From 7db4eb39344c9c0ac1d67bb4e27357bf100d4347 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Mon, 16 Sep 2024 12:48:08 +0200 Subject: [PATCH 01/66] Added function calls to the ktype service --- dsms/knowledge/ktype.py | 4 +- dsms/knowledge/utils.py | 109 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 102 insertions(+), 11 deletions(-) diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index c1a396e..1b48fec 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -15,9 +15,9 @@ class KType(BaseModel): name: Optional[str] = Field( None, description="Human readable name of the KType." ) - webform: Optional[Any] = Field(None, description="Form data of the KItem.") + webform: Optional[Any] = Field(None, description="Form data of the KType.") json_schema: Optional[Any] = Field( - None, description="OpenAPI schema of the KItem." + None, description="OpenAPI schema of the KType." ) def __hash__(self) -> int: diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 277528e..f9a1268 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -192,7 +192,89 @@ def _get_remote_ktypes() -> Enum: logger.debug("Got the following ktypes from backend: `%s`.", list(ktypes)) return ktypes +def _ktype_exists(ktype: Union[Any, str, UUID]) -> bool: + """Check whether the KType exists in the remote backend""" + from dsms.knowledge.ktype import ( # isort:skip + KType, + ) + + if isinstance(ktype, KType): + route = f"api/knowledge-type/{ktype.id}" + else: + route = f"api/knowledge-type/{ktype}" + response = _perform_request(route, "get") + return response.ok + +def _create_new_ktype(ktype: "KType") -> None: + """Create a new KType in the remote backend""" + body = { + "name": ktype.name, + "id": str(ktype.id), + } + logger.debug("Create new KType with body: %s", body) + response = _perform_request("api/knowledge-type/", "post", json=body) + if not response.ok: + raise ValueError( + f"KType with id `{ktype.id}` could not be created in DSMS: {response.text}`" + ) + +def _get_ktype( + uuid: Union[UUID, str], as_json=False +) -> "Union[KItem, Dict[str, Any]]": + """Get the KType for an instance with a certain ID from remote backend""" + + from dsms import Context, KType #why this Ktype/Kitem import in all these methods?? it is already imported at the beginning + + response = _perform_request(f"api/knowledge-type/{uuid}", "get") + if response.status_code == 404: + raise ValueError( + f"""KType with uuid `{uuid}` does not exist in + DSMS-instance `{Context.dsms.config.host_url}`""" + ) + + if not response.ok: + raise ValueError( + f"""An error occured fetching the KType with uuid `{uuid}`: + `{response.text}`""" + ) + + body = response.json() + if as_json: + response = body + else: + response = KType(**body) + return response + +def _update_ktype(ktype: "KType") -> Response: + """Update a KType in the remote backend.""" + payload = ktype.model_dump( + exclude_none=True, + ) + + logger.debug( + "Update KType for `%s` with body: %s", ktype.id, payload + ) + response = _perform_request( + f"api/knowledge-type/{ktype.id}", "put", json=payload + ) + if not response.ok: + raise ValueError( + f"KType with uuid `{ktype.id}` could not be updated in DSMS: {response.text}`" + ) + return response + + +def _delete_ktype(ktype: "KType") -> None: + """Delete a KType in the remote backend""" + logger.debug("Delete KType with id: %s", ktype.id) + response = _perform_request(f"api/knowledge-type/{ktype.id}", "delete") + if not response.ok: + raise ValueError( + f"KItem with uuid `{ktype.id}` could not be deleted from DSMS: `{response.text}`" + ) + + def _get_kitem_list() -> "List[KItem]": """Get all available KItems from the remote backend.""" from dsms.knowledge.kitem import ( # isort:skip @@ -533,9 +615,7 @@ def _commit_created( elif isinstance(obj, AppConfig): _create_or_update_app_spec(obj) elif isinstance(obj, KType): - raise NotImplementedError( - "Committing of KTypes not implemented yet." - ) + _create_new_ktype(obj) else: raise TypeError( f"Object `{obj}` of type {type(obj)} cannot be committed." @@ -554,9 +634,7 @@ def _commit_updated( elif isinstance(obj, AppConfig): _create_or_update_app_spec(obj, overwrite=True) elif isinstance(obj, KType): - raise NotImplementedError( - "Committing of KTypes not implemented yet." - ) + _commit_updated_ktype(obj) else: raise TypeError( f"Object `{obj}` of type {type(obj)} cannot be committed." @@ -593,6 +671,21 @@ def _commit_updated_kitem(new_kitem: "KItem") -> None: new_kitem.refresh() +def _commit_updated_ktype(new_ktype: "KType") -> None: + """Commit the updated KTypes""" + old_ktype = _get_ktype(new_ktype.id, as_json=True) + logger.debug( + "Fetched data from old KType with id `%s`: %s", + new_ktype.id, + old_ktype, + ) + if old_ktype: + _update_ktype(new_ktype) + logger.debug( + "Fetching updated KType from remote backend: %s", new_ktype.id + ) + + def _commit_deleted( buffer: "Dict[str, Union[KItem, KType, AppConfig]]", ) -> None: @@ -606,9 +699,7 @@ def _commit_deleted( elif isinstance(obj, AppConfig): _delete_app_spec(obj.name) elif isinstance(obj, KType): - raise NotImplementedError( - "Deletion of KTypes not implemented yet." - ) + _delete_ktype(obj) else: raise TypeError( f"Object `{obj}` of type {type(obj)} cannot be committed or deleted." From 5b139c06f64b43c8794d6ae101e497db3cae7894 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Wed, 18 Sep 2024 15:41:21 +0200 Subject: [PATCH 02/66] Added backend check for existing ktypes --- dsms/core/dsms.py | 2 +- dsms/knowledge/ktype.py | 74 +++++++++++++++++++++++++++++++++++++++-- examples/testktype.py | 7 ++++ 3 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 examples/testktype.py diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index 3845509..8ae46af 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -112,7 +112,7 @@ def __delitem__(self, obj) -> None: elif isinstance(obj, AppConfig): self.context.buffers.deleted.update({obj.name: obj}) elif isinstance(obj, KType): - raise NotImplementedError("Deletion of KTypes not available yet.") + self.context.buffers.deleted.update({obj.name: obj}) else: raise TypeError( f"Object must be of type {KItem}, {AppConfig} or {KType}, not {type(obj)}. " diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 1b48fec..dccf572 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -1,11 +1,22 @@ """KItem types""" -from typing import Any, Dict, Optional, Union +import logging +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from uuid import UUID -from pydantic import BaseModel, Field, field_validator, model_serializer +from pydantic import BaseModel, Field, field_validator, model_serializer, ValidationInfo -from dsms.knowledge.utils import _create_custom_properties_model +from dsms.knowledge.utils import _create_custom_properties_model, _ktype_exists + +from dsms.core.logging import handler + +if TYPE_CHECKING: + from dsms import Context + from dsms.core.dsms import DSMS + +logger = logging.getLogger(__name__) +logger.addHandler(handler) +logger.propagate = False class KType(BaseModel): @@ -19,9 +30,37 @@ class KType(BaseModel): json_schema: Optional[Any] = Field( None, description="OpenAPI schema of the KType." ) + in_backend: bool = Field( + False, + description="Whether the Ktype was already created in the backend.", + ) def __hash__(self) -> int: return hash(str(self)) + + def __init__(self, **kwargs: "Any") -> None: + """Initialize the KType""" + from dsms import DSMS + + logger.debug("Initialize KType with model data: %s", kwargs) + + # set dsms instance if not already done + if not self.dsms: + self.dsms = DSMS() + + # initialize the kitem + super().__init__(**kwargs) + + # add ktype to buffer + if not self.in_backend and self.id not in self.context.buffers.created: + logger.debug( + "Marking KTpe with ID `%s` as created and updated during KItem initialization.", + self.id, + ) + self.context.buffers.created.update({self.id: self}) + self.context.buffers.updated.update({self.id: self}) + + logger.debug("KType initialization successful.") @field_validator("webform") @classmethod @@ -40,3 +79,32 @@ def serialize(self): ) for key, value in self.items() } + + @field_validator("in_backend") + @classmethod + def validate_in_backend(cls, value: bool, info: ValidationInfo) -> bool: + """Checks whether the KType already exists""" + ktype_id = info.data["id"] + if not value: + value = _ktype_exists(ktype_id) + return value + + + @property + def dsms(cls) -> "DSMS": + """DSMS context getter""" + return cls.context.dsms + + @dsms.setter + def dsms(cls, value: "DSMS") -> None: + """DSMS context setter""" + cls.context.dsms = value + + @property + def context(cls) -> "Context": + """Getter for Context""" + from dsms import ( # isort:skip + Context, + ) + + return Context diff --git a/examples/testktype.py b/examples/testktype.py new file mode 100644 index 0000000..c83fd56 --- /dev/null +++ b/examples/testktype.py @@ -0,0 +1,7 @@ +from dsms import DSMS, KItem, KType + +dsms = DSMS(env="../.env") + +dsms.kitems +for ktype in dsms.ktypes: + print(ktype) \ No newline at end of file From ed618b5d43595b013c7efd4774168d3da4e56de2 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Fri, 20 Sep 2024 16:48:10 +0200 Subject: [PATCH 03/66] Added additional methods to support ktype functionalities --- dsms/core/dsms.py | 9 ++++++++- dsms/knowledge/ktype.py | 44 +++++++++++++++++++---------------------- dsms/knowledge/utils.py | 37 +++++++++++++++++++++++++++++----- examples/testktype.py | 7 ------- testktype.py | 19 ++++++++++++++++++ 5 files changed, 79 insertions(+), 37 deletions(-) delete mode 100644 examples/testktype.py create mode 100644 testktype.py diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index 8ae46af..1183237 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -17,6 +17,7 @@ _get_kitem, _get_kitem_list, _get_remote_ktypes, + _get_ktype_list ) if TYPE_CHECKING: @@ -145,6 +146,11 @@ def sparql_interface(self) -> SparqlInterface: def ktypes(cls) -> "Enum": """ "Enum of the KTypes defined in the DSMS instance.""" return cls._ktypes + + # @property + # def ktypes(cls) -> "List[KType]": + # return _get_ktype_list() + @property def config(cls) -> Configuration: @@ -181,7 +187,8 @@ def kitems(cls) -> "List[KItem]": The default timeout for requests is defined under the `request_timeout`-attribute in the `Configuration`-class.""" return _get_kitem_list() - + + @property def app_configs(cls) -> "List[AppConfig]": """Return available app configs in the DSMS""" diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index dccf572..1dc13d8 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -1,12 +1,17 @@ """KItem types""" import logging +from datetime import datetime from typing import TYPE_CHECKING, Any, Dict, Optional, Union from uuid import UUID from pydantic import BaseModel, Field, field_validator, model_serializer, ValidationInfo -from dsms.knowledge.utils import _create_custom_properties_model, _ktype_exists +from dsms.knowledge.utils import ( + _create_custom_properties_model, + _ktype_exists, + _refresh_ktype +) from dsms.core.logging import handler @@ -30,9 +35,12 @@ class KType(BaseModel): json_schema: Optional[Any] = Field( None, description="OpenAPI schema of the KType." ) - in_backend: bool = Field( - False, - description="Whether the Ktype was already created in the backend.", + rdf_mapping: Optional[str] = Field(None, description="") + created_at: Optional[Union[str, datetime]] = Field( + None, description="Time and date when the KType was created." + ) + updated_at: Optional[Union[str, datetime]] = Field( + None, description="Time and date when the KType was updated." ) def __hash__(self) -> int: @@ -48,7 +56,6 @@ def __init__(self, **kwargs: "Any") -> None: if not self.dsms: self.dsms = DSMS() - # initialize the kitem super().__init__(**kwargs) # add ktype to buffer @@ -68,26 +75,11 @@ def create_model(cls, value: Optional[Dict[str, Any]]) -> Any: """Create the datamodel for the ktype""" return _create_custom_properties_model(value) - @model_serializer - def serialize(self): - """Serialize ktype.""" - return { - key: ( - value - if not isinstance(value, BaseModel) - else value.model_dump_json() - ) - for key, value in self.items() - } - - @field_validator("in_backend") - @classmethod - def validate_in_backend(cls, value: bool, info: ValidationInfo) -> bool: + + @property + def in_backend(self) -> bool: """Checks whether the KType already exists""" - ktype_id = info.data["id"] - if not value: - value = _ktype_exists(ktype_id) - return value + return _ktype_exists(self) @property @@ -108,3 +100,7 @@ def context(cls) -> "Context": ) return Context + + def refresh(self) -> None: + """Refresh the KType""" + _refresh_ktype(self) diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index f9a1268..b84d960 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -192,6 +192,20 @@ def _get_remote_ktypes() -> Enum: logger.debug("Got the following ktypes from backend: `%s`.", list(ktypes)) return ktypes +def _get_ktype_list() -> "List[KType]": + """Get all available KTypes from the remote backend.""" + from dsms import ( # isort:skip + KType + ) + + response = _perform_request("api/knowledge-type/", "get") + if not response.ok: + raise ValueError( + f"Something went wrong fetching the available ktypes: {response.text}" + ) + return [KType(**ktype) for ktype in response.json()] + + def _ktype_exists(ktype: Union[Any, str, UUID]) -> bool: """Check whether the KType exists in the remote backend""" from dsms.knowledge.ktype import ( # isort:skip @@ -219,22 +233,22 @@ def _create_new_ktype(ktype: "KType") -> None: ) def _get_ktype( - uuid: Union[UUID, str], as_json=False -) -> "Union[KItem, Dict[str, Any]]": + id: str, as_json=False +) -> "Union[KType, Dict[str, Any]]": """Get the KType for an instance with a certain ID from remote backend""" from dsms import Context, KType #why this Ktype/Kitem import in all these methods?? it is already imported at the beginning - response = _perform_request(f"api/knowledge-type/{uuid}", "get") + response = _perform_request(f"api/knowledge-type/{id}", "get") if response.status_code == 404: raise ValueError( - f"""KType with uuid `{uuid}` does not exist in + f"""KType with the id `{id}` does not exist in DSMS-instance `{Context.dsms.config.host_url}`""" ) if not response.ok: raise ValueError( - f"""An error occured fetching the KType with uuid `{uuid}`: + f"""An error occured fetching the KType with id `{id}`: `{response.text}`""" ) @@ -684,6 +698,7 @@ def _commit_updated_ktype(new_ktype: "KType") -> None: logger.debug( "Fetching updated KType from remote backend: %s", new_ktype.id ) + new_ktype.refresh() def _commit_deleted( @@ -717,6 +732,18 @@ def _refresh_kitem(kitem: "KItem") -> None: ) setattr(kitem, key, value) kitem.dataframe = _inspect_dataframe(kitem.id) + + +def _refresh_ktype(ktype: "KType") -> None: + """Refresh the KItem""" + for key, value in _get_ktype(ktype.id, as_json=True).items(): + logger.debug( + "Set updated property `%s` for KType with id `%s` after commiting: %s", + key, + ktype.id, + value, + ) + setattr(ktype, key, value) def _split_iri(iri: str) -> List[str]: diff --git a/examples/testktype.py b/examples/testktype.py deleted file mode 100644 index c83fd56..0000000 --- a/examples/testktype.py +++ /dev/null @@ -1,7 +0,0 @@ -from dsms import DSMS, KItem, KType - -dsms = DSMS(env="../.env") - -dsms.kitems -for ktype in dsms.ktypes: - print(ktype) \ No newline at end of file diff --git a/testktype.py b/testktype.py new file mode 100644 index 0000000..e40e2c4 --- /dev/null +++ b/testktype.py @@ -0,0 +1,19 @@ +from dsms import DSMS, KItem, KType + +dsms = DSMS(env=".env") + +dsms.kitems +# type = dsms.context.ktypes() +# type = KType( +# id='testtype20', +# name='newtype15' +# ) +# dsms.commit() +# for i in range(15): +# if(i!=0): +# ktype=dsms.context.ktypes['testtype'+str(i+1)] +# print(ktype) +# del dsms[ktype] +# dsms.commit() +for ktype in dsms.context.ktypes: + print(ktype) \ No newline at end of file From d8d2e671a8285556962a7339bcc5b62318355810 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Tue, 24 Sep 2024 11:42:30 +0200 Subject: [PATCH 04/66] Example for ktype usage has been added. --- examples/ktypes/1_creation.ipynb | 109 ++++++++++++++++++++++++++++++ examples/ktypes/2_updation.ipynb | 96 ++++++++++++++++++++++++++ examples/ktypes/3_deletion.ipynb | 101 ++++++++++++++++++++++++++++ examples/ktypes/4_fetch.ipynb | 112 +++++++++++++++++++++++++++++++ testktype.py | 16 ++--- 5 files changed, 426 insertions(+), 8 deletions(-) create mode 100644 examples/ktypes/1_creation.ipynb create mode 100644 examples/ktypes/2_updation.ipynb create mode 100644 examples/ktypes/3_deletion.ipynb create mode 100644 examples/ktypes/4_fetch.ipynb diff --git a/examples/ktypes/1_creation.ipynb b/examples/ktypes/1_creation.ipynb new file mode 100644 index 0000000..ace06f3 --- /dev/null +++ b/examples/ktypes/1_creation.ipynb @@ -0,0 +1,109 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 1. Create KTypes with the SDK\n", + "\n", + "Steps on how to create new Ktypes." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.1. Setting up\n", + "\n", + "Import the needed classes and functions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KType" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.2. Create KTypes\n", + "\n", + "New KTypes can be created by a simple class inititation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "type = KType( \n", + " id='batch',\n", + " name='Batch'\n", + ")\n", + "\n", + "type" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `commit` method should be executed to synchronize the changes with the DSMS SDK." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `type` object will automatically get updated with after the commit." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "type" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/ktypes/2_updation.ipynb b/examples/ktypes/2_updation.ipynb new file mode 100644 index 0000000..2eae749 --- /dev/null +++ b/examples/ktypes/2_updation.ipynb @@ -0,0 +1,96 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 2. Updating KTypes with the SDK\n", + "\n", + "Steps on how to update the existing ktypes." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1 Setting up" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The ktype object can be fetched using its id." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype = dsms.context.ktypes['batch']\n", + "ktype" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype.name = 'Batches'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/ktypes/3_deletion.ipynb b/examples/ktypes/3_deletion.ipynb new file mode 100644 index 0000000..e2da229 --- /dev/null +++ b/examples/ktypes/3_deletion.ipynb @@ -0,0 +1,101 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 3. Deleting KTypes with the SDK\n", + "\n", + "Steps on how to delete the existing KTypes." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.1. Setting up" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.2. Deleting the KType\n", + "\n", + "The existing Ktypes can be fetched and be deleted as follows." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The ktype object is first fetched using its id." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype = dsms.context.ktypes['batch']\n", + "ktype" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The fetched ktype can be deleted by applying the `del`-operator to the `dsms` object with the individual `KType` object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "del dsms[ktype]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As always, commit the changes." + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/ktypes/4_fetch.ipynb b/examples/ktypes/4_fetch.ipynb new file mode 100644 index 0000000..4814711 --- /dev/null +++ b/examples/ktypes/4_fetch.ipynb @@ -0,0 +1,112 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 4. Fetching KTypes from the SDK\n", + "\n", + "Steps on how to fetch the existing KTypes from the SDK." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.1. Setting up" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.2. Fetching KTypes\n", + "\n", + "The existing KTypes can be fetched as follows." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The available KTypes in the SDK can be fetched from an enum list." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for ktype in dsms.ktypes:\n", + " print(ktype)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The actual KType object can be fetched from the `context` ind the `dsms` object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for ktype in dsms.context.ktypes:\n", + " print(ktype)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The individual KType object can be fetched using its id." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype = dsms.context.ktypes['batch']\n", + "ktype" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/testktype.py b/testktype.py index e40e2c4..781277b 100644 --- a/testktype.py +++ b/testktype.py @@ -2,18 +2,18 @@ dsms = DSMS(env=".env") -dsms.kitems # type = dsms.context.ktypes() -# type = KType( -# id='testtype20', -# name='newtype15' -# ) -# dsms.commit() +type = KType( + id='batch', + name='Batch' +) +dsms.commit() +print(type) # for i in range(15): # if(i!=0): # ktype=dsms.context.ktypes['testtype'+str(i+1)] # print(ktype) # del dsms[ktype] # dsms.commit() -for ktype in dsms.context.ktypes: - print(ktype) \ No newline at end of file +# for ktype in dsms.context.ktypes: +# print(ktype) \ No newline at end of file From c96381e4981e5010d51334b01c79ba32532acc2c Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Tue, 24 Sep 2024 05:26:52 -0500 Subject: [PATCH 05/66] Fixed pylint errors --- dsms/core/dsms.py | 7 ++----- dsms/knowledge/ktype.py | 11 +++++------ dsms/knowledge/utils.py | 16 +++++----------- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index 1183237..e2a90ce 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -17,7 +17,6 @@ _get_kitem, _get_kitem_list, _get_remote_ktypes, - _get_ktype_list ) if TYPE_CHECKING: @@ -146,12 +145,11 @@ def sparql_interface(self) -> SparqlInterface: def ktypes(cls) -> "Enum": """ "Enum of the KTypes defined in the DSMS instance.""" return cls._ktypes - + # @property # def ktypes(cls) -> "List[KType]": # return _get_ktype_list() - @property def config(cls) -> Configuration: """Property returning the DSMS Configuration""" @@ -187,8 +185,7 @@ def kitems(cls) -> "List[KItem]": The default timeout for requests is defined under the `request_timeout`-attribute in the `Configuration`-class.""" return _get_kitem_list() - - + @property def app_configs(cls) -> "List[AppConfig]": """Return available app configs in the DSMS""" diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 1dc13d8..7b51e91 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -5,15 +5,15 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union from uuid import UUID -from pydantic import BaseModel, Field, field_validator, model_serializer, ValidationInfo +from pydantic import BaseModel, Field, field_validator from dsms.knowledge.utils import ( _create_custom_properties_model, - _ktype_exists, + _ktype_exists, _refresh_ktype ) -from dsms.core.logging import handler +from dsms.core.logging import handler if TYPE_CHECKING: from dsms import Context @@ -45,7 +45,7 @@ class KType(BaseModel): def __hash__(self) -> int: return hash(str(self)) - + def __init__(self, **kwargs: "Any") -> None: """Initialize the KType""" from dsms import DSMS @@ -98,9 +98,8 @@ def context(cls) -> "Context": from dsms import ( # isort:skip Context, ) + return Context - return Context - def refresh(self) -> None: """Refresh the KType""" _refresh_ktype(self) diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index b84d960..cf3903d 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -233,25 +233,22 @@ def _create_new_ktype(ktype: "KType") -> None: ) def _get_ktype( - id: str, as_json=False + ktype_id: str, as_json=False ) -> "Union[KType, Dict[str, Any]]": """Get the KType for an instance with a certain ID from remote backend""" - - from dsms import Context, KType #why this Ktype/Kitem import in all these methods?? it is already imported at the beginning + from dsms import Context, KType - response = _perform_request(f"api/knowledge-type/{id}", "get") + response = _perform_request(f"api/knowledge-type/{ktype_id}", "get") if response.status_code == 404: raise ValueError( - f"""KType with the id `{id}` does not exist in + f"""KType with the id `{ktype_id}` does not exist in DSMS-instance `{Context.dsms.config.host_url}`""" ) - if not response.ok: raise ValueError( - f"""An error occured fetching the KType with id `{id}`: + f"""An error occured fetching the KType with id `{ktype_id}`: `{response.text}`""" ) - body = response.json() if as_json: response = body @@ -265,7 +262,6 @@ def _update_ktype(ktype: "KType") -> Response: payload = ktype.model_dump( exclude_none=True, ) - logger.debug( "Update KType for `%s` with body: %s", ktype.id, payload ) @@ -288,7 +284,6 @@ def _delete_ktype(ktype: "KType") -> None: f"KItem with uuid `{ktype.id}` could not be deleted from DSMS: `{response.text}`" ) - def _get_kitem_list() -> "List[KItem]": """Get all available KItems from the remote backend.""" from dsms.knowledge.kitem import ( # isort:skip @@ -732,7 +727,6 @@ def _refresh_kitem(kitem: "KItem") -> None: ) setattr(kitem, key, value) kitem.dataframe = _inspect_dataframe(kitem.id) - def _refresh_ktype(ktype: "KType") -> None: """Refresh the KItem""" From 3d0f850f72d0cb3865bcdd08920c30a92ebbcb72 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Wed, 25 Sep 2024 05:16:17 -0500 Subject: [PATCH 06/66] Fixed pylint errors --- dsms/knowledge/ktype.py | 8 +++----- dsms/knowledge/utils.py | 21 ++++++++++----------- testktype.py | 9 +++------ 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 7b51e91..724d15f 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -7,14 +7,13 @@ from pydantic import BaseModel, Field, field_validator +from dsms.core.logging import handler from dsms.knowledge.utils import ( _create_custom_properties_model, _ktype_exists, - _refresh_ktype + _refresh_ktype, ) -from dsms.core.logging import handler - if TYPE_CHECKING: from dsms import Context from dsms.core.dsms import DSMS @@ -75,13 +74,11 @@ def create_model(cls, value: Optional[Dict[str, Any]]) -> Any: """Create the datamodel for the ktype""" return _create_custom_properties_model(value) - @property def in_backend(self) -> bool: """Checks whether the KType already exists""" return _ktype_exists(self) - @property def dsms(cls) -> "DSMS": """DSMS context getter""" @@ -98,6 +95,7 @@ def context(cls) -> "Context": from dsms import ( # isort:skip Context, ) + return Context def refresh(self) -> None: diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index cf3903d..faf1179 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -192,11 +192,10 @@ def _get_remote_ktypes() -> Enum: logger.debug("Got the following ktypes from backend: `%s`.", list(ktypes)) return ktypes + def _get_ktype_list() -> "List[KType]": """Get all available KTypes from the remote backend.""" - from dsms import ( # isort:skip - KType - ) + from dsms import KType # isort:skip response = _perform_request("api/knowledge-type/", "get") if not response.ok: @@ -219,11 +218,12 @@ def _ktype_exists(ktype: Union[Any, str, UUID]) -> bool: response = _perform_request(route, "get") return response.ok + def _create_new_ktype(ktype: "KType") -> None: """Create a new KType in the remote backend""" body = { - "name": ktype.name, - "id": str(ktype.id), + "name": ktype.name, + "id": str(ktype.id), } logger.debug("Create new KType with body: %s", body) response = _perform_request("api/knowledge-type/", "post", json=body) @@ -232,9 +232,8 @@ def _create_new_ktype(ktype: "KType") -> None: f"KType with id `{ktype.id}` could not be created in DSMS: {response.text}`" ) -def _get_ktype( - ktype_id: str, as_json=False -) -> "Union[KType, Dict[str, Any]]": + +def _get_ktype(ktype_id: str, as_json=False) -> "Union[KType, Dict[str, Any]]": """Get the KType for an instance with a certain ID from remote backend""" from dsms import Context, KType @@ -262,9 +261,7 @@ def _update_ktype(ktype: "KType") -> Response: payload = ktype.model_dump( exclude_none=True, ) - logger.debug( - "Update KType for `%s` with body: %s", ktype.id, payload - ) + logger.debug("Update KType for `%s` with body: %s", ktype.id, payload) response = _perform_request( f"api/knowledge-type/{ktype.id}", "put", json=payload ) @@ -284,6 +281,7 @@ def _delete_ktype(ktype: "KType") -> None: f"KItem with uuid `{ktype.id}` could not be deleted from DSMS: `{response.text}`" ) + def _get_kitem_list() -> "List[KItem]": """Get all available KItems from the remote backend.""" from dsms.knowledge.kitem import ( # isort:skip @@ -728,6 +726,7 @@ def _refresh_kitem(kitem: "KItem") -> None: setattr(kitem, key, value) kitem.dataframe = _inspect_dataframe(kitem.id) + def _refresh_ktype(ktype: "KType") -> None: """Refresh the KItem""" for key, value in _get_ktype(ktype.id, as_json=True).items(): diff --git a/testktype.py b/testktype.py index 781277b..0fc5428 100644 --- a/testktype.py +++ b/testktype.py @@ -1,12 +1,9 @@ -from dsms import DSMS, KItem, KType +from dsms import DSMS, KType dsms = DSMS(env=".env") # type = dsms.context.ktypes() -type = KType( - id='batch', - name='Batch' -) +type = KType(id="batch", name="Batch") dsms.commit() print(type) # for i in range(15): @@ -16,4 +13,4 @@ # del dsms[ktype] # dsms.commit() # for ktype in dsms.context.ktypes: -# print(ktype) \ No newline at end of file +# print(ktype) From 6bd9835756a020c04f4846ed77f01430128c2595 Mon Sep 17 00:00:00 2001 From: Arjun G <65660072+arjungkk@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:48:11 +0200 Subject: [PATCH 07/66] Delete testktype.py --- testktype.py | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 testktype.py diff --git a/testktype.py b/testktype.py deleted file mode 100644 index 0fc5428..0000000 --- a/testktype.py +++ /dev/null @@ -1,16 +0,0 @@ -from dsms import DSMS, KType - -dsms = DSMS(env=".env") - -# type = dsms.context.ktypes() -type = KType(id="batch", name="Batch") -dsms.commit() -print(type) -# for i in range(15): -# if(i!=0): -# ktype=dsms.context.ktypes['testtype'+str(i+1)] -# print(ktype) -# del dsms[ktype] -# dsms.commit() -# for ktype in dsms.context.ktypes: -# print(ktype) From 997549dd5aa98d4c30a05faebad962a8b3019653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Fri, 27 Sep 2024 13:23:13 +0200 Subject: [PATCH 08/66] update ktype retrieval --- docs/dsms_sdk/tutorials/7_ktypes.ipynb | 280 ++++++++++++ dsms/core/dsms.py | 18 +- dsms/knowledge/kitem.py | 21 +- dsms/knowledge/properties/linked_kitems.py | 4 +- dsms/knowledge/utils.py | 10 +- examples/ktypes/1_creation.ipynb | 109 ----- examples/ktypes/2_updation.ipynb | 96 ---- examples/ktypes/3_deletion.ipynb | 101 ----- examples/ktypes/4_fetch.ipynb | 112 ----- examples/tutorials/5_search.ipynb | 484 ++++++++++----------- examples/tutorials/7_ktypes.ipynb | 277 ++++++++++++ 11 files changed, 817 insertions(+), 695 deletions(-) create mode 100644 docs/dsms_sdk/tutorials/7_ktypes.ipynb delete mode 100644 examples/ktypes/1_creation.ipynb delete mode 100644 examples/ktypes/2_updation.ipynb delete mode 100644 examples/ktypes/3_deletion.ipynb delete mode 100644 examples/ktypes/4_fetch.ipynb create mode 100644 examples/tutorials/7_ktypes.ipynb diff --git a/docs/dsms_sdk/tutorials/7_ktypes.ipynb b/docs/dsms_sdk/tutorials/7_ktypes.ipynb new file mode 100644 index 0000000..9f79a15 --- /dev/null +++ b/docs/dsms_sdk/tutorials/7_ktypes.ipynb @@ -0,0 +1,280 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 7. Interact with KTypes through the SDK\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7.1. Setting up\n", + "\n", + "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", + "\n", + "\n", + "Now let us import the needed classes and functions for this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KType" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\".env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7.2. Create KTypes\n", + "\n", + "New KTypes can be created by a simple class inititation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype = KType( \n", + " id='batch',\n", + " name='Batch'\n", + ")\n", + "\n", + "ktype" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `commit` method should be executed to synchronize the changes with the DSMS SDK." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `type` object will automatically get updated with after the commit." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7.3. Update KTypes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The ktype object can be fetched using its id." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype = dsms.ktypes.Batch\n", + "ktype" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can change e.g. the name of the ktype" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype.name = 'Batches'\n", + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After the committing, we can see the changes of the ktype" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7.4. Delete KTypes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The fetched ktype can be deleted by applying the `del`-operator to the `dsms` object with the individual `KType` object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "del dsms[ktype]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As always, commit the changes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7.5. Fetching KTypes\n", + "\n", + "The existing KTypes can be fetched as follows." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The available KTypes in the SDK can be fetched from an enum list." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for ktype in dsms.ktypes:\n", + " print(ktype)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The actual KType object can be fetched from the `context` ind the `dsms` object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for ktype in dsms.context.ktypes:\n", + " print(ktype)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The individual KType object can be fetched using its id." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype = dsms.context.ktypes['batch']\n", + "ktype" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index e2a90ce..d5ced68 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: from enum import Enum - from typing import Optional + from typing import Optional, Union from dsms.apps import AppConfig from dsms.core.context import Buffers @@ -72,6 +72,7 @@ def __init__( """ self._config = None + self._ktype = None self._context.dsms = self if env: @@ -94,7 +95,7 @@ def __init__( ) self._sparql_interface = SparqlInterface(self) - self._ktypes = _get_remote_ktypes() + self.ktypes = _get_remote_ktypes() def __getitem__(self, key: str) -> "KItem": """Get KItem from remote DSMS instance.""" @@ -128,7 +129,7 @@ def commit(self) -> None: def search( self, query: "Optional[str]" = None, - ktypes: "Optional[List[KType]]" = [], + ktypes: "Optional[List[Union[Enum, KType]]]" = [], annotations: "Optional[List[str]]" = [], limit: int = 10, allow_fuzzy: "Optional[bool]" = True, @@ -143,9 +144,18 @@ def sparql_interface(self) -> SparqlInterface: @property def ktypes(cls) -> "Enum": - """ "Enum of the KTypes defined in the DSMS instance.""" + """Getter for the Enum of the KTypes defined in the DSMS instance.""" return cls._ktypes + @ktypes.setter + def ktypes(self, value: "Enum") -> None: + """Setter for the ktypes property of the DSMS instance. + + Args: + value: the Enum object to be set as the ktypes property. + """ + self._ktypes = value + # @property # def ktypes(cls) -> "List[KType]": # return _get_ktype_list() diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 39067a5..c132823 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -433,16 +433,18 @@ def validate_updated(cls, value: str) -> Any: @field_validator("ktype") @classmethod - def validate_ktype(cls, value: KType, info: ValidationInfo) -> KType: + def validate_ktype( + cls, value: Union[str, Enum], info: ValidationInfo + ) -> KType: """Validate the data attribute of the KItem""" from dsms import Context if not value: ktype_id = info.data.get("ktype_id") - if not isinstance(ktype_id, str): - value = Context.ktypes.get(ktype_id.value) - else: + if isinstance(ktype_id, str): value = Context.ktypes.get(ktype_id) + else: + value = ktype_id.value if not value: raise TypeError( @@ -471,10 +473,10 @@ def validate_slug(cls, value: str, info: ValidationInfo) -> str: if not isinstance(kitem_exists, bool): kitem_exists = cls.in_backend - if not isinstance(ktype_id, str): - ktype = ktype_id.value - else: + if isinstance(ktype_id, str): ktype = ktype_id + else: + ktype = ktype_id.value.id name = info.data.get("name") if not value: @@ -618,7 +620,8 @@ def url(cls) -> str: def is_a(self, to_be_compared: KType) -> bool: """Check the KType of the KItem""" return ( - self.ktype_id == to_be_compared.value # pylint: disable=no-member + self.ktype_id + == to_be_compared.value.id # pylint: disable=no-member ) def refresh(self) -> None: @@ -629,7 +632,7 @@ def _get_ktype_as_str(self) -> str: if isinstance(self.ktype_id, str): ktype = self.ktype_id elif isinstance(self.ktype_id, Enum): - ktype = self.ktype_id.value # pylint: disable=no-member + ktype = self.ktype_id.value.id # pylint: disable=no-member else: raise TypeError(f"Datatype for KType is unknown: {type(ktype)}") return ktype diff --git a/dsms/knowledge/properties/linked_kitems.py b/dsms/knowledge/properties/linked_kitems.py index a64b23c..d7a584f 100644 --- a/dsms/knowledge/properties/linked_kitems.py +++ b/dsms/knowledge/properties/linked_kitems.py @@ -148,8 +148,8 @@ def fetch(self) -> "KItem": def is_a(self, to_be_compared: KType) -> bool: """Check the KType of the KItem""" return ( - self.ktype_id.value # pylint: disable=no-member - == to_be_compared.value + self.ktype_id # pylint: disable=no-member + == to_be_compared.value.id ) # OVERRIDE diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index faf1179..53e78cd 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -187,7 +187,8 @@ def _get_remote_ktypes() -> Enum: Context.ktypes = {ktype["id"]: KType(**ktype) for ktype in response.json()} ktypes = Enum( - "KTypes", {_name_to_camel(key): key for key in Context.ktypes} + "KTypes", + {_name_to_camel(key): value for key, value in Context.ktypes.items()}, ) logger.debug("Got the following ktypes from backend: `%s`.", list(ktypes)) return ktypes @@ -680,6 +681,8 @@ def _commit_updated_kitem(new_kitem: "KItem") -> None: def _commit_updated_ktype(new_ktype: "KType") -> None: """Commit the updated KTypes""" + from dsms import Context + old_ktype = _get_ktype(new_ktype.id, as_json=True) logger.debug( "Fetched data from old KType with id `%s`: %s", @@ -692,6 +695,7 @@ def _commit_updated_ktype(new_ktype: "KType") -> None: "Fetching updated KType from remote backend: %s", new_ktype.id ) new_ktype.refresh() + Context.dsms.ktypes = _get_remote_ktypes() def _commit_deleted( @@ -754,7 +758,7 @@ def _make_annotation_schema(iri: str) -> Dict[str, Any]: def _search( query: Optional[str] = None, - ktypes: "Optional[List[KType]]" = [], + ktypes: "Optional[List[Union[Enum, KType]]]" = [], annotations: "Optional[List[str]]" = [], limit: "Optional[int]" = 10, allow_fuzzy: "Optional[bool]" = True, @@ -764,7 +768,7 @@ def _search( payload = { "search_term": query or "", - "ktypes": [ktype.value for ktype in ktypes], + "ktypes": [ktype.value.id for ktype in ktypes], "annotations": [_make_annotation_schema(iri) for iri in annotations], "limit": limit, } diff --git a/examples/ktypes/1_creation.ipynb b/examples/ktypes/1_creation.ipynb deleted file mode 100644 index ace06f3..0000000 --- a/examples/ktypes/1_creation.ipynb +++ /dev/null @@ -1,109 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 1. Create KTypes with the SDK\n", - "\n", - "Steps on how to create new Ktypes." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.1. Setting up\n", - "\n", - "Import the needed classes and functions." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS, KType" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.2. Create KTypes\n", - "\n", - "New KTypes can be created by a simple class inititation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "type = KType( \n", - " id='batch',\n", - " name='Batch'\n", - ")\n", - "\n", - "type" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `commit` method should be executed to synchronize the changes with the DSMS SDK." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `type` object will automatically get updated with after the commit." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "type" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/ktypes/2_updation.ipynb b/examples/ktypes/2_updation.ipynb deleted file mode 100644 index 2eae749..0000000 --- a/examples/ktypes/2_updation.ipynb +++ /dev/null @@ -1,96 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 2. Updating KTypes with the SDK\n", - "\n", - "Steps on how to update the existing ktypes." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.1 Setting up" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The ktype object can be fetched using its id." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ktype = dsms.context.ktypes['batch']\n", - "ktype" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ktype.name = 'Batches'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ktype" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/ktypes/3_deletion.ipynb b/examples/ktypes/3_deletion.ipynb deleted file mode 100644 index e2da229..0000000 --- a/examples/ktypes/3_deletion.ipynb +++ /dev/null @@ -1,101 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 3. Deleting KTypes with the SDK\n", - "\n", - "Steps on how to delete the existing KTypes." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.1. Setting up" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.2. Deleting the KType\n", - "\n", - "The existing Ktypes can be fetched and be deleted as follows." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The ktype object is first fetched using its id." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ktype = dsms.context.ktypes['batch']\n", - "ktype" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The fetched ktype can be deleted by applying the `del`-operator to the `dsms` object with the individual `KType` object." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "del dsms[ktype]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As always, commit the changes." - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/ktypes/4_fetch.ipynb b/examples/ktypes/4_fetch.ipynb deleted file mode 100644 index 4814711..0000000 --- a/examples/ktypes/4_fetch.ipynb +++ /dev/null @@ -1,112 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 4. Fetching KTypes from the SDK\n", - "\n", - "Steps on how to fetch the existing KTypes from the SDK." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.1. Setting up" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.2. Fetching KTypes\n", - "\n", - "The existing KTypes can be fetched as follows." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The available KTypes in the SDK can be fetched from an enum list." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for ktype in dsms.ktypes:\n", - " print(ktype)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The actual KType object can be fetched from the `context` ind the `dsms` object." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for ktype in dsms.context.ktypes:\n", - " print(ktype)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The individual KType object can be fetched using its id." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ktype = dsms.context.ktypes['batch']\n", - "ktype" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/tutorials/5_search.ipynb b/examples/tutorials/5_search.ipynb index d6f8aec..42610d3 100644 --- a/examples/tutorials/5_search.ipynb +++ b/examples/tutorials/5_search.ipynb @@ -38,7 +38,7 @@ "metadata": {}, "outputs": [], "source": [ - "dsms = DSMS(env=\".env\")" + "dsms = DSMS(env=\"../../.env\")" ] }, { @@ -143,29 +143,43 @@ "\n", "\tname = Machine-1, \n", "\n", - "\tid = d1524272-5f34-454b-bb61-7b0ab3a825a7, \n", + "\tid = 1d2c5439-1971-4a92-907b-e30ce00da344, \n", "\n", "\tktype_id = testing-machine, \n", "\n", "\tin_backend = True, \n", "\n", - "\tslug = machine-1-d1524272, \n", + "\tslug = machine-1-1d2c5439, \n", "\n", - "\tannotations = [], \n", + "\tannotations = [\n", + "\t\t{\n", + "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", + "\t\t\tname: TestingMachine,\n", + "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", + "\t\t\tdescription: None\n", + "\t\t}\n", + "\t], \n", "\n", "\tattachments = [], \n", "\n", "\tlinked_kitems = [\n", "\t\t\n", - "\t\t\tid: ba1a7871-ff75-465f-8625-a7a0202edca4\n", - "\t\t\tname: Specimen-1\n", - "\t\t\tslug: specimen-1-ba1a7871\n", - "\t\t\tktype_id: specimen\n", + "\t\t\tid: b77d1d0c-1c30-483c-bb3c-bddbcf3707b7\n", + "\t\t\tname: Research Institute ABC\n", + "\t\t\tslug: researchinstituteabc-b77d1d0c\n", + "\t\t\tktype_id: organization\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: []\n", + "\t\t\tannotations: [{\n", + "\t\t\tiri: www.researchBACiri.org/foo,\n", + "\t\t\tname: foo,\n", + "\t\t\tnamespace: www.researchBACiri.org,\n", + "\t\t\tdescription: None\n", + "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: d1524272-5f34-454b-bb61-7b0ab3a825a7\n", + "\t\t\tid: 1d2c5439-1971-4a92-907b-e30ce00da344\n", + "\t\t}, {\n", + "\t\t\tid: 63c045bd-e0d5-429c-ad92-39e7e778c9a9\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -175,25 +189,25 @@ "\t\t\tlinked_affiliations: []\n", "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", - "\t\t\tcreated_at: 2024-08-23T17:15:46.876917\n", - "\t\t\tupdated_at: 2024-08-23T17:15:46.876917\n", + "\t\t\tcustom_properties: None\n", + "\t\t\tcreated_at: 2024-09-26T14:42:49.728733\n", + "\t\t\tupdated_at: 2024-09-26T14:42:49.728733\n", "\t\t, \n", "\t\t\n", - "\t\t\tid: 2335f57a-efc2-47ce-83b3-866cdeb6d30d\n", - "\t\t\tname: Research Institute ABC\n", - "\t\t\tslug: researchinstituteabc-2335f57a\n", - "\t\t\tktype_id: organization\n", + "\t\t\tid: d68f1a4c-0da4-454b-aaa5-3c927e690c0d\n", + "\t\t\tname: Specimen-1\n", + "\t\t\tslug: specimen-1-d68f1a4c\n", + "\t\t\tktype_id: specimen\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", "\t\t\tannotations: [{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: research ABC Institute,\n", - "\t\t\tnamespace: research,\n", + "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", + "\t\t\tname: TestPiece,\n", + "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: d1524272-5f34-454b-bb61-7b0ab3a825a7\n", + "\t\t\tid: 1d2c5439-1971-4a92-907b-e30ce00da344\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -203,9 +217,9 @@ "\t\t\tlinked_affiliations: []\n", "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: None\n", - "\t\t\tcreated_at: 2024-08-23T17:15:47.998202\n", - "\t\t\tupdated_at: 2024-08-23T17:15:47.998202\n", + "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", + "\t\t\tcreated_at: 2024-09-26T14:42:48.908560\n", + "\t\t\tupdated_at: 2024-09-26T14:42:48.908560\n", "\t\t\n", "\t], \n", "\n", @@ -221,9 +235,9 @@ "\n", "\tcontacts = [], \n", "\n", - "\tcreated_at = 2024-08-23 17:15:45.431622, \n", + "\tcreated_at = 2024-09-26 14:42:48.010709, \n", "\n", - "\tupdated_at = 2024-08-23 17:15:45.431622, \n", + "\tupdated_at = 2024-09-26 14:42:48.010709, \n", "\n", "\texternal_links = [], \n", "\n", @@ -274,19 +288,19 @@ "\n", "\tname = Research Institute ABC, \n", "\n", - "\tid = 2335f57a-efc2-47ce-83b3-866cdeb6d30d, \n", + "\tid = f61b851e-9a5c-456e-b486-488d3d32bd8e, \n", "\n", "\tktype_id = organization, \n", "\n", "\tin_backend = True, \n", "\n", - "\tslug = researchinstituteabc-2335f57a, \n", + "\tslug = researchinstituteabc-f61b851e, \n", "\n", "\tannotations = [\n", "\t\t{\n", "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: research ABC Institute,\n", - "\t\t\tnamespace: research,\n", + "\t\t\tname: foo,\n", + "\t\t\tnamespace: www.researchBACiri.org,\n", "\t\t\tdescription: None\n", "\t\t}\n", "\t], \n", @@ -295,17 +309,52 @@ "\n", "\tlinked_kitems = [\n", "\t\t\n", - "\t\t\tid: d1524272-5f34-454b-bb61-7b0ab3a825a7\n", + "\t\t\tid: 5f93eca8-e076-4d7c-b5fa-57cb226d7aa1\n", + "\t\t\tname: Machine-2\n", + "\t\t\tslug: machine-2-5f93eca8\n", + "\t\t\tktype_id: testing-machine\n", + "\t\t\tsummary: None\n", + "\t\t\tavatar_exists: False\n", + "\t\t\tannotations: [{\n", + "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", + "\t\t\tname: TestingMachine,\n", + "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", + "\t\t\tdescription: None\n", + "\t\t}]\n", + "\t\t\tlinked_kitems: [{\n", + "\t\t\tid: f61b851e-9a5c-456e-b486-488d3d32bd8e\n", + "\t\t}, {\n", + "\t\t\tid: ff58c3ac-9f7d-4aa2-8fc1-9524f324f598\n", + "\t\t}]\n", + "\t\t\texternal_links: []\n", + "\t\t\tcontacts: []\n", + "\t\t\tauthors: [{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}]\n", + "\t\t\tlinked_affiliations: []\n", + "\t\t\tattachments: []\n", + "\t\t\tuser_groups: []\n", + "\t\t\tcustom_properties: {'Producer': 'StressStrain GmBH', 'Room Number': 'B500', 'Description': 'Compression Test Machine'}\n", + "\t\t\tcreated_at: 2024-09-26T14:38:14.079698\n", + "\t\t\tupdated_at: 2024-09-26T14:38:14.079698\n", + "\t\t, \n", + "\t\t\n", + "\t\t\tid: e32870cb-1686-4d76-a51e-7692568c1dc0\n", "\t\t\tname: Machine-1\n", - "\t\t\tslug: machine-1-d1524272\n", + "\t\t\tslug: machine-1-e32870cb\n", "\t\t\tktype_id: testing-machine\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: []\n", + "\t\t\tannotations: [{\n", + "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", + "\t\t\tname: TestingMachine,\n", + "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", + "\t\t\tdescription: None\n", + "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: ba1a7871-ff75-465f-8625-a7a0202edca4\n", + "\t\t\tid: 43a67f5a-fd31-4a39-9697-bfc39aa852b2\n", "\t\t}, {\n", - "\t\t\tid: 2335f57a-efc2-47ce-83b3-866cdeb6d30d\n", + "\t\t\tid: f61b851e-9a5c-456e-b486-488d3d32bd8e\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -316,8 +365,8 @@ "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", "\t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - "\t\t\tcreated_at: 2024-08-23T17:15:45.431622\n", - "\t\t\tupdated_at: 2024-08-23T17:15:45.431622\n", + "\t\t\tcreated_at: 2024-09-26T14:38:13.645998\n", + "\t\t\tupdated_at: 2024-09-26T14:38:13.645998\n", "\t\t\n", "\t], \n", "\n", @@ -333,9 +382,9 @@ "\n", "\tcontacts = [], \n", "\n", - "\tcreated_at = 2024-08-23 17:15:47.998202, \n", + "\tcreated_at = 2024-09-26 14:38:15.465213, \n", "\n", - "\tupdated_at = 2024-08-23 17:15:47.998202, \n", + "\tupdated_at = 2024-09-26 14:38:15.465213, \n", "\n", "\texternal_links = [], \n", "\n", @@ -403,96 +452,13 @@ "\n", "\tname = Research Institute ABC, \n", "\n", - "\tid = 2335f57a-efc2-47ce-83b3-866cdeb6d30d, \n", + "\tid = 7b58fc00-bdb3-4584-a86b-b8f447189429, \n", "\n", "\tktype_id = organization, \n", "\n", "\tin_backend = True, \n", "\n", - "\tslug = researchinstituteabc-2335f57a, \n", - "\n", - "\tannotations = [\n", - "\t\t{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: research ABC Institute,\n", - "\t\t\tnamespace: research,\n", - "\t\t\tdescription: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [\n", - "\t\t\n", - "\t\t\tid: d1524272-5f34-454b-bb61-7b0ab3a825a7\n", - "\t\t\tname: Machine-1\n", - "\t\t\tslug: machine-1-d1524272\n", - "\t\t\tktype_id: testing-machine\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: []\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: ba1a7871-ff75-465f-8625-a7a0202edca4\n", - "\t\t}, {\n", - "\t\t\tid: 2335f57a-efc2-47ce-83b3-866cdeb6d30d\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - "\t\t\tcreated_at: 2024-08-23T17:15:45.431622\n", - "\t\t\tupdated_at: 2024-08-23T17:15:45.431622\n", - "\t\t\n", - "\t], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = 2024-08-23 17:15:47.998202, \n", - "\n", - "\tupdated_at = 2024-08-23 17:15:47.998202, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = None, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")\n", - "\n", - "\n", - "KItem(\n", - "\n", - "\tname = Research Institute ABC, \n", - "\n", - "\tid = 379e07a7-acd3-45dd-8bbd-ad5edb549946, \n", - "\n", - "\tktype_id = organization, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = researchinstituteabc-379e07a7, \n", + "\tslug = researchinstituteabc-7b58fc00, \n", "\n", "\tannotations = [\n", "\t\t{\n", @@ -507,9 +473,9 @@ "\n", "\tlinked_kitems = [\n", "\t\t\n", - "\t\t\tid: 31a7d354-6dfc-4aa5-832c-a1e581757193\n", + "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", "\t\t\tname: Machine-1\n", - "\t\t\tslug: machine-1-31a7d354\n", + "\t\t\tslug: machine-1-6dd446f1\n", "\t\t\tktype_id: testing-machine\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", @@ -520,9 +486,9 @@ "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: af4e0334-6325-4ae7-91fd-cc1b21864cf1\n", + "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", "\t\t}, {\n", - "\t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", + "\t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -533,13 +499,13 @@ "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", "\t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - "\t\t\tcreated_at: 2024-08-23T18:16:40.453930\n", - "\t\t\tupdated_at: 2024-08-23T18:16:40.453930\n", + "\t\t\tcreated_at: 2024-09-26T14:44:24.055330\n", + "\t\t\tupdated_at: 2024-09-26T14:44:24.055330\n", "\t\t, \n", "\t\t\n", - "\t\t\tid: 8155c264-c790-4151-bff5-471e6a64fc21\n", + "\t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", "\t\t\tname: Machine-2\n", - "\t\t\tslug: machine-2-8155c264\n", + "\t\t\tslug: machine-2-fd1a2a9a\n", "\t\t\tktype_id: testing-machine\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", @@ -550,9 +516,9 @@ "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: ce4309d3-cb6b-4fe0-85e6-d74e0b282b5d\n", + "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", "\t\t}, {\n", - "\t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", + "\t\t\tid: b15bc4c8-0180-43ce-8d73-c38e7d9f4aae\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -563,8 +529,8 @@ "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", "\t\t\tcustom_properties: {'Producer': 'StressStrain GmBH', 'Room Number': 'B500', 'Description': 'Compression Test Machine'}\n", - "\t\t\tcreated_at: 2024-08-23T18:16:42.132533\n", - "\t\t\tupdated_at: 2024-08-23T18:16:42.132533\n", + "\t\t\tcreated_at: 2024-09-26T14:44:24.494230\n", + "\t\t\tupdated_at: 2024-09-26T14:44:24.494230\n", "\t\t\n", "\t], \n", "\n", @@ -580,9 +546,9 @@ "\n", "\tcontacts = [], \n", "\n", - "\tcreated_at = 2024-08-23 18:16:46.777379, \n", + "\tcreated_at = 2024-09-26 14:44:25.705701, \n", "\n", - "\tupdated_at = 2024-08-23 18:16:46.777379, \n", + "\tupdated_at = 2024-09-26 14:44:25.705701, \n", "\n", "\texternal_links = [], \n", "\n", @@ -635,9 +601,9 @@ "text/plain": [ "[\n", "\t\t\n", - "\t\t\tid: 31a7d354-6dfc-4aa5-832c-a1e581757193\n", + "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", "\t\t\tname: Machine-1\n", - "\t\t\tslug: machine-1-31a7d354\n", + "\t\t\tslug: machine-1-6dd446f1\n", "\t\t\tktype_id: testing-machine\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", @@ -648,9 +614,9 @@ "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: af4e0334-6325-4ae7-91fd-cc1b21864cf1\n", + "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", "\t\t}, {\n", - "\t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", + "\t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -661,13 +627,13 @@ "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", "\t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - "\t\t\tcreated_at: 2024-08-23T18:16:40.453930\n", - "\t\t\tupdated_at: 2024-08-23T18:16:40.453930\n", + "\t\t\tcreated_at: 2024-09-26T14:44:24.055330\n", + "\t\t\tupdated_at: 2024-09-26T14:44:24.055330\n", "\t\t, \n", "\t\t\n", - "\t\t\tid: 8155c264-c790-4151-bff5-471e6a64fc21\n", + "\t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", "\t\t\tname: Machine-2\n", - "\t\t\tslug: machine-2-8155c264\n", + "\t\t\tslug: machine-2-fd1a2a9a\n", "\t\t\tktype_id: testing-machine\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", @@ -678,9 +644,9 @@ "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: ce4309d3-cb6b-4fe0-85e6-d74e0b282b5d\n", + "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", "\t\t}, {\n", - "\t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", + "\t\t\tid: b15bc4c8-0180-43ce-8d73-c38e7d9f4aae\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -691,8 +657,8 @@ "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", "\t\t\tcustom_properties: {'Producer': 'StressStrain GmBH', 'Room Number': 'B500', 'Description': 'Compression Test Machine'}\n", - "\t\t\tcreated_at: 2024-08-23T18:16:42.132533\n", - "\t\t\tupdated_at: 2024-08-23T18:16:42.132533\n", + "\t\t\tcreated_at: 2024-09-26T14:44:24.494230\n", + "\t\t\tupdated_at: 2024-09-26T14:44:24.494230\n", "\t\t\n", "\t]" ] @@ -725,13 +691,13 @@ "\n", "\tname = Machine-1, \n", "\n", - "\tid = 31a7d354-6dfc-4aa5-832c-a1e581757193, \n", + "\tid = 6dd446f1-cc37-462b-83d5-cc55d46d57b1, \n", "\n", "\tktype_id = testing-machine, \n", "\n", "\tin_backend = True, \n", "\n", - "\tslug = machine-1-31a7d354, \n", + "\tslug = machine-1-6dd446f1, \n", "\n", "\tannotations = [\n", "\t\t{\n", @@ -746,20 +712,22 @@ "\n", "\tlinked_kitems = [\n", "\t\t\n", - "\t\t\tid: af4e0334-6325-4ae7-91fd-cc1b21864cf1\n", - "\t\t\tname: Specimen-1\n", - "\t\t\tslug: specimen-1-af4e0334\n", - "\t\t\tktype_id: specimen\n", + "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", + "\t\t\tname: Research Institute ABC\n", + "\t\t\tslug: researchinstituteabc-7b58fc00\n", + "\t\t\tktype_id: organization\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", - "\t\t\tname: TestPiece,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", + "\t\t\tiri: www.researchBACiri.org/foo,\n", + "\t\t\tname: foo,\n", + "\t\t\tnamespace: www.researchBACiri.org,\n", "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 31a7d354-6dfc-4aa5-832c-a1e581757193\n", + "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", + "\t\t}, {\n", + "\t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -769,27 +737,25 @@ "\t\t\tlinked_affiliations: []\n", "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", - "\t\t\tcreated_at: 2024-08-23T18:16:43.773825\n", - "\t\t\tupdated_at: 2024-08-23T18:16:43.773825\n", + "\t\t\tcustom_properties: None\n", + "\t\t\tcreated_at: 2024-09-26T14:44:25.705701\n", + "\t\t\tupdated_at: 2024-09-26T14:44:25.705701\n", "\t\t, \n", "\t\t\n", - "\t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", - "\t\t\tname: Research Institute ABC\n", - "\t\t\tslug: researchinstituteabc-379e07a7\n", - "\t\t\tktype_id: organization\n", + "\t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", + "\t\t\tname: Specimen-1\n", + "\t\t\tslug: specimen-1-e6ef3088\n", + "\t\t\tktype_id: specimen\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", "\t\t\tannotations: [{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.researchBACiri.org,\n", + "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", + "\t\t\tname: TestPiece,\n", + "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 31a7d354-6dfc-4aa5-832c-a1e581757193\n", - "\t\t}, {\n", - "\t\t\tid: 8155c264-c790-4151-bff5-471e6a64fc21\n", + "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -799,9 +765,9 @@ "\t\t\tlinked_affiliations: []\n", "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: None\n", - "\t\t\tcreated_at: 2024-08-23T18:16:46.777379\n", - "\t\t\tupdated_at: 2024-08-23T18:16:46.777379\n", + "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", + "\t\t\tcreated_at: 2024-09-26T14:44:24.894772\n", + "\t\t\tupdated_at: 2024-09-26T14:44:24.894772\n", "\t\t\n", "\t], \n", "\n", @@ -817,9 +783,9 @@ "\n", "\tcontacts = [], \n", "\n", - "\tcreated_at = 2024-08-23 18:16:40.453930, \n", + "\tcreated_at = 2024-09-26 14:44:24.055330, \n", "\n", - "\tupdated_at = 2024-08-23 18:16:40.453930, \n", + "\tupdated_at = 2024-09-26 14:44:24.055330, \n", "\n", "\texternal_links = [], \n", "\n", @@ -866,9 +832,9 @@ "data": { "text/plain": [ "{'https://w3id.org/steel/ProcessOntology/TestingMachine': [\n", - " \t\t\tid: 31a7d354-6dfc-4aa5-832c-a1e581757193\n", + " \t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", " \t\t\tname: Machine-1\n", - " \t\t\tslug: machine-1-31a7d354\n", + " \t\t\tslug: machine-1-6dd446f1\n", " \t\t\tktype_id: testing-machine\n", " \t\t\tsummary: None\n", " \t\t\tavatar_exists: False\n", @@ -879,9 +845,9 @@ " \t\t\tdescription: None\n", " \t\t}]\n", " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: af4e0334-6325-4ae7-91fd-cc1b21864cf1\n", + " \t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", " \t\t}, {\n", - " \t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", + " \t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", " \t\t}]\n", " \t\t\texternal_links: []\n", " \t\t\tcontacts: []\n", @@ -892,13 +858,13 @@ " \t\t\tattachments: []\n", " \t\t\tuser_groups: []\n", " \t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - " \t\t\tcreated_at: 2024-08-23T18:16:40.453930\n", - " \t\t\tupdated_at: 2024-08-23T18:16:40.453930\n", + " \t\t\tcreated_at: 2024-09-26T14:44:24.055330\n", + " \t\t\tupdated_at: 2024-09-26T14:44:24.055330\n", " \t\t,\n", " \n", - " \t\t\tid: 8155c264-c790-4151-bff5-471e6a64fc21\n", + " \t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", " \t\t\tname: Machine-2\n", - " \t\t\tslug: machine-2-8155c264\n", + " \t\t\tslug: machine-2-fd1a2a9a\n", " \t\t\tktype_id: testing-machine\n", " \t\t\tsummary: None\n", " \t\t\tavatar_exists: False\n", @@ -909,9 +875,9 @@ " \t\t\tdescription: None\n", " \t\t}]\n", " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: ce4309d3-cb6b-4fe0-85e6-d74e0b282b5d\n", + " \t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", " \t\t}, {\n", - " \t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", + " \t\t\tid: b15bc4c8-0180-43ce-8d73-c38e7d9f4aae\n", " \t\t}]\n", " \t\t\texternal_links: []\n", " \t\t\tcontacts: []\n", @@ -922,8 +888,8 @@ " \t\t\tattachments: []\n", " \t\t\tuser_groups: []\n", " \t\t\tcustom_properties: {'Producer': 'StressStrain GmBH', 'Room Number': 'B500', 'Description': 'Compression Test Machine'}\n", - " \t\t\tcreated_at: 2024-08-23T18:16:42.132533\n", - " \t\t\tupdated_at: 2024-08-23T18:16:42.132533\n", + " \t\t\tcreated_at: 2024-09-26T14:44:24.494230\n", + " \t\t\tupdated_at: 2024-09-26T14:44:24.494230\n", " \t\t]}" ] }, @@ -955,13 +921,13 @@ "\n", "\tname = Machine-1, \n", "\n", - "\tid = 31a7d354-6dfc-4aa5-832c-a1e581757193, \n", + "\tid = 6dd446f1-cc37-462b-83d5-cc55d46d57b1, \n", "\n", "\tktype_id = testing-machine, \n", "\n", "\tin_backend = True, \n", "\n", - "\tslug = machine-1-31a7d354, \n", + "\tslug = machine-1-6dd446f1, \n", "\n", "\tannotations = [\n", "\t\t{\n", @@ -976,20 +942,22 @@ "\n", "\tlinked_kitems = [\n", "\t\t\n", - "\t\t\tid: af4e0334-6325-4ae7-91fd-cc1b21864cf1\n", - "\t\t\tname: Specimen-1\n", - "\t\t\tslug: specimen-1-af4e0334\n", - "\t\t\tktype_id: specimen\n", + "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", + "\t\t\tname: Research Institute ABC\n", + "\t\t\tslug: researchinstituteabc-7b58fc00\n", + "\t\t\tktype_id: organization\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", - "\t\t\tname: TestPiece,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", + "\t\t\tiri: www.researchBACiri.org/foo,\n", + "\t\t\tname: foo,\n", + "\t\t\tnamespace: www.researchBACiri.org,\n", "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 31a7d354-6dfc-4aa5-832c-a1e581757193\n", + "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", + "\t\t}, {\n", + "\t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -999,27 +967,25 @@ "\t\t\tlinked_affiliations: []\n", "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", - "\t\t\tcreated_at: 2024-08-23T18:16:43.773825\n", - "\t\t\tupdated_at: 2024-08-23T18:16:43.773825\n", + "\t\t\tcustom_properties: None\n", + "\t\t\tcreated_at: 2024-09-26T14:44:25.705701\n", + "\t\t\tupdated_at: 2024-09-26T14:44:25.705701\n", "\t\t, \n", "\t\t\n", - "\t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", - "\t\t\tname: Research Institute ABC\n", - "\t\t\tslug: researchinstituteabc-379e07a7\n", - "\t\t\tktype_id: organization\n", + "\t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", + "\t\t\tname: Specimen-1\n", + "\t\t\tslug: specimen-1-e6ef3088\n", + "\t\t\tktype_id: specimen\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", "\t\t\tannotations: [{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.researchBACiri.org,\n", + "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", + "\t\t\tname: TestPiece,\n", + "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 31a7d354-6dfc-4aa5-832c-a1e581757193\n", - "\t\t}, {\n", - "\t\t\tid: 8155c264-c790-4151-bff5-471e6a64fc21\n", + "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -1029,9 +995,9 @@ "\t\t\tlinked_affiliations: []\n", "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: None\n", - "\t\t\tcreated_at: 2024-08-23T18:16:46.777379\n", - "\t\t\tupdated_at: 2024-08-23T18:16:46.777379\n", + "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", + "\t\t\tcreated_at: 2024-09-26T14:44:24.894772\n", + "\t\t\tupdated_at: 2024-09-26T14:44:24.894772\n", "\t\t\n", "\t], \n", "\n", @@ -1047,9 +1013,9 @@ "\n", "\tcontacts = [], \n", "\n", - "\tcreated_at = 2024-08-23 18:16:40.453930, \n", + "\tcreated_at = 2024-09-26 14:44:24.055330, \n", "\n", - "\tupdated_at = 2024-08-23 18:16:40.453930, \n", + "\tupdated_at = 2024-09-26 14:44:24.055330, \n", "\n", "\texternal_links = [], \n", "\n", @@ -1095,10 +1061,10 @@ { "data": { "text/plain": [ - "{: [\n", - " \t\t\tid: 31a7d354-6dfc-4aa5-832c-a1e581757193\n", + "{, json_schema=None, rdf_mapping=None, created_at='2024-08-19T17:13:57.534570', updated_at='2024-08-19T18:11:40.465640')>: [\n", + " \t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", " \t\t\tname: Machine-1\n", - " \t\t\tslug: machine-1-31a7d354\n", + " \t\t\tslug: machine-1-6dd446f1\n", " \t\t\tktype_id: testing-machine\n", " \t\t\tsummary: None\n", " \t\t\tavatar_exists: False\n", @@ -1109,9 +1075,9 @@ " \t\t\tdescription: None\n", " \t\t}]\n", " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: af4e0334-6325-4ae7-91fd-cc1b21864cf1\n", + " \t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", " \t\t}, {\n", - " \t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", + " \t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", " \t\t}]\n", " \t\t\texternal_links: []\n", " \t\t\tcontacts: []\n", @@ -1122,13 +1088,13 @@ " \t\t\tattachments: []\n", " \t\t\tuser_groups: []\n", " \t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - " \t\t\tcreated_at: 2024-08-23T18:16:40.453930\n", - " \t\t\tupdated_at: 2024-08-23T18:16:40.453930\n", + " \t\t\tcreated_at: 2024-09-26T14:44:24.055330\n", + " \t\t\tupdated_at: 2024-09-26T14:44:24.055330\n", " \t\t,\n", " \n", - " \t\t\tid: 8155c264-c790-4151-bff5-471e6a64fc21\n", + " \t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", " \t\t\tname: Machine-2\n", - " \t\t\tslug: machine-2-8155c264\n", + " \t\t\tslug: machine-2-fd1a2a9a\n", " \t\t\tktype_id: testing-machine\n", " \t\t\tsummary: None\n", " \t\t\tavatar_exists: False\n", @@ -1139,9 +1105,9 @@ " \t\t\tdescription: None\n", " \t\t}]\n", " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: ce4309d3-cb6b-4fe0-85e6-d74e0b282b5d\n", + " \t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", " \t\t}, {\n", - " \t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", + " \t\t\tid: b15bc4c8-0180-43ce-8d73-c38e7d9f4aae\n", " \t\t}]\n", " \t\t\texternal_links: []\n", " \t\t\tcontacts: []\n", @@ -1152,8 +1118,8 @@ " \t\t\tattachments: []\n", " \t\t\tuser_groups: []\n", " \t\t\tcustom_properties: {'Producer': 'StressStrain GmBH', 'Room Number': 'B500', 'Description': 'Compression Test Machine'}\n", - " \t\t\tcreated_at: 2024-08-23T18:16:42.132533\n", - " \t\t\tupdated_at: 2024-08-23T18:16:42.132533\n", + " \t\t\tcreated_at: 2024-09-26T14:44:24.494230\n", + " \t\t\tupdated_at: 2024-09-26T14:44:24.494230\n", " \t\t]}" ] }, @@ -1185,13 +1151,13 @@ "\n", "\tname = Machine-1, \n", "\n", - "\tid = 31a7d354-6dfc-4aa5-832c-a1e581757193, \n", + "\tid = 6dd446f1-cc37-462b-83d5-cc55d46d57b1, \n", "\n", "\tktype_id = testing-machine, \n", "\n", "\tin_backend = True, \n", "\n", - "\tslug = machine-1-31a7d354, \n", + "\tslug = machine-1-6dd446f1, \n", "\n", "\tannotations = [\n", "\t\t{\n", @@ -1206,20 +1172,22 @@ "\n", "\tlinked_kitems = [\n", "\t\t\n", - "\t\t\tid: af4e0334-6325-4ae7-91fd-cc1b21864cf1\n", - "\t\t\tname: Specimen-1\n", - "\t\t\tslug: specimen-1-af4e0334\n", - "\t\t\tktype_id: specimen\n", + "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", + "\t\t\tname: Research Institute ABC\n", + "\t\t\tslug: researchinstituteabc-7b58fc00\n", + "\t\t\tktype_id: organization\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", - "\t\t\tname: TestPiece,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", + "\t\t\tiri: www.researchBACiri.org/foo,\n", + "\t\t\tname: foo,\n", + "\t\t\tnamespace: www.researchBACiri.org,\n", "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 31a7d354-6dfc-4aa5-832c-a1e581757193\n", + "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", + "\t\t}, {\n", + "\t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -1229,27 +1197,25 @@ "\t\t\tlinked_affiliations: []\n", "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", - "\t\t\tcreated_at: 2024-08-23T18:16:43.773825\n", - "\t\t\tupdated_at: 2024-08-23T18:16:43.773825\n", + "\t\t\tcustom_properties: None\n", + "\t\t\tcreated_at: 2024-09-26T14:44:25.705701\n", + "\t\t\tupdated_at: 2024-09-26T14:44:25.705701\n", "\t\t, \n", "\t\t\n", - "\t\t\tid: 379e07a7-acd3-45dd-8bbd-ad5edb549946\n", - "\t\t\tname: Research Institute ABC\n", - "\t\t\tslug: researchinstituteabc-379e07a7\n", - "\t\t\tktype_id: organization\n", + "\t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", + "\t\t\tname: Specimen-1\n", + "\t\t\tslug: specimen-1-e6ef3088\n", + "\t\t\tktype_id: specimen\n", "\t\t\tsummary: None\n", "\t\t\tavatar_exists: False\n", "\t\t\tannotations: [{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.researchBACiri.org,\n", + "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", + "\t\t\tname: TestPiece,\n", + "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", "\t\t\tdescription: None\n", "\t\t}]\n", "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 31a7d354-6dfc-4aa5-832c-a1e581757193\n", - "\t\t}, {\n", - "\t\t\tid: 8155c264-c790-4151-bff5-471e6a64fc21\n", + "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", "\t\t}]\n", "\t\t\texternal_links: []\n", "\t\t\tcontacts: []\n", @@ -1259,9 +1225,9 @@ "\t\t\tlinked_affiliations: []\n", "\t\t\tattachments: []\n", "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: None\n", - "\t\t\tcreated_at: 2024-08-23T18:16:46.777379\n", - "\t\t\tupdated_at: 2024-08-23T18:16:46.777379\n", + "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", + "\t\t\tcreated_at: 2024-09-26T14:44:24.894772\n", + "\t\t\tupdated_at: 2024-09-26T14:44:24.894772\n", "\t\t\n", "\t], \n", "\n", @@ -1277,9 +1243,9 @@ "\n", "\tcontacts = [], \n", "\n", - "\tcreated_at = 2024-08-23 18:16:40.453930, \n", + "\tcreated_at = 2024-09-26 14:44:24.055330, \n", "\n", - "\tupdated_at = 2024-08-23 18:16:40.453930, \n", + "\tupdated_at = 2024-09-26 14:44:24.055330, \n", "\n", "\texternal_links = [], \n", "\n", diff --git a/examples/tutorials/7_ktypes.ipynb b/examples/tutorials/7_ktypes.ipynb new file mode 100644 index 0000000..a32441d --- /dev/null +++ b/examples/tutorials/7_ktypes.ipynb @@ -0,0 +1,277 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 7. Interact with KTypes through the SDK\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7.1. Setting up\n", + "\n", + "Import the needed classes and functions." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KType" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7.2. Create KTypes\n", + "\n", + "New KTypes can be created by a simple class inititation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype = KType( \n", + " id='batch',\n", + " name='Batch'\n", + ")\n", + "\n", + "ktype" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `commit` method should be executed to synchronize the changes with the DSMS SDK." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `type` object will automatically get updated with after the commit." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7.3. Update KTypes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The ktype object can be fetched using its id." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype = dsms.ktypes.Batch\n", + "ktype" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can change e.g. the name of the ktype" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype.name = 'Batches'\n", + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After the committing, we can see the changes of the ktype" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7.4. Delete KTypes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The fetched ktype can be deleted by applying the `del`-operator to the `dsms` object with the individual `KType` object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "del dsms[ktype]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As always, commit the changes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7.5. Fetching KTypes\n", + "\n", + "The existing KTypes can be fetched as follows." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The available KTypes in the SDK can be fetched from an enum list." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for ktype in dsms.ktypes:\n", + " print(ktype)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The actual KType object can be fetched from the `context` ind the `dsms` object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for ktype in dsms.context.ktypes:\n", + " print(ktype)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The individual KType object can be fetched using its id." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ktype = dsms.context.ktypes['batch']\n", + "ktype" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 46daa6ed72dfd3673c1deb7b7d97e40663d081fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Fri, 27 Sep 2024 13:39:52 +0200 Subject: [PATCH 09/66] udpate pytests --- tests/conftest.py | 51 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e2f16d7..503638f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,6 +49,17 @@ class MockDB: ] } + ktypes = [ + { + "name": "organization", + "id": "organization", + }, + { + "name": "dataset", + "id": "dataset", + }, + ] + @pytest.fixture(scope="function") def custom_address(request) -> str: @@ -89,18 +100,8 @@ def mock_callbacks(custom_address) -> "Dict[str, Any]": ktypes = urljoin(custom_address, "api/knowledge-type/") def return_ktypes(request): - ktypes = [ - { - "name": "organization", - "id": "organization", - }, - { - "name": "dataset", - "id": "dataset", - }, - ] header = {"content_type": "application/json"} - return (200, header, json.dumps(ktypes)) + return (200, header, json.dumps(MockDB.ktypes)) def return_kitems(request): # Extract 'id' parameter from the URL @@ -114,6 +115,19 @@ def return_kitems(request): else: return 200, {}, json.dumps(MockDB.kitems[item_id]) + def return_ktype(request): + # Extract 'id' parameter from the URL + url_parts = request.url.split("/") + ktype_id = url_parts[-1] + ktypes = {ktype["id"]: ktype for ktype in MockDB.ktypes} + + # Your logic to generate a dynamic response based on 'ktype_id' + # This is just a placeholder; you should replace it with your actual logic + if ktype_id not in ktypes: + return 404, {}, "KType does not exist" + else: + return 200, {}, json.dumps(ktypes[ktype_id]) + def return_dataframe(request): # Extract 'id' parameter from the URL url_parts = request.url.split("/") @@ -178,6 +192,20 @@ def _get_slugs() -> "Dict[str, Any]": for slug in MockDB.slugs } + def _get_individual_ktypes() -> "Dict[str, Any]": + return { + urljoin(custom_address, f"api/knowledge-type/{ktype['id']}"): [ + { + "method": responses.GET, + "returns": { + "content_type": "application/json", + "callback": return_ktype, + }, + } + ] + for ktype in MockDB.ktypes + } + return { ktypes: [ { @@ -191,6 +219,7 @@ def _get_slugs() -> "Dict[str, Any]": **_get_kitems(), **_get_dataframe(), **_get_slugs(), + **_get_individual_ktypes(), } From c70ba76b96c2eff2a1a5cccf260613133c127a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Tue, 1 Oct 2024 16:31:41 +0200 Subject: [PATCH 10/66] minorly refactor ktype enum for direct retrieval of ktype model --- .pylintrc | 1 + docs/dsms_sdk/tutorials/1_introduction.ipynb | 115 ++++++-- docs/dsms_sdk/tutorials/2_creation.ipynb | 53 ++-- docs/dsms_sdk/tutorials/7_ktypes.ipynb | 271 ++++++++++++++----- dsms/core/dsms.py | 6 +- dsms/knowledge/kitem.py | 9 +- dsms/knowledge/ktype.py | 41 ++- dsms/knowledge/properties/linked_kitems.py | 5 +- dsms/knowledge/utils.py | 89 +++++- examples/tutorials/1_introduction.ipynb | 108 +++++++- examples/tutorials/2_creation.ipynb | 54 ++-- examples/tutorials/7_ktypes.ipynb | 268 +++++++++++++----- tests/test_kitem.py | 3 +- 13 files changed, 794 insertions(+), 229 deletions(-) diff --git a/.pylintrc b/.pylintrc index 070db46..b42ad41 100644 --- a/.pylintrc +++ b/.pylintrc @@ -77,6 +77,7 @@ disable= no-name-in-module, too-many-public-methods, too-many-locals, no-value-for-parameter, + too-many-lines, diff --git a/docs/dsms_sdk/tutorials/1_introduction.ipynb b/docs/dsms_sdk/tutorials/1_introduction.ipynb index b9bcb50..254ef11 100644 --- a/docs/dsms_sdk/tutorials/1_introduction.ipynb +++ b/docs/dsms_sdk/tutorials/1_introduction.ipynb @@ -13,19 +13,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 1.1. Setting up\n", - "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", - "\n", - "Now let us import the needed classes and functions for this tutorial." + "### 1.1. Setting up" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "from dsms import DSMS, KItem" + "from dsms import DSMS" ] }, { @@ -37,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -60,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -69,7 +66,7 @@ "[]" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -98,16 +95,94 @@ "name": "stdout", "output_type": "stream", "text": [ - "KTypes.Organization\n", - "KTypes.App\n", - "KTypes.Dataset\n", - "KTypes.DatasetCatalog\n", - "KTypes.Expert\n", - "KTypes.Test\n", - "KTypes.Specimen\n", - "KTypes.Batch\n", - "KTypes.Resource\n", - "KTypes.TestingMachine\n" + "Organization(\n", + "\tid=organization,\n", + "\tname=None,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "App(\n", + "\tid=app,\n", + "\tname=None,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "Dataset(\n", + "\tid=dataset,\n", + "\tname=dataset,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "DatasetCatalog(\n", + "\tid=dataset-catalog,\n", + "\tname=dataset catalog,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "Expert(\n", + "\tid=expert,\n", + "\tname=expert,\n", + "\twebform={\n", + "\t\tname=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tskills=dict(annotation=Union[str, NoneType] required=False default=None)\n", + "\t},\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T17:09:46.125058\n", + ")\n", + "Test(\n", + "\tid=test,\n", + "\tname=test,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-14T07:45:38.051796,\n", + "\tupdated_at=2024-05-14T07:45:38.051796\n", + ")\n", + "Specimen(\n", + "\tid=specimen,\n", + "\tname=Specimen,\n", + "\twebform={\n", + "\t\tsampletype=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tsampleinformation=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tsampleproductionprocess=dict(annotation=Union[str, NoneType] required=False default=None)\n", + "\t},\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-22T07:19:24.079365,\n", + "\tupdated_at=2024-05-22T07:20:08.774884\n", + ")\n", + "Resource(\n", + "\tid=resource,\n", + "\tname=Resource,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-06-10T10:17:31.071959,\n", + "\tupdated_at=2024-06-10T10:17:40.156104\n", + ")\n", + "TestingMachine(\n", + "\tid=testing-machine,\n", + "\tname=Testing Machine,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-08-19T17:13:57.534570,\n", + "\tupdated_at=2024-08-19T18:11:40.465640\n", + ")\n" ] } ], @@ -133,7 +208,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/docs/dsms_sdk/tutorials/2_creation.ipynb b/docs/dsms_sdk/tutorials/2_creation.ipynb index e97199e..fcff28b 100644 --- a/docs/dsms_sdk/tutorials/2_creation.ipynb +++ b/docs/dsms_sdk/tutorials/2_creation.ipynb @@ -14,14 +14,13 @@ "metadata": {}, "source": [ "### 2.1. Setting up\n", - "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", "\n", "Now let us import the needed classes and functions for this tutorial." ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -37,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -62,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -72,13 +71,13 @@ "\n", "\tname = Machine-1, \n", "\n", - "\tid = 6a5fd4a4-4dc0-4643-84eb-1e35513974ba, \n", + "\tid = c4128bec-3ecd-4c92-a307-7015bc9f2e86, \n", "\n", "\tktype_id = KTypes.TestingMachine, \n", "\n", "\tin_backend = False, \n", "\n", - "\tslug = machine-1-6a5fd4a4, \n", + "\tslug = machine-1-c4128bec, \n", "\n", "\tannotations = [], \n", "\n", @@ -118,7 +117,7 @@ ")" ] }, - "execution_count": 3, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -145,16 +144,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'https://stahldigital.materials-data.space/knowledge/testing-machine/machine-1-6a5fd4a4'" + "'https://bue.materials-data.space/knowledge/testing-machine/machine-1-c4128bec'" ] }, - "execution_count": 4, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -173,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -183,13 +182,13 @@ "\n", "\tname = Machine-1, \n", "\n", - "\tid = 6a5fd4a4-4dc0-4643-84eb-1e35513974ba, \n", + "\tid = c4128bec-3ecd-4c92-a307-7015bc9f2e86, \n", "\n", "\tktype_id = testing-machine, \n", "\n", "\tin_backend = True, \n", "\n", - "\tslug = machine-1-6a5fd4a4, \n", + "\tslug = machine-1-c4128bec, \n", "\n", "\tannotations = [], \n", "\n", @@ -201,7 +200,7 @@ "\n", "\tauthors = [\n", "\t\t{\n", - "\t\t\tuser_id: aa97bc4c-939e-4142-8f22-c6be8c0df228\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", "\t\t}\n", "\t], \n", "\n", @@ -209,9 +208,9 @@ "\n", "\tcontacts = [], \n", "\n", - "\tcreated_at = 2024-08-23 18:16:24.190604, \n", + "\tcreated_at = 2024-10-01 14:21:54.195585, \n", "\n", - "\tupdated_at = 2024-08-23 18:16:24.190604, \n", + "\tupdated_at = 2024-10-01 14:21:54.195585, \n", "\n", "\texternal_links = [], \n", "\n", @@ -233,7 +232,7 @@ ")" ] }, - "execution_count": 5, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -251,7 +250,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -260,7 +259,7 @@ "'Machine-1'" ] }, - "execution_count": 6, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -278,16 +277,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "UUID('6a5fd4a4-4dc0-4643-84eb-1e35513974ba')" + "UUID('c4128bec-3ecd-4c92-a307-7015bc9f2e86')" ] }, - "execution_count": 7, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -305,16 +304,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "KType(id='testing-machine', name='Testing Machine', webform=, json_schema=None)" + "KType(id='testing-machine', name='Testing Machine', webform=, json_schema=None, rdf_mapping=None, created_at='2024-08-19T17:13:57.534570', updated_at='2024-08-19T18:11:40.465640')" ] }, - "execution_count": 8, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -332,7 +331,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -341,7 +340,7 @@ "True" ] }, - "execution_count": 9, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } diff --git a/docs/dsms_sdk/tutorials/7_ktypes.ipynb b/docs/dsms_sdk/tutorials/7_ktypes.ipynb index 9f79a15..bd39195 100644 --- a/docs/dsms_sdk/tutorials/7_ktypes.ipynb +++ b/docs/dsms_sdk/tutorials/7_ktypes.ipynb @@ -15,13 +15,12 @@ "\n", "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", "\n", - "\n", - "Now let us import the needed classes and functions for this tutorial." + "Import the needed classes and functions." ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -37,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -55,9 +54,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Batch(\n", + "\tid=batch,\n", + "\tname=Batch,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=None,\n", + "\tupdated_at=None\n", + ")" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ktype = KType( \n", " id='batch',\n", @@ -76,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -92,9 +110,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Batch(\n", + "\tid=batch,\n", + "\tname=Batch,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-10-01T14:21:01.164121,\n", + "\tupdated_at=2024-10-01T14:21:01.164121\n", + ")" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ktype" ] @@ -115,9 +152,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Batch(\n", + "\tid=batch,\n", + "\tname=Batch,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-10-01T14:21:01.164121,\n", + "\tupdated_at=2024-10-01T14:21:01.164121\n", + ")" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ktype = dsms.ktypes.Batch\n", "ktype" @@ -132,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -149,9 +205,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Batch(\n", + "\tid=batch,\n", + "\tname=Batches,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-10-01T14:21:01.164121,\n", + "\tupdated_at=2024-10-01T14:21:12.169973\n", + ")" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ktype" ] @@ -160,99 +235,169 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 7.4. Delete KTypes" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The fetched ktype can be deleted by applying the `del`-operator to the `dsms` object with the individual `KType` object." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "del dsms[ktype]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As always, commit the changes." + "## 7.5. Fetching KTypes\n", + "\n", + "The existing KTypes can be fetched as follows." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Organization(\n", + "\tid=organization,\n", + "\tname=None,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "App(\n", + "\tid=app,\n", + "\tname=None,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "Dataset(\n", + "\tid=dataset,\n", + "\tname=dataset,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "DatasetCatalog(\n", + "\tid=dataset-catalog,\n", + "\tname=dataset catalog,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "Expert(\n", + "\tid=expert,\n", + "\tname=expert,\n", + "\twebform={\n", + "\t\tname=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tskills=dict(annotation=Union[str, NoneType] required=False default=None)\n", + "\t},\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T17:09:46.125058\n", + ")\n", + "Test(\n", + "\tid=test,\n", + "\tname=test,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-14T07:45:38.051796,\n", + "\tupdated_at=2024-05-14T07:45:38.051796\n", + ")\n", + "Specimen(\n", + "\tid=specimen,\n", + "\tname=Specimen,\n", + "\twebform={\n", + "\t\tsampletype=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tsampleinformation=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tsampleproductionprocess=dict(annotation=Union[str, NoneType] required=False default=None)\n", + "\t},\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-22T07:19:24.079365,\n", + "\tupdated_at=2024-05-22T07:20:08.774884\n", + ")\n", + "Resource(\n", + "\tid=resource,\n", + "\tname=Resource,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-06-10T10:17:31.071959,\n", + "\tupdated_at=2024-06-10T10:17:40.156104\n", + ")\n", + "TestingMachine(\n", + "\tid=testing-machine,\n", + "\tname=Testing Machine,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-08-19T17:13:57.534570,\n", + "\tupdated_at=2024-08-19T18:11:40.465640\n", + ")\n", + "Batch(\n", + "\tid=batch,\n", + "\tname=Batches,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-10-01T14:21:01.164121,\n", + "\tupdated_at=2024-10-01T14:21:12.169973\n", + ")\n" + ] + } + ], "source": [ - "dsms.commit()" + "for ktype in dsms.ktypes:\n", + " print(ktype)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 7.5. Fetching KTypes\n", - "\n", - "The existing KTypes can be fetched as follows." + "## 7.4. Delete KTypes" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The available KTypes in the SDK can be fetched from an enum list." + "The fetched ktype can be deleted by applying the `del`-operator to the `dsms` object with the individual `KType` object." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ - "for ktype in dsms.ktypes:\n", - " print(ktype)" + "del dsms[ktype]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The actual KType object can be fetched from the `context` ind the `dsms` object." + "As always, commit the changes." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ - "for ktype in dsms.context.ktypes:\n", - " print(ktype)" + "dsms.commit()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The individual KType object can be fetched using its id." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ktype = dsms.context.ktypes['batch']\n", - "ktype" + "The available KTypes in the SDK can be fetched from an enum list." ] } ], diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index d5ced68..219a3d6 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -1,6 +1,7 @@ """DSMS connection module""" import os +from enum import Enum from typing import TYPE_CHECKING, Any, Dict, List from dotenv import load_dotenv @@ -20,7 +21,6 @@ ) if TYPE_CHECKING: - from enum import Enum from typing import Optional, Union from dsms.apps import AppConfig @@ -112,7 +112,9 @@ def __delitem__(self, obj) -> None: self.context.buffers.deleted.update({obj.id: obj}) elif isinstance(obj, AppConfig): self.context.buffers.deleted.update({obj.name: obj}) - elif isinstance(obj, KType): + elif isinstance(obj, KType) or ( + isinstance(obj, Enum) and isinstance(obj.value, KType) + ): self.context.buffers.deleted.update({obj.name: obj}) else: raise TypeError( diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index c132823..fd39600 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -444,7 +444,7 @@ def validate_ktype( if isinstance(ktype_id, str): value = Context.ktypes.get(ktype_id) else: - value = ktype_id.value + value = ktype_id if not value: raise TypeError( @@ -476,7 +476,7 @@ def validate_slug(cls, value: str, info: ValidationInfo) -> str: if isinstance(ktype_id, str): ktype = ktype_id else: - ktype = ktype_id.value.id + ktype = ktype_id.id name = info.data.get("name") if not value: @@ -619,10 +619,7 @@ def url(cls) -> str: def is_a(self, to_be_compared: KType) -> bool: """Check the KType of the KItem""" - return ( - self.ktype_id - == to_be_compared.value.id # pylint: disable=no-member - ) + return self.ktype_id == to_be_compared.id # pylint: disable=no-member def refresh(self) -> None: """Refresh the KItem""" diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 724d15f..e722fee 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -1,17 +1,19 @@ """KItem types""" import logging +import warnings from datetime import datetime from typing import TYPE_CHECKING, Any, Dict, Optional, Union from uuid import UUID -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_serializer from dsms.core.logging import handler from dsms.knowledge.utils import ( _create_custom_properties_model, _ktype_exists, _refresh_ktype, + print_ktype, ) if TYPE_CHECKING: @@ -68,6 +70,28 @@ def __init__(self, **kwargs: "Any") -> None: logger.debug("KType initialization successful.") + def __setattr__(self, name, value) -> None: + """Add ktype to updated-buffer if an attribute is set""" + super().__setattr__(name, value) + logger.debug( + "Setting property with key `%s` on KType level: %s.", name, value + ) + + if self.id not in self.context.buffers.updated: + logger.debug( + "Setting KType with ID `%s` as updated during KType.__setattr__", + self.id, + ) + self.context.buffers.updated.update({self.id: self}) + + def __repr__(self) -> str: + """Print the KType""" + return print_ktype(self) + + def __str__(self) -> str: + """Print the KType""" + return print_ktype(self) + @field_validator("webform") @classmethod def create_model(cls, value: Optional[Dict[str, Any]]) -> Any: @@ -101,3 +125,18 @@ def context(cls) -> "Context": def refresh(self) -> None: """Refresh the KType""" _refresh_ktype(self) + + @model_serializer + def serialize(self): + """Serialize ktype.""" + return { + key: ( + value + if key != "webform" + else warnings.warn( + """Commiting `webform` is not supported yet. + Will commit the changes in the ktype without it.""" + ) + ) + for key, value in self.__dict__.items() + } diff --git a/dsms/knowledge/properties/linked_kitems.py b/dsms/knowledge/properties/linked_kitems.py index d7a584f..baf03d5 100644 --- a/dsms/knowledge/properties/linked_kitems.py +++ b/dsms/knowledge/properties/linked_kitems.py @@ -147,10 +147,7 @@ def fetch(self) -> "KItem": def is_a(self, to_be_compared: KType) -> bool: """Check the KType of the KItem""" - return ( - self.ktype_id # pylint: disable=no-member - == to_be_compared.value.id - ) + return self.ktype_id == to_be_compared.id # pylint: disable=no-member # OVERRIDE def __str__(self) -> str: diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 53e78cd..347fb72 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -171,6 +171,47 @@ def _validate_model( return values +def print_webform(webform: BaseModel) -> str: + """ + Helper function to pretty print a webform. + + Args: + webform (BaseModel): The webform to print. + + Returns: + str: A string representation of the webform. + """ + if hasattr(webform, "model_fields"): + fields = [ + f"\t\t{model_key}=dict({model_value})" + for model_key, model_value in webform.model_fields.items() + if model_key != "kitem" + ] + if fields: + fields = "\twebform={\n" + ",\n".join(fields) + "\n\t}" + else: + fields = "\twebform=None" + else: + fields = "\twebform=None" + return fields + + +def print_ktype(self) -> str: + """Pretty print the ktype fields""" + if hasattr(self, "value"): + fields = [ + f"\t{key}={value}" if key != "webform" else print_webform(value) + for key, value in self.value.__dict__.items() + ] + else: + fields = [ + f"\t{key}={value}" if key != "webform" else print_webform(value) + for key, value in self.__dict__.items() + ] + fields = ",\n".join(fields) + return f"{self.name}(\n{fields}\n)" + + def _get_remote_ktypes() -> Enum: """Get the KTypes from the remote backend""" from dsms import ( # isort:skip @@ -190,6 +231,47 @@ def _get_remote_ktypes() -> Enum: "KTypes", {_name_to_camel(key): value for key, value in Context.ktypes.items()}, ) + + def custom_getattr(self, name) -> None: + """ + Custom getattr method for the Enum of KTypes. + + When a KType field is accessed, first check if the field is an attribute of the + underlying KType object (self.value). If it is, return that. + Otherwise, call the super method to access the Enum field. + + This is needed because the Enum object is not a KType object, but has all the same + fields. This allows us to access the fields of the KType object as if it were an + Enum. + """ + if hasattr(self.value, name): + return getattr(self.value, name) + return super(ktypes, self).__getattr__(name) + + def custom_setattr(self, name, value) -> None: + """ + Custom setattr method for the Enum of KTypes. + + When a KType field is set, first check if the field is an attribute of the + underlying KType object (self.value). If it is, set that. + Otherwise, call the super method to set the Enum field. + + This is needed because the Enum object is not a KType object, but has all the same + fields. This allows us to set the fields of the KType object as if it were an + Enum. + """ + + if hasattr(self.value, name): + setattr(self.value, name, value) + else: + super(ktypes, self).__setattr__(name, value) + + # Attach methods to the dynamically created Enum class + setattr(ktypes, "__getattr__", custom_getattr) + setattr(ktypes, "__setattr__", custom_setattr) + setattr(ktypes, "__str__", print_ktype) + setattr(ktypes, "__repr__", print_ktype) + logger.debug("Got the following ktypes from backend: `%s`.", list(ktypes)) return ktypes @@ -275,12 +357,15 @@ def _update_ktype(ktype: "KType") -> Response: def _delete_ktype(ktype: "KType") -> None: """Delete a KType in the remote backend""" + from dsms import Context + logger.debug("Delete KType with id: %s", ktype.id) response = _perform_request(f"api/knowledge-type/{ktype.id}", "delete") if not response.ok: raise ValueError( f"KItem with uuid `{ktype.id}` could not be deleted from DSMS: `{response.text}`" ) + Context.dsms.ktypes = _get_remote_ktypes() def _get_kitem_list() -> "List[KItem]": @@ -710,7 +795,9 @@ def _commit_deleted( _delete_kitem(obj) elif isinstance(obj, AppConfig): _delete_app_spec(obj.name) - elif isinstance(obj, KType): + elif isinstance(obj, KType) or ( + isinstance(obj, Enum) and isinstance(obj.value, KType) + ): _delete_ktype(obj) else: raise TypeError( diff --git a/examples/tutorials/1_introduction.ipynb b/examples/tutorials/1_introduction.ipynb index a727932..1665bb0 100644 --- a/examples/tutorials/1_introduction.ipynb +++ b/examples/tutorials/1_introduction.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -34,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -66,7 +66,7 @@ "[]" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -95,16 +95,94 @@ "name": "stdout", "output_type": "stream", "text": [ - "KTypes.Organization\n", - "KTypes.App\n", - "KTypes.Dataset\n", - "KTypes.DatasetCatalog\n", - "KTypes.Expert\n", - "KTypes.Test\n", - "KTypes.Specimen\n", - "KTypes.Batch\n", - "KTypes.Resource\n", - "KTypes.TestingMachine\n" + "Organization(\n", + "\tid=organization,\n", + "\tname=None,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "App(\n", + "\tid=app,\n", + "\tname=None,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "Dataset(\n", + "\tid=dataset,\n", + "\tname=dataset,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "DatasetCatalog(\n", + "\tid=dataset-catalog,\n", + "\tname=dataset catalog,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "Expert(\n", + "\tid=expert,\n", + "\tname=expert,\n", + "\twebform={\n", + "\t\tname=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tskills=dict(annotation=Union[str, NoneType] required=False default=None)\n", + "\t},\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T17:09:46.125058\n", + ")\n", + "Test(\n", + "\tid=test,\n", + "\tname=test,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-14T07:45:38.051796,\n", + "\tupdated_at=2024-05-14T07:45:38.051796\n", + ")\n", + "Specimen(\n", + "\tid=specimen,\n", + "\tname=Specimen,\n", + "\twebform={\n", + "\t\tsampletype=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tsampleinformation=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tsampleproductionprocess=dict(annotation=Union[str, NoneType] required=False default=None)\n", + "\t},\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-22T07:19:24.079365,\n", + "\tupdated_at=2024-05-22T07:20:08.774884\n", + ")\n", + "Resource(\n", + "\tid=resource,\n", + "\tname=Resource,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-06-10T10:17:31.071959,\n", + "\tupdated_at=2024-06-10T10:17:40.156104\n", + ")\n", + "TestingMachine(\n", + "\tid=testing-machine,\n", + "\tname=Testing Machine,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-08-19T17:13:57.534570,\n", + "\tupdated_at=2024-08-19T18:11:40.465640\n", + ")\n" ] } ], @@ -130,7 +208,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/examples/tutorials/2_creation.ipynb b/examples/tutorials/2_creation.ipynb index 2f37561..5cb7539 100644 --- a/examples/tutorials/2_creation.ipynb +++ b/examples/tutorials/2_creation.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -36,11 +36,11 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ - "dsms = DSMS(env=\".env\")" + "dsms = DSMS(env=\"../../.env\")" ] }, { @@ -61,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -71,13 +71,13 @@ "\n", "\tname = Machine-1, \n", "\n", - "\tid = 6a5fd4a4-4dc0-4643-84eb-1e35513974ba, \n", + "\tid = c4128bec-3ecd-4c92-a307-7015bc9f2e86, \n", "\n", "\tktype_id = KTypes.TestingMachine, \n", "\n", "\tin_backend = False, \n", "\n", - "\tslug = machine-1-6a5fd4a4, \n", + "\tslug = machine-1-c4128bec, \n", "\n", "\tannotations = [], \n", "\n", @@ -117,7 +117,7 @@ ")" ] }, - "execution_count": 3, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -144,16 +144,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'https://stahldigital.materials-data.space/knowledge/testing-machine/machine-1-6a5fd4a4'" + "'https://bue.materials-data.space/knowledge/testing-machine/machine-1-c4128bec'" ] }, - "execution_count": 4, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -172,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -182,13 +182,13 @@ "\n", "\tname = Machine-1, \n", "\n", - "\tid = 6a5fd4a4-4dc0-4643-84eb-1e35513974ba, \n", + "\tid = c4128bec-3ecd-4c92-a307-7015bc9f2e86, \n", "\n", "\tktype_id = testing-machine, \n", "\n", "\tin_backend = True, \n", "\n", - "\tslug = machine-1-6a5fd4a4, \n", + "\tslug = machine-1-c4128bec, \n", "\n", "\tannotations = [], \n", "\n", @@ -200,7 +200,7 @@ "\n", "\tauthors = [\n", "\t\t{\n", - "\t\t\tuser_id: aa97bc4c-939e-4142-8f22-c6be8c0df228\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", "\t\t}\n", "\t], \n", "\n", @@ -208,9 +208,9 @@ "\n", "\tcontacts = [], \n", "\n", - "\tcreated_at = 2024-08-23 18:16:24.190604, \n", + "\tcreated_at = 2024-10-01 14:21:54.195585, \n", "\n", - "\tupdated_at = 2024-08-23 18:16:24.190604, \n", + "\tupdated_at = 2024-10-01 14:21:54.195585, \n", "\n", "\texternal_links = [], \n", "\n", @@ -232,7 +232,7 @@ ")" ] }, - "execution_count": 5, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -250,7 +250,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -259,7 +259,7 @@ "'Machine-1'" ] }, - "execution_count": 6, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -277,16 +277,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "UUID('6a5fd4a4-4dc0-4643-84eb-1e35513974ba')" + "UUID('c4128bec-3ecd-4c92-a307-7015bc9f2e86')" ] }, - "execution_count": 7, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -304,16 +304,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "KType(id='testing-machine', name='Testing Machine', webform=, json_schema=None)" + "KType(id='testing-machine', name='Testing Machine', webform=, json_schema=None, rdf_mapping=None, created_at='2024-08-19T17:13:57.534570', updated_at='2024-08-19T18:11:40.465640')" ] }, - "execution_count": 8, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -331,7 +331,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -340,7 +340,7 @@ "True" ] }, - "execution_count": 9, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } diff --git a/examples/tutorials/7_ktypes.ipynb b/examples/tutorials/7_ktypes.ipynb index a32441d..633ef62 100644 --- a/examples/tutorials/7_ktypes.ipynb +++ b/examples/tutorials/7_ktypes.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -34,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -52,9 +52,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Batch(\n", + "\tid=batch,\n", + "\tname=Batch,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=None,\n", + "\tupdated_at=None\n", + ")" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ktype = KType( \n", " id='batch',\n", @@ -73,7 +92,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -89,9 +108,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Batch(\n", + "\tid=batch,\n", + "\tname=Batch,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-10-01T14:21:01.164121,\n", + "\tupdated_at=2024-10-01T14:21:01.164121\n", + ")" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ktype" ] @@ -112,9 +150,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Batch(\n", + "\tid=batch,\n", + "\tname=Batch,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-10-01T14:21:01.164121,\n", + "\tupdated_at=2024-10-01T14:21:01.164121\n", + ")" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ktype = dsms.ktypes.Batch\n", "ktype" @@ -129,7 +186,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -146,9 +203,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Batch(\n", + "\tid=batch,\n", + "\tname=Batches,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-10-01T14:21:01.164121,\n", + "\tupdated_at=2024-10-01T14:21:12.169973\n", + ")" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ktype" ] @@ -157,99 +233,169 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 7.4. Delete KTypes" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The fetched ktype can be deleted by applying the `del`-operator to the `dsms` object with the individual `KType` object." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "del dsms[ktype]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As always, commit the changes." + "## 7.5. Fetching KTypes\n", + "\n", + "The existing KTypes can be fetched as follows." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Organization(\n", + "\tid=organization,\n", + "\tname=None,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "App(\n", + "\tid=app,\n", + "\tname=None,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "Dataset(\n", + "\tid=dataset,\n", + "\tname=dataset,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "DatasetCatalog(\n", + "\tid=dataset-catalog,\n", + "\tname=dataset catalog,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T13:01:42.756035\n", + ")\n", + "Expert(\n", + "\tid=expert,\n", + "\tname=expert,\n", + "\twebform={\n", + "\t\tname=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tskills=dict(annotation=Union[str, NoneType] required=False default=None)\n", + "\t},\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-06T13:01:42.756035,\n", + "\tupdated_at=2024-05-06T17:09:46.125058\n", + ")\n", + "Test(\n", + "\tid=test,\n", + "\tname=test,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-14T07:45:38.051796,\n", + "\tupdated_at=2024-05-14T07:45:38.051796\n", + ")\n", + "Specimen(\n", + "\tid=specimen,\n", + "\tname=Specimen,\n", + "\twebform={\n", + "\t\tsampletype=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tsampleinformation=dict(annotation=Union[str, NoneType] required=False default=None),\n", + "\t\tsampleproductionprocess=dict(annotation=Union[str, NoneType] required=False default=None)\n", + "\t},\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-05-22T07:19:24.079365,\n", + "\tupdated_at=2024-05-22T07:20:08.774884\n", + ")\n", + "Resource(\n", + "\tid=resource,\n", + "\tname=Resource,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-06-10T10:17:31.071959,\n", + "\tupdated_at=2024-06-10T10:17:40.156104\n", + ")\n", + "TestingMachine(\n", + "\tid=testing-machine,\n", + "\tname=Testing Machine,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-08-19T17:13:57.534570,\n", + "\tupdated_at=2024-08-19T18:11:40.465640\n", + ")\n", + "Batch(\n", + "\tid=batch,\n", + "\tname=Batches,\n", + "\twebform=None,\n", + "\tjson_schema=None,\n", + "\trdf_mapping=None,\n", + "\tcreated_at=2024-10-01T14:21:01.164121,\n", + "\tupdated_at=2024-10-01T14:21:12.169973\n", + ")\n" + ] + } + ], "source": [ - "dsms.commit()" + "for ktype in dsms.ktypes:\n", + " print(ktype)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 7.5. Fetching KTypes\n", - "\n", - "The existing KTypes can be fetched as follows." + "## 7.4. Delete KTypes" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The available KTypes in the SDK can be fetched from an enum list." + "The fetched ktype can be deleted by applying the `del`-operator to the `dsms` object with the individual `KType` object." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ - "for ktype in dsms.ktypes:\n", - " print(ktype)" + "del dsms[ktype]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The actual KType object can be fetched from the `context` ind the `dsms` object." + "As always, commit the changes." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ - "for ktype in dsms.context.ktypes:\n", - " print(ktype)" + "dsms.commit()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The individual KType object can be fetched using its id." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ktype = dsms.context.ktypes['batch']\n", - "ktype" + "The available KTypes in the SDK can be fetched from an enum list." ] } ], diff --git a/tests/test_kitem.py b/tests/test_kitem.py index 6eb399d..9ed8e19 100644 --- a/tests/test_kitem.py +++ b/tests/test_kitem.py @@ -145,7 +145,6 @@ def test_kitem_default_ktypes(custom_address): @responses.activate def test_ktype_property(get_mock_kitem_ids, custom_address): - from dsms.core.context import Context from dsms.core.dsms import DSMS from dsms.knowledge.kitem import KItem @@ -158,7 +157,7 @@ def test_ktype_property(get_mock_kitem_ids, custom_address): ktype_id=dsms.ktypes.Organization, ) - assert kitem.ktype == Context.ktypes.get(dsms.ktypes.Organization.value) + assert kitem.ktype == dsms.ktypes.Organization # @responses.activate From 5ade86071dcc88e1470a8cf492ad2c6adc88d504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Tue, 1 Oct 2024 17:11:51 +0200 Subject: [PATCH 11/66] update ktype validation --- dsms/knowledge/kitem.py | 19 +++++++++++++++++-- tests/test_kitem.py | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index fd39600..1546495 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -431,12 +431,27 @@ def validate_updated(cls, value: str) -> Any: ) return value + @field_validator("ktype_id") + @classmethod + def validate_ktype_id(cls, value: Union[str, Enum]) -> KType: + """Validate the ktype id of the KItem""" + from dsms import Context + + if isinstance(value, str): + value = Context.ktypes.get(value) + if not value: + raise TypeError( + f"KType for `ktype_id={value}` does not exist." + ) + + return value.id + @field_validator("ktype") @classmethod def validate_ktype( cls, value: Union[str, Enum], info: ValidationInfo ) -> KType: - """Validate the data attribute of the KItem""" + """Validate the ktype of the KItem""" from dsms import Context if not value: @@ -629,7 +644,7 @@ def _get_ktype_as_str(self) -> str: if isinstance(self.ktype_id, str): ktype = self.ktype_id elif isinstance(self.ktype_id, Enum): - ktype = self.ktype_id.value.id # pylint: disable=no-member + ktype = self.ktype_id.id # pylint: disable=no-member else: raise TypeError(f"Datatype for KType is unknown: {type(ktype)}") return ktype diff --git a/tests/test_kitem.py b/tests/test_kitem.py index 9ed8e19..9cd4981 100644 --- a/tests/test_kitem.py +++ b/tests/test_kitem.py @@ -157,7 +157,7 @@ def test_ktype_property(get_mock_kitem_ids, custom_address): ktype_id=dsms.ktypes.Organization, ) - assert kitem.ktype == dsms.ktypes.Organization + assert kitem.is_a(dsms.ktypes.Organization) # @responses.activate From 42842564fa738b1f6a05682eee2b5cdecc5977c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 2 Oct 2024 19:15:27 +0200 Subject: [PATCH 12/66] update setup.cfg and bring back underscore for private variable --- dsms/core/dsms.py | 2 +- setup.cfg | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index 219a3d6..493002b 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -95,7 +95,7 @@ def __init__( ) self._sparql_interface = SparqlInterface(self) - self.ktypes = _get_remote_ktypes() + self._ktypes = _get_remote_ktypes() def __getitem__(self, key: str) -> "KItem": """Get KItem from remote DSMS instance.""" diff --git a/setup.cfg b/setup.cfg index 2fdb9b2..3d2eee7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,8 +5,8 @@ description = Python SDK core-package for working with the Dataspace Management long_description = file: README.md long_description_content_type = text/markdown url = https://github.com/MI-FraunhoferIWM/dsms-python-sdk -author = Matthias Büschelberger, Yoav Nahshon, Pablo De Andres -author_email = matthias.bueschelberger@iwm.fraunhofer.de, yoav.nahshon@iwm.fraunhofer.de, pablo.de.andres@iwm.fraunhofer.de +author = Matthias Büschelberger, Yoav Nahshon, Pablo De Andres, Arjun Gopalakrishnan, Priyabrat Mishra +author_email = matthias.bueschelberger@iwm.fraunhofer.de, yoav.nahshon@iwm.fraunhofer.de, pablo.de.andres@iwm.fraunhofer.de, arjun.gopalakrishnan@iwm.fraunhofer.de, priyabrat.mishra@iwm.fraunhofer.de license = BSD-3-Clause license_files = LICENSE classifiers = From 9c0b6c410e2ea351f4750e45842cf6d5b1f76793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 2 Oct 2024 19:19:04 +0200 Subject: [PATCH 13/66] remove unneeded utils --- dsms/core/dsms.py | 13 ------------- dsms/knowledge/utils.py | 12 ------------ 2 files changed, 25 deletions(-) diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index 493002b..078ae30 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -149,19 +149,6 @@ def ktypes(cls) -> "Enum": """Getter for the Enum of the KTypes defined in the DSMS instance.""" return cls._ktypes - @ktypes.setter - def ktypes(self, value: "Enum") -> None: - """Setter for the ktypes property of the DSMS instance. - - Args: - value: the Enum object to be set as the ktypes property. - """ - self._ktypes = value - - # @property - # def ktypes(cls) -> "List[KType]": - # return _get_ktype_list() - @property def config(cls) -> Configuration: """Property returning the DSMS Configuration""" diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 347fb72..797bb1c 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -276,18 +276,6 @@ def custom_setattr(self, name, value) -> None: return ktypes -def _get_ktype_list() -> "List[KType]": - """Get all available KTypes from the remote backend.""" - from dsms import KType # isort:skip - - response = _perform_request("api/knowledge-type/", "get") - if not response.ok: - raise ValueError( - f"Something went wrong fetching the available ktypes: {response.text}" - ) - return [KType(**ktype) for ktype in response.json()] - - def _ktype_exists(ktype: Union[Any, str, UUID]) -> bool: """Check whether the KType exists in the remote backend""" from dsms.knowledge.ktype import ( # isort:skip From 6dc696f112e4dc34f554dcf7476909550c2af1b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 2 Oct 2024 19:21:00 +0200 Subject: [PATCH 14/66] update jupyter notebook for docs --- docs/dsms_sdk/tutorials/1_introduction.ipynb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/dsms_sdk/tutorials/1_introduction.ipynb b/docs/dsms_sdk/tutorials/1_introduction.ipynb index 254ef11..25a8fc1 100644 --- a/docs/dsms_sdk/tutorials/1_introduction.ipynb +++ b/docs/dsms_sdk/tutorials/1_introduction.ipynb @@ -13,7 +13,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 1.1. Setting up" + "### 1.1. Setting up\n", + "\n", + "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))" ] }, { From a70ad66993eb2cdb2a34d057cc7ab453a9d6baa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 2 Oct 2024 19:22:27 +0200 Subject: [PATCH 15/66] update jupyter notebook for docs --- docs/dsms_sdk/tutorials/1_introduction.ipynb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/dsms_sdk/tutorials/1_introduction.ipynb b/docs/dsms_sdk/tutorials/1_introduction.ipynb index 25a8fc1..a9bdbac 100644 --- a/docs/dsms_sdk/tutorials/1_introduction.ipynb +++ b/docs/dsms_sdk/tutorials/1_introduction.ipynb @@ -15,7 +15,9 @@ "source": [ "### 1.1. Setting up\n", "\n", - "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))" + "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", + "\n", + "Now let us import the needed classes and functions for this tutorial." ] }, { From db4509befddbaef6eb92ddb64e29bc96207d39bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 2 Oct 2024 19:24:20 +0200 Subject: [PATCH 16/66] remove unneeded private property assignment --- dsms/core/dsms.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index 078ae30..ae0dd0a 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -72,7 +72,6 @@ def __init__( """ self._config = None - self._ktype = None self._context.dsms = self if env: From ffdac4ad0c6cf7c168cc26a53ef1a0875b8deb92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 2 Oct 2024 19:35:56 +0200 Subject: [PATCH 17/66] update ktype validator --- dsms/knowledge/kitem.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 1546495..540200b 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -175,7 +175,7 @@ class KItem(BaseModel): custom_properties: Optional[Any] = Field( None, description="Custom properties associated to the KItem" ) - ktype: Optional[KType] = Field( + ktype: Optional[Union[str, Enum, KType]] = Field( None, description="KType of the KItem", exclude=True ) @@ -449,22 +449,21 @@ def validate_ktype_id(cls, value: Union[str, Enum]) -> KType: @field_validator("ktype") @classmethod def validate_ktype( - cls, value: Union[str, Enum], info: ValidationInfo + cls, value: Optional[Union[KType, str, Enum]], info: ValidationInfo ) -> KType: """Validate the ktype of the KItem""" from dsms import Context if not value: - ktype_id = info.data.get("ktype_id") - if isinstance(ktype_id, str): - value = Context.ktypes.get(ktype_id) - else: - value = ktype_id + value = info.data.get("ktype_id") + if isinstance(value, str): + value = Context.ktypes.get(value) if not value: raise TypeError( - f"KType for `ktype_id={ktype_id}` does not exist." + f"KType for `ktype_id={value}` does not exist." ) + return value @field_validator("in_backend") From a0e1e693df79d472810db8529694b3db4a09c5e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 2 Oct 2024 19:37:28 +0200 Subject: [PATCH 18/66] update jupyter notebooks for docs --- docs/dsms_sdk/tutorials/1_introduction.ipynb | 1 - docs/dsms_sdk/tutorials/2_creation.ipynb | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dsms_sdk/tutorials/1_introduction.ipynb b/docs/dsms_sdk/tutorials/1_introduction.ipynb index a9bdbac..77e1095 100644 --- a/docs/dsms_sdk/tutorials/1_introduction.ipynb +++ b/docs/dsms_sdk/tutorials/1_introduction.ipynb @@ -14,7 +14,6 @@ "metadata": {}, "source": [ "### 1.1. Setting up\n", - "\n", "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", "\n", "Now let us import the needed classes and functions for this tutorial." diff --git a/docs/dsms_sdk/tutorials/2_creation.ipynb b/docs/dsms_sdk/tutorials/2_creation.ipynb index fcff28b..b075b7c 100644 --- a/docs/dsms_sdk/tutorials/2_creation.ipynb +++ b/docs/dsms_sdk/tutorials/2_creation.ipynb @@ -14,6 +14,7 @@ "metadata": {}, "source": [ "### 2.1. Setting up\n", + "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", "\n", "Now let us import the needed classes and functions for this tutorial." ] From cffaeb3c35317e753dd12161a32bd9e278985a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 2 Oct 2024 19:43:14 +0200 Subject: [PATCH 19/66] bring accidentally deleted ktype.setter --- dsms/core/dsms.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index ae0dd0a..9a0f92a 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -72,6 +72,7 @@ def __init__( """ self._config = None + self._ktypes = None self._context.dsms = self if env: @@ -94,7 +95,7 @@ def __init__( ) self._sparql_interface = SparqlInterface(self) - self._ktypes = _get_remote_ktypes() + self.ktypes = _get_remote_ktypes() def __getitem__(self, key: str) -> "KItem": """Get KItem from remote DSMS instance.""" @@ -148,6 +149,15 @@ def ktypes(cls) -> "Enum": """Getter for the Enum of the KTypes defined in the DSMS instance.""" return cls._ktypes + @ktypes.setter + def ktypes(self, value: "Enum") -> None: + """Setter for the ktypes property of the DSMS instance. + + Args: + value: the Enum object to be set as the ktypes property. + """ + self._ktypes = value + @property def config(cls) -> Configuration: """Property returning the DSMS Configuration""" From 901dc737367e28a39edc9ed43c0816166bb8393a Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Mon, 21 Oct 2024 17:16:27 +0200 Subject: [PATCH 20/66] Mapped webform to pydantic model --- dsms/knowledge/ktype.py | 23 ++++++++++++++------ dsms/knowledge/utils.py | 17 +++++++++------ dsms/knowledge/webform.py | 33 ++++++++++++++++++++++++++++ test.py | 45 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 13 deletions(-) create mode 100644 dsms/knowledge/webform.py create mode 100644 test.py diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index e722fee..4f8919d 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -2,11 +2,12 @@ import logging import warnings +import json from datetime import datetime from typing import TYPE_CHECKING, Any, Dict, Optional, Union from uuid import UUID -from pydantic import BaseModel, Field, field_validator, model_serializer +from pydantic import BaseModel, Field, field_validator, model_serializer, model_validator from dsms.core.logging import handler from dsms.knowledge.utils import ( @@ -15,6 +16,7 @@ _refresh_ktype, print_ktype, ) +from dsms.knowledge.webform import Webform if TYPE_CHECKING: from dsms import Context @@ -32,7 +34,7 @@ class KType(BaseModel): name: Optional[str] = Field( None, description="Human readable name of the KType." ) - webform: Optional[Any] = Field(None, description="Form data of the KType.") + webform: Optional[Webform] = Field(None, description="Form data of the KType.") json_schema: Optional[Any] = Field( None, description="OpenAPI schema of the KType." ) @@ -43,6 +45,7 @@ class KType(BaseModel): updated_at: Optional[Union[str, datetime]] = Field( None, description="Time and date when the KType was updated." ) + custom_properties: Optional[Any] = Field(None, description="") def __hash__(self) -> int: return hash(str(self)) @@ -133,10 +136,18 @@ def serialize(self): key: ( value if key != "webform" - else warnings.warn( - """Commiting `webform` is not supported yet. - Will commit the changes in the ktype without it.""" - ) + else json.dumps(value) ) for key, value in self.__dict__.items() } + + @model_validator(mode="after") + @classmethod + def validate_ktype(cls, self: "KType") -> "KType": + if self.webform is not None: + self.custom_properties = _create_custom_properties_model(self.webform) + return self + + + + diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 797bb1c..cb9db09 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -56,15 +56,17 @@ def _create_custom_properties_model( value: Optional[Dict[str, Any]] ) -> BaseModel: """Convert the dict with the model schema into a pydantic model.""" - from dsms import KItem + from dsms import KItem, KType + from dsms.knowledge.webform import Webform + print(type(value)) fields = {} - if isinstance(value, dict): - for item in value.get("sections"): - for form_input in item.get("inputs"): - label = form_input.get("label") - dtype = form_input.get("widget") - default = form_input.get("defaultValue") + if isinstance(value, Webform): + for item in value.sections: + for form_input in item.inputs: + label = form_input.label + dtype = form_input.widget + default = form_input.default_value slug = _slugify(label) if dtype in ("Text", "File", "Textarea", "Vocabulary term"): dtype = Optional[str] @@ -108,6 +110,7 @@ def _create_custom_properties_model( setattr(model, "__str__", _print_properties) setattr(model, "__repr__", _print_properties) setattr(model, "__setattr__", __setattr_property__) + setattr(model, "serialize", KType.serialize) logger.debug("Create custom properties model with fields: %s", fields) return model diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py new file mode 100644 index 0000000..efc9e4a --- /dev/null +++ b/dsms/knowledge/webform.py @@ -0,0 +1,33 @@ +from typing import List, Any, Optional, Union +from uuid import UUID +from pydantic import BaseModel, Field + +class Inputs(BaseModel): + id: Union[UUID, str] = Field(None, description="") + label: Optional[str] = Field(None, description="") + widget: Optional[str] = Field(None, description="") + default_value: Optional[str] = Field(None, description="") + value: Optional[Any] = Field(None, description="") + check: Optional[str] = Field(None, description="") + error: Optional[str] = Field(None, description="") + feedback: Optional[str] = Field(None, description="") + hint: Optional[str] = Field(None, description="") + measurement_unit: Optional[str] = Field(None, description="") + mapping: Optional[str] = Field(None, description="") + knowledge_type: Optional[str] = Field(None, description="") + knowledge_service_url: Optional[str] = Field(None, description="") + vocabulary_service_url: Optional[str] = Field(None, description="") + hidden: Optional[bool] = Field(False, description="") + ignore: Optional[bool] = Field(False, description="") + extra: dict = Field({}, description="") + +class Sections(BaseModel): + id: Union[UUID, str] = Field(None, description="") + name: Optional[str] = Field(None, description="") + inputs: List[Inputs] = Field([], description="") + hidden: Optional[bool] = Field(False, description="") + + +class Webform(BaseModel): + semantics_enabled: Optional[bool] = Field(False, description="") + sections: List[Sections] = Field([], description="") \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..b0832d8 --- /dev/null +++ b/test.py @@ -0,0 +1,45 @@ +import json +from dsms import DSMS, KType +dsms = DSMS(env=".env") + +# for ktype in dsms.ktypes: +# print(ktype) + # if ktype.webform is not None: + # print(ktype.webform) +webform = { + "semantics_enabled": "true", + "sections": [ + { + "id": "id167cc4c989cd8", + "name": "testsection", + "inputs": [ + { + "id": "id74979e625fa1a", + "label": "test box", + "widget": "Checkbox", + "defaultValue": "", + "value": "true", + "check": "null", + "error": "null", + "feedback": "null", + "hint": "a", + "measurementUnit": "Pa", + "mapping": "null", + "knowledgeType": "null", + "knowledgeServiceUrl": "null", + "vocabularyServiceUrl": "null", + "hidden": "false", + "ignore": "false", + "extra": {} + } + ], + "hidden": "false" + } + ] +} +ktype = KType( + id='testabc', + name='ABC', + webform=webform +) +dsms.commit() From 2b81fae0984e280552629ed3da9c174e1d0f3ce2 Mon Sep 17 00:00:00 2001 From: Arjun G <65660072+arjungkk@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:44:54 +0200 Subject: [PATCH 21/66] Delete test.py Deleted a test file --- test.py | 45 --------------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index b0832d8..0000000 --- a/test.py +++ /dev/null @@ -1,45 +0,0 @@ -import json -from dsms import DSMS, KType -dsms = DSMS(env=".env") - -# for ktype in dsms.ktypes: -# print(ktype) - # if ktype.webform is not None: - # print(ktype.webform) -webform = { - "semantics_enabled": "true", - "sections": [ - { - "id": "id167cc4c989cd8", - "name": "testsection", - "inputs": [ - { - "id": "id74979e625fa1a", - "label": "test box", - "widget": "Checkbox", - "defaultValue": "", - "value": "true", - "check": "null", - "error": "null", - "feedback": "null", - "hint": "a", - "measurementUnit": "Pa", - "mapping": "null", - "knowledgeType": "null", - "knowledgeServiceUrl": "null", - "vocabularyServiceUrl": "null", - "hidden": "false", - "ignore": "false", - "extra": {} - } - ], - "hidden": "false" - } - ] -} -ktype = KType( - id='testabc', - name='ABC', - webform=webform -) -dsms.commit() From 2af08a7a14488d2e2fa235796debea0166d64572 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Thu, 24 Oct 2024 11:58:13 +0200 Subject: [PATCH 22/66] Added alias generator to webform inputs --- .gitignore | 3 ++ README.md | 15 ++++--- dsms/knowledge/ktype.py | 53 +++++++++++++------------ dsms/knowledge/utils.py | 5 +-- dsms/knowledge/webform.py | 82 ++++++++++++++++++++++++++------------- test.py | 45 --------------------- 6 files changed, 95 insertions(+), 108 deletions(-) delete mode 100644 test.py diff --git a/.gitignore b/.gitignore index 981818a..cf10da8 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,6 @@ cython_debug/ .vscode/ examples/basic_usage.ipynb + +#Test files +test.py diff --git a/README.md b/README.md index ee26a21..4be7a96 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,13 @@ Please have a look at our tutorials on _readthedocs_: * [6. Apps](https://dsms-python-sdk.readthedocs.io/en/latest/dsms_sdk/tutorials/6_apps.html) Or try our Jupyter Notebooks: -* [1. Introduction](examples/tutorials/1_introduction.ipynb) -* [2. Creation](examples/tutorials/2_creation.ipynb) -* [3. Updation](examples/tutorials/3_updation.ipynb) -* [4. Deletion](examples/tutorials/4_deletion.ipynb) -* [5. Search](examples/tutorials/5_search.ipynb) -* [6. Apps](examples/tutorials/6_apps.ipynb) +* [1. Introduction](docs\dsms_sdk\tutorials\1_introduction.ipynb) +* [2. Creation](docs\dsms_sdk\tutorials\2_creation.ipynb) +* [3. Updation](docs\dsms_sdk\tutorials\3_updation.ipynb) +* [4. Deletion](docs\dsms_sdk\tutorials\4_deletion.ipynb) +* [5. Search](docs\dsms_sdk\tutorials\5_search.ipynb) +* [6. Apps](docs\dsms_sdk\tutorials\6_apps.ipynb) +* [7. KTypes](docs\dsms_sdk\tutorials\7_ktypes.ipynb) ## Authors @@ -65,6 +66,8 @@ Or try our Jupyter Notebooks: [Priyabrat Mishra](mailto:priyabrat.mishra@iwm.fraunhofer.de) (Fraunhofer Institute for Mechanics of Materials IWM) +[Arjun Gopalakrishnan](mailto:arjun.gopalakrishnan@iwm.fraunhofer.de) (Fraunhofer Institute for Mechanics of Materials IWM) + ## License This project is licensed under the BSD 3-Clause. See the LICENSE file for more information. diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 4f8919d..1a2c7a7 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -1,13 +1,11 @@ """KItem types""" import logging -import warnings -import json from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from uuid import UUID -from pydantic import BaseModel, Field, field_validator, model_serializer, model_validator +from pydantic import BaseModel, Field, model_serializer, model_validator from dsms.core.logging import handler from dsms.knowledge.utils import ( @@ -34,7 +32,9 @@ class KType(BaseModel): name: Optional[str] = Field( None, description="Human readable name of the KType." ) - webform: Optional[Webform] = Field(None, description="Form data of the KType.") + webform: Optional[Webform] = Field( + None, description="Form data of the KType." + ) json_schema: Optional[Any] = Field( None, description="OpenAPI schema of the KType." ) @@ -76,16 +76,20 @@ def __init__(self, **kwargs: "Any") -> None: def __setattr__(self, name, value) -> None: """Add ktype to updated-buffer if an attribute is set""" super().__setattr__(name, value) - logger.debug( - "Setting property with key `%s` on KType level: %s.", name, value - ) - if self.id not in self.context.buffers.updated: + if name != "custom_properties": logger.debug( - "Setting KType with ID `%s` as updated during KType.__setattr__", - self.id, + "Setting property with key `%s` on KType level: %s.", + name, + value, ) - self.context.buffers.updated.update({self.id: self}) + + if self.id not in self.context.buffers.updated: + logger.debug( + "Setting KType with ID `%s` as updated during KType.__setattr__", + self.id, + ) + self.context.buffers.updated.update({self.id: self}) def __repr__(self) -> str: """Print the KType""" @@ -95,12 +99,6 @@ def __str__(self) -> str: """Print the KType""" return print_ktype(self) - @field_validator("webform") - @classmethod - def create_model(cls, value: Optional[Dict[str, Any]]) -> Any: - """Create the datamodel for the ktype""" - return _create_custom_properties_model(value) - @property def in_backend(self) -> bool: """Checks whether the KType already exists""" @@ -134,20 +132,21 @@ def serialize(self): """Serialize ktype.""" return { key: ( - value - if key != "webform" - else json.dumps(value) + value.dict(exclude_none=False, by_alias=False) + if key == "webform" and not isinstance(value, dict) + else value ) for key, value in self.__dict__.items() + if key != "custom_properties" } - + @model_validator(mode="after") @classmethod def validate_ktype(cls, self: "KType") -> "KType": + """Model validator for ktype""" if self.webform is not None: - self.custom_properties = _create_custom_properties_model(self.webform) + self.webform.model_validate(self.webform) + self.custom_properties = _create_custom_properties_model( + self.webform + ) return self - - - - diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index cb9db09..d3f50d5 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -59,7 +59,6 @@ def _create_custom_properties_model( from dsms import KItem, KType from dsms.knowledge.webform import Webform - print(type(value)) fields = {} if isinstance(value, Webform): for item in value.sections: @@ -78,8 +77,8 @@ def _create_custom_properties_model( choices = Enum( _name_to_camel(label) + "Choices", { - _name_to_camel(choice["value"]): choice["value"] - for choice in form_input.get("choices") + _name_to_camel(choice.value): choice.value + for choice in form_input.choices }, ) dtype = Optional[choices] diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index efc9e4a..d0fcf96 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -1,33 +1,61 @@ -from typing import List, Any, Optional, Union -from uuid import UUID -from pydantic import BaseModel, Field +"""Webform model""" + +from typing import Any, List, Optional + +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel + + +class WebformSelectOption(BaseModel): + """Choices in webform""" + + label: Optional[str] = Field(None) + value: Optional[Any] = Field(None) + disabled: Optional[bool] = Field(False) + class Inputs(BaseModel): - id: Union[UUID, str] = Field(None, description="") - label: Optional[str] = Field(None, description="") - widget: Optional[str] = Field(None, description="") - default_value: Optional[str] = Field(None, description="") - value: Optional[Any] = Field(None, description="") - check: Optional[str] = Field(None, description="") - error: Optional[str] = Field(None, description="") - feedback: Optional[str] = Field(None, description="") - hint: Optional[str] = Field(None, description="") - measurement_unit: Optional[str] = Field(None, description="") - mapping: Optional[str] = Field(None, description="") - knowledge_type: Optional[str] = Field(None, description="") - knowledge_service_url: Optional[str] = Field(None, description="") - vocabulary_service_url: Optional[str] = Field(None, description="") - hidden: Optional[bool] = Field(False, description="") - ignore: Optional[bool] = Field(False, description="") - extra: dict = Field({}, description="") - + """Input fields in the sections in webform""" + + model_config = ConfigDict( + alias_generator=lambda field_name: to_camel(field_name) + ) + + id: Optional[str] = Field(None) + label: Optional[str] = Field(None) + widget: Optional[str] = Field(None) + default_value: Optional[Any] = Field(None) + value: Optional[Any] = Field(None) + choices: List[WebformSelectOption] = Field([]) + on_change: Optional[Any] = Field(None) + check: Optional[Any] = Field(None) + error: Optional[str] = Field(None) + feedback: Optional[str] = Field(None) + hint: Optional[str] = Field(None) + measurement_unit: Optional[str] = Field(None) + mapping: Optional[str] = Field(None) + mapping_name: Optional[str] = Field(None) + mapping_kitem_id: Optional[str] = Field(None) + knowledge_type: Optional[str] = Field(None) + knowledge_service_url: Optional[str] = Field(None) + vocabulary_service_url: Optional[str] = Field(None) + hidden: Optional[bool] = Field(False) + ignore: Optional[bool] = Field(False) + extra: dict = Field({}) + + class Sections(BaseModel): - id: Union[UUID, str] = Field(None, description="") - name: Optional[str] = Field(None, description="") - inputs: List[Inputs] = Field([], description="") - hidden: Optional[bool] = Field(False, description="") + """Sections in webform""" + + id: Optional[str] = Field(None) + name: Optional[str] = Field(None) + inputs: List[Inputs] = Field([]) + hidden: Optional[bool] = Field(False) class Webform(BaseModel): - semantics_enabled: Optional[bool] = Field(False, description="") - sections: List[Sections] = Field([], description="") \ No newline at end of file + """User defined webform for ktype""" + + semantics_enabled: Optional[bool] = Field(False) + rdf_type: Optional[str] = Field(None) + sections: List[Sections] = Field([]) diff --git a/test.py b/test.py deleted file mode 100644 index b0832d8..0000000 --- a/test.py +++ /dev/null @@ -1,45 +0,0 @@ -import json -from dsms import DSMS, KType -dsms = DSMS(env=".env") - -# for ktype in dsms.ktypes: -# print(ktype) - # if ktype.webform is not None: - # print(ktype.webform) -webform = { - "semantics_enabled": "true", - "sections": [ - { - "id": "id167cc4c989cd8", - "name": "testsection", - "inputs": [ - { - "id": "id74979e625fa1a", - "label": "test box", - "widget": "Checkbox", - "defaultValue": "", - "value": "true", - "check": "null", - "error": "null", - "feedback": "null", - "hint": "a", - "measurementUnit": "Pa", - "mapping": "null", - "knowledgeType": "null", - "knowledgeServiceUrl": "null", - "vocabularyServiceUrl": "null", - "hidden": "false", - "ignore": "false", - "extra": {} - } - ], - "hidden": "false" - } - ] -} -ktype = KType( - id='testabc', - name='ABC', - webform=webform -) -dsms.commit() From d2b4c5f44c44eea6cac60195715fcdf0b46a7813 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Thu, 24 Oct 2024 12:05:31 +0200 Subject: [PATCH 23/66] Updated README and removed duplicate files --- README.md | 14 +- examples/basic_usage.ipynb | 1588 ----------------------- examples/tutorials/1_introduction.ipynb | 216 --- examples/tutorials/2_creation.ipynb | 386 ------ examples/tutorials/3_updation.ipynb | 287 ---- examples/tutorials/4_deletion.ipynb | 404 ------ examples/tutorials/5_search.ipynb | 1323 ------------------- examples/tutorials/6_apps.ipynb | 1031 --------------- examples/tutorials/7_ktypes.ipynb | 423 ------ examples/tutorials/testfile.txt | 1 - 10 files changed, 7 insertions(+), 5666 deletions(-) delete mode 100644 examples/basic_usage.ipynb delete mode 100644 examples/tutorials/1_introduction.ipynb delete mode 100644 examples/tutorials/2_creation.ipynb delete mode 100644 examples/tutorials/3_updation.ipynb delete mode 100644 examples/tutorials/4_deletion.ipynb delete mode 100644 examples/tutorials/5_search.ipynb delete mode 100644 examples/tutorials/6_apps.ipynb delete mode 100644 examples/tutorials/7_ktypes.ipynb delete mode 100644 examples/tutorials/testfile.txt diff --git a/README.md b/README.md index 4be7a96..a96e6cc 100644 --- a/README.md +++ b/README.md @@ -48,13 +48,13 @@ Please have a look at our tutorials on _readthedocs_: * [6. Apps](https://dsms-python-sdk.readthedocs.io/en/latest/dsms_sdk/tutorials/6_apps.html) Or try our Jupyter Notebooks: -* [1. Introduction](docs\dsms_sdk\tutorials\1_introduction.ipynb) -* [2. Creation](docs\dsms_sdk\tutorials\2_creation.ipynb) -* [3. Updation](docs\dsms_sdk\tutorials\3_updation.ipynb) -* [4. Deletion](docs\dsms_sdk\tutorials\4_deletion.ipynb) -* [5. Search](docs\dsms_sdk\tutorials\5_search.ipynb) -* [6. Apps](docs\dsms_sdk\tutorials\6_apps.ipynb) -* [7. KTypes](docs\dsms_sdk\tutorials\7_ktypes.ipynb) +* [1. Introduction](docs/dsms_sdk/tutorials/1_introduction.ipynb) +* [2. Creation](docs/dsms_sdk/tutorials/2_creation.ipynb) +* [3. Updation](docs/dsms_sdk/tutorials/3_updation.ipynb) +* [4. Deletion](docs/dsms_sdk/tutorials/4_deletion.ipynb) +* [5. Search](docs/dsms_sdk/tutorials/5_search.ipynb) +* [6. Apps](docs/dsms_sdk/tutorials/6_apps.ipynb) +* [7. KTypes](docs/dsms_sdk/tutorials/7_ktypes.ipynb) ## Authors diff --git a/examples/basic_usage.ipynb b/examples/basic_usage.ipynb deleted file mode 100644 index d66b53c..0000000 --- a/examples/basic_usage.ipynb +++ /dev/null @@ -1,1588 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Basic useage of the DSMS-Python-SDK\n", - "\n", - "Before you run this tutorial: make sure to have access to an DSMS-instance of your interest, that you have installed this package and that you have copied the needed variables such as the `DSMS_HOST_URL` and `DSMS_TOKEN` into an `.env`-file." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First of all, make let us import the needed classes and functions for this tutortial." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS, KItem" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1: Introduction" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see which kind of DSMS-object we own as a user:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can investigate what a KItem needs in order to be created. KItems are entirely based on [`Pydantic`](https://docs.pydantic.dev/latest/)-Models (v2), hence the properties (in `Pydantic` called `Fields`) are automatically validated once we set them. \n", - "\n", - "The schema of the KItem itself is a JSON schema which is machine-readable and can be directly incorporated into [Swagger](https://swagger.io/tools/swagger-ui/)-supported APIs like e.g. [`FastAPI`](https://fastapi.tiangolo.com/)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can investigate the KTypes defined in the remote instance:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "KTypes.Organization\n", - "KTypes.App\n", - "KTypes.Dataset\n", - "KTypes.DatasetCatalog\n", - "KTypes.Expert\n", - "KTypes.Test\n", - "KTypes.Specimen\n", - "KTypes.Batch\n", - "KTypes.Resource\n", - "KTypes.TestingMachine\n" - ] - } - ], - "source": [ - "for ktype in dsms.ktypes:\n", - " print(ktype)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2: Create KItems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can make new KItems by simple class-initiation:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = foo123, \n", - "\n", - "\tid = 617cd64f-1a4d-48d5-aed6-ff9d9e3ddb20, \n", - "\n", - "\tktype_id = KTypes.Dataset, \n", - "\n", - "\tin_backend = False, \n", - "\n", - "\tslug = foo123-617cd64f, \n", - "\n", - "\tannotations = [], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = None, \n", - "\n", - "\tupdated_at = None, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tfoo: bar\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item = KItem(\n", - " name=\"foo123\",\n", - " ktype_id=dsms.ktypes.Dataset,\n", - " custom_properties={\"foo\": \"bar\"},\n", - ")\n", - "\n", - "item" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Remember: changes are only syncronized with the DSMS when you call the `commit`-method:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'https://bue.materials-data.space/knowledge/dataset/foo123-617cd64f'" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.commit()\n", - "item.url" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As we can see, the object we created before running the `commit`-method has automatically been updated, e.g. with the creation- and update-timestamp:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = foo123, \n", - "\n", - "\tid = 617cd64f-1a4d-48d5-aed6-ff9d9e3ddb20, \n", - "\n", - "\tktype_id = dataset, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = foo123-617cd64f, \n", - "\n", - "\tannotations = [], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = 2024-08-20 08:01:50.536406, \n", - "\n", - "\tupdated_at = 2024-08-20 08:01:50.536406, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tfoo: bar\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3. Update KItems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, we would like to update the properties of our KItem we created previously.\n", - "\n", - "Depending on the schema of each property (see `KItem.model_schema_json()` in the **Introduction** of this tutorial), we can simply use the standard `list`-method as we know them from basic Python (e.g. for the `annotations`, `attachments`, `external_link`, etc). \n", - "\n", - "\n", - "Other properties which are not `list`-like can be simply set by attribute-assignment (e.g. `name`, `slug`, `ktype_id`, etc)." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "item.name = \"foobar\"\n", - "item.custom_properties.foobar = \"foobar\"\n", - "item.attachments.append(\"../README.md\")\n", - "item.annotations.append(\"www.example.org/foo\")\n", - "item.external_links.append(\n", - " {\"url\": \"http://example.org\", \"label\": \"example link\"}\n", - ")\n", - "item.contacts.append({\"name\": \"foo\", \"email\": \"foo@bar.mail\"})\n", - "item.affiliations.append(\"foobar team\")\n", - "item.user_groups.append({\"name\": \"foogroup\", \"group_id\": \"123\"})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Changes are sent to the DSMS through the `commit`-method again." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = foobar, \n", - "\n", - "\tid = 617cd64f-1a4d-48d5-aed6-ff9d9e3ddb20, \n", - "\n", - "\tktype_id = dataset, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = foo123-617cd64f, \n", - "\n", - "\tannotations = [\n", - "\t\t{\n", - "\t\t\tiri: www.example.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.example.org,\n", - "\t\t\tdescription: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tattachments = [\n", - "\t\t{\n", - "\t\t\tname: README.md\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tlinked_kitems = [], \n", - "\n", - "\taffiliations = [\n", - "\t\t{\n", - "\t\t\tname: foobar team\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [\n", - "\t\t{\n", - "\t\t\tname: foo,\n", - "\t\t\temail: foo@bar.mail,\n", - "\t\t\tuser_id: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tcreated_at = 2024-08-20 08:01:50.536406, \n", - "\n", - "\tupdated_at = 2024-08-20 08:01:54.745601, \n", - "\n", - "\texternal_links = [\n", - "\t\t{\n", - "\t\t\tlabel: example link,\n", - "\t\t\turl: http://example.org/\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [\n", - "\t\t{\n", - "\t\t\tname: foogroup,\n", - "\t\t\tgroup_id: 123\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tfoo: bar\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see now that e.g. the local system path of the attachment is changed to a simply file name, which means that the upload was successful. If not so, an error would have beem thrown during the `commit`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Furthermore we can also download the file we uploaded again:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\t Downloaded file: {\n", - "\t\t\tname: README.md\n", - "\t\t}\n", - "|------------------------------------Beginning of file------------------------------------|\n", - "# DSMS-SDK\n", - "Python SDK core-package for interacting with the Dataspace Management System (DSMS)\n", - "\n", - "\n", - "## Authors\n", - "\n", - "[Matthias Büschelberger](mailto:matthias.bueschelberger@iwm.fraunhofer.de) (Fraunhofer Institute for Mechanics of Materials IWM)\n", - "\n", - "[Yoav Nahshon](mailto:yoav.nahshon@iwm.fraunhofer.de) (Fraunhofer Institute for Mechanics of Materials IWM)\n", - "\n", - "[Pablo De Andres](mailto:pablo.de.andres@iwm.fraunhofer.de) (Fraunhofer Institute for Mechanics of Materials IWM)\n", - "\n", - "## License\n", - "\n", - "This project is licensed under the BSD 3-Clause. See the LICENSE file for more information.\n", - "\n", - "## Usage\n", - "\n", - "The SDK provides a general Python interface to a remote DSMS deployment, allowing users to access, store and link data in a DSMS instance easily and safely. The package provides the following main capabilities:\n", - "\n", - "- Managing Knowledge-Items (KItems), which are data instances of an explicitly defined semantic class type (KType)\n", - " - Creating, updating and deleting meta data and properties, e.g. date, operator, material response data for a conducted tensile test\n", - " - Administrating authorship, contact information and supplementary information upon making changes or adding KItems\n", - " - Semantic annotation of KItems\n", - "- Conduct simple free-text searches within the DSMS instance including filters (e.g. limiting the search for certain materials) as well as a more experts-aware SPARQL interface\n", - "- Linking KItems to other KItems\n", - "- Linking Apps to KItems, triggererd, for example, during a file upload\n", - "- Performing simple file upload and download using attachments to KItems\n", - "- Export of a knowledge (sub) graph as common serializations (.ttl, .json)\n", - "\n", - "For the basic usage, please have a look on the Jupyter Notebook under `examples/basic_usage.ipynb`. This tutorial provides a basic overview of using the dsms package to interact with Knowledge Items.\n", - "\n", - "## Disclaimer\n", - "\n", - "Copyright (c) 2014-2024, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. acting on behalf of its Fraunhofer IWM.\n", - "\n", - "Contact: [Matthias Büschelberger](mailto:matthias.bueschelberger@iwm.fraunhofer.de)\n", - "\n", - "|---------------------------------------End of file---------------------------------------|\n" - ] - } - ], - "source": [ - "for file in item.attachments:\n", - " download = file.download()\n", - "\n", - " print(\"\\t\\t\\t Downloaded file:\", file)\n", - " print(\"|------------------------------------Beginning of file------------------------------------|\")\n", - " print(download)\n", - " print(\"|---------------------------------------End of file---------------------------------------|\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4: Delete KItems and their properties" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also remove properties from the KItem without deleting the KItem itself.\n", - "\n", - "For the `list`-like properties, we can use the standard `list`-methods from basic Python again (e.g. `pop`, `remove`, etc. or the `del`-operator).\n", - "\n", - "For the other, non-`list`-like properties, we can simply use the attribute-assignment again.\n", - "\n", - "When we only want single parts of the properties in the KItem, we can do it like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{\n", - "\t\t\tiri: www.example.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.example.org,\n", - "\t\t\tdescription: None\n", - "\t\t}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item.attachments.pop(0)\n", - "item.annotations.pop(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, we can also reset the entire property by setting it to e.g. an empty list again:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "item.user_groups = []" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "See the changes:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Send the changes to the DSMS with the `commit`-method:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = foobar, \n", - "\n", - "\tid = 617cd64f-1a4d-48d5-aed6-ff9d9e3ddb20, \n", - "\n", - "\tktype_id = dataset, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = foo123-617cd64f, \n", - "\n", - "\tannotations = [], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [], \n", - "\n", - "\taffiliations = [\n", - "\t\t{\n", - "\t\t\tname: foobar team\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [\n", - "\t\t{\n", - "\t\t\tname: foo,\n", - "\t\t\temail: foo@bar.mail,\n", - "\t\t\tuser_id: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tcreated_at = 2024-08-20 08:01:50.536406, \n", - "\n", - "\tupdated_at = 2024-08-20 08:01:54.745601, \n", - "\n", - "\texternal_links = [\n", - "\t\t{\n", - "\t\t\tlabel: example link,\n", - "\t\t\turl: http://example.org/\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tfoo: bar\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, we can also delete the whole KItem from the DSMS by applying the `del`-operator to the `dsms`-object with the individual `KItem`-object:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "del dsms[item]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Commit the changes:" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5: Search for KItems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the last unit of this tutorial, we would like to search for specfic KItems we created in the DSMS.\n", - "\n", - "For this purpose, we will firstly create some KItems and apply the `search`-method on the `DSMS`-object later on in order to find them again in the DSMS.\n", - "\n", - "We also wnat to demonstrate here, that we can link KItems to each other in order to find e.g. a related item of type `DatasetCatalog`. For this strategy, we are using the `linked_kitems`-attribute and the `id` of the item which we would like to link.\n", - "\n", - "The procedure looks like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "item = KItem(\n", - " name=\"foo 1\",\n", - " ktype_id=dsms.ktypes.DatasetCatalog\n", - ")\n", - "\n", - "item2 = KItem(\n", - " name=\"foo 2\",\n", - " ktype_id=dsms.ktypes.Organization,\n", - " linked_kitems=[item],\n", - " annotations=[\"www.example.org/foo\"]\n", - ")\n", - "item3 = KItem(\n", - " name=\"foo 3\", \n", - " ktype_id=dsms.ktypes.Organization\n", - ")\n", - "item4 = KItem(\n", - " name=\"foo 4\",\n", - " ktype_id=dsms.ktypes.Organization,\n", - " annotations=[\"www.example.org/bar\"],\n", - ")\n", - "\n", - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, we are apply to search for e.g. kitems of type `DatasetCatalog`:" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[SearchResult(hit=KItem(\n", - " \n", - " \tname = foo 1, \n", - " \n", - " \tid = 966beb55-6eb5-422e-b8c1-84d65b8cf50d, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tin_backend = True, \n", - " \n", - " \tslug = foo1-966beb55, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [\n", - " \t\t\n", - " \t\t\tid: 3ca3c293-8845-4f0e-afcb-6c680c07239f\n", - " \t\t\tname: foo 2\n", - " \t\t\tslug: foo2-3ca3c293\n", - " \t\t\tktype_id: organization\n", - " \t\t\tsummary: None\n", - " \t\t\tavatar_exists: False\n", - " \t\t\tannotations: [{\n", - " \t\t\tiri: www.example.org/foo,\n", - " \t\t\tname: foo,\n", - " \t\t\tnamespace: www.example.org,\n", - " \t\t\tdescription: None\n", - " \t\t}]\n", - " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: 966beb55-6eb5-422e-b8c1-84d65b8cf50d\n", - " \t\t}]\n", - " \t\t\texternal_links: []\n", - " \t\t\tcontacts: []\n", - " \t\t\tauthors: [{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}]\n", - " \t\t\tlinked_affiliations: []\n", - " \t\t\tattachments: []\n", - " \t\t\tuser_groups: []\n", - " \t\t\tcustom_properties: None\n", - " \t\t\tcreated_at: 2024-08-20T08:02:09.024038\n", - " \t\t\tupdated_at: 2024-08-20T08:02:09.024038\n", - " \t\t\n", - " \t], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\t{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-08-20 08:02:08.491699, \n", - " \n", - " \tupdated_at = 2024-08-20 08:02:08.491699, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = None, \n", - " \n", - " \tdataframe = None, \n", - " \n", - " \trdf_exists = False\n", - " ), fuzzy=False)]" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.search(ktypes=[dsms.ktypes.DatasetCatalog])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... and for all of type `Organization` and `DatasetCatalog`:" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[SearchResult(hit=KItem(\n", - " \n", - " \tname = foo 1, \n", - " \n", - " \tid = 966beb55-6eb5-422e-b8c1-84d65b8cf50d, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tin_backend = True, \n", - " \n", - " \tslug = foo1-966beb55, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [\n", - " \t\t\n", - " \t\t\tid: 3ca3c293-8845-4f0e-afcb-6c680c07239f\n", - " \t\t\tname: foo 2\n", - " \t\t\tslug: foo2-3ca3c293\n", - " \t\t\tktype_id: organization\n", - " \t\t\tsummary: None\n", - " \t\t\tavatar_exists: False\n", - " \t\t\tannotations: [{\n", - " \t\t\tiri: www.example.org/foo,\n", - " \t\t\tname: foo,\n", - " \t\t\tnamespace: www.example.org,\n", - " \t\t\tdescription: None\n", - " \t\t}]\n", - " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: 966beb55-6eb5-422e-b8c1-84d65b8cf50d\n", - " \t\t}]\n", - " \t\t\texternal_links: []\n", - " \t\t\tcontacts: []\n", - " \t\t\tauthors: [{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}]\n", - " \t\t\tlinked_affiliations: []\n", - " \t\t\tattachments: []\n", - " \t\t\tuser_groups: []\n", - " \t\t\tcustom_properties: None\n", - " \t\t\tcreated_at: 2024-08-20T08:02:09.024038\n", - " \t\t\tupdated_at: 2024-08-20T08:02:09.024038\n", - " \t\t\n", - " \t], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\t{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-08-20 08:02:08.491699, \n", - " \n", - " \tupdated_at = 2024-08-20 08:02:08.491699, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = None, \n", - " \n", - " \tdataframe = None, \n", - " \n", - " \trdf_exists = False\n", - " ), fuzzy=False),\n", - " SearchResult(hit=KItem(\n", - " \n", - " \tname = foo 2, \n", - " \n", - " \tid = 3ca3c293-8845-4f0e-afcb-6c680c07239f, \n", - " \n", - " \tktype_id = organization, \n", - " \n", - " \tin_backend = True, \n", - " \n", - " \tslug = foo2-3ca3c293, \n", - " \n", - " \tannotations = [\n", - " \t\t{\n", - " \t\t\tiri: www.example.org/foo,\n", - " \t\t\tname: foo,\n", - " \t\t\tnamespace: www.example.org,\n", - " \t\t\tdescription: None\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [\n", - " \t\t\n", - " \t\t\tid: 966beb55-6eb5-422e-b8c1-84d65b8cf50d\n", - " \t\t\tname: foo 1\n", - " \t\t\tslug: foo1-966beb55\n", - " \t\t\tktype_id: dataset-catalog\n", - " \t\t\tsummary: None\n", - " \t\t\tavatar_exists: False\n", - " \t\t\tannotations: []\n", - " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: 3ca3c293-8845-4f0e-afcb-6c680c07239f\n", - " \t\t}]\n", - " \t\t\texternal_links: []\n", - " \t\t\tcontacts: []\n", - " \t\t\tauthors: [{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}]\n", - " \t\t\tlinked_affiliations: []\n", - " \t\t\tattachments: []\n", - " \t\t\tuser_groups: []\n", - " \t\t\tcustom_properties: None\n", - " \t\t\tcreated_at: 2024-08-20T08:02:08.491699\n", - " \t\t\tupdated_at: 2024-08-20T08:02:08.491699\n", - " \t\t\n", - " \t], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\t{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-08-20 08:02:09.024038, \n", - " \n", - " \tupdated_at = 2024-08-20 08:02:09.024038, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = None, \n", - " \n", - " \tdataframe = None, \n", - " \n", - " \trdf_exists = False\n", - " ), fuzzy=False),\n", - " SearchResult(hit=KItem(\n", - " \n", - " \tname = foo 3, \n", - " \n", - " \tid = 31a58a53-8f50-4f18-93ee-90ff5a806e14, \n", - " \n", - " \tktype_id = organization, \n", - " \n", - " \tin_backend = True, \n", - " \n", - " \tslug = foo3-31a58a53, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\t{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-08-20 08:02:09.532080, \n", - " \n", - " \tupdated_at = 2024-08-20 08:02:09.532080, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = None, \n", - " \n", - " \tdataframe = None, \n", - " \n", - " \trdf_exists = False\n", - " ), fuzzy=False),\n", - " SearchResult(hit=KItem(\n", - " \n", - " \tname = foo 4, \n", - " \n", - " \tid = 552ab1b9-64df-4343-9e4e-e5c292c3999f, \n", - " \n", - " \tktype_id = organization, \n", - " \n", - " \tin_backend = True, \n", - " \n", - " \tslug = foo4-552ab1b9, \n", - " \n", - " \tannotations = [\n", - " \t\t{\n", - " \t\t\tiri: www.example.org/bar,\n", - " \t\t\tname: bar,\n", - " \t\t\tnamespace: www.example.org,\n", - " \t\t\tdescription: None\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\t{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-08-20 08:02:10.062595, \n", - " \n", - " \tupdated_at = 2024-08-20 08:02:10.062595, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = None, \n", - " \n", - " \tdataframe = None, \n", - " \n", - " \trdf_exists = False\n", - " ), fuzzy=False),\n", - " SearchResult(hit=KItem(\n", - " \n", - " \tname = Research Institute ABC, \n", - " \n", - " \tid = 21aa50c3-5ec2-4ac3-aba8-69071a4287e2, \n", - " \n", - " \tktype_id = organization, \n", - " \n", - " \tin_backend = True, \n", - " \n", - " \tslug = researchinstituteabc-21aa50c3, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\t{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-08-19 18:26:00.740761, \n", - " \n", - " \tupdated_at = 2024-08-19 18:26:00.740761, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = None, \n", - " \n", - " \tdataframe = None, \n", - " \n", - " \trdf_exists = False\n", - " ), fuzzy=False)]" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.search(ktypes=[dsms.ktypes.Organization, dsms.ktypes.DatasetCatalog])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... or for all of type `DatasetCatalog` with `foo` in the name:" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[SearchResult(hit=KItem(\n", - " \n", - " \tname = foo 1, \n", - " \n", - " \tid = 966beb55-6eb5-422e-b8c1-84d65b8cf50d, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tin_backend = True, \n", - " \n", - " \tslug = foo1-966beb55, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [\n", - " \t\t\n", - " \t\t\tid: 3ca3c293-8845-4f0e-afcb-6c680c07239f\n", - " \t\t\tname: foo 2\n", - " \t\t\tslug: foo2-3ca3c293\n", - " \t\t\tktype_id: organization\n", - " \t\t\tsummary: None\n", - " \t\t\tavatar_exists: False\n", - " \t\t\tannotations: [{\n", - " \t\t\tiri: www.example.org/foo,\n", - " \t\t\tname: foo,\n", - " \t\t\tnamespace: www.example.org,\n", - " \t\t\tdescription: None\n", - " \t\t}]\n", - " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: 966beb55-6eb5-422e-b8c1-84d65b8cf50d\n", - " \t\t}]\n", - " \t\t\texternal_links: []\n", - " \t\t\tcontacts: []\n", - " \t\t\tauthors: [{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}]\n", - " \t\t\tlinked_affiliations: []\n", - " \t\t\tattachments: []\n", - " \t\t\tuser_groups: []\n", - " \t\t\tcustom_properties: None\n", - " \t\t\tcreated_at: 2024-08-20T08:02:09.024038\n", - " \t\t\tupdated_at: 2024-08-20T08:02:09.024038\n", - " \t\t\n", - " \t], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\t{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-08-20 08:02:08.491699, \n", - " \n", - " \tupdated_at = 2024-08-20 08:02:08.491699, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = None, \n", - " \n", - " \tdataframe = None, \n", - " \n", - " \trdf_exists = False\n", - " ), fuzzy=False)]" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.search(query=\"foo\", ktypes=[dsms.ktypes.DatasetCatalog])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... and for all of type `Organization` with the annotation `www.example.org/foo`:" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[SearchResult(hit=KItem(\n", - " \n", - " \tname = foo 2, \n", - " \n", - " \tid = 3ca3c293-8845-4f0e-afcb-6c680c07239f, \n", - " \n", - " \tktype_id = organization, \n", - " \n", - " \tin_backend = True, \n", - " \n", - " \tslug = foo2-3ca3c293, \n", - " \n", - " \tannotations = [\n", - " \t\t{\n", - " \t\t\tiri: www.example.org/foo,\n", - " \t\t\tname: foo,\n", - " \t\t\tnamespace: www.example.org,\n", - " \t\t\tdescription: None\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [\n", - " \t\t\n", - " \t\t\tid: 966beb55-6eb5-422e-b8c1-84d65b8cf50d\n", - " \t\t\tname: foo 1\n", - " \t\t\tslug: foo1-966beb55\n", - " \t\t\tktype_id: dataset-catalog\n", - " \t\t\tsummary: None\n", - " \t\t\tavatar_exists: False\n", - " \t\t\tannotations: []\n", - " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: 3ca3c293-8845-4f0e-afcb-6c680c07239f\n", - " \t\t}]\n", - " \t\t\texternal_links: []\n", - " \t\t\tcontacts: []\n", - " \t\t\tauthors: [{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}]\n", - " \t\t\tlinked_affiliations: []\n", - " \t\t\tattachments: []\n", - " \t\t\tuser_groups: []\n", - " \t\t\tcustom_properties: None\n", - " \t\t\tcreated_at: 2024-08-20T08:02:08.491699\n", - " \t\t\tupdated_at: 2024-08-20T08:02:08.491699\n", - " \t\t\n", - " \t], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\t{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-08-20 08:02:09.024038, \n", - " \n", - " \tupdated_at = 2024-08-20 08:02:09.024038, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = None, \n", - " \n", - " \tdataframe = None, \n", - " \n", - " \trdf_exists = False\n", - " ), fuzzy=False)]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.search(\n", - " ktypes=[dsms.ktypes.Organization], annotations=[\"www.example.org/foo\"]\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Clean up the DSMS from the tutortial:" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "del dsms[item]\n", - "del dsms[item2]\n", - "del dsms[item3]\n", - "del dsms[item4]\n", - "\n", - "dsms.commit()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6. Apps" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can investigate which apps are available through JupyterLab:" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[AppConfig(name=ckan-fetch, specification={'metadata': {'generateName': 'ckan-resource-request-'}}),\n", - " AppConfig(name=csv_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", - " AppConfig(name=csv_tensile_test_f2, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", - " AppConfig(name=excel_notched_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", - " AppConfig(name=excel_shear_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", - " AppConfig(name=excel_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", - " AppConfig(name=ternary-plot, specification={'metadata': {'generateName': 'ckan-tenary-app-'}})]" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.app_configs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "7. HDF5 " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We are also able to upload dataframes or time series data and investigate them:" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Column-wise:\n", - "column: a ,\n", - " data: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]\n", - "column: b ,\n", - " data: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100]\n", - "\n", - "As data frame:\n", - " a b\n", - "0 0 1\n", - "1 1 2\n", - "2 2 3\n", - "3 3 4\n", - "4 4 5\n", - ".. .. ...\n", - "95 95 96\n", - "96 96 97\n", - "97 97 98\n", - "98 98 99\n", - "99 99 100\n", - "\n", - "[100 rows x 2 columns]\n" - ] - } - ], - "source": [ - "data = {\"a\": list(range(100)), \"b\": list(range(1,101))}\n", - "\n", - "\n", - "item = KItem(name=\"testdata1234\", ktype_id=dsms.ktypes.DatasetCatalog, dataframe=data)\n", - "dsms.commit()\n", - "\n", - "print(\"Column-wise:\")\n", - "for column in item.dataframe:\n", - " print(\"column:\", column.name, \",\\n\", \"data:\", column.get())\n", - "\n", - "df = item.dataframe.to_df()\n", - "print(\"\\nAs data frame:\")\n", - "print(df)\n", - "\n", - "new_df = df.drop(['a'], axis=1)\n", - "item.dataframe = new_df\n", - "\n", - "dsms.commit()" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "del dsms[item]" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sdk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/tutorials/1_introduction.ipynb b/examples/tutorials/1_introduction.ipynb deleted file mode 100644 index 1665bb0..0000000 --- a/examples/tutorials/1_introduction.ipynb +++ /dev/null @@ -1,216 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 1. Connecting with the SDK to DSMS\n", - "\n", - "In this tutorial we see the overview on how to setup and basic use DSMS-SDK\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.1. Setting up" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.2. Introduction to KItems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see which kind of DSMS-object we own as a user (in the beginning, we own none):" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.kitems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can investigate what a KItem needs in order to be created. KItems are entirely based on Pydantic-Models (v2), hence the properties (in Pydantic called Fields) are automatically validated once we set them.\n", - "\n", - "The schema of the KItem itself is a JSON schema which is machine-readable and can be directly incorporated into Swagger-supported APIs like e.g. FastAPI.\n", - "\n", - "We can investigate the KTypes defined in the remote instance:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Organization(\n", - "\tid=organization,\n", - "\tname=None,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-06T13:01:42.756035,\n", - "\tupdated_at=2024-05-06T13:01:42.756035\n", - ")\n", - "App(\n", - "\tid=app,\n", - "\tname=None,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-06T13:01:42.756035,\n", - "\tupdated_at=2024-05-06T13:01:42.756035\n", - ")\n", - "Dataset(\n", - "\tid=dataset,\n", - "\tname=dataset,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-06T13:01:42.756035,\n", - "\tupdated_at=2024-05-06T13:01:42.756035\n", - ")\n", - "DatasetCatalog(\n", - "\tid=dataset-catalog,\n", - "\tname=dataset catalog,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-06T13:01:42.756035,\n", - "\tupdated_at=2024-05-06T13:01:42.756035\n", - ")\n", - "Expert(\n", - "\tid=expert,\n", - "\tname=expert,\n", - "\twebform={\n", - "\t\tname=dict(annotation=Union[str, NoneType] required=False default=None),\n", - "\t\tskills=dict(annotation=Union[str, NoneType] required=False default=None)\n", - "\t},\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-06T13:01:42.756035,\n", - "\tupdated_at=2024-05-06T17:09:46.125058\n", - ")\n", - "Test(\n", - "\tid=test,\n", - "\tname=test,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-14T07:45:38.051796,\n", - "\tupdated_at=2024-05-14T07:45:38.051796\n", - ")\n", - "Specimen(\n", - "\tid=specimen,\n", - "\tname=Specimen,\n", - "\twebform={\n", - "\t\tsampletype=dict(annotation=Union[str, NoneType] required=False default=None),\n", - "\t\tsampleinformation=dict(annotation=Union[str, NoneType] required=False default=None),\n", - "\t\tsampleproductionprocess=dict(annotation=Union[str, NoneType] required=False default=None)\n", - "\t},\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-22T07:19:24.079365,\n", - "\tupdated_at=2024-05-22T07:20:08.774884\n", - ")\n", - "Resource(\n", - "\tid=resource,\n", - "\tname=Resource,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-06-10T10:17:31.071959,\n", - "\tupdated_at=2024-06-10T10:17:40.156104\n", - ")\n", - "TestingMachine(\n", - "\tid=testing-machine,\n", - "\tname=Testing Machine,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-08-19T17:13:57.534570,\n", - "\tupdated_at=2024-08-19T18:11:40.465640\n", - ")\n" - ] - } - ], - "source": [ - "for ktype in dsms.ktypes:\n", - " print(ktype)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sdk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/tutorials/2_creation.ipynb b/examples/tutorials/2_creation.ipynb deleted file mode 100644 index 5cb7539..0000000 --- a/examples/tutorials/2_creation.ipynb +++ /dev/null @@ -1,386 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 2. Create KItems with the SDK\n", - "\n", - "In this tutorial we see how to create new Kitems." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.1. Setting up\n", - "\n", - "Now let us import the needed classes and functions for this tutorial." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS, KItem" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "### 2.2: Create KItems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can make new KItems by simple class-initiation: (Make sure existing KItems are not given as input). \n", - "#" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = Machine-1, \n", - "\n", - "\tid = c4128bec-3ecd-4c92-a307-7015bc9f2e86, \n", - "\n", - "\tktype_id = KTypes.TestingMachine, \n", - "\n", - "\tin_backend = False, \n", - "\n", - "\tslug = machine-1-c4128bec, \n", - "\n", - "\tannotations = [], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = None, \n", - "\n", - "\tupdated_at = None, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tProducer: TestingLab GmBH, \n", - "\t\tLocation: A404, \n", - "\t\tModel Number: Bending Test Machine No 777\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item = KItem(\n", - " name=\"Machine-1\",\n", - " ktype_id=dsms.ktypes.TestingMachine,\n", - " custom_properties={\"Producer\": \"TestingLab GmBH\",\n", - " \"Location\": \"A404\",\n", - " \"Model Number\" : \"Bending Test Machine No 777\"\n", - " },\n", - ")\n", - "\n", - "item" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Remember: changes are only syncronized with the DSMS when you call the `commit`-method:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'https://bue.materials-data.space/knowledge/testing-machine/machine-1-c4128bec'" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.commit()\n", - "item.url" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As we can see, the object we created before running the `commit`-method has automatically been updated, e.g. with the creation- and update-timestamp. We can check this with the below command:" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = Machine-1, \n", - "\n", - "\tid = c4128bec-3ecd-4c92-a307-7015bc9f2e86, \n", - "\n", - "\tktype_id = testing-machine, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = machine-1-c4128bec, \n", - "\n", - "\tannotations = [], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = 2024-10-01 14:21:54.195585, \n", - "\n", - "\tupdated_at = 2024-10-01 14:21:54.195585, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tProducer: TestingLab GmBH, \n", - "\t\tLocation: A404, \n", - "\t\tModel Number: Bending Test Machine No 777\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To just get the name of the item, we can do it as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Machine-1'" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item.name" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As well as the id of the kitem we can do it as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "UUID('c4128bec-3ecd-4c92-a307-7015bc9f2e86')" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item.id" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To check the KType of the item newly created we can use the following:" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KType(id='testing-machine', name='Testing Machine', webform=, json_schema=None, rdf_mapping=None, created_at='2024-08-19T17:13:57.534570', updated_at='2024-08-19T18:11:40.465640')" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item.ktype" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... and also check the KType:" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item.is_a(dsms.ktypes.TestingMachine)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "Now you can check if the particular kitem is in the list of KItems. This can be done either by using the command:\n", - " `\n", - " dsms.kitems\n", - " `\n", - " or by logging into the frontend dsms instance." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sdk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/tutorials/3_updation.ipynb b/examples/tutorials/3_updation.ipynb deleted file mode 100644 index 3bf4d84..0000000 --- a/examples/tutorials/3_updation.ipynb +++ /dev/null @@ -1,287 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 3. Updating KItems with the SDK\n", - "\n", - "In this tutorial we see how to update existing Kitems." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.1. Setting up\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now lets get the kitem we created in the [2nd tutorial : Creation of Kitems](2_creation.ipynb)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Machine-1\n" - ] - } - ], - "source": [ - "item = dsms.kitems[-1]\n", - "print(item.name)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.2. Updating Kitems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, we would like to update the properties of our KItem we created previously.\n", - "\n", - "Depending on the schema of each property (see [DSMS KItem Schema](../dsms_kitem_schema.md)), we can simply use the standard `list`-method as we know them from basic Python (e.g. for the `annotations`, `attachments`, `external_link`, etc). \n", - "\n", - "Other properties which are not `list`-like can be simply set by attribute-assignment (e.g. `name`, `slug`, `ktype_id`, etc)." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "item.name = \"Machine-1\"\n", - "item.custom_properties.Producer = \"Machinery GmBH\"\n", - "item.attachments.append(\"testfile.txt\")\n", - "item.annotations.append(\"www.machinery.org/\")\n", - "item.external_links.append(\n", - " {\"url\": \"http://machine.org\", \"label\": \"machine-link\"}\n", - ")\n", - "item.contacts.append({\"name\": \"machinesupport\", \"email\": \"machinesupport@group.mail\"})\n", - "item.affiliations.append(\"machine-team\")\n", - "item.user_groups.append({\"name\": \"machinegroup\", \"group_id\": \"123\"})" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see now that the local system path of the attachment is changed to a simply file name, which means that the upload was successful. If not so, an error would have been thrown during the `commit`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see the updates when we print the item:" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = Machine-1, \n", - "\n", - "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", - "\n", - "\tktype_id = testing-machine, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = machine-1-dd091666, \n", - "\n", - "\tannotations = [\n", - "\t\t{\n", - "\t\t\tiri: www.machinery.org/,\n", - "\t\t\tname: ,\n", - "\t\t\tnamespace: www.machinery.org,\n", - "\t\t\tdescription: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tattachments = [\n", - "\t\t{\n", - "\t\t\tname: testfile.txt\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tlinked_kitems = [], \n", - "\n", - "\taffiliations = [\n", - "\t\t{\n", - "\t\t\tname: machine-team\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [\n", - "\t\t{\n", - "\t\t\tname: machinesupport,\n", - "\t\t\temail: machinesupport@group.mail,\n", - "\t\t\tuser_id: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tcreated_at = 2024-08-19 18:12:11.338394, \n", - "\n", - "\tupdated_at = 2024-08-19 18:12:11.338394, \n", - "\n", - "\texternal_links = [\n", - "\t\t{\n", - "\t\t\tlabel: machine-link,\n", - "\t\t\turl: http://machine.org/\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [\n", - "\t\t{\n", - "\t\t\tname: machinegroup,\n", - "\t\t\tgroup_id: 123\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tProducer: Machinery GmBH, \n", - "\t\tLocation: A404, \n", - "\t\tModel Number: Bending Test Machine No 777\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Furthermore we can also download the file we uploaded again:" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\t Downloaded file: {\n", - "\t\t\tname: testfile.txt\n", - "\t\t}\n", - "|------------------------------------Beginning of file------------------------------------|\n", - "This is a calibration protocol!\n", - "|---------------------------------------End of file---------------------------------------|\n" - ] - } - ], - "source": [ - "for file in item.attachments:\n", - " download = file.download()\n", - "\n", - " print(\"\\t\\t\\t Downloaded file:\", file)\n", - " print(\"|------------------------------------Beginning of file------------------------------------|\")\n", - " print(download)\n", - " print(\"|---------------------------------------End of file---------------------------------------|\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sdk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/tutorials/4_deletion.ipynb b/examples/tutorials/4_deletion.ipynb deleted file mode 100644 index 210df6b..0000000 --- a/examples/tutorials/4_deletion.ipynb +++ /dev/null @@ -1,404 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 4. Deleting KItems with the SDK\n", - "\n", - "In this tutorial we see how to delete new Kitems and their properties." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.1. Setting up\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Then lets see the Kitem we are interested in to remove." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "KItem(\n", - "\n", - "\tname = Machine-1, \n", - "\n", - "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", - "\n", - "\tktype_id = testing-machine, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = machine-1-dd091666, \n", - "\n", - "\tannotations = [\n", - "\t\t{\n", - "\t\t\tiri: www.machinery.org/,\n", - "\t\t\tname: ,\n", - "\t\t\tnamespace: www.machinery.org,\n", - "\t\t\tdescription: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tattachments = [\n", - "\t\t{\n", - "\t\t\tname: testfile.txt\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tlinked_kitems = [], \n", - "\n", - "\taffiliations = [\n", - "\t\t{\n", - "\t\t\tname: machine-team\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [\n", - "\t\t{\n", - "\t\t\tname: machinesupport,\n", - "\t\t\temail: machinesupport@group.mail,\n", - "\t\t\tuser_id: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tcreated_at = 2024-08-19 18:12:11.338394, \n", - "\n", - "\tupdated_at = 2024-08-19 18:12:11.338394, \n", - "\n", - "\texternal_links = [\n", - "\t\t{\n", - "\t\t\tlabel: machine-link,\n", - "\t\t\turl: http://machine.org/\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [\n", - "\t\t{\n", - "\t\t\tname: machinegroup,\n", - "\t\t\tgroup_id: 123\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tProducer: Machinery GmBH, \n", - "\t\tLocation: A404, \n", - "\t\tModel Number: Bending Test Machine No 777\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")\n" - ] - } - ], - "source": [ - "item = dsms.kitems[-1]\n", - "print(item)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.2. Deletion of KItems and their properties" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also remove properties from the KItem without deleting the KItem itself.\n", - "\n", - "For the `list`-like properties, we can use the standard `list`-methods from basic Python again (e.g. `pop`, `remove`, etc. or the `del`-operator).\n", - "\n", - "For the other, non-`list`-like properties, we can simply use the attribute-assignment again.\n", - "\n", - "When we only want single parts of the properties in the KItem, we can do it like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{\n", - "\t\t\tname: machinegroup,\n", - "\t\t\tgroup_id: 123\n", - "\t\t}" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item.attachments.pop(0)\n", - "item.annotations.pop(0)\n", - "item.external_links.pop(0)\n", - "item.contacts.pop(0)\n", - "item.user_groups.pop(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, we can also reset the entire property by setting it to e.g. an empty list again:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "item.affiliations = []" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can delete the custom properties by setting the property to an empty dict:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "item.custom_properties = {}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Send the changes to the DSMS with the `commit`-method:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "See the changes:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = Machine-1, \n", - "\n", - "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", - "\n", - "\tktype_id = testing-machine, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = machine-1-dd091666, \n", - "\n", - "\tannotations = [], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [], \n", - "\n", - "\taffiliations = [\n", - "\t\t{\n", - "\t\t\tname: machine-team\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [\n", - "\t\t{\n", - "\t\t\tname: machinesupport,\n", - "\t\t\temail: machinesupport@group.mail,\n", - "\t\t\tuser_id: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tcreated_at = 2024-08-19 18:12:11.338394, \n", - "\n", - "\tupdated_at = 2024-08-19 18:12:11.338394, \n", - "\n", - "\texternal_links = [\n", - "\t\t{\n", - "\t\t\tlabel: machine-link,\n", - "\t\t\turl: http://machine.org/\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tid: dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", - "\t\tcontent: {}\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, we can also delete the whole KItem from the DSMS by applying the `del`-operator to the `dsms`-object with the individual `KItem`-object:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "del dsms[item]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "Commit the changes:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now to check if the particular kitem was removed, we can do this by using the command:\n", - " `\n", - " dsms.kitems\n", - " `\n", - " or by logging into the frontend dsms instance." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sdk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/tutorials/5_search.ipynb b/examples/tutorials/5_search.ipynb deleted file mode 100644 index 42610d3..0000000 --- a/examples/tutorials/5_search.ipynb +++ /dev/null @@ -1,1323 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 5. Searching KItems with the SDK\n", - "\n", - "In this tutorial we see how to search existing Kitems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5.1. Setting up\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS, KItem" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5.2. Searching for KItems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this section, we would like to search for specfic KItems we created in the DSMS.\n", - "\n", - "For this purpose, we will firstly create some KItems and apply the `search`-method on the `DSMS`-object later on in order to find them again in the DSMS.\n", - "\n", - "We also want to demonstrate here, that we can link KItems to each other in order to find e.g. a related item of type `DatasetCatalog`. For this strategy, we are using the `linked_kitems`- attribute and the `id` of the item which we would like to link.\n", - "\n", - "The procedure looks like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "item1 = KItem(\n", - " name=\"Machine-1\",\n", - " ktype_id=dsms.ktypes.TestingMachine,\n", - " annotations=[\"https://w3id.org/steel/ProcessOntology/TestingMachine\"],\n", - " custom_properties={\"Producer\": \"TestingLab GmBH\",\n", - " \"Room Number\": \"A404\",\n", - " \"Description\": \"Bending Test Machine\"\n", - " }\n", - ")\n", - "\n", - "item2 = KItem(\n", - " name=\"Machine-2\",\n", - " ktype_id=dsms.ktypes.TestingMachine,\n", - " annotations=[\"https://w3id.org/steel/ProcessOntology/TestingMachine\"],\n", - " custom_properties={\"Producer\": \"StressStrain GmBH\",\n", - " \"Room Number\": \"B500\",\n", - " \"Description\": \"Compression Test Machine\"\n", - " }\n", - ")\n", - "\n", - "item3 = KItem(\n", - " name=\"Specimen-1\", \n", - " ktype_id=dsms.ktypes.Specimen,\n", - " linked_kitems=[item1],\n", - " annotations=[\"https://w3id.org/steel/ProcessOntology/TestPiece\"],\n", - " custom_properties={\"Geometry\": \"Cylindrical 150mm x 20mm x 40mm\",\n", - " \"Material\": \"Concrete\",\n", - " \"Project ID\": \"ConstructionProject2024\"\n", - " }\n", - "\n", - ")\n", - "item4 = KItem(\n", - " name=\"Specimen-2\",\n", - " ktype_id=dsms.ktypes.Specimen,\n", - " linked_kitems=[item2],\n", - " annotations=[\"https://w3id.org/steel/ProcessOntology/TestPiece\"],\n", - " custom_properties={\"Geometry\": \"Rectangular 200mm x 30mm x 20mm\",\n", - " \"Material\": \"Metal\",\n", - " \"Project ID\": \"MetalBlenders2024\"\n", - " }\n", - ")\n", - "\n", - "item5 = KItem(\n", - " name=\"Research Institute ABC\",\n", - " ktype_id=dsms.ktypes.Organization,\n", - " linked_kitems=[item1,item2],\n", - " annotations=[\"www.researchBACiri.org/foo\"],\n", - ")\n", - "\n", - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "

Note : Here in this tutorial, we use dsms.search with `limit=1` to maintain readability but the user can adjust the variable `limit` as per requirement.

\n", - "\n", - "\n", - "Now, we are apply to search for e.g. kitems of type `TestingMachine`:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "KItem(\n", - "\n", - "\tname = Machine-1, \n", - "\n", - "\tid = 1d2c5439-1971-4a92-907b-e30ce00da344, \n", - "\n", - "\tktype_id = testing-machine, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = machine-1-1d2c5439, \n", - "\n", - "\tannotations = [\n", - "\t\t{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - "\t\t\tname: TestingMachine,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [\n", - "\t\t\n", - "\t\t\tid: b77d1d0c-1c30-483c-bb3c-bddbcf3707b7\n", - "\t\t\tname: Research Institute ABC\n", - "\t\t\tslug: researchinstituteabc-b77d1d0c\n", - "\t\t\tktype_id: organization\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.researchBACiri.org,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 1d2c5439-1971-4a92-907b-e30ce00da344\n", - "\t\t}, {\n", - "\t\t\tid: 63c045bd-e0d5-429c-ad92-39e7e778c9a9\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: None\n", - "\t\t\tcreated_at: 2024-09-26T14:42:49.728733\n", - "\t\t\tupdated_at: 2024-09-26T14:42:49.728733\n", - "\t\t, \n", - "\t\t\n", - "\t\t\tid: d68f1a4c-0da4-454b-aaa5-3c927e690c0d\n", - "\t\t\tname: Specimen-1\n", - "\t\t\tslug: specimen-1-d68f1a4c\n", - "\t\t\tktype_id: specimen\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", - "\t\t\tname: TestPiece,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 1d2c5439-1971-4a92-907b-e30ce00da344\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", - "\t\t\tcreated_at: 2024-09-26T14:42:48.908560\n", - "\t\t\tupdated_at: 2024-09-26T14:42:48.908560\n", - "\t\t\n", - "\t], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = 2024-09-26 14:42:48.010709, \n", - "\n", - "\tupdated_at = 2024-09-26 14:42:48.010709, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tProducer: TestingLab GmBH, \n", - "\t\tRoom Number: A404, \n", - "\t\tDescription: Bending Test Machine\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")\n", - "\n", - "\n" - ] - } - ], - "source": [ - "for result in dsms.search(ktypes=[dsms.ktypes.TestingMachine], limit=1):\n", - " print(result.hit)\n", - " print(\"\\n\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... and for all of type `Organization` and `DatasetCatalog`:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "KItem(\n", - "\n", - "\tname = Research Institute ABC, \n", - "\n", - "\tid = f61b851e-9a5c-456e-b486-488d3d32bd8e, \n", - "\n", - "\tktype_id = organization, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = researchinstituteabc-f61b851e, \n", - "\n", - "\tannotations = [\n", - "\t\t{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.researchBACiri.org,\n", - "\t\t\tdescription: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [\n", - "\t\t\n", - "\t\t\tid: 5f93eca8-e076-4d7c-b5fa-57cb226d7aa1\n", - "\t\t\tname: Machine-2\n", - "\t\t\tslug: machine-2-5f93eca8\n", - "\t\t\tktype_id: testing-machine\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - "\t\t\tname: TestingMachine,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: f61b851e-9a5c-456e-b486-488d3d32bd8e\n", - "\t\t}, {\n", - "\t\t\tid: ff58c3ac-9f7d-4aa2-8fc1-9524f324f598\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'Producer': 'StressStrain GmBH', 'Room Number': 'B500', 'Description': 'Compression Test Machine'}\n", - "\t\t\tcreated_at: 2024-09-26T14:38:14.079698\n", - "\t\t\tupdated_at: 2024-09-26T14:38:14.079698\n", - "\t\t, \n", - "\t\t\n", - "\t\t\tid: e32870cb-1686-4d76-a51e-7692568c1dc0\n", - "\t\t\tname: Machine-1\n", - "\t\t\tslug: machine-1-e32870cb\n", - "\t\t\tktype_id: testing-machine\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - "\t\t\tname: TestingMachine,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 43a67f5a-fd31-4a39-9697-bfc39aa852b2\n", - "\t\t}, {\n", - "\t\t\tid: f61b851e-9a5c-456e-b486-488d3d32bd8e\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - "\t\t\tcreated_at: 2024-09-26T14:38:13.645998\n", - "\t\t\tupdated_at: 2024-09-26T14:38:13.645998\n", - "\t\t\n", - "\t], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = 2024-09-26 14:38:15.465213, \n", - "\n", - "\tupdated_at = 2024-09-26 14:38:15.465213, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = None, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")\n", - "fuzziness: False\n", - "\n", - "\n" - ] - } - ], - "source": [ - "for result in dsms.search(ktypes=[dsms.ktypes.Organization, dsms.ktypes.DatasetCatalog], limit=1):\n", - " print(result.hit)\n", - " print(\"fuzziness: \", result.fuzzy)\n", - " print(\"\\n\")\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... or for all of type `Dataset` with `Specimen-1` in the name:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "for result in dsms.search(query=\"Specimen-1\", ktypes=[dsms.ktypes.Dataset], limit=1):\n", - " print(result.hit)\n", - " print(\"\\n\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... and for all of type `Organization` with the annotation `www.researchBACiri.org/foo`:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "KItem(\n", - "\n", - "\tname = Research Institute ABC, \n", - "\n", - "\tid = 7b58fc00-bdb3-4584-a86b-b8f447189429, \n", - "\n", - "\tktype_id = organization, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = researchinstituteabc-7b58fc00, \n", - "\n", - "\tannotations = [\n", - "\t\t{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.researchBACiri.org,\n", - "\t\t\tdescription: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [\n", - "\t\t\n", - "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", - "\t\t\tname: Machine-1\n", - "\t\t\tslug: machine-1-6dd446f1\n", - "\t\t\tktype_id: testing-machine\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - "\t\t\tname: TestingMachine,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - "\t\t}, {\n", - "\t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - "\t\t\tcreated_at: 2024-09-26T14:44:24.055330\n", - "\t\t\tupdated_at: 2024-09-26T14:44:24.055330\n", - "\t\t, \n", - "\t\t\n", - "\t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", - "\t\t\tname: Machine-2\n", - "\t\t\tslug: machine-2-fd1a2a9a\n", - "\t\t\tktype_id: testing-machine\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - "\t\t\tname: TestingMachine,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - "\t\t}, {\n", - "\t\t\tid: b15bc4c8-0180-43ce-8d73-c38e7d9f4aae\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'Producer': 'StressStrain GmBH', 'Room Number': 'B500', 'Description': 'Compression Test Machine'}\n", - "\t\t\tcreated_at: 2024-09-26T14:44:24.494230\n", - "\t\t\tupdated_at: 2024-09-26T14:44:24.494230\n", - "\t\t\n", - "\t], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = 2024-09-26 14:44:25.705701, \n", - "\n", - "\tupdated_at = 2024-09-26 14:44:25.705701, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = None, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")\n", - "\n", - "\n" - ] - } - ], - "source": [ - "for result in dsms.search(\n", - " ktypes=[dsms.ktypes.Organization], annotations=[\"www.researchBACiri.org/foo\"], limit=1\n", - " ):\n", - " print(result.hit)\n", - " print(\"\\n\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5.3. Fetching linked KItems from a KItem" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the beginning under **5.1** we created some kitems and linked each other. Now we want to fetch the linked kitems and display them to the user. For this we use the `linked_kitems` attribute." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[\n", - "\t\t\n", - "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", - "\t\t\tname: Machine-1\n", - "\t\t\tslug: machine-1-6dd446f1\n", - "\t\t\tktype_id: testing-machine\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - "\t\t\tname: TestingMachine,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - "\t\t}, {\n", - "\t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - "\t\t\tcreated_at: 2024-09-26T14:44:24.055330\n", - "\t\t\tupdated_at: 2024-09-26T14:44:24.055330\n", - "\t\t, \n", - "\t\t\n", - "\t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", - "\t\t\tname: Machine-2\n", - "\t\t\tslug: machine-2-fd1a2a9a\n", - "\t\t\tktype_id: testing-machine\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - "\t\t\tname: TestingMachine,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - "\t\t}, {\n", - "\t\t\tid: b15bc4c8-0180-43ce-8d73-c38e7d9f4aae\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'Producer': 'StressStrain GmBH', 'Room Number': 'B500', 'Description': 'Compression Test Machine'}\n", - "\t\t\tcreated_at: 2024-09-26T14:44:24.494230\n", - "\t\t\tupdated_at: 2024-09-26T14:44:24.494230\n", - "\t\t\n", - "\t]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item5.linked_kitems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, this linked KItem is not a \"real\" KItem. Due to performance reasons, we only display a slim representation of the linked KItem. Imagine if the linked KItem also has a linked KItem... this might lead to an infinite recursion. We can fetch the \"real\" KItem by using the `fetch` method of the linked KItem:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = Machine-1, \n", - "\n", - "\tid = 6dd446f1-cc37-462b-83d5-cc55d46d57b1, \n", - "\n", - "\tktype_id = testing-machine, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = machine-1-6dd446f1, \n", - "\n", - "\tannotations = [\n", - "\t\t{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - "\t\t\tname: TestingMachine,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [\n", - "\t\t\n", - "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - "\t\t\tname: Research Institute ABC\n", - "\t\t\tslug: researchinstituteabc-7b58fc00\n", - "\t\t\tktype_id: organization\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.researchBACiri.org,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", - "\t\t}, {\n", - "\t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: None\n", - "\t\t\tcreated_at: 2024-09-26T14:44:25.705701\n", - "\t\t\tupdated_at: 2024-09-26T14:44:25.705701\n", - "\t\t, \n", - "\t\t\n", - "\t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", - "\t\t\tname: Specimen-1\n", - "\t\t\tslug: specimen-1-e6ef3088\n", - "\t\t\tktype_id: specimen\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", - "\t\t\tname: TestPiece,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", - "\t\t\tcreated_at: 2024-09-26T14:44:24.894772\n", - "\t\t\tupdated_at: 2024-09-26T14:44:24.894772\n", - "\t\t\n", - "\t], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = 2024-09-26 14:44:24.055330, \n", - "\n", - "\tupdated_at = 2024-09-26 14:44:24.055330, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tProducer: TestingLab GmBH, \n", - "\t\tRoom Number: A404, \n", - "\t\tDescription: Bending Test Machine\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item5.linked_kitems[0].fetch()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also get the linked KItems grouped by annotation:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'https://w3id.org/steel/ProcessOntology/TestingMachine': [\n", - " \t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", - " \t\t\tname: Machine-1\n", - " \t\t\tslug: machine-1-6dd446f1\n", - " \t\t\tktype_id: testing-machine\n", - " \t\t\tsummary: None\n", - " \t\t\tavatar_exists: False\n", - " \t\t\tannotations: [{\n", - " \t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - " \t\t\tname: TestingMachine,\n", - " \t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - " \t\t\tdescription: None\n", - " \t\t}]\n", - " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - " \t\t}, {\n", - " \t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", - " \t\t}]\n", - " \t\t\texternal_links: []\n", - " \t\t\tcontacts: []\n", - " \t\t\tauthors: [{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}]\n", - " \t\t\tlinked_affiliations: []\n", - " \t\t\tattachments: []\n", - " \t\t\tuser_groups: []\n", - " \t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - " \t\t\tcreated_at: 2024-09-26T14:44:24.055330\n", - " \t\t\tupdated_at: 2024-09-26T14:44:24.055330\n", - " \t\t,\n", - " \n", - " \t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", - " \t\t\tname: Machine-2\n", - " \t\t\tslug: machine-2-fd1a2a9a\n", - " \t\t\tktype_id: testing-machine\n", - " \t\t\tsummary: None\n", - " \t\t\tavatar_exists: False\n", - " \t\t\tannotations: [{\n", - " \t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - " \t\t\tname: TestingMachine,\n", - " \t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - " \t\t\tdescription: None\n", - " \t\t}]\n", - " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - " \t\t}, {\n", - " \t\t\tid: b15bc4c8-0180-43ce-8d73-c38e7d9f4aae\n", - " \t\t}]\n", - " \t\t\texternal_links: []\n", - " \t\t\tcontacts: []\n", - " \t\t\tauthors: [{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}]\n", - " \t\t\tlinked_affiliations: []\n", - " \t\t\tattachments: []\n", - " \t\t\tuser_groups: []\n", - " \t\t\tcustom_properties: {'Producer': 'StressStrain GmBH', 'Room Number': 'B500', 'Description': 'Compression Test Machine'}\n", - " \t\t\tcreated_at: 2024-09-26T14:44:24.494230\n", - " \t\t\tupdated_at: 2024-09-26T14:44:24.494230\n", - " \t\t]}" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item5.linked_kitems.by_annotation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Again, we are able to fetch the \"real\" KItem by using the `fetch` method of the linked KItem:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = Machine-1, \n", - "\n", - "\tid = 6dd446f1-cc37-462b-83d5-cc55d46d57b1, \n", - "\n", - "\tktype_id = testing-machine, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = machine-1-6dd446f1, \n", - "\n", - "\tannotations = [\n", - "\t\t{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - "\t\t\tname: TestingMachine,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [\n", - "\t\t\n", - "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - "\t\t\tname: Research Institute ABC\n", - "\t\t\tslug: researchinstituteabc-7b58fc00\n", - "\t\t\tktype_id: organization\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.researchBACiri.org,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", - "\t\t}, {\n", - "\t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: None\n", - "\t\t\tcreated_at: 2024-09-26T14:44:25.705701\n", - "\t\t\tupdated_at: 2024-09-26T14:44:25.705701\n", - "\t\t, \n", - "\t\t\n", - "\t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", - "\t\t\tname: Specimen-1\n", - "\t\t\tslug: specimen-1-e6ef3088\n", - "\t\t\tktype_id: specimen\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", - "\t\t\tname: TestPiece,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", - "\t\t\tcreated_at: 2024-09-26T14:44:24.894772\n", - "\t\t\tupdated_at: 2024-09-26T14:44:24.894772\n", - "\t\t\n", - "\t], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = 2024-09-26 14:44:24.055330, \n", - "\n", - "\tupdated_at = 2024-09-26 14:44:24.055330, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tProducer: TestingLab GmBH, \n", - "\t\tRoom Number: A404, \n", - "\t\tDescription: Bending Test Machine\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item5.linked_kitems.by_annotation[\"https://w3id.org/steel/ProcessOntology/TestingMachine\"][0].fetch()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Additionally, we can retrieve the linked KItems grouped by ktype:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{, json_schema=None, rdf_mapping=None, created_at='2024-08-19T17:13:57.534570', updated_at='2024-08-19T18:11:40.465640')>: [\n", - " \t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", - " \t\t\tname: Machine-1\n", - " \t\t\tslug: machine-1-6dd446f1\n", - " \t\t\tktype_id: testing-machine\n", - " \t\t\tsummary: None\n", - " \t\t\tavatar_exists: False\n", - " \t\t\tannotations: [{\n", - " \t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - " \t\t\tname: TestingMachine,\n", - " \t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - " \t\t\tdescription: None\n", - " \t\t}]\n", - " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - " \t\t}, {\n", - " \t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", - " \t\t}]\n", - " \t\t\texternal_links: []\n", - " \t\t\tcontacts: []\n", - " \t\t\tauthors: [{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}]\n", - " \t\t\tlinked_affiliations: []\n", - " \t\t\tattachments: []\n", - " \t\t\tuser_groups: []\n", - " \t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", - " \t\t\tcreated_at: 2024-09-26T14:44:24.055330\n", - " \t\t\tupdated_at: 2024-09-26T14:44:24.055330\n", - " \t\t,\n", - " \n", - " \t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", - " \t\t\tname: Machine-2\n", - " \t\t\tslug: machine-2-fd1a2a9a\n", - " \t\t\tktype_id: testing-machine\n", - " \t\t\tsummary: None\n", - " \t\t\tavatar_exists: False\n", - " \t\t\tannotations: [{\n", - " \t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - " \t\t\tname: TestingMachine,\n", - " \t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - " \t\t\tdescription: None\n", - " \t\t}]\n", - " \t\t\tlinked_kitems: [{\n", - " \t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - " \t\t}, {\n", - " \t\t\tid: b15bc4c8-0180-43ce-8d73-c38e7d9f4aae\n", - " \t\t}]\n", - " \t\t\texternal_links: []\n", - " \t\t\tcontacts: []\n", - " \t\t\tauthors: [{\n", - " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - " \t\t}]\n", - " \t\t\tlinked_affiliations: []\n", - " \t\t\tattachments: []\n", - " \t\t\tuser_groups: []\n", - " \t\t\tcustom_properties: {'Producer': 'StressStrain GmBH', 'Room Number': 'B500', 'Description': 'Compression Test Machine'}\n", - " \t\t\tcreated_at: 2024-09-26T14:44:24.494230\n", - " \t\t\tupdated_at: 2024-09-26T14:44:24.494230\n", - " \t\t]}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item5.linked_kitems.by_ktype" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... and retrieve them by type:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "KItem(\n", - "\n", - "\tname = Machine-1, \n", - "\n", - "\tid = 6dd446f1-cc37-462b-83d5-cc55d46d57b1, \n", - "\n", - "\tktype_id = testing-machine, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = machine-1-6dd446f1, \n", - "\n", - "\tannotations = [\n", - "\t\t{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestingMachine,\n", - "\t\t\tname: TestingMachine,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tattachments = [], \n", - "\n", - "\tlinked_kitems = [\n", - "\t\t\n", - "\t\t\tid: 7b58fc00-bdb3-4584-a86b-b8f447189429\n", - "\t\t\tname: Research Institute ABC\n", - "\t\t\tslug: researchinstituteabc-7b58fc00\n", - "\t\t\tktype_id: organization\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: www.researchBACiri.org/foo,\n", - "\t\t\tname: foo,\n", - "\t\t\tnamespace: www.researchBACiri.org,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", - "\t\t}, {\n", - "\t\t\tid: fd1a2a9a-7353-476c-a8b5-19fe6e6d75e1\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: None\n", - "\t\t\tcreated_at: 2024-09-26T14:44:25.705701\n", - "\t\t\tupdated_at: 2024-09-26T14:44:25.705701\n", - "\t\t, \n", - "\t\t\n", - "\t\t\tid: e6ef3088-bb47-471b-bd31-ffb1b06cd184\n", - "\t\t\tname: Specimen-1\n", - "\t\t\tslug: specimen-1-e6ef3088\n", - "\t\t\tktype_id: specimen\n", - "\t\t\tsummary: None\n", - "\t\t\tavatar_exists: False\n", - "\t\t\tannotations: [{\n", - "\t\t\tiri: https://w3id.org/steel/ProcessOntology/TestPiece,\n", - "\t\t\tname: TestPiece,\n", - "\t\t\tnamespace: https://w3id.org/steel/ProcessOntology,\n", - "\t\t\tdescription: None\n", - "\t\t}]\n", - "\t\t\tlinked_kitems: [{\n", - "\t\t\tid: 6dd446f1-cc37-462b-83d5-cc55d46d57b1\n", - "\t\t}]\n", - "\t\t\texternal_links: []\n", - "\t\t\tcontacts: []\n", - "\t\t\tauthors: [{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}]\n", - "\t\t\tlinked_affiliations: []\n", - "\t\t\tattachments: []\n", - "\t\t\tuser_groups: []\n", - "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", - "\t\t\tcreated_at: 2024-09-26T14:44:24.894772\n", - "\t\t\tupdated_at: 2024-09-26T14:44:24.894772\n", - "\t\t\n", - "\t], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = False, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = 2024-09-26 14:44:24.055330, \n", - "\n", - "\tupdated_at = 2024-09-26 14:44:24.055330, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = {\n", - "\t\tProducer: TestingLab GmBH, \n", - "\t\tRoom Number: A404, \n", - "\t\tDescription: Bending Test Machine\n", - "\t}, \n", - "\n", - "\tdataframe = None, \n", - "\n", - "\trdf_exists = False\n", - ")" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item5.linked_kitems.by_ktype[dsms.ktypes.TestingMachine][0].fetch()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Clean up the DSMS from the tutortial:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "del dsms[item1]\n", - "del dsms[item2]\n", - "del dsms[item3]\n", - "del dsms[item4]\n", - "del dsms[item5]\n", - "\n", - "dsms.commit()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sdk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/tutorials/6_apps.ipynb b/examples/tutorials/6_apps.ipynb deleted file mode 100644 index b658a96..0000000 --- a/examples/tutorials/6_apps.ipynb +++ /dev/null @@ -1,1031 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 6. DSMS Apps and Pipelines\n", - "\n", - "In this tutorial we see how to create apps and run them manually" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6.1: Setting up\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS, KItem, AppConfig\n", - "import time" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6.1. Investigating Available Apps" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can investigate which apps are available:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[AppConfig(name=ckan-fetch, specification={'metadata': {'generateName': 'ckan-resource-request-'}}),\n", - " AppConfig(name=csv_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", - " AppConfig(name=csv_tensile_test_f2, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", - " AppConfig(name=excel_notched_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", - " AppConfig(name=excel_shear_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", - " AppConfig(name=excel_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", - " AppConfig(name=ternary-plot, specification={'metadata': {'generateName': 'ckan-tenary-app-'}})]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.app_configs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6.2 Create a new app config and apply it to a KItem" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6.2.1 Arbitrary python code" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To be defined." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6.2.2 - Data2RDF" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 6.2.2.1 Prepare app and its config" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the following example, we would like to upload some csv with some arbitrary data and describe it through an RDF. This will give us the opportunity to harmonize the entities of the data file through ontological concepts and allow us to convert values of the data frame columns in to any compatible unit we would like to have.\n", - "\n", - "Fist of all, let us define the data:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "data = \"\"\"A,B,C\n", - "1.2,1.3,1.5\n", - "1.7,1.8,1.9\n", - "2.0,2.1,2.3\n", - "2.5,2.6,2.8\n", - "3.0,3.2,3.4\n", - "3.6,3.7,3.9\n", - "4.1,4.3,4.4\n", - "4.7,4.8,5.0\n", - "5.2,5.3,5.5\n", - "5.8,6.0,6.1\n", - "\"\"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will also give the config a defined name:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "configname = \"testapp2\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As a next step, we want to create a new app specification. The specification is following the definition of an [**Argo Workflow**](https://argo-workflows.readthedocs.io/en/latest/). The workflow shall trigger a pipeline from a docker image with the [**Data2RDF package**](https://data2rdf.readthedocs.io/en/latest/). \n", - "\n", - "The image has already been deployed on the k8s cluster of the DSMS and the workflow template with the name `dsms-data2rdf` has been implemented previously. Hence we only need to configure our pipeline for our data shown above, which we would like to upload and describe through an RDF.\n", - "\n", - "For more details about the data2rdf package, please refer to the documentation of Data2RDF mentioned above.\n", - "\n", - "The parameters of the app config are defining the inputs for our Data2RDF pipeline. This e.g. are: \n", - "\n", - "* the parser kind (`csv` here)\n", - "* the time series header length (`1` here)\n", - "* the metadata length (`0` here)\n", - "* the time series separator (`,` here)\n", - "* the log level (`DEBUG` here)\n", - "* the mapping \n", - " * `A` is the test time and has a unit in seconds\n", - " * `B` is the standard force in kilonewtons\n", - " * `C` is the absolut cross head travel in millimeters" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "parameters = [\n", - " {\"name\": \"parser\", \"value\": \"csv\"},\n", - " {\"name\": \"time_series_header_length\", \"value\": 1},\n", - " {\"name\": \"metadata_length\", \"value\": 0},\n", - " {\"name\": \"time_series_sep\", \"value\": \",\"},\n", - " {\"name\": \"log_level\", \"value\": \"DEBUG\"},\n", - " {\n", - " \"name\": \"mapping\",\n", - " \"value\": \"\"\"\n", - " [\n", - " {\n", - " \"key\": \"A\",\n", - " \"iri\": \"https://w3id.org/steel/ProcessOntology/TestTime\",\n", - " \"unit\": \"s\"\n", - " },\n", - " {\n", - " \"key\": \"B\",\n", - " \"iri\": \"https://w3id.org/steel/ProcessOntology/StandardForce\",\n", - " \"unit\": \"kN\"\n", - " },\n", - " {\n", - " \"key\": \"C\",\n", - " \"iri\": \"https://w3id.org/steel/ProcessOntology/AbsoluteCrossheadTravel\",\n", - " \"unit\": \"mm\"\n", - " }\n", - " ]\n", - " \"\"\",\n", - " },\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we add the parameters to our app specification. We assign a prefix `datardf-` which shall generate a new name with some random characters as suffix. The workflow template with the Docker image we want to run is called `dsms-data2rdf` and its `entrypoint` is `execute_pipeline`." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "# Define app specification\n", - "specification = {\n", - " \"apiVersion\": \"argoproj.io/v1alpha1\",\n", - " \"kind\": \"Workflow\",\n", - " \"metadata\": {\"generateName\": \"data2rdf-\"},\n", - " \"spec\": {\n", - " \"entrypoint\": \"execute_pipeline\",\n", - " \"workflowTemplateRef\": {\"name\": \"dsms-data2rdf\"},\n", - " \"arguments\": {\"parameters\": parameters},\n", - " },\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we instanciate the new app config:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "appspec = AppConfig(\n", - " name=configname,\n", - " specification=specification, # this can also be a file path to a yaml file instead of a dict\n", - " expose_sdk_config=True,\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Please note that the `expose_sdk_config` is important here, since the pipeline needs to be aware of the settings of our platform and our current SDK-session." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We commit the new app config:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we would like to apply the app config to a KItem. The set the `triggerUponUpload` must be set to `True` so that the app is triggered automatically when we upload an attachment.\n", - "\n", - "Additionally, we must tell the file extension for which the upload shall be triggered. Here it is `.csv`.\n", - "\n", - "We also want to generate a qr code as avatar for the KItem with `avatar={\"include_qr\": True}`." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "item = KItem(\n", - " name=\"my tensile test experiment\",\n", - " ktype_id=dsms.ktypes.Dataset,\n", - " kitem_apps=[\n", - " {\n", - " \"executable\": appspec.name,\n", - " \"title\": \"data2rdf\",\n", - " \"additional_properties\": {\n", - " \"triggerUponUpload\": True,\n", - " \"triggerUponUploadFileExtensions\": [\".csv\"],\n", - " },\n", - " }\n", - " ],\n", - " avatar={\"include_qr\": True},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We commit the KItem:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we add our data with our attachment:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "item.attachments = [{\"name\": \"dummy_data.csv\", \"content\": data}]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And we commit again:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 6.2.2.2 Get results" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we can verify that the data extraction was successful:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "KItem(\n", - "\n", - "\tname = my tensile test experiment, \n", - "\n", - "\tid = 58683a2d-7823-4638-bc8e-91b461afa593, \n", - "\n", - "\tktype_id = dataset, \n", - "\n", - "\tin_backend = True, \n", - "\n", - "\tslug = mytensiletestexperiment-58683a2d, \n", - "\n", - "\tannotations = [], \n", - "\n", - "\tattachments = [\n", - "\t\t{\n", - "\t\t\tname: dummy_data.csv\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tlinked_kitems = [], \n", - "\n", - "\taffiliations = [], \n", - "\n", - "\tauthors = [\n", - "\t\t{\n", - "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tavatar_exists = True, \n", - "\n", - "\tcontacts = [], \n", - "\n", - "\tcreated_at = 2024-08-20 08:04:25.756982, \n", - "\n", - "\tupdated_at = 2024-08-20 08:04:25.756982, \n", - "\n", - "\texternal_links = [], \n", - "\n", - "\tkitem_apps = [\n", - "\t\t{\n", - "\t\t\tkitem_app_id: 22,\n", - "\t\t\texecutable: testapp2,\n", - "\t\t\ttitle: data2rdf,\n", - "\t\t\tdescription: None,\n", - "\t\t\ttags: None,\n", - "\t\t\tadditional_properties: {triggerUponUpload: True, triggerUponUploadFileExtensions: ['.csv']}\n", - "\t\t}\n", - "\t], \n", - "\n", - "\tsummary = None, \n", - "\n", - "\tuser_groups = [], \n", - "\n", - "\tcustom_properties = None, \n", - "\n", - "\tdataframe = [\n", - "\t\t{\n", - "\t\t\tcolumn_id: 0,\n", - "\t\t\tname: TestTime\n", - "\t\t}, \n", - "\t\t{\n", - "\t\t\tcolumn_id: 1,\n", - "\t\t\tname: StandardForce\n", - "\t\t}, \n", - "\t\t{\n", - "\t\t\tcolumn_id: 2,\n", - "\t\t\tname: AbsoluteCrossheadTravel\n", - "\t\t}\n", - "\t], \n", - "\n", - "\trdf_exists = True\n", - ")\n" - ] - } - ], - "source": [ - "print(item)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And also that the RDF generation was successful:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "@prefix csvw: .\n", - "@prefix dcat: .\n", - "@prefix dcterms: .\n", - "@prefix ns1: .\n", - "@prefix ns2: .\n", - "@prefix rdfs: .\n", - "@prefix xsd: .\n", - "\n", - " a dcat:Dataset ;\n", - " dcterms:hasPart ;\n", - " dcat:distribution [ a dcat:Distribution ;\n", - " dcat:accessURL \"https://bue.materials-data.space/api/knowledge/data_api/58683a2d-7823-4638-bc8e-91b461afa593\"^^xsd:anyURI ;\n", - " dcat:mediaType \"http://www.iana.org/assignments/media-types/text/csv\"^^xsd:anyURI ] .\n", - "\n", - " a ;\n", - " ns1:hasUnit \"http://qudt.org/vocab/unit/MilliM\"^^xsd:anyURI .\n", - "\n", - " a ;\n", - " ns1:hasUnit \"http://qudt.org/vocab/unit/KiloN\"^^xsd:anyURI .\n", - "\n", - " a ;\n", - " ns1:hasUnit \"http://qudt.org/vocab/unit/SEC\"^^xsd:anyURI .\n", - "\n", - " a csvw:TableGroup ;\n", - " csvw:table [ a csvw:Table ;\n", - " rdfs:label \"Time series data\" ;\n", - " csvw:tableSchema [ a csvw:Schema ;\n", - " csvw:column [ a csvw:Column ;\n", - " ns1:quantity ;\n", - " csvw:titles \"C\"^^xsd:string ;\n", - " ns2:page [ a ns2:Document ;\n", - " dcterms:format \"https://www.iana.org/assignments/media-types/application/json\"^^xsd:anyURI ;\n", - " dcterms:identifier \"https://bue.materials-data.space/api/knowledge/data_api/column-2\"^^xsd:anyURI ;\n", - " dcterms:type \"http://purl.org/dc/terms/Dataset\"^^xsd:anyURI ] ],\n", - " [ a csvw:Column ;\n", - " ns1:quantity ;\n", - " csvw:titles \"B\"^^xsd:string ;\n", - " ns2:page [ a ns2:Document ;\n", - " dcterms:format \"https://www.iana.org/assignments/media-types/application/json\"^^xsd:anyURI ;\n", - " dcterms:identifier \"https://bue.materials-data.space/api/knowledge/data_api/column-1\"^^xsd:anyURI ;\n", - " dcterms:type \"http://purl.org/dc/terms/Dataset\"^^xsd:anyURI ] ],\n", - " [ a csvw:Column ;\n", - " ns1:quantity ;\n", - " csvw:titles \"A\"^^xsd:string ;\n", - " ns2:page [ a ns2:Document ;\n", - " dcterms:format \"https://www.iana.org/assignments/media-types/application/json\"^^xsd:anyURI ;\n", - " dcterms:identifier \"https://bue.materials-data.space/api/knowledge/data_api/column-0\"^^xsd:anyURI ;\n", - " dcterms:type \"http://purl.org/dc/terms/Dataset\"^^xsd:anyURI ] ] ] ] .\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(item.subgraph.serialize())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And now we are able to convert our data into any compatiable unit we want. For the `StandardForce`, it was previously `kN`, but we want to have it in `N` now:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[1300.0,\n", - " 1800.0,\n", - " 2100.0,\n", - " 2600.0,\n", - " 3200.0,\n", - " 3700.0,\n", - " 4300.0,\n", - " 4800.0,\n", - " 5300.0,\n", - " 6000.0]" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item.dataframe.StandardForce.convert_to(\"N\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 6.2.2.3 Manipulate dataframe" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We are able to retrieve the dataframe as pd.DataFrame:" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
TestTimeStandardForceAbsoluteCrossheadTravel
01.21.31.5
11.71.81.9
22.02.12.3
32.52.62.8
43.03.23.4
53.63.73.9
64.14.34.4
74.74.85.0
85.25.35.5
95.86.06.1
\n", - "
" - ], - "text/plain": [ - " TestTime StandardForce AbsoluteCrossheadTravel\n", - "0 1.2 1.3 1.5\n", - "1 1.7 1.8 1.9\n", - "2 2.0 2.1 2.3\n", - "3 2.5 2.6 2.8\n", - "4 3.0 3.2 3.4\n", - "5 3.6 3.7 3.9\n", - "6 4.1 4.3 4.4\n", - "7 4.7 4.8 5.0\n", - "8 5.2 5.3 5.5\n", - "9 5.8 6.0 6.1" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "item.dataframe.to_df()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We are able to overwrite the dataframe with new data:" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "item.dataframe = {\n", - " \"TestTime\": list(range(100)),\n", - " \"StandardForce\": list(range(1,101)),\n", - " \"AbsoluteCrossheadTravel\": list(range(2,102))\n", - "}\n", - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We are able to retrieve the data colum-wise:" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "column: TestTime ,\n", - " data: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]\n", - "column: StandardForce ,\n", - " data: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100]\n", - "column: AbsoluteCrossheadTravel ,\n", - " data: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101]\n" - ] - } - ], - "source": [ - "for column in item.dataframe:\n", - " print(\"column:\", column.name, \",\\n\", \"data:\", column.get())\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... and also to modify the dataframe directly as we need:" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "new_df = item.dataframe.to_df().drop(['TestTime'], axis=1)\n", - "item.dataframe = new_df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 6.2.2.4 Run app on demand" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We are able to run the app on demand, not being triggered automatically during the upload of an attachment every time. For this purpose, we just need to refer to the name of the app we assigned during the KItem creation ( here it is simply `data2rdf`).\n", - "\n", - "Additionally, we need to tell the `attachment_name` and hand over the access token and host url to the app by explicitly setting `expose_sdk_config` to `True`. This is basically telling the SDK that the app is using the SDK also internally and that the app should receive its parameters from the current SDK session.\n", - "\n", - "The app is running synchronously, hence the `job` is created when the pipeline run finished." - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "job = item.kitem_apps.by_title[\"data2rdf\"].run(\n", - " attachment_name=item.attachments[0].name,\n", - " expose_sdk_config=True\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We are able to retrieve the job status:" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "JobStatus(phase='Succeeded', estimated_duration=None, finished_at='08/20/2024, 08:05:35', started_at='08/20/2024, 08:05:15', message=None, progress='1/1')" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "job.status" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... and the job logs:" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\"[2024-08-20 08:05:23,481 - dsms_data2rdf.main - INFO]: Fetch KItem: \\n KItem(\\n\\n\\tname = my tensile test experiment, \\n\\n\\tid = 58683a2d-7823-4638-bc8e-91b461afa593, \\n\\n\\tktype_id = dataset, \\n\\n\\tin_backend = True, \\n\\n\\tslug = mytensiletestexperiment-58683a2d, \\n\\n\\tannotations = [], \\n\\n\\tattachments = [\\n\\t\\t{\\n\\t\\t\\tname: dummy_data.csv\\n\\t\\t}\\n\\t], \\n\\n\\tlinked_kitems = [], \\n\\n\\taffiliations = [], \\n\\n\\tauthors = [\\n\\t\\t{\\n\\t\\t\\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\\n\\t\\t}\\n\\t], \\n\\n\\tavatar_exists = True, \\n\\n\\tcontacts = [], \\n\\n\\tcreated_at = 2024-08-20 08:04:25.756982, \\n\\n\\tupdated_at = 2024-08-20 08:04:25.756982, \\n\\n\\texternal_links = [], \\n\\n\\tkitem_apps = [\\n\\t\\t{\\n\\t\\t\\tkitem_app_id: 22,\\n\\t\\t\\texecutable: testapp2,\\n\\t\\t\\ttitle: data2rdf,\\n\\t\\t\\tdescription: None,\\n\\t\\t\\ttags: None,\\n\\t\\t\\tadditional_properties: {triggerUponUpload: True, triggerUponUploadFileExtensions: ['.csv']}\\n\\t\\t}\\n\\t], \\n\\n\\tsummary = None, \\n\\n\\tuser_groups = [], \\n\\n\\tcustom_properties = None, \\n\\n\\tdataframe = [\\n\\t\\t{\\n\\t\\t\\tcolumn_id: 0,\\n\\t\\t\\tname: TestTime\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 1,\\n\\t\\t\\tname: StandardForce\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 2,\\n\\t\\t\\tname: AbsoluteCrossheadTravel\\n\\t\\t}\\n\\t], \\n\\n\\trdf_exists = True\\n)\\n[2024-08-20 08:05:23,494 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser arguments: {'metadata_sep': ',', 'metadata_length': '0', 'time_series_sep': ',', 'time_series_header_length': '1', 'drop_na': 'false', 'fillna': ''}\\n[2024-08-20 08:05:23,494 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser: Parser.csv\\n[2024-08-20 08:05:23,494 - dsms_data2rdf.main - INFO]: Run pipeline with the following config: {'base_iri': 'https://bue.materials-data.space/58683a2d-7823-4638-bc8e-91b461afa593', 'data_download_uri': 'https://bue.materials-data.space/api/knowledge/data_api/58683a2d-7823-4638-bc8e-91b461afa593', 'graph_identifier': 'https://bue.materials-data.space/58683a2d-7823-4638-bc8e-91b461afa593', 'separator': '/', 'encoding': 'utf-8'}\\n[2024-08-20 08:05:28,419 - dsms_data2rdf.main - INFO]: Pipeline did detect any metadata. Will not make annotations for KItem\\n[2024-08-20 08:05:28,810 - dsms_data2rdf.main - INFO]: Checking that dataframe is up to date.\\n[2024-08-20 08:05:28,810 - dsms_data2rdf.main - INFO]: Dataframe upload was successful after 0 retries.\\n[2024-08-20 08:05:28,810 - dsms_data2rdf.main - INFO]: Done!\\n\"\n" - ] - } - ], - "source": [ - "print(job.logs)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In case we would like to run the job in the background, we simply add a `wait=False`:" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "job = item.kitem_apps.by_title[\"data2rdf\"].run(\n", - " attachment_name=item.attachments[0].name,\n", - " expose_sdk_config=True,\n", - " wait=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We are able to monitor the job status and logs asynchronously:" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " Current status:\n", - "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", - "\n", - " Current logs:\n", - "\"\"\n", - "\n", - " Current status:\n", - "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", - "\n", - " Current logs:\n", - "\"\"\n", - "\n", - " Current status:\n", - "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", - "\n", - " Current logs:\n", - "\"\"\n", - "\n", - " Current status:\n", - "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", - "\n", - " Current logs:\n", - "\"\"\n", - "\n", - " Current status:\n", - "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", - "\n", - " Current logs:\n", - "\"\"\n", - "\n", - " Current status:\n", - "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", - "\n", - " Current logs:\n", - "\"\"\n", - "\n", - " Current status:\n", - "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", - "\n", - " Current logs:\n", - "\"\"\n", - "\n", - " Current status:\n", - "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", - "\n", - " Current logs:\n", - "\"\"\n", - "\n", - " Current status:\n", - "phase='Succeeded' estimated_duration=None finished_at='08/20/2024, 08:05:59' started_at='08/20/2024, 08:05:39' message=None progress='1/1'\n", - "\n", - " Current logs:\n", - "\"[2024-08-20 08:05:45,472 - dsms_data2rdf.main - INFO]: Fetch KItem: \\n KItem(\\n\\n\\tname = my tensile test experiment, \\n\\n\\tid = 58683a2d-7823-4638-bc8e-91b461afa593, \\n\\n\\tktype_id = dataset, \\n\\n\\tin_backend = True, \\n\\n\\tslug = mytensiletestexperiment-58683a2d, \\n\\n\\tannotations = [], \\n\\n\\tattachments = [\\n\\t\\t{\\n\\t\\t\\tname: dummy_data.csv\\n\\t\\t}\\n\\t], \\n\\n\\tlinked_kitems = [], \\n\\n\\taffiliations = [], \\n\\n\\tauthors = [\\n\\t\\t{\\n\\t\\t\\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\\n\\t\\t}\\n\\t], \\n\\n\\tavatar_exists = True, \\n\\n\\tcontacts = [], \\n\\n\\tcreated_at = 2024-08-20 08:04:25.756982, \\n\\n\\tupdated_at = 2024-08-20 08:04:25.756982, \\n\\n\\texternal_links = [], \\n\\n\\tkitem_apps = [\\n\\t\\t{\\n\\t\\t\\tkitem_app_id: 22,\\n\\t\\t\\texecutable: testapp2,\\n\\t\\t\\ttitle: data2rdf,\\n\\t\\t\\tdescription: None,\\n\\t\\t\\ttags: None,\\n\\t\\t\\tadditional_properties: {triggerUponUpload: True, triggerUponUploadFileExtensions: ['.csv']}\\n\\t\\t}\\n\\t], \\n\\n\\tsummary = None, \\n\\n\\tuser_groups = [], \\n\\n\\tcustom_properties = None, \\n\\n\\tdataframe = [\\n\\t\\t{\\n\\t\\t\\tcolumn_id: 0,\\n\\t\\t\\tname: TestTime\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 1,\\n\\t\\t\\tname: StandardForce\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 2,\\n\\t\\t\\tname: AbsoluteCrossheadTravel\\n\\t\\t}\\n\\t], \\n\\n\\trdf_exists = True\\n)\\n[2024-08-20 08:05:45,485 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser arguments: {'metadata_sep': ',', 'metadata_length': '0', 'time_series_sep': ',', 'time_series_header_length': '1', 'drop_na': 'false', 'fillna': ''}\\n[2024-08-20 08:05:45,485 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser: Parser.csv\\n[2024-08-20 08:05:45,485 - dsms_data2rdf.main - INFO]: Run pipeline with the following config: {'base_iri': 'https://bue.materials-data.space/58683a2d-7823-4638-bc8e-91b461afa593', 'data_download_uri': 'https://bue.materials-data.space/api/knowledge/data_api/58683a2d-7823-4638-bc8e-91b461afa593', 'graph_identifier': 'https://bue.materials-data.space/58683a2d-7823-4638-bc8e-91b461afa593', 'separator': '/', 'encoding': 'utf-8'}\\n[2024-08-20 08:05:50,422 - dsms_data2rdf.main - INFO]: Pipeline did detect any metadata. Will not make annotations for KItem\\n[2024-08-20 08:05:50,816 - dsms_data2rdf.main - INFO]: Checking that dataframe is up to date.\\n[2024-08-20 08:05:50,816 - dsms_data2rdf.main - INFO]: Dataframe upload was successful after 0 retries.\\n[2024-08-20 08:05:50,816 - dsms_data2rdf.main - INFO]: Done!\\n\"\n" - ] - } - ], - "source": [ - "while True:\n", - " time.sleep(1)\n", - " print(\"\\n Current status:\")\n", - " print(job.status)\n", - " print(\"\\n Current logs:\")\n", - " print(job.logs)\n", - " if job.status.phase != \"Running\":\n", - " break" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**IMPORTANT**: When job has run asychronously (in the background), we need to manually refresh the KItem afterwards:" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "item.refresh()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Clean up the DSMS from the tutorial" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [], - "source": [ - "del dsms[item]\n", - "del dsms[appspec]\n", - "dsms.commit()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sdk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/tutorials/7_ktypes.ipynb b/examples/tutorials/7_ktypes.ipynb deleted file mode 100644 index 633ef62..0000000 --- a/examples/tutorials/7_ktypes.ipynb +++ /dev/null @@ -1,423 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 7. Interact with KTypes through the SDK\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7.1. Setting up\n", - "\n", - "Import the needed classes and functions." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "from dsms import DSMS, KType" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now source the environmental variables from an `.env` file and start the DSMS-session." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "dsms = DSMS(env=\"../../.env\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7.2. Create KTypes\n", - "\n", - "New KTypes can be created by a simple class inititation." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Batch(\n", - "\tid=batch,\n", - "\tname=Batch,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=None,\n", - "\tupdated_at=None\n", - ")" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ktype = KType( \n", - " id='batch',\n", - " name='Batch'\n", - ")\n", - "\n", - "ktype" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `commit` method should be executed to synchronize the changes with the DSMS SDK." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `type` object will automatically get updated with after the commit." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Batch(\n", - "\tid=batch,\n", - "\tname=Batch,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-10-01T14:21:01.164121,\n", - "\tupdated_at=2024-10-01T14:21:01.164121\n", - ")" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ktype" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7.3. Update KTypes" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The ktype object can be fetched using its id." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Batch(\n", - "\tid=batch,\n", - "\tname=Batch,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-10-01T14:21:01.164121,\n", - "\tupdated_at=2024-10-01T14:21:01.164121\n", - ")" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ktype = dsms.ktypes.Batch\n", - "ktype" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can change e.g. the name of the ktype" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "ktype.name = 'Batches'\n", - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After the committing, we can see the changes of the ktype" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Batch(\n", - "\tid=batch,\n", - "\tname=Batches,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-10-01T14:21:01.164121,\n", - "\tupdated_at=2024-10-01T14:21:12.169973\n", - ")" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ktype" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7.5. Fetching KTypes\n", - "\n", - "The existing KTypes can be fetched as follows." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Organization(\n", - "\tid=organization,\n", - "\tname=None,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-06T13:01:42.756035,\n", - "\tupdated_at=2024-05-06T13:01:42.756035\n", - ")\n", - "App(\n", - "\tid=app,\n", - "\tname=None,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-06T13:01:42.756035,\n", - "\tupdated_at=2024-05-06T13:01:42.756035\n", - ")\n", - "Dataset(\n", - "\tid=dataset,\n", - "\tname=dataset,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-06T13:01:42.756035,\n", - "\tupdated_at=2024-05-06T13:01:42.756035\n", - ")\n", - "DatasetCatalog(\n", - "\tid=dataset-catalog,\n", - "\tname=dataset catalog,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-06T13:01:42.756035,\n", - "\tupdated_at=2024-05-06T13:01:42.756035\n", - ")\n", - "Expert(\n", - "\tid=expert,\n", - "\tname=expert,\n", - "\twebform={\n", - "\t\tname=dict(annotation=Union[str, NoneType] required=False default=None),\n", - "\t\tskills=dict(annotation=Union[str, NoneType] required=False default=None)\n", - "\t},\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-06T13:01:42.756035,\n", - "\tupdated_at=2024-05-06T17:09:46.125058\n", - ")\n", - "Test(\n", - "\tid=test,\n", - "\tname=test,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-14T07:45:38.051796,\n", - "\tupdated_at=2024-05-14T07:45:38.051796\n", - ")\n", - "Specimen(\n", - "\tid=specimen,\n", - "\tname=Specimen,\n", - "\twebform={\n", - "\t\tsampletype=dict(annotation=Union[str, NoneType] required=False default=None),\n", - "\t\tsampleinformation=dict(annotation=Union[str, NoneType] required=False default=None),\n", - "\t\tsampleproductionprocess=dict(annotation=Union[str, NoneType] required=False default=None)\n", - "\t},\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-05-22T07:19:24.079365,\n", - "\tupdated_at=2024-05-22T07:20:08.774884\n", - ")\n", - "Resource(\n", - "\tid=resource,\n", - "\tname=Resource,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-06-10T10:17:31.071959,\n", - "\tupdated_at=2024-06-10T10:17:40.156104\n", - ")\n", - "TestingMachine(\n", - "\tid=testing-machine,\n", - "\tname=Testing Machine,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-08-19T17:13:57.534570,\n", - "\tupdated_at=2024-08-19T18:11:40.465640\n", - ")\n", - "Batch(\n", - "\tid=batch,\n", - "\tname=Batches,\n", - "\twebform=None,\n", - "\tjson_schema=None,\n", - "\trdf_mapping=None,\n", - "\tcreated_at=2024-10-01T14:21:01.164121,\n", - "\tupdated_at=2024-10-01T14:21:12.169973\n", - ")\n" - ] - } - ], - "source": [ - "for ktype in dsms.ktypes:\n", - " print(ktype)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7.4. Delete KTypes" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The fetched ktype can be deleted by applying the `del`-operator to the `dsms` object with the individual `KType` object." - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "del dsms[ktype]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As always, commit the changes." - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The available KTypes in the SDK can be fetched from an enum list." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "sdk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/tutorials/testfile.txt b/examples/tutorials/testfile.txt deleted file mode 100644 index 7976166..0000000 --- a/examples/tutorials/testfile.txt +++ /dev/null @@ -1 +0,0 @@ -This is a calibration protocol! From a3b7c2e9fc3c198933f40c43781769e367e4fce7 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Thu, 24 Oct 2024 12:23:48 +0200 Subject: [PATCH 24/66] Added pylint fixes --- dsms/knowledge/webform.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index d0fcf96..b77782f 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -18,7 +18,9 @@ class Inputs(BaseModel): """Input fields in the sections in webform""" model_config = ConfigDict( - alias_generator=lambda field_name: to_camel(field_name) + alias_generator=lambda field_name: to_camel( # pylint: disable=W0108 + field_name + ) ) id: Optional[str] = Field(None) @@ -56,6 +58,11 @@ class Sections(BaseModel): class Webform(BaseModel): """User defined webform for ktype""" + model_config = ConfigDict( + alias_generator=lambda field_name: to_camel( # pylint: disable=W0108 + field_name + ) + ) semantics_enabled: Optional[bool] = Field(False) rdf_type: Optional[str] = Field(None) sections: List[Sections] = Field([]) From aac1be43a6891161b8680c9c2872efe3724d215c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Thu, 21 Nov 2024 16:17:04 +0100 Subject: [PATCH 25/66] adapt sdk to new platform backend --- dsms/knowledge/kitem.py | 10 +++++++++- dsms/knowledge/properties/annotations.py | 2 +- dsms/knowledge/utils.py | 13 +++++++------ tests/conftest.py | 2 +- tests/test_utils.py | 8 ++++---- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 39067a5..c280486 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -191,6 +191,14 @@ class KItem(BaseModel): default_factory=Avatar, description="KItem avatar interface" ) + access_url: Optional[str] = Field( + None, description="Access URL of the KItem" + ) + + context_id: Optional[Union[UUID, str]] = Field( + None, description="Context ID of the KItem" + ) + model_config = ConfigDict( extra="forbid", validate_assignment=True, @@ -610,7 +618,7 @@ def context(cls) -> "Context": @property def url(cls) -> str: """URL of the KItem""" - return urljoin( + return cls.access_url or urljoin( str(cls.context.dsms.config.host_url), f"knowledge/{cls._get_ktype_as_str()}/{cls.slug}", ) diff --git a/dsms/knowledge/properties/annotations.py b/dsms/knowledge/properties/annotations.py index b3be8f2..a04d699 100644 --- a/dsms/knowledge/properties/annotations.py +++ b/dsms/knowledge/properties/annotations.py @@ -15,7 +15,7 @@ class Annotation(KItemProperty): """KItem annotation model""" iri: str = Field(..., description="IRI of the annotation") - name: str = Field(..., description="Name of the annotation") + label: str = Field(..., description="Label of the annotation") namespace: str = Field(..., description="Namespace of the annotation") description: Optional[str] = Field( None, description="Description of the annotation" diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 277528e..ddd763c 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -284,6 +284,7 @@ def _update_kitem(new_kitem: "KItem", old_kitem: "Dict[str, Any]") -> Response: "created_at", "external_links", "dataframe", + "access_url", }, exclude_none=True, ) @@ -637,8 +638,8 @@ def _split_iri(iri: str) -> List[str]: def _make_annotation_schema(iri: str) -> Dict[str, Any]: - namespace, name = _split_iri(iri) - return {"namespace": namespace, "name": name, "iri": iri} + namespace, label = _split_iri(iri) + return {"namespace": namespace, "label": label, "iri": iri} def _search( @@ -703,7 +704,7 @@ def _get_dataframe_column(kitem_id: str, column_id: int) -> List[Any]: """Download the column of a dataframe container of a certain kitem""" response = _perform_request( - f"api/knowledge/data_api/{kitem_id}/column-{column_id}", "get" + f"api/knowledge/data/{kitem_id}/column-{column_id}", "get" ) if not response.ok: message = f"""Something went wrong fetch column id `{column_id}` @@ -714,7 +715,7 @@ def _get_dataframe_column(kitem_id: str, column_id: int) -> List[Any]: def _inspect_dataframe(kitem_id: str) -> Optional[List[Dict[str, Any]]]: """Get column info for the dataframe container of a certain kitem""" - response = _perform_request(f"api/knowledge/data_api/{kitem_id}", "get") + response = _perform_request(f"api/knowledge/data/{kitem_id}", "get") if not response.ok and response.status_code == 404: dataframe = None elif not response.ok and response.status_code != 404: @@ -734,7 +735,7 @@ def _update_dataframe(kitem_id: str, data: pd.DataFrame): data.to_json(buffer, indent=2) buffer.seek(0) response = _perform_request( - f"api/knowledge/data_api/{kitem_id}", "put", files={"data": buffer} + f"api/knowledge/data/{kitem_id}", "put", files={"data": buffer} ) if not response.ok: raise RuntimeError( @@ -744,7 +745,7 @@ def _update_dataframe(kitem_id: str, data: pd.DataFrame): def _delete_dataframe(kitem_id: str) -> Response: logger.debug("Delete DataFrame for kitem with id `%s`.", kitem_id) - return _perform_request(f"api/knowledge/data_api/{kitem_id}", "delete") + return _perform_request(f"api/knowledge/data/{kitem_id}", "delete") def _commit_avatar(kitem) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index e2f16d7..14c858a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -150,7 +150,7 @@ def _get_kitems() -> "Dict[str, Any]": def _get_dataframe() -> "Dict[str, Any]": return { - urljoin(custom_address, f"api/knowledge/data_api/{uid}"): [ + urljoin(custom_address, f"api/knowledge/data/{uid}"): [ { "method": responses.GET, "returns": { diff --git a/tests/test_utils.py b/tests/test_utils.py index 485c474..1d7d848 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -61,7 +61,7 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address): "annotations": [ { "iri": "http://example.org/", - "name": "bar", + "label": "bar", "namespace": "example", "description": None, } @@ -99,7 +99,7 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address): annotations=[ { "iri": "http://example.org/", - "name": "foo", + "label": "foo", "namespace": "example", } ], @@ -115,7 +115,7 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address): "annotations_to_link": [ { "iri": "http://example.org/", - "name": "foo", + "label": "foo", "namespace": "example", "description": None, } @@ -136,7 +136,7 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address): "annotations_to_unlink": [ { "iri": "http://example.org/", - "name": "bar", + "label": "bar", "namespace": "example", "description": None, } From aa4f3e6d611b40dd81d81a6774e124558adbfd61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Thu, 21 Nov 2024 16:27:07 +0100 Subject: [PATCH 26/66] Bump version v2.0.4 -> v2.1.0dev0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 2fdb9b2..e4bb6fa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.0.4 +version = v2.1.0dev0 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From 7f4bff9fe2bf6e9576c6bf6d27319fdc93d13f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Thu, 21 Nov 2024 16:33:14 +0100 Subject: [PATCH 27/66] make upper restriction for pydantic --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e4bb6fa..406700f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ install_requires = html5lib>=1,<2 lru-cache<1 pandas>=2,<3 - pydantic>=2 + pydantic>=2,<2.8 pydantic-settings python-dotenv qrcode-artistic>=3,<4 From 761c3e31a5e810289093a174e0cb291b89d838ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 27 Nov 2024 13:33:01 +0100 Subject: [PATCH 28/66] set max length of string-field names --- dsms/knowledge/kitem.py | 9 +++++++-- dsms/knowledge/ktype.py | 6 ++++-- dsms/knowledge/properties/affiliations.py | 4 +++- dsms/knowledge/properties/annotations.py | 13 +++++++------ dsms/knowledge/properties/apps.py | 12 +++++++++--- dsms/knowledge/properties/attachments.py | 4 +++- dsms/knowledge/properties/authors.py | 5 ++++- dsms/knowledge/properties/external_links.py | 4 +++- dsms/knowledge/properties/summary.py | 5 ++++- dsms/knowledge/properties/user_groups.py | 8 ++++++-- tests/test_utils.py | 3 --- 11 files changed, 50 insertions(+), 23 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index c280486..fe12e95 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -116,7 +116,9 @@ class KItem(BaseModel): """ # public - name: str = Field(..., description="Human readable name of the KItem") + name: str = Field( + ..., description="Human readable name of the KItem", max_length=300 + ) id: Optional[UUID] = Field( default_factory=uuid4, description="ID of the KItem", @@ -127,7 +129,10 @@ class KItem(BaseModel): description="Whether the KItem was already created in the backend.", ) slug: Optional[str] = Field( - None, description="Slug of the KContext.dsms", min_length=4 + None, + description="Slug of the KContext.dsms", + min_length=4, + max_length=1000, ) annotations: List[Union[str, Annotation]] = Field( [], description="Annotations of the KItem" diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index c1a396e..b6d95db 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -11,9 +11,11 @@ class KType(BaseModel): """Knowledge type of the knowledge item.""" - id: Union[UUID, str] = Field(..., description="ID of the KType.") + id: Union[UUID, str] = Field( + ..., description="ID of the KType.", max_length=50 + ) name: Optional[str] = Field( - None, description="Human readable name of the KType." + None, description="Human readable name of the KType.", max_length=50 ) webform: Optional[Any] = Field(None, description="Form data of the KItem.") json_schema: Optional[Any] = Field( diff --git a/dsms/knowledge/properties/affiliations.py b/dsms/knowledge/properties/affiliations.py index bb0297c..1153cad 100644 --- a/dsms/knowledge/properties/affiliations.py +++ b/dsms/knowledge/properties/affiliations.py @@ -14,7 +14,9 @@ class Affiliation(KItemProperty): """Affiliation of a KItem.""" - name: str = Field(..., description="Name of the affiliation") + name: str = Field( + ..., description="Name of the affiliation", max_length=100 + ) class AffiliationsProperty(KItemPropertyList): diff --git a/dsms/knowledge/properties/annotations.py b/dsms/knowledge/properties/annotations.py index a04d699..0f7a563 100644 --- a/dsms/knowledge/properties/annotations.py +++ b/dsms/knowledge/properties/annotations.py @@ -1,6 +1,6 @@ """Annotation property of a KItem""" -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from pydantic import Field @@ -14,11 +14,12 @@ class Annotation(KItemProperty): """KItem annotation model""" - iri: str = Field(..., description="IRI of the annotation") - label: str = Field(..., description="Label of the annotation") - namespace: str = Field(..., description="Namespace of the annotation") - description: Optional[str] = Field( - None, description="Description of the annotation" + iri: str = Field(..., description="IRI of the annotation", max_length=200) + label: str = Field( + ..., description="Label of the annotation", max_length=100 + ) + namespace: str = Field( + ..., description="Namespace of the annotation", max_length=100 ) diff --git a/dsms/knowledge/properties/apps.py b/dsms/knowledge/properties/apps.py index d0253e1..872ee86 100644 --- a/dsms/knowledge/properties/apps.py +++ b/dsms/knowledge/properties/apps.py @@ -64,11 +64,17 @@ class App(KItemProperty): None, description="ID of the KItem App" ) executable: str = Field( - ..., description="Name of the executable related to the app" + ..., + description="Name of the executable related to the app", + max_length=400, + ) + title: str = Field( + ..., description="Title of the appilcation", max_length=50 ) - title: str = Field(..., description="Title of the appilcation") description: Optional[str] = Field( - None, description="Description of the appilcation" + None, + description="Description of the appilcation", + max_length=1000, ) tags: Optional[dict] = Field( None, description="Tags related to the appilcation" diff --git a/dsms/knowledge/properties/attachments.py b/dsms/knowledge/properties/attachments.py index 10df1e7..175d858 100644 --- a/dsms/knowledge/properties/attachments.py +++ b/dsms/knowledge/properties/attachments.py @@ -16,7 +16,9 @@ class Attachment(KItemProperty): """Attachment uploaded by a certain user.""" - name: str = Field(..., description="File name of the attachment") + name: str = Field( + ..., description="File name of the attachment", max_length=100 + ) content: Optional[Union[str, bytes]] = Field( None, description="Content of the file" diff --git a/dsms/knowledge/properties/authors.py b/dsms/knowledge/properties/authors.py index 66738f3..473297d 100644 --- a/dsms/knowledge/properties/authors.py +++ b/dsms/knowledge/properties/authors.py @@ -14,7 +14,10 @@ class Author(KItemProperty): """Author of a KItem.""" - user_id: UUID = Field(..., description="ID of the DSMS User") + user_id: UUID = Field( + ..., + description="ID of the DSMS User", + ) # OVERRIDE @model_serializer diff --git a/dsms/knowledge/properties/external_links.py b/dsms/knowledge/properties/external_links.py index 937d6a7..45ab338 100644 --- a/dsms/knowledge/properties/external_links.py +++ b/dsms/knowledge/properties/external_links.py @@ -13,7 +13,9 @@ class ExternalLink(KItemProperty): """External link of a KItem.""" - label: str = Field(..., description="Label of the external link") + label: str = Field( + ..., description="Label of the external link", max_length=50 + ) url: AnyUrl = Field(..., description="URL of the external link") diff --git a/dsms/knowledge/properties/summary.py b/dsms/knowledge/properties/summary.py index ae694a0..834a8c7 100644 --- a/dsms/knowledge/properties/summary.py +++ b/dsms/knowledge/properties/summary.py @@ -15,7 +15,10 @@ class Summary(BaseModel): """Model for the custom properties of the KItem""" - id: Optional[UUID] = Field(None, description="ID of the KItem") + id: Optional[UUID] = Field( + None, + description="ID of the KItem", + ) text: str = Field(..., description="Summary text of the KItem") kitem: Optional[Any] = Field( None, description="KItem related to the summary", exclude=True diff --git a/dsms/knowledge/properties/user_groups.py b/dsms/knowledge/properties/user_groups.py index 4c5fe7a..930b162 100644 --- a/dsms/knowledge/properties/user_groups.py +++ b/dsms/knowledge/properties/user_groups.py @@ -13,8 +13,12 @@ class UserGroup(KItemProperty): """Users groups related to a KItem.""" - name: str = Field(..., description="Name of the user group") - group_id: str = Field(..., description="ID of the user group") + name: str = Field( + ..., description="Name of the user group", max_length=100 + ) + group_id: str = Field( + ..., description="ID of the user group", max_length=100 + ) class UserGroupsProperty(KItemPropertyList): diff --git a/tests/test_utils.py b/tests/test_utils.py index 1d7d848..9535a90 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -63,7 +63,6 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address): "iri": "http://example.org/", "label": "bar", "namespace": "example", - "description": None, } ], "linked_kitems": [ @@ -117,7 +116,6 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address): "iri": "http://example.org/", "label": "foo", "namespace": "example", - "description": None, } ], "user_groups_to_add": [], @@ -138,7 +136,6 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address): "iri": "http://example.org/", "label": "bar", "namespace": "example", - "description": None, } ], "user_groups_to_remove": [], From 8796d2ed31e7a3001d1cee86b1a35e44d5bff639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 27 Nov 2024 13:51:46 +0100 Subject: [PATCH 29/66] bump dev version tag --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 406700f..4a64858 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev0 +version = v2.1.0dev1 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From 02ab43e86c264a293c7b066b48b4923b090fe2e1 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Mon, 2 Dec 2024 10:01:43 +0100 Subject: [PATCH 30/66] Support for webform changes --- dsms/__init__.py | 4 +- dsms/apps/config.py | 38 ++++----- dsms/core/__init__.py | 4 +- dsms/core/dsms.py | 16 ++-- dsms/core/{context.py => session.py} | 2 +- dsms/core/utils.py | 14 ++-- dsms/knowledge/kitem.py | 30 ++++---- dsms/knowledge/ktype.py | 37 ++++----- dsms/knowledge/properties/base.py | 18 ++--- dsms/knowledge/properties/linked_kitems.py | 4 +- dsms/knowledge/properties/summary.py | 10 +-- dsms/knowledge/semantics/queries/base.py | 4 +- dsms/knowledge/semantics/units/conversion.py | 8 +- dsms/knowledge/semantics/units/utils.py | 4 +- dsms/knowledge/utils.py | 37 ++++++--- dsms/knowledge/webform.py | 81 +++++++++++++++----- tests/conftest.py | 6 +- tests/test_kitem.py | 12 +-- 18 files changed, 191 insertions(+), 138 deletions(-) rename dsms/core/{context.py => session.py} (97%) diff --git a/dsms/__init__.py b/dsms/__init__.py index 1ab4066..a8e2834 100644 --- a/dsms/__init__.py +++ b/dsms/__init__.py @@ -2,9 +2,9 @@ from dsms.apps import AppConfig from dsms.core.configuration import Configuration -from dsms.core.context import Context +from dsms.core.session import Session from dsms.core.dsms import DSMS from dsms.knowledge.kitem import KItem from dsms.knowledge.ktype import KType -__all__ = ["DSMS", "Configuration", "Context", "KItem", "KType", "AppConfig"] +__all__ = ["DSMS", "Configuration", "Session", "KItem", "KType", "AppConfig"] diff --git a/dsms/apps/config.py b/dsms/apps/config.py index 946a178..13cecab 100644 --- a/dsms/apps/config.py +++ b/dsms/apps/config.py @@ -27,7 +27,7 @@ logger.propagate = False if TYPE_CHECKING: - from dsms import DSMS, Context + from dsms import DSMS, Session class AppConfig(BaseModel): @@ -76,15 +76,15 @@ def __init__(self, **kwargs: "Any") -> None: # add app config to buffer if ( not self.in_backend - and self.name not in self.context.buffers.created + and self.name not in self.session.buffers.created ): logger.debug( """Marking AppConfig with name `%s` as created and updated during AppConfig initialization.""", self.name, ) - self.context.buffers.created.update({self.name: self}) - self.context.buffers.updated.update({self.name: self}) + self.session.buffers.created.update({self.name: self}) + self.session.buffers.updated.update({self.name: self}) logger.debug("AppConfig initialization successful.") @@ -94,12 +94,12 @@ def __setattr__(self, name, value) -> None: logger.debug( "Setting property with key `%s` on KItem level: %s.", name, value ) - if self.name not in self.context.buffers.updated: + if self.name not in self.session.buffers.updated: logger.debug( "Setting AppConfig with name `%s` as updated during AppConfig.__setattr__", self.name, ) - self.context.buffers.updated.update({self.name: self}) + self.session.buffers.updated.update({self.name: self}) def __str__(self) -> str: """Pretty print the app config fields""" @@ -156,20 +156,20 @@ def validate_specification(cls, self: "AppConfig") -> str: raise RuntimeError( f"Invalid yaml specification path: `{error.args[0]}`" ) from error - self.context.buffers.updated.update({self.name: self}) + self.session.buffers.updated.update({self.name: self}) elif isinstance(self.specification, dict) and self.in_backend: spec = _get_app_specification(self.name) if ( not yaml.safe_load(spec) == self.specification - and self.name not in self.context.buffers.updated + and self.name not in self.session.buffers.updated ): - self.context.buffers.updated.update({self.name: self}) + self.session.buffers.updated.update({self.name: self}) elif ( isinstance(self.specification, dict) and not self.in_backend - and self.name not in self.context.buffers.updated + and self.name not in self.session.buffers.updated ): - self.context.buffers.updated.update({self.name: self}) + self.session.buffers.updated.update({self.name: self}) if self.expose_sdk_config: self.specification["spec"]["arguments"]["parameters"] += [ { @@ -190,20 +190,20 @@ def in_backend(self) -> bool: return _app_spec_exists(self.name) @property - def context(cls) -> "Context": - """Getter for Context""" + def session(cls) -> "Session": + """Getter for Session""" from dsms import ( # isort:skip - Context, + Session, ) - return Context + return Session @property def dsms(self) -> "DSMS": - """DSMS context getter""" - return self.context.dsms + """DSMS session getter""" + return self.session.dsms @dsms.setter def dsms(self, value: "DSMS") -> None: - """DSMS context setter""" - self.context.dsms = value + """DSMS session setter""" + self.session.dsms = value diff --git a/dsms/core/__init__.py b/dsms/core/__init__.py index 7705c83..ba8493f 100644 --- a/dsms/core/__init__.py +++ b/dsms/core/__init__.py @@ -1,6 +1,6 @@ """DSMS core module """ from dsms.core.configuration import Configuration -from dsms.core.context import Context +from dsms.core.session import Session -__all__ = ["Context", "Configuration"] +__all__ = ["Session", "Configuration"] diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index 9a0f92a..573c336 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -8,7 +8,7 @@ from dsms.apps.utils import _get_available_apps_specs from dsms.core.configuration import Configuration -from dsms.core.context import Context +from dsms.core.session import Session from dsms.core.utils import _ping_dsms from dsms.knowledge.sparql_interface import SparqlInterface from dsms.knowledge.utils import _search @@ -24,7 +24,7 @@ from typing import Optional, Union from dsms.apps import AppConfig - from dsms.core.context import Buffers + from dsms.core.session import Buffers from dsms.knowledge.kitem import KItem from dsms.knowledge.ktype import KType from dsms.knowledge.search import SearchResult @@ -49,7 +49,7 @@ class DSMS: """ - _context = Context + _session = Session def __init__( self, @@ -73,7 +73,7 @@ def __init__( self._config = None self._ktypes = None - self._context.dsms = self + self._session.dsms = self if env: if not os.path.exists(env): @@ -207,12 +207,12 @@ def app_configs(cls) -> "List[AppConfig]": @property def buffers(cls) -> "Buffers": """Return buffers of the DSMS session""" - return cls._context.buffers + return cls._session.buffers @property - def context(cls) -> "Context": - """Return DSMS context""" - return cls._context + def context(cls) -> "Session": + """Return DSMS session""" + return cls._session @classmethod def __get_pydantic_core_schema__(cls): diff --git a/dsms/core/context.py b/dsms/core/session.py similarity index 97% rename from dsms/core/context.py rename to dsms/core/session.py index d206526..447dcf3 100644 --- a/dsms/core/context.py +++ b/dsms/core/session.py @@ -20,7 +20,7 @@ class Buffers: deleted: "Dict[UUID, KItem]" = {} -class Context: +class Session: """Object giving the current DSMS context.""" dsms: "Optional[DSMS]" = None diff --git a/dsms/core/utils.py b/dsms/core/utils.py index d2382cb..9409d26 100644 --- a/dsms/core/utils.py +++ b/dsms/core/utils.py @@ -14,16 +14,16 @@ def _kitem_id2uri(kitem_id: UUID) -> str: "Convert a kitem id in the DSMS to the full resolvable URI" - from dsms import Context + from dsms import Session - return urljoin(str(Context.dsms.config.host_url), str(kitem_id)) + return urljoin(str(Session.dsms.config.host_url), str(kitem_id)) def _uri2kitem_idi(uri: str) -> str: "Extract the kitem id from an URI of the DSMS" - from dsms import Context + from dsms import Session - return uri.replace(f"{Context.dsms.config.host_url}/", "").split("/")[0] + return uri.replace(f"{Session.dsms.config.host_url}/", "").split("/")[0] def _ping_dsms(): @@ -35,9 +35,9 @@ def _perform_request(route: str, method: str, **kwargs: "Any") -> Response: """Perform a general request for a certain route and with a certain method. Kwargs are general arguments which can be passed to the `requests.request`-function. """ - from dsms import Context + from dsms import Session - dsms = Context.dsms + dsms = Session.dsms response = requests.request( method, url=urljoin(str(dsms.config.host_url), route), @@ -46,7 +46,7 @@ def _perform_request(route: str, method: str, **kwargs: "Any") -> Response: verify=dsms.config.ssl_verify, **kwargs, ) - response.encoding = Context.dsms.config.encoding + response.encoding = Session.dsms.config.encoding return response diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 540200b..bfaa076 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -63,7 +63,7 @@ from dsms.knowledge.sparql_interface.utils import _get_subgraph # isort:skip if TYPE_CHECKING: - from dsms import Context + from dsms import Session from dsms.core.dsms import DSMS logger = logging.getLogger(__name__) @@ -411,11 +411,11 @@ def validate_user_groups( @classmethod def validate_created(cls, value: str) -> Any: """Convert the str for `created_at` in to a `datetime`-object""" - from dsms import Context + from dsms import Session if isinstance(value, str): value = datetime.strptime( - value, Context.dsms.config.datetime_format + value, Session.dsms.config.datetime_format ) return value @@ -423,11 +423,11 @@ def validate_created(cls, value: str) -> Any: @classmethod def validate_updated(cls, value: str) -> Any: """Convert the str for `created_at` in to a `datetime`-object""" - from dsms import Context + from dsms import Session if isinstance(value, str): value = datetime.strptime( - value, Context.dsms.config.datetime_format + value, Session.dsms.config.datetime_format ) return value @@ -435,10 +435,10 @@ def validate_updated(cls, value: str) -> Any: @classmethod def validate_ktype_id(cls, value: Union[str, Enum]) -> KType: """Validate the ktype id of the KItem""" - from dsms import Context + from dsms import Session if isinstance(value, str): - value = Context.ktypes.get(value) + value = Session.ktypes.get(value) if not value: raise TypeError( f"KType for `ktype_id={value}` does not exist." @@ -452,13 +452,13 @@ def validate_ktype( cls, value: Optional[Union[KType, str, Enum]], info: ValidationInfo ) -> KType: """Validate the ktype of the KItem""" - from dsms import Context + from dsms import Session if not value: value = info.data.get("ktype_id") if isinstance(value, str): - value = Context.ktypes.get(value) + value = Session.ktypes.get(value) if not value: raise TypeError( f"KType for `ktype_id={value}` does not exist." @@ -479,7 +479,7 @@ def validate_in_backend(cls, value: bool, info: ValidationInfo) -> bool: @classmethod def validate_slug(cls, value: str, info: ValidationInfo) -> str: """Validate slug""" - from dsms import Context + from dsms import Session ktype_id = info.data["ktype_id"] kitem_id = info.data.get("id") @@ -499,7 +499,7 @@ def validate_slug(cls, value: str, info: ValidationInfo) -> str: raise ValueError( "Slug length must have a minimum length of 4." ) - if Context.dsms.config.individual_slugs: + if Session.dsms.config.individual_slugs: value += f"-{str(kitem_id).split('-', maxsplit=1)[0]}" if not kitem_exists and not _slug_is_available(ktype, value): raise ValueError(f"Slug for `{value}` is already taken.") @@ -615,13 +615,13 @@ def subgraph(cls) -> Optional[Graph]: ) @property - def context(cls) -> "Context": - """Getter for Context""" + def context(cls) -> "Session": + """Getter for Session""" from dsms import ( # isort:skip - Context, + Session, ) - return Context + return Session @property def url(cls) -> str: diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 1a2c7a7..6ef3e6c 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -2,7 +2,7 @@ import logging from datetime import datetime -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, List, Optional, Union from uuid import UUID from pydantic import BaseModel, Field, model_serializer, model_validator @@ -17,7 +17,7 @@ from dsms.knowledge.webform import Webform if TYPE_CHECKING: - from dsms import Context + from dsms import Session from dsms.core.dsms import DSMS logger = logging.getLogger(__name__) @@ -32,20 +32,21 @@ class KType(BaseModel): name: Optional[str] = Field( None, description="Human readable name of the KType." ) + context: Optional[bool] = Field(False, description="Identifies if the ktype is a context ktype") + context_schema: Optional[list] = Field([], description="Schema of the context ktype") webform: Optional[Webform] = Field( None, description="Form data of the KType." ) json_schema: Optional[Any] = Field( None, description="OpenAPI schema of the KType." ) - rdf_mapping: Optional[str] = Field(None, description="") created_at: Optional[Union[str, datetime]] = Field( None, description="Time and date when the KType was created." ) updated_at: Optional[Union[str, datetime]] = Field( None, description="Time and date when the KType was updated." ) - custom_properties: Optional[Any] = Field(None, description="") + custom_properties: Optional[Any] = Field(None, description="Additional custom properties for the KType.") def __hash__(self) -> int: return hash(str(self)) @@ -63,13 +64,13 @@ def __init__(self, **kwargs: "Any") -> None: super().__init__(**kwargs) # add ktype to buffer - if not self.in_backend and self.id not in self.context.buffers.created: + if not self.in_backend and self.id not in self.session.buffers.created: logger.debug( "Marking KTpe with ID `%s` as created and updated during KItem initialization.", self.id, ) - self.context.buffers.created.update({self.id: self}) - self.context.buffers.updated.update({self.id: self}) + self.session.buffers.created.update({self.id: self}) + self.session.buffers.updated.update({self.id: self}) logger.debug("KType initialization successful.") @@ -84,12 +85,12 @@ def __setattr__(self, name, value) -> None: value, ) - if self.id not in self.context.buffers.updated: + if self.id not in self.session.buffers.updated: logger.debug( "Setting KType with ID `%s` as updated during KType.__setattr__", self.id, ) - self.context.buffers.updated.update({self.id: self}) + self.session.buffers.updated.update({self.id: self}) def __repr__(self) -> str: """Print the KType""" @@ -106,22 +107,22 @@ def in_backend(self) -> bool: @property def dsms(cls) -> "DSMS": - """DSMS context getter""" - return cls.context.dsms + """DSMS session getter""" + return cls.session.dsms @dsms.setter def dsms(cls, value: "DSMS") -> None: - """DSMS context setter""" - cls.context.dsms = value + """DSMS session setter""" + cls.session.dsms = value @property - def context(cls) -> "Context": - """Getter for Context""" + def session(cls) -> "Session": + """Getter for Session""" from dsms import ( # isort:skip - Context, + Session, ) - return Context + return Session def refresh(self) -> None: """Refresh the KType""" @@ -133,7 +134,7 @@ def serialize(self): return { key: ( value.dict(exclude_none=False, by_alias=False) - if key == "webform" and not isinstance(value, dict) + if key == "webform" and value is not None and not isinstance(value, dict) else value ) for key, value in self.__dict__.items() diff --git a/dsms/knowledge/properties/base.py b/dsms/knowledge/properties/base.py index f764ae5..25076b7 100644 --- a/dsms/knowledge/properties/base.py +++ b/dsms/knowledge/properties/base.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: from typing import Any, Callable, Dict, Iterable, List, Set, Union - from dsms import Context, KItem + from dsms import Session, KItem class KItemProperty(BaseModel): @@ -94,13 +94,13 @@ def exclude(cls) -> "Optional[Set[str]]": return cls.model_config.get("exclude") @property - def context(cls) -> "Context": - """Getter for Context""" + def context(cls) -> "Session": + """Getter for Session""" from dsms import ( # isort:skip - Context, + Session, ) - return Context + return Session @model_serializer def serialize(self): @@ -270,13 +270,13 @@ def _mark_as_updated(self) -> None: self.context.buffers.updated.update({self._kitem.id: self._kitem}) @property - def context(cls) -> "Context": - """Getter for Context""" + def context(cls) -> "Session": + """Getter for Session""" from dsms import ( # isort:skip - Context, + Session, ) - return Context + return Session @property def kitem(cls) -> "KItem": diff --git a/dsms/knowledge/properties/linked_kitems.py b/dsms/knowledge/properties/linked_kitems.py index baf03d5..108a166 100644 --- a/dsms/knowledge/properties/linked_kitems.py +++ b/dsms/knowledge/properties/linked_kitems.py @@ -260,11 +260,11 @@ def by_annotation(self) -> "Dict[str, List[KItem]]": @property def by_ktype(self) -> "Dict[KType, List[KItem]]": """Get the kitems grouped by ktype""" - from dsms import Context + from dsms import Session grouped = {} for linked in self: - ktype = Context.dsms.ktypes[_name_to_camel(linked.ktype_id)] + ktype = Session.dsms.ktypes[_name_to_camel(linked.ktype_id)] if not ktype in grouped: grouped[ktype] = [] if not linked in grouped[ktype]: diff --git a/dsms/knowledge/properties/summary.py b/dsms/knowledge/properties/summary.py index ae694a0..40973fa 100644 --- a/dsms/knowledge/properties/summary.py +++ b/dsms/knowledge/properties/summary.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from typing import Set - from dsms import Context + from dsms import Session class Summary(BaseModel): @@ -60,13 +60,13 @@ def id(cls) -> Optional[UUID]: return cls.kitem.id # pylint: disable=E1101 @property - def context(cls) -> "Context": - """Getter for Context""" + def context(cls) -> "Session": + """Getter for Session""" from dsms import ( # isort:skip - Context, + Session, ) - return Context + return Session @property def exclude(cls) -> "Optional[Set[str]]": diff --git a/dsms/knowledge/semantics/queries/base.py b/dsms/knowledge/semantics/queries/base.py index 37f1b58..4cfdf1c 100644 --- a/dsms/knowledge/semantics/queries/base.py +++ b/dsms/knowledge/semantics/queries/base.py @@ -15,11 +15,11 @@ class BaseSparqlQuery(ABC): """Abstract class for DSMS Sparql Query""" def __init__(self, **kwargs: "Dict[str, Any]") -> None: - from dsms import Context + from dsms import Session self._kwargs = kwargs self._results: "Optional[Dict[str, Any]]" = None - self._dsms: "DSMS" = Context.dsms + self._dsms: "DSMS" = Session.dsms self.execute() diff --git a/dsms/knowledge/semantics/units/conversion.py b/dsms/knowledge/semantics/units/conversion.py index f6f0e98..9373699 100644 --- a/dsms/knowledge/semantics/units/conversion.py +++ b/dsms/knowledge/semantics/units/conversion.py @@ -119,13 +119,13 @@ def _get_factor_from_uri(uri: str) -> int: @lru_cache def _get_qudt_graph(ontology_ref: str) -> Graph: - from dsms import Context + from dsms import Session - url = getattr(Context.dsms.config, ontology_ref) - encoding = Context.dsms.config.encoding + url = getattr(Session.dsms.config, ontology_ref) + encoding = Session.dsms.config.encoding graph = Graph() - response = requests.get(url, timeout=Context.dsms.config.request_timeout) + response = requests.get(url, timeout=Session.dsms.config.request_timeout) if response.status_code != 200: raise RuntimeError( f"Could not download QUDT ontology. Please check URI: {url}" diff --git a/dsms/knowledge/semantics/units/utils.py b/dsms/knowledge/semantics/units/utils.py index 6870cca..0916da6 100644 --- a/dsms/knowledge/semantics/units/utils.py +++ b/dsms/knowledge/semantics/units/utils.py @@ -92,9 +92,9 @@ def get_property_unit( ValueError: If unable to retrieve the unit for the property due to any errors or if the property does not have a unit or has more than one unit associated with it. """ - from dsms import Context + from dsms import Session - units_sparql_object = Context.dsms.config.units_sparql_object + units_sparql_object = Session.dsms.config.units_sparql_object if not issubclass(units_sparql_object, BaseUnitSparqlQuery): raise TypeError( f"´{units_sparql_object}´ must be a subclass of `{BaseUnitSparqlQuery}`" diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index d3f50d5..6d57d1d 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -4,6 +4,9 @@ import logging import re import warnings +import time +import random +import string from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union @@ -35,7 +38,7 @@ if TYPE_CHECKING: from dsms.apps import AppConfig - from dsms.core.context import Buffers + from dsms.core.session import Buffers from dsms.knowledge import KItem, KType from dsms.knowledge.properties import Attachment @@ -51,6 +54,16 @@ def _is_number(value): except Exception: return False +def id_generator(prefix: str = "id") -> str: + """Generates a random string as id""" + + # Generate a unique part using time and random characters + unique_part = f"{int(time.time() * 1000)}" # Milliseconds since epoch + random_part = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) + # Combine prefix, unique part, and random part + generated_id = f"{prefix}{unique_part}{random_part}" + return generated_id + def _create_custom_properties_model( value: Optional[Dict[str, Any]] @@ -217,7 +230,7 @@ def print_ktype(self) -> str: def _get_remote_ktypes() -> Enum: """Get the KTypes from the remote backend""" from dsms import ( # isort:skip - Context, + Session, KType, ) @@ -227,11 +240,11 @@ def _get_remote_ktypes() -> Enum: f"Something went wrong fetching the remote ktypes: {response.text}" ) - Context.ktypes = {ktype["id"]: KType(**ktype) for ktype in response.json()} + Session.ktypes = {ktype["id"]: KType(**ktype) for ktype in response.json()} ktypes = Enum( "KTypes", - {_name_to_camel(key): value for key, value in Context.ktypes.items()}, + {_name_to_camel(key): value for key, value in Session.ktypes.items()}, ) def custom_getattr(self, name) -> None: @@ -308,13 +321,13 @@ def _create_new_ktype(ktype: "KType") -> None: def _get_ktype(ktype_id: str, as_json=False) -> "Union[KType, Dict[str, Any]]": """Get the KType for an instance with a certain ID from remote backend""" - from dsms import Context, KType + from dsms import Session, KType response = _perform_request(f"api/knowledge-type/{ktype_id}", "get") if response.status_code == 404: raise ValueError( f"""KType with the id `{ktype_id}` does not exist in - DSMS-instance `{Context.dsms.config.host_url}`""" + DSMS-instance `{Session.dsms.config.host_url}`""" ) if not response.ok: raise ValueError( @@ -347,7 +360,7 @@ def _update_ktype(ktype: "KType") -> Response: def _delete_ktype(ktype: "KType") -> None: """Delete a KType in the remote backend""" - from dsms import Context + from dsms import Session logger.debug("Delete KType with id: %s", ktype.id) response = _perform_request(f"api/knowledge-type/{ktype.id}", "delete") @@ -355,7 +368,7 @@ def _delete_ktype(ktype: "KType") -> None: raise ValueError( f"KItem with uuid `{ktype.id}` could not be deleted from DSMS: `{response.text}`" ) - Context.dsms.ktypes = _get_remote_ktypes() + Session.dsms.ktypes = _get_remote_ktypes() def _get_kitem_list() -> "List[KItem]": @@ -390,13 +403,13 @@ def _get_kitem( uuid: Union[str, UUID], as_json=False ) -> "Union[KItem, Dict[str, Any]]": """Get the KItem for a instance with a certain ID from remote backend""" - from dsms import Context, KItem + from dsms import Session, KItem response = _perform_request(f"api/knowledge/kitems/{uuid}", "get") if response.status_code == 404: raise ValueError( f"""KItem with uuid `{uuid}` does not exist in - DSMS-instance `{Context.dsms.config.host_url}`""" + DSMS-instance `{Session.dsms.config.host_url}`""" ) if not response.ok: raise ValueError( @@ -756,7 +769,7 @@ def _commit_updated_kitem(new_kitem: "KItem") -> None: def _commit_updated_ktype(new_ktype: "KType") -> None: """Commit the updated KTypes""" - from dsms import Context + from dsms import Session old_ktype = _get_ktype(new_ktype.id, as_json=True) logger.debug( @@ -770,7 +783,7 @@ def _commit_updated_ktype(new_ktype: "KType") -> None: "Fetching updated KType from remote backend: %s", new_ktype.id ) new_ktype.refresh() - Context.dsms.ktypes = _get_remote_ktypes() + Session.dsms.ktypes = _get_remote_ktypes() def _commit_deleted( diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index b77782f..9913f5e 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -2,9 +2,27 @@ from typing import Any, List, Optional +from enum import Enum + from pydantic import BaseModel, ConfigDict, Field from pydantic.alias_generators import to_camel +from dsms.knowledge.utils import id_generator + +class Widget(Enum): + """Enum for widgets""" + + TEXT = "Text" + FILE = "File" + TEXTAREA = "Textarea" + VOCABULARY_TERM = "Vocabulary term" + NUMBER = "Number" + SLIDER = "Slider" + CHECKBOX = "Checkbox" + SELECT = "Select" + RADIO = "Radio" + KNOWLEDGE_ITEM = "Knowledge item" + class WebformSelectOption(BaseModel): """Choices in webform""" @@ -14,7 +32,25 @@ class WebformSelectOption(BaseModel): disabled: Optional[bool] = Field(False) -class Inputs(BaseModel): +class WebformMeasurementUnit(BaseModel): + """Measurement unit""" + + label: Optional[Any] = Field(None) + iri: Optional[str] = Field(None) + symbol: Optional[str] = Field(None) + namespace: Optional[str] = Field(None) + + +class WebformRangeOptions(BaseModel): + """Range options""" + + min: Optional[int] = Field(0) + max: Optional[int] = Field(0) + step: Optional[int] = Field(0) + range: Optional[bool] = Field(False) + + +class Input(BaseModel): """Input fields in the sections in webform""" model_config = ConfigDict( @@ -23,35 +59,37 @@ class Inputs(BaseModel): ) ) - id: Optional[str] = Field(None) + id: Optional[str] = Field(default_factory=id_generator) label: Optional[str] = Field(None) - widget: Optional[str] = Field(None) + widget: Optional[Widget] = Field(None) default_value: Optional[Any] = Field(None) + required: Optional[bool] = Field(False) value: Optional[Any] = Field(None) - choices: List[WebformSelectOption] = Field([]) - on_change: Optional[Any] = Field(None) - check: Optional[Any] = Field(None) - error: Optional[str] = Field(None) - feedback: Optional[str] = Field(None) hint: Optional[str] = Field(None) - measurement_unit: Optional[str] = Field(None) - mapping: Optional[str] = Field(None) - mapping_name: Optional[str] = Field(None) - mapping_kitem_id: Optional[str] = Field(None) + hidden: Optional[bool] = Field(False) + ignore: Optional[bool] = Field(False) + select_options: List[WebformSelectOption] = Field([]) + onKItemAdd: Optional[Any] = Field(None) + measurement_unit: Optional[WebformMeasurementUnit] = Field(None) + relation_mapping: Optional[str] = Field(None) + relation_mapping_name: Optional[str] = Field(None) + relation_mapping_type: Optional[str] = Field(None) + class_mapping: Optional[str] = Field(None) + multiple_selection: Optional[bool] = Field(False) + mappingKitemId: List[str] = Field([]) + previouslyMappedKitemId: List[str] = Field([]) knowledge_type: Optional[str] = Field(None) knowledge_service_url: Optional[str] = Field(None) vocabulary_service_url: Optional[str] = Field(None) - hidden: Optional[bool] = Field(False) - ignore: Optional[bool] = Field(False) - extra: dict = Field({}) + range_options: Optional[WebformRangeOptions] = Field(None) -class Sections(BaseModel): - """Sections in webform""" +class Section(BaseModel): + """Section in webform""" - id: Optional[str] = Field(None) + id: Optional[str] = Field(default_factory=id_generator) name: Optional[str] = Field(None) - inputs: List[Inputs] = Field([]) + inputs: List[Input] = Field([]) hidden: Optional[bool] = Field(False) @@ -64,5 +102,6 @@ class Webform(BaseModel): ) ) semantics_enabled: Optional[bool] = Field(False) - rdf_type: Optional[str] = Field(None) - sections: List[Sections] = Field([]) + sections_enabled: Optional[bool] = Field(False) + class_mapping: Optional[str] = Field(None) + sections: List[Section] = Field([]) diff --git a/tests/conftest.py b/tests/conftest.py index 503638f..36dc889 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -247,10 +247,10 @@ def register_mocks( @pytest.fixture(autouse=True, scope="function") -def reset_dsms_context(): - from dsms.core.context import Context +def reset_dsms_session(): + from dsms.core.session import Session - Context.dsms = None + Session.dsms = None @pytest.fixture(scope="function") diff --git a/tests/test_kitem.py b/tests/test_kitem.py index 9cd4981..3108491 100644 --- a/tests/test_kitem.py +++ b/tests/test_kitem.py @@ -8,11 +8,11 @@ def test_kitem_basic(custom_address, get_mock_kitem_ids): """Test KItem properties""" from dsms.core.configuration import Configuration - from dsms.core.context import Context + from dsms.core.session import Session from dsms.core.dsms import DSMS from dsms.knowledge.kitem import KItem - assert Context.dsms is None + assert Session.dsms is None with pytest.warns(UserWarning, match="No authentication details"): dsms = DSMS(host_url=custom_address) @@ -25,7 +25,7 @@ def test_kitem_basic(custom_address, get_mock_kitem_ids): assert isinstance(instance.dsms, DSMS) assert isinstance(instance.dsms.config, Configuration) - assert Context.dsms == instance.dsms + assert Session.dsms == instance.dsms @responses.activate @@ -33,11 +33,11 @@ def test_kitem_config_class(custom_address, get_mock_kitem_ids): """Test KItem properties""" from dsms.core.configuration import Configuration - from dsms.core.context import Context + from dsms.core.session import Session from dsms.core.dsms import DSMS from dsms.knowledge.kitem import KItem - assert Context.dsms is None + assert Session.dsms is None with pytest.warns(UserWarning, match="No authentication details"): config = Configuration(host_url=custom_address) @@ -51,7 +51,7 @@ def test_kitem_config_class(custom_address, get_mock_kitem_ids): assert isinstance(instance.dsms, DSMS) assert config == instance.dsms.config - assert Context.dsms == instance.dsms + assert Session.dsms == instance.dsms @responses.activate From aab928f6cf890ee117c2233714970b13d2ac9f4b Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Mon, 2 Dec 2024 10:13:58 +0100 Subject: [PATCH 31/66] Support for webform --- dsms/__init__.py | 2 +- dsms/core/session.py | 2 +- dsms/knowledge/ktype.py | 18 +++++++++++++----- dsms/knowledge/properties/base.py | 2 +- dsms/knowledge/utils.py | 15 +++++++++------ dsms/knowledge/webform.py | 6 +++--- tests/test_kitem.py | 4 ++-- 7 files changed, 30 insertions(+), 19 deletions(-) diff --git a/dsms/__init__.py b/dsms/__init__.py index a8e2834..24b3344 100644 --- a/dsms/__init__.py +++ b/dsms/__init__.py @@ -2,8 +2,8 @@ from dsms.apps import AppConfig from dsms.core.configuration import Configuration -from dsms.core.session import Session from dsms.core.dsms import DSMS +from dsms.core.session import Session from dsms.knowledge.kitem import KItem from dsms.knowledge.ktype import KType diff --git a/dsms/core/session.py b/dsms/core/session.py index 447dcf3..13e5f5f 100644 --- a/dsms/core/session.py +++ b/dsms/core/session.py @@ -21,7 +21,7 @@ class Buffers: class Session: - """Object giving the current DSMS context.""" + """Object giving the current DSMS session.""" dsms: "Optional[DSMS]" = None diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 6ef3e6c..7fead54 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -2,7 +2,7 @@ import logging from datetime import datetime -from typing import TYPE_CHECKING, Any, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from uuid import UUID from pydantic import BaseModel, Field, model_serializer, model_validator @@ -32,8 +32,12 @@ class KType(BaseModel): name: Optional[str] = Field( None, description="Human readable name of the KType." ) - context: Optional[bool] = Field(False, description="Identifies if the ktype is a context ktype") - context_schema: Optional[list] = Field([], description="Schema of the context ktype") + context: Optional[bool] = Field( + False, description="Identifies if the ktype is a context ktype" + ) + context_schema: Optional[list] = Field( + [], description="Schema of the context ktype" + ) webform: Optional[Webform] = Field( None, description="Form data of the KType." ) @@ -46,7 +50,9 @@ class KType(BaseModel): updated_at: Optional[Union[str, datetime]] = Field( None, description="Time and date when the KType was updated." ) - custom_properties: Optional[Any] = Field(None, description="Additional custom properties for the KType.") + custom_properties: Optional[Any] = Field( + None, description="Additional custom properties for the KType." + ) def __hash__(self) -> int: return hash(str(self)) @@ -134,7 +140,9 @@ def serialize(self): return { key: ( value.dict(exclude_none=False, by_alias=False) - if key == "webform" and value is not None and not isinstance(value, dict) + if key == "webform" + and value is not None + and not isinstance(value, dict) else value ) for key, value in self.__dict__.items() diff --git a/dsms/knowledge/properties/base.py b/dsms/knowledge/properties/base.py index 25076b7..700a543 100644 --- a/dsms/knowledge/properties/base.py +++ b/dsms/knowledge/properties/base.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: from typing import Any, Callable, Dict, Iterable, List, Set, Union - from dsms import Session, KItem + from dsms import KItem, Session class KItemProperty(BaseModel): diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 6d57d1d..2f10cc3 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -2,11 +2,11 @@ import base64 import io import logging -import re -import warnings -import time import random +import re import string +import time +import warnings from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union @@ -54,12 +54,15 @@ def _is_number(value): except Exception: return False + def id_generator(prefix: str = "id") -> str: """Generates a random string as id""" # Generate a unique part using time and random characters unique_part = f"{int(time.time() * 1000)}" # Milliseconds since epoch - random_part = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) + random_part = "".join( + random.choices(string.ascii_lowercase + string.digits, k=6) + ) # Combine prefix, unique part, and random part generated_id = f"{prefix}{unique_part}{random_part}" return generated_id @@ -321,7 +324,7 @@ def _create_new_ktype(ktype: "KType") -> None: def _get_ktype(ktype_id: str, as_json=False) -> "Union[KType, Dict[str, Any]]": """Get the KType for an instance with a certain ID from remote backend""" - from dsms import Session, KType + from dsms import KType, Session response = _perform_request(f"api/knowledge-type/{ktype_id}", "get") if response.status_code == 404: @@ -403,7 +406,7 @@ def _get_kitem( uuid: Union[str, UUID], as_json=False ) -> "Union[KItem, Dict[str, Any]]": """Get the KItem for a instance with a certain ID from remote backend""" - from dsms import Session, KItem + from dsms import KItem, Session response = _perform_request(f"api/knowledge/kitems/{uuid}", "get") if response.status_code == 404: diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index 9913f5e..b85b4af 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -1,17 +1,17 @@ """Webform model""" -from typing import Any, List, Optional - from enum import Enum +from typing import Any, List, Optional from pydantic import BaseModel, ConfigDict, Field from pydantic.alias_generators import to_camel from dsms.knowledge.utils import id_generator + class Widget(Enum): """Enum for widgets""" - + TEXT = "Text" FILE = "File" TEXTAREA = "Textarea" diff --git a/tests/test_kitem.py b/tests/test_kitem.py index 3108491..e8169fc 100644 --- a/tests/test_kitem.py +++ b/tests/test_kitem.py @@ -8,8 +8,8 @@ def test_kitem_basic(custom_address, get_mock_kitem_ids): """Test KItem properties""" from dsms.core.configuration import Configuration - from dsms.core.session import Session from dsms.core.dsms import DSMS + from dsms.core.session import Session from dsms.knowledge.kitem import KItem assert Session.dsms is None @@ -33,8 +33,8 @@ def test_kitem_config_class(custom_address, get_mock_kitem_ids): """Test KItem properties""" from dsms.core.configuration import Configuration - from dsms.core.session import Session from dsms.core.dsms import DSMS + from dsms.core.session import Session from dsms.knowledge.kitem import KItem assert Session.dsms is None From 6efc6e516e1440d77c7da5e0287a2156cfe67766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Mon, 2 Dec 2024 22:20:03 +0100 Subject: [PATCH 32/66] add schema transformation function --- .pylintrc | 1 + dsms/knowledge/utils.py | 141 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) diff --git a/.pylintrc b/.pylintrc index 070db46..6ede681 100644 --- a/.pylintrc +++ b/.pylintrc @@ -77,6 +77,7 @@ disable= no-name-in-module, too-many-public-methods, too-many-locals, no-value-for-parameter, + too-many-branches, diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index ddd763c..2d019b1 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -2,7 +2,10 @@ import base64 import io import logging +import random import re +import string +import time import warnings from enum import Enum from pathlib import Path @@ -10,6 +13,7 @@ from uuid import UUID import pandas as pd +import requests import segno import yaml from PIL import Image @@ -844,3 +848,140 @@ def _delete_app_spec(name: str) -> None: message = f"Something went wrong deleting app spec with name `{name}`: {response.text}" raise RuntimeError(message) return response.text + + +def _get_ktype(ktype_id: str) -> Optional[Dict[str, Any]]: + try: + response = _perform_request(f"api/knowledge-type/{ktype_id}", "get") + + if not response.status_code == 200: + logger.error( + "An error occurred while retrieving RDF mapping: %s, %s", + response.status_code, + response.content, + ) + except requests.RequestException as e: + logger.error("KType mapping connection failed: %s", e) + except Exception as ex: + logger.error("An error occurred while retrieving Mapping: %s", ex) + return response.json() + + +def _transform_custom_properties_schema( + custom_properties: Any, ktype_id: str, from_context: bool = False +): + """ + Given a ktype_id and a custom_properties dictionary, + transform the input into the format expected by the frontend. + If the ktype_id is not found, just return the custom_properties dictionary + as is. + """ + + if from_context: + from dsms import Context + + ktype_spec = Context.ktypes.get(ktype_id) + else: + ktype_spec = _get_ktype(ktype_id) + + if ktype_spec: + webform = ktype_spec.get("webform") + else: + webform = None + + if webform and isinstance(custom_properties, dict): + copy_properties = custom_properties.copy() + transformed_sections = {} + for section_def in webform["sections"]: + for input_def in section_def["inputs"]: + label = input_def["label"] + if label in copy_properties: + if input_def.get("classMapping"): + class_mapping = { + "classMapping": { + "iri": input_def.get("classMapping") + } + } + else: + class_mapping = {} + + entry = { + "id": input_def["id"], + "label": label, + "value": copy_properties.pop(label), + "measurementUnit": input_def.get("measurementUnit"), + **class_mapping, + } + section_name = section_def["name"] + if section_name not in transformed_sections: + section = { + "id": section_def["id"], + "name": section_name, + "entries": [], + } + transformed_sections[section_name] = section + transformed_sections[section_name]["entries"].append(entry) + if copy_properties: + logger.info( + "Some custom properties were not found in the webform: %s for ktype: %s", + copy_properties, + ktype_id, + ) + transformed_sections["General"] = _make_misc_section( + copy_properties + ) + response = {"sections": list(transformed_sections.values())} + elif not webform and isinstance(custom_properties, dict): + response = _transform_from_flat_schema(custom_properties) + elif isinstance(custom_properties, dict) and custom_properties.get( + "sections" + ): + response = custom_properties + else: + raise TypeError( + f"Invalid custom properties type: {type(custom_properties)}" + ) + return response + + +def _transform_from_flat_schema( + custom_properties: Dict[str, Any] +) -> Dict[str, Any]: + return {"sections": [_make_misc_section(custom_properties)]} + + +def _make_misc_section(custom_properties: dict): + """ + If the ktype_id is not found, return the custom_properties dictionary + as is, wrapped in a section named "Misc". + """ + section = {"id": id_generator(), "name": "Misc", "entries": []} + for key, value in custom_properties.items(): + section["entries"].append( + { + "id": id_generator(), + "label": key, + "value": value, + } + ) + return section + + +def id_generator(prefix: str = "id") -> str: + # Generate a unique part using time and random characters + """ + Generates a unique id using a combination of the current time and 6 random characters. + + Args: + prefix (str): The prefix to use for the generated id. Defaults to "id". + + Returns: + str: The generated id. + """ + unique_part = f"{int(time.time() * 1000)}" # Milliseconds since epoch + random_part = "".join( + random.choices(string.ascii_lowercase + string.digits, k=6) # nosec + ) + # Combine prefix, unique part, and random part + generated_id = f"{prefix}{unique_part}{random_part}" + return generated_id From 1977ed9c9decc0ba0f8b71a9e44a3aedaf10463c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Mon, 2 Dec 2024 22:28:42 +0100 Subject: [PATCH 33/66] bump dev version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 4a64858..0f3d733 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev1 +version = v2.1.0dev2 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From 63d7be2bbd1a66baccff4e16759548d825a9bb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Mon, 9 Dec 2024 22:26:19 +0100 Subject: [PATCH 34/66] make temporary compatibility with old webform model --- dsms/knowledge/utils.py | 65 +++++++++++++++++++++++------------------ setup.cfg | 2 +- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 2d019b1..4bff81a 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -64,34 +64,43 @@ def _create_custom_properties_model( fields = {} if isinstance(value, dict): - for item in value.get("sections"): - for form_input in item.get("inputs"): - label = form_input.get("label") - dtype = form_input.get("widget") - default = form_input.get("defaultValue") - slug = _slugify(label) - if dtype in ("Text", "File", "Textarea", "Vocabulary term"): - dtype = Optional[str] - elif dtype in ("Number", "Slider"): - dtype = Optional[NumericalDataType] - elif dtype == "Checkbox": - dtype = Optional[bool] - elif dtype in ("Select", "Radio"): - choices = Enum( - _name_to_camel(label) + "Choices", - { - _name_to_camel(choice["value"]): choice["value"] - for choice in form_input.get("choices") - }, - ) - dtype = Optional[choices] - elif dtype == "Knowledge item": - warnings.warn( - "knowledge item not fully supported for KTypes yet." - ) - dtype = Optional[str] - - fields[slug] = (dtype, default or None) + for item in value.get("sections", []): + if isinstance(item, dict): + for form_input in item.get("inputs", []): + if isinstance(form_input, dict): + label = form_input.get("label") + dtype = form_input.get("widget") + default = form_input.get("defaultValue") + slug = _slugify(label) + if dtype in ( + "Text", + "File", + "Textarea", + "Vocabulary term", + ): + dtype = Optional[str] + elif dtype in ("Number", "Slider"): + dtype = Optional[NumericalDataType] + elif dtype == "Checkbox": + dtype = Optional[bool] + elif dtype in ("Select", "Radio"): + choices = Enum( + _name_to_camel(label) + "Choices", + { + _name_to_camel(choice["value"]): choice[ + "value" + ] + for choice in form_input.get("choices") + }, + ) + dtype = Optional[choices] + elif dtype == "Knowledge item": + warnings.warn( + "knowledge item not fully supported for KTypes yet." + ) + dtype = Optional[str] + + fields[slug] = (dtype, default or None) fields["kitem"] = ( Optional[KItem], Field(None, exclude=True), diff --git a/setup.cfg b/setup.cfg index 0f3d733..151bc8d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev2 +version = v2.1.0dev3 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From 0d069068702632f869d6263fc1fcd6893249e1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Tue, 10 Dec 2024 11:53:13 +0100 Subject: [PATCH 35/66] temporary fix for custom properties --- dsms/knowledge/kitem.py | 6 ++++++ dsms/knowledge/utils.py | 2 +- setup.cfg | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index fe12e95..a1c43d5 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -58,6 +58,7 @@ _inspect_dataframe, _make_annotation_schema, _refresh_kitem, + _transform_from_flat_schema, ) from dsms.knowledge.sparql_interface.utils import _get_subgraph # isort:skip @@ -567,6 +568,11 @@ def validate_custom_properties(cls, self) -> "KItem": raise TypeError( f"Invalid type: {type(content)}" ) from error + # transform from flat dict. fix: replace with `_transform_from_flat_schema` + # content = _transform_from_flat_schema( + # content, self.ktype_id, from_context=True + # ) + content = _transform_from_flat_schema(content) was_in_buffer = self.id in self.context.buffers.updated self.custom_properties = self.ktype.webform(**content) # fix: find a better way to prehebit that properties are diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 4bff81a..17a0a84 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -894,7 +894,7 @@ def _transform_custom_properties_schema( ktype_spec = _get_ktype(ktype_id) if ktype_spec: - webform = ktype_spec.get("webform") + webform = ktype_spec.webform else: webform = None diff --git a/setup.cfg b/setup.cfg index 151bc8d..573b124 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev3 +version = v2.1.0dev4 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From 79a7f0d4b925a755beee4c31b58dc7be5c937168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Tue, 10 Dec 2024 13:04:11 +0100 Subject: [PATCH 36/66] set upper limit for pydantic --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 573b124..835d7e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev4 +version = v2.1.0dev5 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown @@ -24,7 +24,7 @@ install_requires = html5lib>=1,<2 lru-cache<1 pandas>=2,<3 - pydantic>=2,<2.8 + pydantic>=2,<3 pydantic-settings python-dotenv qrcode-artistic>=3,<4 From d6c23b93a1f4d3cad039c223624c06eb8b1f9664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Thu, 12 Dec 2024 23:47:54 +0100 Subject: [PATCH 37/66] update custom properties model --- dsms/knowledge/kitem.py | 48 +-- dsms/knowledge/ktype.py | 26 +- dsms/knowledge/utils.py | 138 -------- dsms/knowledge/webform.py | 700 +++++++++++++++++++++++++++++++++++--- 4 files changed, 685 insertions(+), 227 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index f967dd0..5ccef3f 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -2,6 +2,7 @@ import json import logging +import warnings from datetime import datetime from enum import Enum from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union @@ -58,11 +59,13 @@ _inspect_dataframe, _make_annotation_schema, _refresh_kitem, - _transform_from_flat_schema, + _transform_custom_properties_schema, ) from dsms.knowledge.sparql_interface.utils import _get_subgraph # isort:skip +from dsms.knowledge.webform import KItemCustomPropertiesModel # isort:skip + if TYPE_CHECKING: from dsms import Session from dsms.core.dsms import DSMS @@ -178,9 +181,9 @@ class KItem(BaseModel): [], description="User groups able to access the KItem.", ) - custom_properties: Optional[Any] = Field( - None, description="Custom properties associated to the KItem" - ) + custom_properties: Optional[ + Union[KItemCustomPropertiesModel, Dict] + ] = Field(None, description="Custom properties associated to the KItem") ktype: Optional[Union[str, Enum, KType]] = Field( None, description="KType of the KItem", exclude=True ) @@ -197,10 +200,6 @@ class KItem(BaseModel): default_factory=Avatar, description="KItem avatar interface" ) - access_url: Optional[str] = Field( - None, description="Access URL of the KItem" - ) - context_id: Optional[Union[UUID, str]] = Field( None, description="Context ID of the KItem" ) @@ -572,8 +571,8 @@ def validate_custom_properties(cls, self) -> "KItem": f"""Custom properties must be one of the following types: {(BaseModel, dict, type(None))}. Not {type(self.custom_properties)}""" ) - # validate content with webform model - if self.ktype.webform and isinstance(self.custom_properties, dict): + # transform custom properties + if isinstance(self.custom_properties, dict): content = ( self.custom_properties.get("content") or self.custom_properties ) @@ -584,20 +583,17 @@ def validate_custom_properties(cls, self) -> "KItem": raise TypeError( f"Invalid type: {type(content)}" ) from error - # transform from flat dict. fix: replace with `_transform_from_flat_schema` - # content = _transform_from_flat_schema( - # content, self.ktype_id, from_context=True - # ) - content = _transform_from_flat_schema(content) + content = _transform_custom_properties_schema( + content, self.ktype_id, from_context=True + ) was_in_buffer = self.id in self.context.buffers.updated - self.custom_properties = self.ktype.webform(**content) - # fix: find a better way to prehebit that properties are - # set in the buffer + self.custom_properties = KItemCustomPropertiesModel(**content) if not was_in_buffer: self.context.buffers.updated.pop(self.id) - # set kitem id for custom properties - if isinstance(self.custom_properties, BaseModel): - self.custom_properties.kitem = self + warnings.warn( + "A flat dict was provided for custom properties." + "Will transform to `KItemCustomPropertiesModel`.", + ) return self def _set_kitem_for_properties(self) -> None: @@ -606,7 +602,15 @@ def _set_kitem_for_properties(self) -> None: """ for prop in self.__dict__.values(): if ( - isinstance(prop, (KItemPropertyList, Summary, Avatar)) + isinstance( + prop, + ( + KItemPropertyList, + Summary, + Avatar, + KItemCustomPropertiesModel, + ), + ) and not prop.kitem ): logger.debug( diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 457f8c0..2da9f5a 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -5,15 +5,10 @@ from typing import TYPE_CHECKING, Any, Optional, Union from uuid import UUID -from pydantic import BaseModel, Field, model_serializer, model_validator +from pydantic import BaseModel, Field, model_serializer from dsms.core.logging import handler -from dsms.knowledge.utils import ( - _create_custom_properties_model, - _ktype_exists, - _refresh_ktype, - print_ktype, -) +from dsms.knowledge.utils import _ktype_exists, _refresh_ktype, print_ktype from dsms.knowledge.webform import Webform if TYPE_CHECKING: @@ -52,9 +47,6 @@ class KType(BaseModel): updated_at: Optional[Union[str, datetime]] = Field( None, description="Time and date when the KType was updated." ) - custom_properties: Optional[Any] = Field( - None, description="Additional custom properties for the KType." - ) def __hash__(self) -> int: return hash(str(self)) @@ -74,7 +66,7 @@ def __init__(self, **kwargs: "Any") -> None: # add ktype to buffer if not self.in_backend and self.id not in self.session.buffers.created: logger.debug( - "Marking KTpe with ID `%s` as created and updated during KItem initialization.", + "Marking KType with ID `%s` as created and updated during KItem initialization.", self.id, ) self.session.buffers.created.update({self.id: self}) @@ -150,16 +142,4 @@ def serialize(self): else value ) for key, value in self.__dict__.items() - if key != "custom_properties" } - - @model_validator(mode="after") - @classmethod - def validate_ktype(cls, self: "KType") -> "KType": - """Model validator for ktype""" - if self.webform is not None: - self.webform.model_validate(self.webform) - self.custom_properties = _create_custom_properties_model( - self.webform - ) - return self diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 1df53cc..a9e1fd1 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -6,7 +6,6 @@ import re import string import time -import warnings from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union @@ -20,20 +19,12 @@ from pydantic import ( # isort: skip BaseModel, - ConfigDict, - Field, - create_model, - model_validator, ) from dsms.core.logging import handler # isort:skip from dsms.core.utils import _name_to_camel, _perform_request # isort:skip -from dsms.knowledge.properties.custom_datatype import ( # isort:skip - NumericalDataType, -) - from dsms.knowledge.search import SearchResult # isort:skip if TYPE_CHECKING: @@ -47,135 +38,6 @@ logger.propagate = False -def _is_number(value): - try: - float(value) - return True - except Exception: - return False - - -def _create_custom_properties_model( - value: Optional[Dict[str, Any]] -) -> BaseModel: - """Convert the dict with the model schema into a pydantic model.""" - from dsms import KItem, KType - from dsms.knowledge.webform import Webform - - fields = {} - if isinstance(value, Webform): - for item in value.sections: - for form_input in item.inputs: - label = form_input.label - dtype = form_input.widget - default = form_input.default_value - slug = _slugify(label) - if dtype in ("Text", "File", "Textarea", "Vocabulary term"): - dtype = Optional[str] - elif dtype in ("Number", "Slider"): - dtype = Optional[NumericalDataType] - elif dtype == "Checkbox": - dtype = Optional[bool] - elif dtype in ("Select", "Radio"): - choices = Enum( - _name_to_camel(label) + "Choices", - { - _name_to_camel(choice.value): choice.value - for choice in form_input.choices - }, - ) - dtype = Optional[choices] - elif dtype == "Knowledge item": - warnings.warn( - "knowledge item not fully supported for KTypes yet." - ) - dtype = Optional[str] - - fields[slug] = (dtype, default or None) - fields["kitem"] = ( - Optional[KItem], - Field(None, exclude=True), - ) - - config = ConfigDict( - extra="allow", arbitrary_types_allowed=True, exclude={"kitem"} - ) - validators = { - "validate_model": model_validator(mode="before")(_validate_model) - } - model = create_model( - "CustomPropertiesModel", - __config__=config, - __validators__=validators, - **fields, - ) - setattr(model, "__str__", _print_properties) - setattr(model, "__repr__", _print_properties) - setattr(model, "__setattr__", __setattr_property__) - setattr(model, "serialize", KType.serialize) - logger.debug("Create custom properties model with fields: %s", fields) - return model - - -def _print_properties(self: Any) -> str: - fields = ", \n".join( - [ - f"\t\t{key}: {value}" - for key, value in self.model_dump().items() - if key not in self.model_config["exclude"] - ] - ) - return f"{{\n{fields}\n\t}}" - - -def __setattr_property__(self, key, value) -> None: - logger.debug( - "Setting property for custom property with key `%s` with value `%s`.", - key, - value, - ) - if _is_number(value): - # convert to convertable numeric object - value = _create_numerical_dtype(key, value, self.kitem) - # mark as updated - if key != "kitem" and self.kitem: - logger.debug( - "Setting related kitem for custom properties with id `%s` as updated", - self.kitem.id, - ) - self.kitem.context.buffers.updated.update({self.kitem.id: self.kitem}) - elif key == "kitem": - # set kitem for convertable numeric datatype - for prop in self.model_dump().values(): - if isinstance(prop, NumericalDataType) and not prop.kitem: - prop.kitem = value - # reassignment of extra fields does not work, seems to be a bug? - # have this workaround: - if key in self.__pydantic_extra__: - self.__pydantic_extra__[key] = value - super(BaseModel, self).__setattr__(key, value) - - -def _create_numerical_dtype( - key: str, value: Union[int, float], kitem: "KItem" -) -> NumericalDataType: - value = NumericalDataType(value) - value.name = key - value.kitem = kitem - return value - - -def _validate_model( - cls, values: Dict[str, Any] # pylint: disable=unused-argument -) -> Dict[str, Any]: - for key, value in values.items(): - if _is_number(value): - values[key] = _create_numerical_dtype( - key, value, values.get("kitem") - ) - return values - - def print_webform(webform: BaseModel) -> str: """ Helper function to pretty print a webform. diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index b85b4af..8c6af7b 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -1,13 +1,26 @@ """Webform model""" +import logging from enum import Enum -from typing import Any, List, Optional +from typing import TYPE_CHECKING, Any, List, Optional, Union -from pydantic import BaseModel, ConfigDict, Field +from pydantic import AnyUrl, BaseModel, ConfigDict, Field, PrivateAttr, model_validator from pydantic.alias_generators import to_camel from dsms.knowledge.utils import id_generator +from dsms.knowledge.properties.custom_datatype import ( # isort:skip + NumericalDataType, +) +from dsms.core.logging import handler # isort:skip + +if TYPE_CHECKING: + from dsms.knowledge.kitem import KItem + +logger = logging.getLogger(__name__) +logger.addHandler(handler) +logger.propagate = False + class Widget(Enum): """Enum for widgets""" @@ -27,27 +40,107 @@ class Widget(Enum): class WebformSelectOption(BaseModel): """Choices in webform""" - label: Optional[str] = Field(None) - value: Optional[Any] = Field(None) - disabled: Optional[bool] = Field(False) + label: Optional[str] = Field(None, description="Label of the option") + value: Optional[Any] = Field(None, description="Value of the option") + disabled: Optional[bool] = Field(False, description="Disabled option") class WebformMeasurementUnit(BaseModel): """Measurement unit""" - label: Optional[Any] = Field(None) - iri: Optional[str] = Field(None) - symbol: Optional[str] = Field(None) - namespace: Optional[str] = Field(None) + label: Optional[str] = Field( + None, description="Label of the measurement unit" + ) + iri: Optional[AnyUrl] = Field( + None, description="IRI of the measurement unit" + ) + symbol: Optional[str] = Field( + None, description="Symbol of the measurement unit" + ) + namespace: Optional[AnyUrl] = Field( + None, description="Namespace of the measurement unit" + ) class WebformRangeOptions(BaseModel): """Range options""" - min: Optional[int] = Field(0) - max: Optional[int] = Field(0) - step: Optional[int] = Field(0) - range: Optional[bool] = Field(False) + min: Optional[int] = Field(0, description="Minimum value") + max: Optional[int] = Field(0, description="Maximum value") + step: Optional[int] = Field(0, description="Step value") + range: Optional[bool] = Field(False, description="Range value") + + +class RelationMappingType(Enum): + """ + Relation mapping type + """ + + OBJECT_PROPERY = "object_property" + DATA_PROPERTY = "data_property" + ANNOTATION_PROPERTY = "annotation_property" + + +class RelationMapping(BaseModel): + """Relation mapping""" + + iri: str = Field(..., description="IRI of the annotation", max_length=200) + type: Optional[RelationMappingType] = Field( + None, description="Type of the annotation" + ) + class_iri: Optional[str] = Field( + None, + description="Target class IRI if the type of relation is an object property", + ) + _kitem = PrivateAttr(default=None) + + model_config = ConfigDict( + alias_generator=lambda field_name: to_camel( # pylint: disable=W0108 + field_name + ) + ) + + def __setattr__(self, key, value) -> None: + logger.debug( + "Setting property for relation mapping `%s` with key `%s` with value `%s`.", + self.iri, + key, + value, + ) + + # Set kitem as updated + if key != "kitem" and self.kitem: + logger.debug( + "Setting related kitem with id `%s` as updated", + self.kitem.id, + ) + self.kitem.context.buffers.updated.update( + {self.kitem.id: self.kitem} + ) + elif key == "kitem": + self.kitem = value + + super().__setattr__(key, value) + + @property + def kitem(self) -> "KItem": + """ + KItem instance the entry is related to + + Returns: + KItem: KItem instance + """ + return self._kitem + + @kitem.setter + def kitem(self, value: "KItem") -> None: + """ + Setter for the KItem instance related to the relation mapping. + + Args: + value (KItem): The KItem instance to associate with the relation mapping. + """ + self._kitem = value class Input(BaseModel): @@ -59,38 +152,51 @@ class Input(BaseModel): ) ) - id: Optional[str] = Field(default_factory=id_generator) - label: Optional[str] = Field(None) - widget: Optional[Widget] = Field(None) - default_value: Optional[Any] = Field(None) - required: Optional[bool] = Field(False) - value: Optional[Any] = Field(None) - hint: Optional[str] = Field(None) - hidden: Optional[bool] = Field(False) - ignore: Optional[bool] = Field(False) - select_options: List[WebformSelectOption] = Field([]) - onKItemAdd: Optional[Any] = Field(None) - measurement_unit: Optional[WebformMeasurementUnit] = Field(None) - relation_mapping: Optional[str] = Field(None) - relation_mapping_name: Optional[str] = Field(None) - relation_mapping_type: Optional[str] = Field(None) - class_mapping: Optional[str] = Field(None) - multiple_selection: Optional[bool] = Field(False) - mappingKitemId: List[str] = Field([]) - previouslyMappedKitemId: List[str] = Field([]) - knowledge_type: Optional[str] = Field(None) - knowledge_service_url: Optional[str] = Field(None) - vocabulary_service_url: Optional[str] = Field(None) - range_options: Optional[WebformRangeOptions] = Field(None) + id: Optional[str] = Field( + default_factory=id_generator, description="ID of the input" + ) + label: Optional[str] = Field(None, description="Label of the input") + widget: Optional[Widget] = Field(None, description="Widget of the input") + required: Optional[bool] = Field(False, description="Required input") + value: Optional[Any] = Field(None, description="Value of the input") + hint: Optional[str] = Field(None, description="Hint of the input") + hidden: Optional[bool] = Field(False, description="Hidden input") + ignore: Optional[bool] = Field(False, description="Ignore input") + select_options: List[WebformSelectOption] = Field( + [], description="List of select options" + ) + measurement_unit: Optional[WebformMeasurementUnit] = Field( + None, description="Measurement unit" + ) + relation_mapping: Optional[RelationMapping] = Field( + None, description="Relation mapping" + ) + relation_mapping_extra: Optional[RelationMapping] = Field( + None, description="Relation mapping extra" + ) + multiple_selection: Optional[bool] = Field( + False, description="Multiple selection" + ) + knowledge_type: Optional[str] = Field(None, description="Knowledge type") + range_options: Optional[WebformRangeOptions] = Field( + None, description="Range options" + ) + placeholder: Optional[str] = Field( + None, description="Placeholder for the input" + ) class Section(BaseModel): """Section in webform""" - id: Optional[str] = Field(default_factory=id_generator) - name: Optional[str] = Field(None) - inputs: List[Input] = Field([]) - hidden: Optional[bool] = Field(False) + id: Optional[str] = Field( + default_factory=id_generator, description="ID of the section" + ) + name: Optional[str] = Field(None, description="Name of the section") + inputs: List[Input] = Field( + [], description="List of inputs in the section" + ) + hidden: Optional[bool] = Field(False, description="Hidden section") class Webform(BaseModel): @@ -101,7 +207,513 @@ class Webform(BaseModel): field_name ) ) - semantics_enabled: Optional[bool] = Field(False) - sections_enabled: Optional[bool] = Field(False) - class_mapping: Optional[str] = Field(None) - sections: List[Section] = Field([]) + semantics_enabled: Optional[bool] = Field( + False, description="Semantics enabled" + ) + sections_enabled: Optional[bool] = Field( + False, description="Sections enabled" + ) + class_mapping: Optional[str] = Field(None, description="Class mapping") + sections: List[Section] = Field([], description="List of sections") + + +class MeasurementUnit(BaseModel): + """Measurement unit""" + + iri: Optional[AnyUrl] = Field( + None, description="IRI of the annotation", max_length=200 + ) + label: Optional[str] = Field( + None, description="Label of the measurement unit", max_length=100 + ) + symbol: Optional[str] = Field( + None, description="Symbol of the measurement unit", max_length=100 + ) + namespace: Optional[AnyUrl] = Field( + None, description="Namespace of the measurement unit" + ) + _kitem = PrivateAttr(default=None) + + def __setattr__(self, key, value) -> None: + """ + Set an attribute of the MeasurementUnit instance. + + This method overrides the default behavior of setting an attribute. + It logs the action and updates the related KItem in the buffer if + applicable. If the key is 'kitem', it sets the KItem instance directly. + + Args: + key (str): The name of the attribute to set. + value (Any): The value to set for the attribute. + """ + logger.debug( + "Setting property for measurement unit `%s` with key `%s` with value `%s`.", + self.iri, + key, + value, + ) + + # Set kitem as updated + if key != "kitem" and self.kitem: + logger.debug( + "Setting related kitem with id `%s` as updated", + self.kitem.id, + ) + self.kitem.context.buffers.updated.update( + {self.kitem.id: self.kitem} + ) + + elif key == "kitem": + self.kitem = value + + super().__setattr__(key, value) + + @property + def kitem(self) -> "KItem": + """ + KItem instance the entry is related to + + Returns: + KItem: KItem instance + """ + return self._kitem + + @kitem.setter + def kitem(self, value: "KItem") -> None: + """ + Setter for the KItem instance related to the measurement unit. + + Args: + value (KItem): The KItem instance to associate with the measurement unit. + """ + self._kitem = value + + +class KnowledgeItemReference(BaseModel): + """Reference to a knowledge item if linked in the custom properties""" + + id: str = Field(..., description="ID of the knowledge item") + name: str = Field(..., description="Name of the knowledge item") + + +class Entry(BaseModel): + """ + Entry in a custom properties section + """ + + id: str = Field(default_factory=id_generator) + type: Widget = Field(..., description="Type of the entry") + label: str = Field(..., description="Label of the entry") + value: Optional[ + Union[ + str, + int, + KnowledgeItemReference, + float, + bool, + List, + NumericalDataType, + ] + ] = Field(None, description="Value of the entry") + measurementUnit: Optional[MeasurementUnit] = Field( + None, description="Measurement unit of the entry" + ) + relationMapping: Optional[RelationMapping] = Field( + None, description="Relation mapping of the entry" + ) + _kitem = PrivateAttr(default=None) + + def __setattr__(self, key, value) -> None: + logger.debug( + "Setting property for entry `%s` with key `%s` with value `%s`.", + self.name, + key, + value, + ) + + # Set kitem as updated + if key != "kitem" and self.kitem: + logger.debug( + "Setting related kitem with id `%s` as updated", + self.kitem.id, + ) + self.kitem.context.buffers.updated.update( + {self.kitem.id: self.kitem} + ) + elif key == "kitem": + self.kitem = value + if self.measurementUnit: + self.measurementUnit.kitem = value + if self.relationMapping: + self.relationMapping.kitem = value + + super().__setattr__(key, value) + + @property + def kitem(self) -> "KItem": + """ + KItem instance the entry is related to + + Returns: + KItem: KItem instance + """ + return self._kitem + + @kitem.setter + def kitem(self, kitem: "KItem"): + """ + Set KItem instance the entry is related to + + Args: + kitem (KItem): KItem instance + """ + self._kitem = kitem + + @property + def ktype(self) -> "KItem": + """ + KType instance the entry is related to + + Returns: + KType: KType instance + """ + return self.kitem.ktype + + @property + def webform(self) -> "Webform": + """ + Retrieve the webform associated with the KType of this entry. + + Returns: + Webform: The webform instance related to the KType. + """ + return self.ktype.webform + + @model_validator(mode="after") + @classmethod + def _validate_inputs(cls, self: "Entry") -> "Entry": + spec = cls._get_input_spec(self) + choices = None + + # check if widget is mapped to a data type + if spec.widget in ( + Widget.TEXT, + Widget.FILE, + Widget.TEXTAREA, + Widget.VOCABULARY_TERM, + ): + dtype = str + elif spec.widget in (Widget.NUMBER, Widget.SLIDER): + dtype = NumericalDataType + elif spec.widget == Widget.CHECKBOX: + dtype = bool + elif spec.widget in (Widget.SELECT, Widget.RADIO, Widget.MULTI_SELECT): + if spec.widget == Widget.MULTI_SELECT: + dtype = list + else: + dtype = str + choices = [choice.value for choice in spec.select_options] + elif spec.widget == Widget.KNOWLEDGE_ITEM: + dtype = KnowledgeItemReference + else: + raise ValueError( + f"Widget type is not mapped to a data type: {self.widget}" + ) + + # check if value is set + if self.value is None and spec.value is not None: + self.value = spec.value + + # check if value is of correct type + if self.value is not None and not isinstance(self.value, dtype): + raise ValueError( + f"Value of type {type(self.value)} is not of type {dtype}" + ) + if ( + self.value is not None + and choices is not None + and self.value not in choices + ): + raise ValueError( + f"Value {self.value} is not a valid choice for entry {self.label}" + ) + if self.value is None and spec.value is None and self.required: + raise ValueError(f"Value for entry {self.label} is required") + + # set name and kitem of numerical data type + if isinstance(self.value, NumericalDataType): + self.value.name = self.label + self.value.kitem = self.kitem + + return self + + @classmethod + def _get_input_spec(cls, self: "Entry"): + potential_spec = [] + spec = None + for section in self.webform.sections: + for inp in section.inputs: + if inp.id == self.id: + potential_spec.append(inp) + if len(potential_spec) == 0: + raise ValueError( + f"Could not find input spec for entry {self.label}" + ) + if len(potential_spec) > 1: + raise ValueError( + f"Found multiple input specs for entry {self.label}" + ) + spec = potential_spec.pop() + return spec + + +class CustomPropertiesSection(BaseModel): + """ + Section for custom properties + """ + + id: Optional[str] = Field(default_factory=id_generator) + name: str = Field(..., description="Name of the section") + entries: List[Entry] = Field([], description="Entries of the section") + _kitem = PrivateAttr(default=None) + + @property + def kitem(self) -> "KItem": + """ + KItem instance the section is related to + + Returns: + KItem: KItem instance + """ + return self._kitem + + @kitem.setter + def kitem(self, kitem: "KItem"): + """ + Set KItem instance the section is related to + + Args: + kitem (KItem): KItem instance + """ + self._kitem = kitem + + def __setattr__(self, key, value) -> None: + """ + Set an attribute of the section. + + This method is overridden to intercept setting of properties. If the key is + not a valid attribute of the section, it is interpreted as a label of an + entry of the section. The value is then set for the entry. + + Args: + key: The key of the attribute to be set. + value: The value of the attribute to be set. + """ + logger.debug( + "Setting property for section `%s` with key `%s` with value `%s`.", + self.name, + key, + value, + ) + + # Set kitem as updated + if key != "kitem" and self.kitem: + logger.debug( + "Setting related kitem with id `%s` as updated", + self.kitem.id, + ) + self.kitem.context.buffers.updated.update( + {self.kitem.id: self.kitem} + ) + elif key == "kitem": + self.kitem = value + for entry in self.entries: + entry.kitem = value + + # Set value + if key not in self.model_dump().keys(): + to_be_updated = [] + for entry in self.entries: + if entry.label == key: + to_be_updated.append(entry) + if len(to_be_updated) == 0: + raise AttributeError( + f"Section with name '{self.name}' has no attribute '{key}'" + ) + if len(to_be_updated) > 1: + raise AttributeError( + f"""Section with name '{self.name}' + has multiple attributes '{key}'. Please specify section!""" + ) + + to_be_updated = to_be_updated.pop() + to_be_updated.value = value + else: + super().__setattr__(key, value) + + def __getattr__(self, key) -> Any: + """ + Retrieve an entry from the section by its label. + + This method searches through the entries of the section to find an entry whose + label matches the given key. If the entry is found and is unique, it returns the entry. + If the entry is not found, or if multiple entries with the same label are found, it raises + an AttributeError. If the attribute exists on the BaseModel, it is retrieved using the + superclass's __getattr__ method. + + Args: + key (str): The label of the entry to retrieve. + + Returns: + Entry: The entry matching the given label. + + Raises: + AttributeError: If no entry or multiple entries with the given label are found. + """ + target = [] + if not hasattr(self, key): + for entry in self.entries: + if entry.label == key: + target.append(entry) + if len(target) == 0: + raise AttributeError( + f"Section with name `{self.name}` has no attribute '{key}'" + ) + if len(target) > 1: + raise AttributeError( + f"""Section with name `{self.name}` + has multiple attributes '{key}'. Please specify section!""" + ) + + target = target.pop() + else: + target = super().__getattr__(key) + return target + + +class KItemCustomPropertiesModel(BaseModel): + """ + A custom properties model for a KItem. + """ + + sections: List[CustomPropertiesSection] = Field( + [], description="Sections of custom properties" + ) + _kitem = PrivateAttr(default=None) + + @property + def kitem(self) -> "KItem": + """ + KItem instance the custom properties is related to + + Returns: + KItem: KItem instance + """ + return self._kitem + + @kitem.setter + def kitem(self, kitem: "KItem"): + """ + Set KItem instance the custom properties is related to + + Args: + kitem (KItem): KItem instance + """ + self._kitem = kitem + + def __getattr__(self, key): + """ + Retrieve an entry from the custom properties sections by its label. + + This method searches through the entries of each section within the custom properties + model to find an entry whose label matches the given key. If the entry is found and + is unique, it returns the entry. If the entry is not found, or if multiple entries + with the same label are found, it raises an AttributeError. If the attribute exists + on the BaseModel, it is retrieved using the superclass's __getattr__ method. + + Args: + key (str): The label of the entry to retrieve. + + Returns: + Entry: The entry matching the given label. + + Raises: + AttributeError: If no entry or multiple entries with the given label are found. + """ + target = [] + if not hasattr(self, key): + for section in self.sections: + for entry in section.entries: + if entry.label == key: + target.append(entry) + if len(target) == 0: + raise AttributeError( + f"Custom properties model has no attribute '{key}'" + ) + if len(target) > 1: + raise AttributeError( + f"""Custom properties model has multiple attributes + '{key}'. Please specify section!""" + ) + target = target.pop() + else: + target = super().__getattr__(key) + return target + + def __setattr__(self, key, value) -> None: + """ + Set a custom property's value by its label. + + This method searches through the entries of each section within the custom properties + model to find an entry whose label matches the given key. If the entry is found and is + unique, it sets the value of that entry. If the entry is not found, or if multiple + entries with the same label are found, it raises an AttributeError. If the attribute + exists on the BaseModel, it is set using the superclass's __setattr__ method. + + Args: + key (str): The label of the entry to set. + value (Any): The new value of the entry. + + Raises: + AttributeError: If no entry or multiple entries with the given label are found. + """ + + logger.debug( + "Setting property for custom properties model with key `%s` with value `%s`.", + key, + value, + ) + + # Set kitem as updated + if key != "kitem" and self.kitem: + logger.debug( + "Setting related kitem for custom properties with id `%s` as updated", + self.kitem.id, + ) + self.kitem.context.buffers.updated.update( + {self.kitem.id: self.kitem} + ) + elif key == "kitem": + self.kitem = value + self.sections.kitem = value + + # Set value in model + if key not in self.model_dump().keys(): + to_be_updated = [] + for section in self.sections: + for entry in section.entries: + if entry.label == key: + to_be_updated.append(entry) + if len(to_be_updated) == 0: + raise AttributeError( + f"Custom properties model has no attribute '{key}'" + ) + if len(to_be_updated) > 1: + raise AttributeError( + f"""Custom properties model has multiple attributes + '{key}'. Please specify section!""" + ) + to_be_updated = to_be_updated.pop() + to_be_updated.value = value + else: + super().__setattr__(key, value) From c06d45dd9a0136bd3dfac1d49fc8e516effc505c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Fri, 13 Dec 2024 17:28:30 +0100 Subject: [PATCH 38/66] update webform kitem assignment and validation --- dsms/knowledge/kitem.py | 61 ++- .../properties/custom_datatype/numerical.py | 24 +- dsms/knowledge/utils.py | 87 ++-- dsms/knowledge/webform.py | 493 ++++++++---------- 4 files changed, 310 insertions(+), 355 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 5ccef3f..892870f 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -1,6 +1,5 @@ """Knowledge Item implementation of the DSMS""" -import json import logging import warnings from datetime import datetime @@ -181,9 +180,10 @@ class KItem(BaseModel): [], description="User groups able to access the KItem.", ) - custom_properties: Optional[ - Union[KItemCustomPropertiesModel, Dict] - ] = Field(None, description="Custom properties associated to the KItem") + custom_properties: Optional[Union[Any]] = Field( + None, description="Custom properties associated to the KItem" + ) + ktype: Optional[Union[str, Enum, KType]] = Field( None, description="KType of the KItem", exclude=True ) @@ -204,6 +204,10 @@ class KItem(BaseModel): None, description="Context ID of the KItem" ) + access_url: Optional[str] = Field( + None, description="Access URL of the KItem", exclude=True + ) + model_config = ConfigDict( extra="forbid", validate_assignment=True, @@ -562,37 +566,32 @@ def validate_dataframe( @model_validator(mode="after") @classmethod - def validate_custom_properties(cls, self) -> "KItem": - """Validate the custom properties with respect to the KType of the KItem""" - if not isinstance( - self.custom_properties, (BaseModel, dict, type(None)) - ): - raise TypeError( - f"""Custom properties must be one of the following types: - {(BaseModel, dict, type(None))}. Not {type(self.custom_properties)}""" - ) - # transform custom properties + def validate_custom_properties(cls, self: "KItem") -> "KItem": + """Validate custom properties""" + if isinstance(self.custom_properties, dict): - content = ( + value = ( self.custom_properties.get("content") or self.custom_properties ) - if isinstance(content, str): - try: - content = json.loads(content) - except Exception as error: - raise TypeError( - f"Invalid type: {type(content)}" - ) from error - content = _transform_custom_properties_schema( - content, self.ktype_id, from_context=True + + if not value.get("sections"): + value = _transform_custom_properties_schema( + value, self.ktype.webform + ) + warnings.warn( + """A flat dictionary was provided for custom properties. + Will be transformed into `KItemCustomPropertiesModel`.""" + ) + + self.custom_properties = KItemCustomPropertiesModel( + **value, kitem=self ) - was_in_buffer = self.id in self.context.buffers.updated - self.custom_properties = KItemCustomPropertiesModel(**content) - if not was_in_buffer: - self.context.buffers.updated.pop(self.id) - warnings.warn( - "A flat dict was provided for custom properties." - "Will transform to `KItemCustomPropertiesModel`.", + elif not isinstance( + self.custom_properties, KItemCustomPropertiesModel + ): + raise TypeError( + "Custom properties must be either a dictionary or a " + "KItemCustomPropertiesModel." ) return self diff --git a/dsms/knowledge/properties/custom_datatype/numerical.py b/dsms/knowledge/properties/custom_datatype/numerical.py index 65c6736..8216c46 100644 --- a/dsms/knowledge/properties/custom_datatype/numerical.py +++ b/dsms/knowledge/properties/custom_datatype/numerical.py @@ -9,7 +9,7 @@ ) if TYPE_CHECKING: - from typing import Any, Dict, Optional + from typing import Any, Dict, Generator, Optional from dsms import KItem @@ -38,6 +38,28 @@ def __str__(self) -> str: string = str(self.__float__()) return string + @classmethod + def __get_validators__(cls) -> "Generator": + yield cls.validate + + @classmethod + def validate(cls, v: "Any") -> "NumericalDataType": + """ + Validate the input value as a valid NumericalDataType. + + Args: + v (Any): The value to be validated. + + Returns: + NumericalDataType: An instance of NumericalDataType if validation is successful. + + Raises: + TypeError: If the input value is not a float or int. + """ + if not isinstance(v, (float, int)): + raise TypeError(f"Expected float or int, got {type(v)}") + return cls(v) + def __repr__(self) -> str: return str(self) diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index a9e1fd1..2766a81 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -38,6 +38,14 @@ logger.propagate = False +def _is_number(value): + try: + float(value) + return True + except Exception: + return False + + def print_webform(webform: BaseModel) -> str: """ Helper function to pretty print a webform. @@ -903,55 +911,32 @@ def _delete_app_spec(name: str) -> None: return response.text -def _transform_custom_properties_schema( - custom_properties: Any, ktype_id: str, from_context: bool = False -): - """ - Given a ktype_id and a custom_properties dictionary, - transform the input into the format expected by the frontend. - If the ktype_id is not found, just return the custom_properties dictionary - as is. - """ - - if from_context: - from dsms import Context - - ktype_spec = Context.ktypes.get(ktype_id) - else: - ktype_spec = _get_ktype(ktype_id) - - if ktype_spec: - webform = ktype_spec.webform - else: - webform = None - - if webform and isinstance(custom_properties, dict): +def _transform_custom_properties_schema(custom_properties: Any, webform: Any): + if webform: copy_properties = custom_properties.copy() transformed_sections = {} - for section_def in webform["sections"]: - for input_def in section_def["inputs"]: - label = input_def["label"] - if label in copy_properties: - if input_def.get("classMapping"): + for section_def in webform.sections: + for input_def in section_def.inputs: + if input_def.label in copy_properties: + if input_def.class_mapping: class_mapping = { - "classMapping": { - "iri": input_def.get("classMapping") - } + "classMapping": {"iri": input_def.class_mapping} } else: class_mapping = {} entry = { - "id": input_def["id"], - "label": label, - "value": copy_properties.pop(label), - "measurementUnit": input_def.get("measurementUnit"), + "id": input_def.id, + "label": input_def.label, + "value": copy_properties.pop(input_def.label), + "measurementUnit": input_def.measurement_unit, + "type": input_def.widget, **class_mapping, } - section_name = section_def["name"] + section_name = section_def.name if section_name not in transformed_sections: section = { - "id": section_def["id"], + "id": section_def.id, "name": section_name, "entries": [], } @@ -959,24 +944,15 @@ def _transform_custom_properties_schema( transformed_sections[section_name]["entries"].append(entry) if copy_properties: logger.info( - "Some custom properties were not found in the webform: %s for ktype: %s", + "Some custom properties were not found in the webform: %s", copy_properties, - ktype_id, ) transformed_sections["General"] = _make_misc_section( copy_properties ) response = {"sections": list(transformed_sections.values())} - elif not webform and isinstance(custom_properties, dict): - response = _transform_from_flat_schema(custom_properties) - elif isinstance(custom_properties, dict) and custom_properties.get( - "sections" - ): - response = custom_properties else: - raise TypeError( - f"Invalid custom properties type: {type(custom_properties)}" - ) + response = _transform_from_flat_schema(custom_properties) return response @@ -998,11 +974,26 @@ def _make_misc_section(custom_properties: dict): "id": id_generator(), "label": key, "value": value, + "type": _map_data_type_to_widget(value), } ) return section +def _map_data_type_to_widget(value): + from dsms.knowledge.webform import Widget + + if isinstance(value, str): + widget = Widget.TEXT + elif isinstance(value, (int, float)): + widget = Widget.NUMBER + elif isinstance(value, bool): + widget = Widget.CHECKBOX + else: + raise ValueError(f"Unsupported data type: {type(value)}") + return widget + + def id_generator(prefix: str = "id") -> str: # Generate a unique part using time and random characters """ diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index 8c6af7b..a74e3a7 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -2,9 +2,16 @@ import logging from enum import Enum -from typing import TYPE_CHECKING, Any, List, Optional, Union - -from pydantic import AnyUrl, BaseModel, ConfigDict, Field, PrivateAttr, model_validator +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + Field, + model_serializer, + model_validator, +) from pydantic.alias_generators import to_camel from dsms.knowledge.utils import id_generator @@ -15,7 +22,7 @@ from dsms.core.logging import handler # isort:skip if TYPE_CHECKING: - from dsms.knowledge.kitem import KItem + from dsms.knowledge.ktype import KType logger = logging.getLogger(__name__) logger.addHandler(handler) @@ -35,6 +42,7 @@ class Widget(Enum): SELECT = "Select" RADIO = "Radio" KNOWLEDGE_ITEM = "Knowledge item" + MULTI_SELECT = "Multi-select" class WebformSelectOption(BaseModel): @@ -65,10 +73,12 @@ class WebformMeasurementUnit(BaseModel): class WebformRangeOptions(BaseModel): """Range options""" - min: Optional[int] = Field(0, description="Minimum value") - max: Optional[int] = Field(0, description="Maximum value") - step: Optional[int] = Field(0, description="Step value") - range: Optional[bool] = Field(False, description="Range value") + min: Optional[Union[int, float]] = Field(0, description="Minimum value") + max: Optional[Union[int, float]] = Field(0, description="Maximum value") + step: Optional[Union[int, float]] = Field(0, description="Step value") + range: Optional[Union[int, float]] = Field( + False, description="Range value" + ) class RelationMappingType(Enum): @@ -81,29 +91,87 @@ class RelationMappingType(Enum): ANNOTATION_PROPERTY = "annotation_property" -class RelationMapping(BaseModel): - """Relation mapping""" - - iri: str = Field(..., description="IRI of the annotation", max_length=200) - type: Optional[RelationMappingType] = Field( - None, description="Type of the annotation" - ) - class_iri: Optional[str] = Field( - None, - description="Target class IRI if the type of relation is an object property", - ) - _kitem = PrivateAttr(default=None) +class BaseWebformModel(BaseModel): + """Base webform model""" model_config = ConfigDict( alias_generator=lambda field_name: to_camel( # pylint: disable=W0108 field_name - ) + ), + exclude={"kitem"}, + validate_assignment=True, + extra="forbid", + ) + kitem: Optional[Any] = Field( + None, description="Associated KItem instance", exclude=True, hide=True ) + @property + def ktype(self) -> "KType": + """ + KType instance the entry is related to + + Returns: + KType: KType instance + """ + return self.kitem.ktype + + @property + def webform(self) -> "Webform": + """ + Retrieve the webform associated with the KType of this entry. + + Returns: + Webform: The webform instance related to the KType. + """ + return self.ktype.webform + + @property + def dsms(self): + """ + Get the DSMS instance associated with the kitem. + + This property retrieves the DSMS instance related to the kitem + of the custom properties section. + + Returns: + DSMS: The DSMS instance associated with the kitem. + """ + return self.kitem.dsms + + def __str__(self) -> str: + """Pretty print the model fields""" + fields = ", \n".join( + [ + f"\n\t{key} = {value}" + for key, value in self.__dict__.items() + if ( + key not in self.model_config["exclude"] + and key not in self.dsms.config.hide_properties + ) + ] + ) + return f"{self.__class__.__name__}(\n{fields}\n)" + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + def __setattr__(self, key, value) -> None: + """ + Set an attribute of the model. + + This method sets an attribute of the model and logs the operation. + If the attribute being set is `kitem`, it directly assigns the value. + For other attributes, it marks the associated `kitem` as updated in the + context buffers if it exists. + + Args: + key (str): The name of the attribute to set. + value (Any): The value to set for the attribute. + """ logger.debug( - "Setting property for relation mapping `%s` with key `%s` with value `%s`.", - self.iri, + "Setting property for model attribute with key `%s` with value `%s`.", key, value, ) @@ -117,41 +185,29 @@ def __setattr__(self, key, value) -> None: self.kitem.context.buffers.updated.update( {self.kitem.id: self.kitem} ) + elif key == "kitem": self.kitem = value super().__setattr__(key, value) - @property - def kitem(self) -> "KItem": - """ - KItem instance the entry is related to - - Returns: - KItem: KItem instance - """ - return self._kitem - @kitem.setter - def kitem(self, value: "KItem") -> None: - """ - Setter for the KItem instance related to the relation mapping. +class RelationMapping(BaseWebformModel): + """Relation mapping""" - Args: - value (KItem): The KItem instance to associate with the relation mapping. - """ - self._kitem = value + iri: str = Field(..., description="IRI of the annotation", max_length=200) + type: Optional[RelationMappingType] = Field( + None, description="Type of the annotation" + ) + class_iri: Optional[str] = Field( + None, + description="Target class IRI if the type of relation is an object property", + ) -class Input(BaseModel): +class Input(BaseWebformModel): """Input fields in the sections in webform""" - model_config = ConfigDict( - alias_generator=lambda field_name: to_camel( # pylint: disable=W0108 - field_name - ) - ) - id: Optional[str] = Field( default_factory=id_generator, description="ID of the input" ) @@ -186,7 +242,7 @@ class Input(BaseModel): ) -class Section(BaseModel): +class Section(BaseWebformModel): """Section in webform""" id: Optional[str] = Field( @@ -199,25 +255,22 @@ class Section(BaseModel): hidden: Optional[bool] = Field(False, description="Hidden section") -class Webform(BaseModel): +class Webform(BaseWebformModel): """User defined webform for ktype""" - model_config = ConfigDict( - alias_generator=lambda field_name: to_camel( # pylint: disable=W0108 - field_name - ) - ) semantics_enabled: Optional[bool] = Field( False, description="Semantics enabled" ) sections_enabled: Optional[bool] = Field( False, description="Sections enabled" ) - class_mapping: Optional[str] = Field(None, description="Class mapping") + class_mapping: Optional[Union[List[str], str]] = Field( + [], description="Class mapping" + ) sections: List[Section] = Field([], description="List of sections") -class MeasurementUnit(BaseModel): +class MeasurementUnit(BaseWebformModel): """Measurement unit""" iri: Optional[AnyUrl] = Field( @@ -232,61 +285,6 @@ class MeasurementUnit(BaseModel): namespace: Optional[AnyUrl] = Field( None, description="Namespace of the measurement unit" ) - _kitem = PrivateAttr(default=None) - - def __setattr__(self, key, value) -> None: - """ - Set an attribute of the MeasurementUnit instance. - - This method overrides the default behavior of setting an attribute. - It logs the action and updates the related KItem in the buffer if - applicable. If the key is 'kitem', it sets the KItem instance directly. - - Args: - key (str): The name of the attribute to set. - value (Any): The value to set for the attribute. - """ - logger.debug( - "Setting property for measurement unit `%s` with key `%s` with value `%s`.", - self.iri, - key, - value, - ) - - # Set kitem as updated - if key != "kitem" and self.kitem: - logger.debug( - "Setting related kitem with id `%s` as updated", - self.kitem.id, - ) - self.kitem.context.buffers.updated.update( - {self.kitem.id: self.kitem} - ) - - elif key == "kitem": - self.kitem = value - - super().__setattr__(key, value) - - @property - def kitem(self) -> "KItem": - """ - KItem instance the entry is related to - - Returns: - KItem: KItem instance - """ - return self._kitem - - @kitem.setter - def kitem(self, value: "KItem") -> None: - """ - Setter for the KItem instance related to the measurement unit. - - Args: - value (KItem): The KItem instance to associate with the measurement unit. - """ - self._kitem = value class KnowledgeItemReference(BaseModel): @@ -296,13 +294,13 @@ class KnowledgeItemReference(BaseModel): name: str = Field(..., description="Name of the knowledge item") -class Entry(BaseModel): +class Entry(BaseWebformModel): """ Entry in a custom properties section """ id: str = Field(default_factory=id_generator) - type: Widget = Field(..., description="Type of the entry") + type: Optional[Widget] = Field(None, description="Type of the entry") label: str = Field(..., description="Label of the entry") value: Optional[ Union[ @@ -321,27 +319,20 @@ class Entry(BaseModel): relationMapping: Optional[RelationMapping] = Field( None, description="Relation mapping of the entry" ) - _kitem = PrivateAttr(default=None) def __setattr__(self, key, value) -> None: - logger.debug( - "Setting property for entry `%s` with key `%s` with value `%s`.", - self.name, - key, - value, - ) + """ + Set an attribute of the Entry instance. - # Set kitem as updated - if key != "kitem" and self.kitem: - logger.debug( - "Setting related kitem with id `%s` as updated", - self.kitem.id, - ) - self.kitem.context.buffers.updated.update( - {self.kitem.id: self.kitem} - ) - elif key == "kitem": - self.kitem = value + This method overrides the default behavior of setting an attribute. + It sets the KItem instance of the measurement unit and relation mapping + if the key is 'kitem'. + + Args: + key (str): The name of the attribute to set. + value (Any): The value to set for the attribute. + """ + if key == "kitem": if self.measurementUnit: self.measurementUnit.kitem = value if self.relationMapping: @@ -349,71 +340,50 @@ def __setattr__(self, key, value) -> None: super().__setattr__(key, value) - @property - def kitem(self) -> "KItem": - """ - KItem instance the entry is related to - - Returns: - KItem: KItem instance - """ - return self._kitem - - @kitem.setter - def kitem(self, kitem: "KItem"): - """ - Set KItem instance the entry is related to - - Args: - kitem (KItem): KItem instance - """ - self._kitem = kitem - - @property - def ktype(self) -> "KItem": - """ - KType instance the entry is related to - - Returns: - KType: KType instance - """ - return self.kitem.ktype - - @property - def webform(self) -> "Webform": - """ - Retrieve the webform associated with the KType of this entry. - - Returns: - Webform: The webform instance related to the KType. - """ - return self.ktype.webform - @model_validator(mode="after") @classmethod def _validate_inputs(cls, self: "Entry") -> "Entry": spec = cls._get_input_spec(self) + print("###", self.type, spec, self.value) + if not self.type: + if len(spec) == 0: + raise ValueError( + f"Could not find input spec for entry {self.label}" + ) + if len(spec) > 1: + raise ValueError( + f"Found multiple input specs for entry {self.label}" + ) + spec = spec.pop() + self.type = spec.widget + default_value = spec.value + select_options = spec.select_options + else: + default_value = None + select_options = [] + + dtype = None choices = None # check if widget is mapped to a data type - if spec.widget in ( + if self.type in ( Widget.TEXT, Widget.FILE, Widget.TEXTAREA, Widget.VOCABULARY_TERM, ): dtype = str - elif spec.widget in (Widget.NUMBER, Widget.SLIDER): + elif self.type in (Widget.NUMBER, Widget.SLIDER): dtype = NumericalDataType - elif spec.widget == Widget.CHECKBOX: + elif self.type == Widget.CHECKBOX: dtype = bool - elif spec.widget in (Widget.SELECT, Widget.RADIO, Widget.MULTI_SELECT): - if spec.widget == Widget.MULTI_SELECT: + elif self.type in (Widget.SELECT, Widget.RADIO, Widget.MULTI_SELECT): + if self.type == Widget.MULTI_SELECT: dtype = list else: dtype = str - choices = [choice.value for choice in spec.select_options] - elif spec.widget == Widget.KNOWLEDGE_ITEM: + choices = [choice.value for choice in select_options] + elif self.type == Widget.KNOWLEDGE_ITEM: dtype = KnowledgeItemReference else: raise ValueError( @@ -421,8 +391,8 @@ def _validate_inputs(cls, self: "Entry") -> "Entry": ) # check if value is set - if self.value is None and spec.value is not None: - self.value = spec.value + if self.value is None and default_value is not None: + self.value = default_value # check if value is of correct type if self.value is not None and not isinstance(self.value, dtype): @@ -437,7 +407,7 @@ def _validate_inputs(cls, self: "Entry") -> "Entry": raise ValueError( f"Value {self.value} is not a valid choice for entry {self.label}" ) - if self.value is None and spec.value is None and self.required: + if self.value is None and default_value is None and self.required: raise ValueError(f"Value for entry {self.label} is required") # set name and kitem of numerical data type @@ -447,27 +417,36 @@ def _validate_inputs(cls, self: "Entry") -> "Entry": return self + @model_serializer + def serialize(self) -> Dict[str, Any]: + """ + Serialize the Entry object to a dictionary representation. + + This method transforms the Entry instance into a dictionary, where the keys + are the attribute names and the values are the corresponding attribute values. + The "type" attribute is treated specially by storing its `value` instead of + the object itself. + + Returns: + Dict[str, Any]: A dictionary representation of the Entry object. + """ + return { + key: (value if key != "type" else value.value) + for key, value in self.__dict__.items() + if key != "kitem" + } + @classmethod def _get_input_spec(cls, self: "Entry"): - potential_spec = [] - spec = None + spec = [] for section in self.webform.sections: for inp in section.inputs: if inp.id == self.id: - potential_spec.append(inp) - if len(potential_spec) == 0: - raise ValueError( - f"Could not find input spec for entry {self.label}" - ) - if len(potential_spec) > 1: - raise ValueError( - f"Found multiple input specs for entry {self.label}" - ) - spec = potential_spec.pop() + spec.append(inp) return spec -class CustomPropertiesSection(BaseModel): +class CustomPropertiesSection(BaseWebformModel): """ Section for custom properties """ @@ -475,27 +454,6 @@ class CustomPropertiesSection(BaseModel): id: Optional[str] = Field(default_factory=id_generator) name: str = Field(..., description="Name of the section") entries: List[Entry] = Field([], description="Entries of the section") - _kitem = PrivateAttr(default=None) - - @property - def kitem(self) -> "KItem": - """ - KItem instance the section is related to - - Returns: - KItem: KItem instance - """ - return self._kitem - - @kitem.setter - def kitem(self, kitem: "KItem"): - """ - Set KItem instance the section is related to - - Args: - kitem (KItem): KItem instance - """ - self._kitem = kitem def __setattr__(self, key, value) -> None: """ @@ -509,29 +467,12 @@ def __setattr__(self, key, value) -> None: key: The key of the attribute to be set. value: The value of the attribute to be set. """ - logger.debug( - "Setting property for section `%s` with key `%s` with value `%s`.", - self.name, - key, - value, - ) - - # Set kitem as updated - if key != "kitem" and self.kitem: - logger.debug( - "Setting related kitem with id `%s` as updated", - self.kitem.id, - ) - self.kitem.context.buffers.updated.update( - {self.kitem.id: self.kitem} - ) - elif key == "kitem": - self.kitem = value + if key == "kitem": for entry in self.entries: entry.kitem = value # Set value - if key not in self.model_dump().keys(): + if key not in self.model_dump() and key != "kitem": to_be_updated = [] for entry in self.entries: if entry.label == key: @@ -571,7 +512,7 @@ def __getattr__(self, key) -> Any: AttributeError: If no entry or multiple entries with the given label are found. """ target = [] - if not hasattr(self, key): + if not key in self.model_dump() and key != "kitem": for entry in self.entries: if entry.label == key: target.append(entry) @@ -590,8 +531,27 @@ def __getattr__(self, key) -> Any: target = super().__getattr__(key) return target + @model_validator(mode="before") + @classmethod + def set_kitem(cls, self: Dict[str, Any]) -> Dict[str, Any]: + """ + Set kitem for all entries of the section. + + This validator is called before the model is validated. It sets the kitem + for all entries of the section if the kitem is set. -class KItemCustomPropertiesModel(BaseModel): + Args: + self (CustomPropertiesSection): The section to set the kitem for. + """ + kitem = self.get("kitem") + if kitem: + for entry in self.get("entries"): + if not "kitem" in entry: + entry["kitem"] = kitem + return self + + +class KItemCustomPropertiesModel(BaseWebformModel): """ A custom properties model for a KItem. """ @@ -599,27 +559,6 @@ class KItemCustomPropertiesModel(BaseModel): sections: List[CustomPropertiesSection] = Field( [], description="Sections of custom properties" ) - _kitem = PrivateAttr(default=None) - - @property - def kitem(self) -> "KItem": - """ - KItem instance the custom properties is related to - - Returns: - KItem: KItem instance - """ - return self._kitem - - @kitem.setter - def kitem(self, kitem: "KItem"): - """ - Set KItem instance the custom properties is related to - - Args: - kitem (KItem): KItem instance - """ - self._kitem = kitem def __getattr__(self, key): """ @@ -641,7 +580,8 @@ def __getattr__(self, key): AttributeError: If no entry or multiple entries with the given label are found. """ target = [] - if not hasattr(self, key): + + if not key in self.model_dump() and key != "kitem": for section in self.sections: for entry in section.entries: if entry.label == key: @@ -678,23 +618,7 @@ def __setattr__(self, key, value) -> None: AttributeError: If no entry or multiple entries with the given label are found. """ - logger.debug( - "Setting property for custom properties model with key `%s` with value `%s`.", - key, - value, - ) - - # Set kitem as updated - if key != "kitem" and self.kitem: - logger.debug( - "Setting related kitem for custom properties with id `%s` as updated", - self.kitem.id, - ) - self.kitem.context.buffers.updated.update( - {self.kitem.id: self.kitem} - ) - elif key == "kitem": - self.kitem = value + if key == "kitem": self.sections.kitem = value # Set value in model @@ -717,3 +641,22 @@ def __setattr__(self, key, value) -> None: to_be_updated.value = value else: super().__setattr__(key, value) + + @model_validator(mode="before") + @classmethod + def set_kitem(cls, self: Dict[str, Any]) -> Dict[str, Any]: + """ + Set kitem for all sections of the custom properties model. + + This validator is called before the model is validated. It sets the kitem + for all sections of the custom properties model if the kitem is set. + + Args: + self (KItemCustomPropertiesModel): The custom properties model to set the kitem for. + """ + kitem = self.get("kitem") + if kitem: + for section in self.get("sections"): + if not "kitem" in section: + section["kitem"] = kitem + return self From ddf2458005db32bdcc364314d55784d72be1fc07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Mon, 16 Dec 2024 17:16:46 +0100 Subject: [PATCH 39/66] fix printing of values --- .pylintrc | 1 + dsms/apps/config.py | 48 ++++---- dsms/knowledge/kitem.py | 18 +-- dsms/knowledge/ktype.py | 2 +- dsms/knowledge/properties/affiliations.py | 5 + dsms/knowledge/properties/annotations.py | 6 +- dsms/knowledge/properties/apps.py | 17 +-- dsms/knowledge/properties/attachments.py | 17 +-- dsms/knowledge/properties/authors.py | 5 + dsms/knowledge/properties/avatar.py | 6 +- dsms/knowledge/properties/base.py | 20 +-- dsms/knowledge/properties/contacts.py | 5 + .../properties/custom_datatype/numerical.py | 13 +- dsms/knowledge/properties/dataframe.py | 19 +-- dsms/knowledge/properties/external_links.py | 5 + dsms/knowledge/properties/linked_kitems.py | 16 +-- dsms/knowledge/properties/summary.py | 15 +-- dsms/knowledge/properties/user_groups.py | 5 + dsms/knowledge/utils.py | 65 ++++------ dsms/knowledge/webform.py | 116 ++++++++++-------- setup.cfg | 1 + 21 files changed, 187 insertions(+), 218 deletions(-) diff --git a/.pylintrc b/.pylintrc index a2fec08..a17714e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -79,6 +79,7 @@ disable= no-name-in-module, no-value-for-parameter, too-many-branches, too-many-lines, + too-many-statements, diff --git a/dsms/apps/config.py b/dsms/apps/config.py index 13cecab..a0d27a2 100644 --- a/dsms/apps/config.py +++ b/dsms/apps/config.py @@ -101,30 +101,30 @@ def __setattr__(self, name, value) -> None: ) self.session.buffers.updated.update({self.name: self}) - def __str__(self) -> str: - """Pretty print the app config fields""" - fields = ", ".join( - [ - "{key}={value}".format( # pylint: disable=consider-using-f-string - key=key, - value=( - value - if key != "specification" - else { - "metadata": value.get( # pylint: disable=no-member - "metadata" - ) - } - ), - ) - for key, value in self.__dict__.items() - ] - ) - return f"{self.__class__.__name__}({fields})" - - def __repr__(self) -> str: - """Pretty print the kitem Fields""" - return str(self) + # def __str__(self) -> str: + # """Pretty print the app config fields""" + # fields = ", ".join( + # [ + # "{key}={value}".format( # pylint: disable=consider-using-f-string + # key=key, + # value=( + # value + # if key != "specification" + # else { + # "metadata": value.get( # pylint: disable=no-member + # "metadata" + # ) + # } + # ), + # ) + # for key, value in self.__dict__.items() + # ] + # ) + # return f"{self.__class__.__name__}({fields})" + + # def __repr__(self) -> str: + # """Pretty print the kitem Fields""" + # return str(self) @field_validator("name") @classmethod diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 892870f..1e87a91 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -59,6 +59,7 @@ _make_annotation_schema, _refresh_kitem, _transform_custom_properties_schema, + print_model, ) from dsms.knowledge.sparql_interface.utils import _get_subgraph # isort:skip @@ -262,17 +263,7 @@ def __setattr__(self, name, value) -> None: def __str__(self) -> str: """Pretty print the kitem fields""" - fields = ", \n".join( - [ - f"\n\t{key} = {value}" - for key, value in self.__dict__.items() - if ( - key not in self.model_config["exclude"] - and key not in self.dsms.config.hide_properties - ) - ] - ) - return f"{self.__class__.__name__}(\n{fields}\n)" + return print_model(self, "kitem") def __repr__(self) -> str: """Pretty print the kitem Fields""" @@ -587,11 +578,12 @@ def validate_custom_properties(cls, self: "KItem") -> "KItem": **value, kitem=self ) elif not isinstance( - self.custom_properties, KItemCustomPropertiesModel + self.custom_properties, (KItemCustomPropertiesModel, type(None)) ): raise TypeError( "Custom properties must be either a dictionary or a " - "KItemCustomPropertiesModel." + "KItemCustomPropertiesModel. Not a " + f"{type(self.custom_properties)}: {self.custom_properties}" ) return self diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 2da9f5a..ba9300e 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -94,7 +94,7 @@ def __setattr__(self, name, value) -> None: def __repr__(self) -> str: """Print the KType""" - return print_ktype(self) + return str(self) def __str__(self) -> str: """Print the KType""" diff --git a/dsms/knowledge/properties/affiliations.py b/dsms/knowledge/properties/affiliations.py index 1153cad..2e262c0 100644 --- a/dsms/knowledge/properties/affiliations.py +++ b/dsms/knowledge/properties/affiliations.py @@ -6,6 +6,7 @@ from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList from dsms.knowledge.properties.utils import _str_to_dict +from dsms.knowledge.utils import print_model if TYPE_CHECKING: from typing import Callable @@ -18,6 +19,10 @@ class Affiliation(KItemProperty): ..., description="Name of the affiliation", max_length=100 ) + # OVERRIDE + def __str__(self) -> str: + return print_model(self, "affiliation") + class AffiliationsProperty(KItemPropertyList): """Affiliations property""" diff --git a/dsms/knowledge/properties/annotations.py b/dsms/knowledge/properties/annotations.py index 0f7a563..f058648 100644 --- a/dsms/knowledge/properties/annotations.py +++ b/dsms/knowledge/properties/annotations.py @@ -5,7 +5,7 @@ from pydantic import Field from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList -from dsms.knowledge.utils import _make_annotation_schema +from dsms.knowledge.utils import _make_annotation_schema, print_model if TYPE_CHECKING: from typing import Any, Callable, Dict @@ -22,6 +22,10 @@ class Annotation(KItemProperty): ..., description="Namespace of the annotation", max_length=100 ) + # OVERRIDE + def __str__(self) -> str: + return print_model(self, "annotation") + class AnnotationsProperty(KItemPropertyList): """KItemPropertyList for annotations""" diff --git a/dsms/knowledge/properties/apps.py b/dsms/knowledge/properties/apps.py index 872ee86..ee844ce 100644 --- a/dsms/knowledge/properties/apps.py +++ b/dsms/knowledge/properties/apps.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, Field, model_serializer from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList -from dsms.knowledge.utils import _perform_request +from dsms.knowledge.utils import _perform_request, print_model if TYPE_CHECKING: from typing import Callable @@ -24,17 +24,6 @@ class AdditionalProperties(BaseModel): description="File extensions for which the upload shall be triggered.", ) - def __str__(self) -> str: - """Pretty print the KItemPropertyList""" - values = ", ".join( - [f"{key}: {value}" for key, value in self.__dict__.items()] - ) - return f"{{{values}}}" - - def __repr__(self) -> str: - """Pretty print the Apps""" - return str(self) - class JobStatus(BaseModel): """Status of a job""" @@ -151,6 +140,10 @@ def inputs(self) -> Dict[str, Any]: ) return response.json() + # OVERRIDE + def __str__(self): + return print_model(self, "app") + class Job(BaseModel): """Job running an app""" diff --git a/dsms/knowledge/properties/attachments.py b/dsms/knowledge/properties/attachments.py index 175d858..12494c8 100644 --- a/dsms/knowledge/properties/attachments.py +++ b/dsms/knowledge/properties/attachments.py @@ -7,7 +7,7 @@ from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList from dsms.knowledge.properties.utils import _str_to_dict -from dsms.knowledge.utils import _get_attachment +from dsms.knowledge.utils import _get_attachment, print_model if TYPE_CHECKING: from typing import Any, Callable, Dict, Iterable, List @@ -33,20 +33,7 @@ class Attachment(KItemProperty): # OVERRIDE def __str__(self) -> str: - """Pretty print the Attachment""" - values = ",\n\t\t\t".join( - [ - f"{key}: {value}" - for key, value in self.__dict__.items() - if key not in self.exclude - ] - ) - return f"{{\n\t\t\t{values}\n\t\t}}" - - # OVERRIDE - def __repr__(self) -> str: - """Pretty print the Attachment""" - return str(self) + return print_model(self, "attachment") def download(self, as_bytes: bool = False) -> "Union[str, bytes]": """Download attachment file""" diff --git a/dsms/knowledge/properties/authors.py b/dsms/knowledge/properties/authors.py index 473297d..31667ce 100644 --- a/dsms/knowledge/properties/authors.py +++ b/dsms/knowledge/properties/authors.py @@ -6,6 +6,7 @@ from pydantic import Field, model_serializer from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList +from dsms.knowledge.utils import print_model if TYPE_CHECKING: from typing import Callable @@ -29,6 +30,10 @@ def serialize_author(self) -> Dict[str, Any]: if key != "id" } + # OVERRIDE + def __str__(self): + return print_model(self, "author") + class AuthorsProperty(KItemPropertyList): """KItemPropertyList for authors""" diff --git a/dsms/knowledge/properties/avatar.py b/dsms/knowledge/properties/avatar.py index f2fe1e9..05c40d2 100644 --- a/dsms/knowledge/properties/avatar.py +++ b/dsms/knowledge/properties/avatar.py @@ -6,7 +6,7 @@ from pydantic import ConfigDict, Field from dsms.knowledge.properties.base import KItemProperty -from dsms.knowledge.utils import _get_avatar, _make_avatar +from dsms.knowledge.utils import _get_avatar, _make_avatar, print_model class Avatar(KItemProperty): @@ -31,3 +31,7 @@ def download(self) -> "Image": def generate(self) -> "Image": """Generate avatar as PIL Image""" return _make_avatar(self.kitem, self.file, self.include_qr) + + # OVERRIDE + def __str__(self): + return print_model(self, "avatar") diff --git a/dsms/knowledge/properties/base.py b/dsms/knowledge/properties/base.py index 700a543..07c1c2b 100644 --- a/dsms/knowledge/properties/base.py +++ b/dsms/knowledge/properties/base.py @@ -43,16 +43,9 @@ class KItemProperty(BaseModel): _kitem = PrivateAttr(default=None) + @abstractmethod def __str__(self) -> str: """Pretty print the KItemProperty""" - values = ",\n\t\t\t".join( - [ - f"{key}: {value}" - for key, value in self.__dict__.items() - if key not in self.exclude - ] - ) - return f"{{\n\t\t\t{values}\n\t\t}}" def __repr__(self) -> str: """Pretty print the KItemProperty""" @@ -129,17 +122,6 @@ def k_property_helper(cls) -> "Optional[Callable]": """Optional helper for transforming a given input into the k property item""" - def __str__(self) -> str: - """Pretty print the KItemPropertyList""" - values = ", \n".join(["\t\t" + repr(value) for value in self]) - if values: - values = f"\n{values}\n\t" - return f"[{values}]" - - def __repr__(self) -> str: - """Pretty print the KItemPropertyList""" - return str(self) - def __hash__(self) -> int: return hash(str(self)) diff --git a/dsms/knowledge/properties/contacts.py b/dsms/knowledge/properties/contacts.py index 70353cb..a004677 100644 --- a/dsms/knowledge/properties/contacts.py +++ b/dsms/knowledge/properties/contacts.py @@ -7,6 +7,7 @@ from pydantic import Field from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList +from dsms.knowledge.utils import print_model if TYPE_CHECKING: from typing import Callable @@ -21,6 +22,10 @@ class ContactInfo(KItemProperty): None, description="User ID of the contact person" ) + # OVERRIDE + def __str__(self) -> str: + return print_model(self, "contact") + class ContactsProperty(KItemPropertyList): """KItemPropertyList for contacts""" diff --git a/dsms/knowledge/properties/custom_datatype/numerical.py b/dsms/knowledge/properties/custom_datatype/numerical.py index 8216c46..1dc0fed 100644 --- a/dsms/knowledge/properties/custom_datatype/numerical.py +++ b/dsms/knowledge/properties/custom_datatype/numerical.py @@ -20,9 +20,11 @@ class NumericalDataType(float): """Custom Base data type for custom properties""" - def __init__(self, value) -> None: # pylint: disable=unused-argument - self._kitem: "Optional[KItem]" = None - self._name: "Optional[str]" = None + def __new__(cls, value): + obj = super().__new__(cls, value) + obj._kitem = None + obj._name = None + return obj def __str__(self) -> str: """Pretty print the numerical datatype""" @@ -58,7 +60,10 @@ def validate(cls, v: "Any") -> "NumericalDataType": """ if not isinstance(v, (float, int)): raise TypeError(f"Expected float or int, got {type(v)}") - return cls(v) + obj = super().__new__(cls, v) + obj._kitem = None + obj._name = None + return obj def __repr__(self) -> str: return str(self) diff --git a/dsms/knowledge/properties/dataframe.py b/dsms/knowledge/properties/dataframe.py index b994b4a..ffd3b86 100644 --- a/dsms/knowledge/properties/dataframe.py +++ b/dsms/knowledge/properties/dataframe.py @@ -6,7 +6,7 @@ from pydantic import Field from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList -from dsms.knowledge.utils import _get_dataframe_column, _is_number +from dsms.knowledge.utils import _get_dataframe_column, _is_number, print_model from dsms.knowledge.semantics.units import ( # isort:skip get_conversion_factor, @@ -34,21 +34,8 @@ class Column(KItemProperty): ..., description="Name of the column in the data series." ) - def __repr__(self) -> str: - """Pretty print the numerical datatype""" - if self.kitem and self.kitem.dsms.config.display_units: - try: - unit = f"\tunit={self.get_unit().get('symbol')}\n\t\t" - string = str(self) - string = string[:-1] + unit + string[-1:] - except Exception as error: - logger.debug( - "Could not fetch unit from `%i`: %i", self.name, error.args - ) - string = str(self) - else: - string = str(self) - return string + def __str__(self) -> str: + return print_model(self, "column") def get(self) -> "List[Any]": """ diff --git a/dsms/knowledge/properties/external_links.py b/dsms/knowledge/properties/external_links.py index 45ab338..60b48a7 100644 --- a/dsms/knowledge/properties/external_links.py +++ b/dsms/knowledge/properties/external_links.py @@ -5,6 +5,7 @@ from pydantic import AnyUrl, Field from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList +from dsms.knowledge.utils import print_model if TYPE_CHECKING: from typing import Callable @@ -18,6 +19,10 @@ class ExternalLink(KItemProperty): ) url: AnyUrl = Field(..., description="URL of the external link") + # OVERRIDE + def __str__(self): + return print_model(self, "external_link") + class ExternalLinksProperty(KItemPropertyList): """KItemPropertyList for external links""" diff --git a/dsms/knowledge/properties/linked_kitems.py b/dsms/knowledge/properties/linked_kitems.py index 108a166..447f028 100644 --- a/dsms/knowledge/properties/linked_kitems.py +++ b/dsms/knowledge/properties/linked_kitems.py @@ -26,7 +26,7 @@ from dsms.knowledge.properties.contacts import ContactInfo # isort:skip from dsms.knowledge.properties.external_links import ExternalLink # isort:skip from dsms.knowledge.properties.user_groups import UserGroup # isort:skip -from dsms.knowledge.utils import _get_kitem # isort:skip +from dsms.knowledge.utils import _get_kitem, print_model # isort:skip if TYPE_CHECKING: @@ -50,10 +50,7 @@ class LinkedLinkedKItem(BaseModel): def __str__(self) -> str: """Pretty print the linked KItems of the linked KItem""" - values = ",\n\t\t\t".join( - [f"{key}: {value}" for key, value in self.__dict__.items()] - ) - return f"{{\n\t\t\t{values}\n\t\t}}" + return print_model(self, "linked_kitem") def __repr__(self) -> str: """Pretty print the linked KItems of the linked KItem""" @@ -152,14 +149,7 @@ def is_a(self, to_be_compared: KType) -> bool: # OVERRIDE def __str__(self) -> str: """Pretty print the linked KItem""" - values = "\n\t\t\t".join( - [ - f"{key}: {value}" - for key, value in self.__dict__.items() - if key not in self.exclude - ] - ) - return f"\n\t\t\t{values}\n\t\t" + return print_model(self, "linked_kitem") # OVERRIDE def __repr__(self) -> str: diff --git a/dsms/knowledge/properties/summary.py b/dsms/knowledge/properties/summary.py index bf255dd..75edfb5 100644 --- a/dsms/knowledge/properties/summary.py +++ b/dsms/knowledge/properties/summary.py @@ -6,6 +6,8 @@ from pydantic import BaseModel, ConfigDict, Field, model_serializer +from dsms.knowledge.utils import print_model + if TYPE_CHECKING: from typing import Set @@ -33,16 +35,9 @@ def __setattr__(self, name, value) -> None: super().__setattr__(name, value) self._mark_as_updated() - def __str__(self) -> str: - """Pretty print the custom properties""" - fields = ", ".join( - [ - f"{key}={value}" - for key, value in self.__dict__.items() - if key not in self.exclude - ] - ) - return f"{self.__class__.__name__}({fields})" + # OVERIDE + def __str__(self): + return print_model(self, "summary") def __repr__(self) -> str: """Pretty print the custom properties""" diff --git a/dsms/knowledge/properties/user_groups.py b/dsms/knowledge/properties/user_groups.py index 930b162..af2868b 100644 --- a/dsms/knowledge/properties/user_groups.py +++ b/dsms/knowledge/properties/user_groups.py @@ -5,6 +5,7 @@ from pydantic import Field from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList +from dsms.knowledge.utils import print_model if TYPE_CHECKING: from typing import Callable @@ -20,6 +21,10 @@ class UserGroup(KItemProperty): ..., description="ID of the user group", max_length=100 ) + # OVERWRITE + def __str__(self): + return print_model(self, "user_group") + class UserGroupsProperty(KItemPropertyList): """KItemPropertyList for user_groups""" diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 2766a81..7e3b932 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -11,16 +11,12 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from uuid import UUID +import oyaml as yaml import pandas as pd import segno -import yaml from PIL import Image from requests import Response -from pydantic import ( # isort: skip - BaseModel, -) - from dsms.core.logging import handler # isort:skip from dsms.core.utils import _name_to_camel, _perform_request # isort:skip @@ -46,45 +42,28 @@ def _is_number(value): return False -def print_webform(webform: BaseModel) -> str: - """ - Helper function to pretty print a webform. - - Args: - webform (BaseModel): The webform to print. - - Returns: - str: A string representation of the webform. - """ - if hasattr(webform, "model_fields"): - fields = [ - f"\t\t{model_key}=dict({model_value})" - for model_key, model_value in webform.model_fields.items() - if model_key != "kitem" - ] - if fields: - fields = "\twebform={\n" + ",\n".join(fields) + "\n\t}" - else: - fields = "\twebform=None" - else: - fields = "\twebform=None" - return fields +def print_model(self, key, exclude_extra: set = set()) -> str: + """Pretty print the ktype fields""" + exclude = ( + self.model_config.get("exclude", set()) + | exclude_extra + | self.dsms.config.hide_properties + ) + dumped = self.model_dump( + exclude_none=True, + exclude_unset=True, + exclude=exclude, + ) + dumped = { + key: (str(value) if isinstance(value, UUID) else value) + for key, value in dumped.items() + } + return yaml.dump({key: dumped}) def print_ktype(self) -> str: """Pretty print the ktype fields""" - if hasattr(self, "value"): - fields = [ - f"\t{key}={value}" if key != "webform" else print_webform(value) - for key, value in self.value.__dict__.items() - ] - else: - fields = [ - f"\t{key}={value}" if key != "webform" else print_webform(value) - for key, value in self.__dict__.items() - ] - fields = ",\n".join(fields) - return f"{self.name}(\n{fields}\n)" + return print_model(self, "ktype") def _get_remote_ktypes() -> Enum: @@ -989,8 +968,12 @@ def _map_data_type_to_widget(value): widget = Widget.NUMBER elif isinstance(value, bool): widget = Widget.CHECKBOX + elif isinstance(value, list): + widget = Widget.MULTI_SELECT else: - raise ValueError(f"Unsupported data type: {type(value)}") + raise ValueError( + f"Unsupported data type: {type(value)}. Value: {value}" + ) return widget diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index a74e3a7..245a6d2 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -4,17 +4,23 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from pydantic import ( +from pydantic.alias_generators import to_camel + +from pydantic import ( # isort:skip AnyUrl, BaseModel, ConfigDict, Field, + field_validator, model_serializer, model_validator, ) -from pydantic.alias_generators import to_camel -from dsms.knowledge.utils import id_generator +from dsms.knowledge.utils import ( # isort:skip + _map_data_type_to_widget, + id_generator, + print_model, +) from dsms.knowledge.properties.custom_datatype import ( # isort:skip NumericalDataType, @@ -99,8 +105,6 @@ class BaseWebformModel(BaseModel): field_name ), exclude={"kitem"}, - validate_assignment=True, - extra="forbid", ) kitem: Optional[Any] = Field( None, description="Associated KItem instance", exclude=True, hide=True @@ -141,17 +145,7 @@ def dsms(self): def __str__(self) -> str: """Pretty print the model fields""" - fields = ", \n".join( - [ - f"\n\t{key} = {value}" - for key, value in self.__dict__.items() - if ( - key not in self.model_config["exclude"] - and key not in self.dsms.config.hide_properties - ) - ] - ) - return f"{self.__class__.__name__}(\n{fields}\n)" + return print_model(self, "webform") def __repr__(self) -> str: """Pretty print the model fields""" @@ -195,7 +189,7 @@ def __setattr__(self, key, value) -> None: class RelationMapping(BaseWebformModel): """Relation mapping""" - iri: str = Field(..., description="IRI of the annotation", max_length=200) + iri: str = Field(..., description="IRI of the annotation") type: Optional[RelationMappingType] = Field( None, description="Type of the annotation" ) @@ -274,13 +268,16 @@ class MeasurementUnit(BaseWebformModel): """Measurement unit""" iri: Optional[AnyUrl] = Field( - None, description="IRI of the annotation", max_length=200 + None, + description="IRI of the annotation", ) label: Optional[str] = Field( - None, description="Label of the measurement unit", max_length=100 + None, + description="Label of the measurement unit", ) symbol: Optional[str] = Field( - None, description="Symbol of the measurement unit", max_length=100 + None, + description="Symbol of the measurement unit", ) namespace: Optional[AnyUrl] = Field( None, description="Namespace of the measurement unit" @@ -334,18 +331,28 @@ def __setattr__(self, key, value) -> None: """ if key == "kitem": if self.measurementUnit: - self.measurementUnit.kitem = value + self.measurementUnit.kitem = ( # pylint: disable=assigning-non-slot + value + ) if self.relationMapping: - self.relationMapping.kitem = value + self.relationMapping.kitem = ( # pylint: disable=assigning-non-slot + value + ) super().__setattr__(key, value) + @field_validator("value") + @classmethod + def _validate_value(cls, value: Any) -> Any: + if isinstance(value, (int, float)): + value = NumericalDataType(value) + return value + @model_validator(mode="after") @classmethod def _validate_inputs(cls, self: "Entry") -> "Entry": spec = cls._get_input_spec(self) - print("###", self.type, spec, self.value) - if not self.type: + if spec: if len(spec) == 0: raise ValueError( f"Could not find input spec for entry {self.label}" @@ -358,7 +365,11 @@ def _validate_inputs(cls, self: "Entry") -> "Entry": self.type = spec.widget default_value = spec.value select_options = spec.select_options + elif self.type and not spec: + default_value = None + select_options = [] else: + self.type = _map_data_type_to_widget(self.value) default_value = None select_options = [] @@ -399,14 +410,18 @@ def _validate_inputs(cls, self: "Entry") -> "Entry": raise ValueError( f"Value of type {type(self.value)} is not of type {dtype}" ) - if ( - self.value is not None - and choices is not None - and self.value not in choices - ): - raise ValueError( - f"Value {self.value} is not a valid choice for entry {self.label}" - ) + if self.value is not None and choices is not None: + error_message = f"""Value {self.value} is not a valid choice for entry {self.label}. + Valid choices are: {choices}""" + # in case of multi-select + if isinstance(self.value, list): + for value in self.value: + if value not in choices: + raise ValueError(error_message) + # in case of single-select + else: + if self.value not in choices: + raise ValueError(error_message) if self.value is None and default_value is None and self.required: raise ValueError(f"Value for entry {self.label} is required") @@ -430,19 +445,24 @@ def serialize(self) -> Dict[str, Any]: Returns: Dict[str, Any]: A dictionary representation of the Entry object. """ - return { - key: (value if key != "type" else value.value) - for key, value in self.__dict__.items() - if key != "kitem" - } + dumped = {} + for key, value in self.__dict__.items(): + if key != "kitem": + if isinstance(value, NumericalDataType): + value = float(value) + if key == "type": + value = value.value + dumped[key] = value + return dumped @classmethod def _get_input_spec(cls, self: "Entry"): spec = [] - for section in self.webform.sections: - for inp in section.inputs: - if inp.id == self.id: - spec.append(inp) + if self.webform: + for section in self.webform.sections: + for inp in section.inputs: + if inp.id == self.id: + spec.append(inp) return spec @@ -468,13 +488,13 @@ def __setattr__(self, key, value) -> None: value: The value of the attribute to be set. """ if key == "kitem": - for entry in self.entries: - entry.kitem = value + for entry in self.entries: # pylint: disable=not-an-iterable + entry.kitem = value # pylint: disable=assigning-non-slot # Set value if key not in self.model_dump() and key != "kitem": to_be_updated = [] - for entry in self.entries: + for entry in self.entries: # pylint: disable=not-an-iterable if entry.label == key: to_be_updated.append(entry) if len(to_be_updated) == 0: @@ -513,7 +533,7 @@ def __getattr__(self, key) -> Any: """ target = [] if not key in self.model_dump() and key != "kitem": - for entry in self.entries: + for entry in self.entries: # pylint: disable=not-an-iterable if entry.label == key: target.append(entry) if len(target) == 0: @@ -582,7 +602,7 @@ def __getattr__(self, key): target = [] if not key in self.model_dump() and key != "kitem": - for section in self.sections: + for section in self.sections: # pylint: disable=not-an-iterable for entry in section.entries: if entry.label == key: target.append(entry) @@ -619,12 +639,12 @@ def __setattr__(self, key, value) -> None: """ if key == "kitem": - self.sections.kitem = value + self.sections.kitem = value # pylint: disable=assigning-non-slot # Set value in model if key not in self.model_dump().keys(): to_be_updated = [] - for section in self.sections: + for section in self.sections: # pylint: disable=not-an-iterable for entry in section.entries: if entry.label == key: to_be_updated.append(entry) diff --git a/setup.cfg b/setup.cfg index f200f36..5fbab39 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,7 @@ install_requires = click>=8,<9 html5lib>=1,<2 lru-cache<1 + oyaml==1.1 pandas>=2,<3 pydantic>=2,<3 pydantic-settings From 78c534b28ccee543b2c6a42d9ff113ccdb509690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Mon, 16 Dec 2024 18:58:39 +0100 Subject: [PATCH 40/66] debug custom properties and dataframe --- dsms/knowledge/properties/base.py | 5 ++ dsms/knowledge/webform.py | 127 ++++++++++++++++++++++++++++-- 2 files changed, 125 insertions(+), 7 deletions(-) diff --git a/dsms/knowledge/properties/base.py b/dsms/knowledge/properties/base.py index 07c1c2b..0deccc0 100644 --- a/dsms/knowledge/properties/base.py +++ b/dsms/knowledge/properties/base.py @@ -75,6 +75,11 @@ def kitem(cls) -> "KItem": """KItem related to the KItemProperty""" return cls._kitem + @property + def dsms(cls) -> "KItem": + """DSMS instance related to the KItemProperty""" + return cls.kitem.dsms + @kitem.setter def kitem(cls, item: "KItem") -> None: """Set KItem related to the KItemProperty""" diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index 245a6d2..9d028b5 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -65,16 +65,36 @@ class WebformMeasurementUnit(BaseModel): label: Optional[str] = Field( None, description="Label of the measurement unit" ) - iri: Optional[AnyUrl] = Field( + iri: Optional[Union[str, AnyUrl]] = Field( None, description="IRI of the measurement unit" ) symbol: Optional[str] = Field( None, description="Symbol of the measurement unit" ) - namespace: Optional[AnyUrl] = Field( + namespace: Optional[Union[str, AnyUrl]] = Field( None, description="Namespace of the measurement unit" ) + @model_validator(mode="after") + def check_measurement_unit(cls, self) -> "MeasurementUnit": + """ + Validate and convert IRI and namespace fields to AnyUrl type. + + This method is a model validator that runs after the model is initialized. + It ensures that the `iri` and `namespace` fields of the `MeasurementUnit` + are of type `AnyUrl`. If they are not, it attempts to convert them to + `AnyUrl`. + + Returns: + MeasurementUnit: The validated and potentially modified instance. + """ + + if not isinstance(self.iri, AnyUrl): + self.iri = str(self.iri) + if not isinstance(self.namespace, AnyUrl): + self.namespace = str(self.namespace) + return self + class WebformRangeOptions(BaseModel): """Range options""" @@ -198,6 +218,25 @@ class RelationMapping(BaseWebformModel): description="Target class IRI if the type of relation is an object property", ) + @model_serializer + def serialize(self) -> Dict[str, Any]: + """ + Serialize the Input object to a dictionary representation. + + This method transforms the Input instance into a dictionary, where the keys + are the attribute names and the values are the corresponding attribute values. + The "type" attribute is treated specially by storing its `value` instead of + the object itself. + + Returns: + Dict[str, Any]: A dictionary representation of the Input object. + """ + return { + key: (value.value if isinstance(value, Enum) else value) + for key, value in self.__dict__.items() + if key not in self.model_config["exclude"] + } + class Input(BaseWebformModel): """Input fields in the sections in webform""" @@ -235,6 +274,25 @@ class Input(BaseWebformModel): None, description="Placeholder for the input" ) + @model_serializer + def serialize(self) -> Dict[str, Any]: + """ + Serialize the Input object to a dictionary representation. + + This method transforms the Input instance into a dictionary, where the keys + are the attribute names and the values are the corresponding attribute values. + The "type" attribute is treated specially by storing its `value` instead of + the object itself. + + Returns: + Dict[str, Any]: A dictionary representation of the Input object. + """ + return { + key: (value.value if isinstance(value, Enum) else value) + for key, value in self.__dict__.items() + if key not in self.model_config["exclude"] + } + class Section(BaseWebformModel): """Section in webform""" @@ -267,7 +325,7 @@ class Webform(BaseWebformModel): class MeasurementUnit(BaseWebformModel): """Measurement unit""" - iri: Optional[AnyUrl] = Field( + iri: Optional[Union[str, AnyUrl]] = Field( None, description="IRI of the annotation", ) @@ -279,10 +337,30 @@ class MeasurementUnit(BaseWebformModel): None, description="Symbol of the measurement unit", ) - namespace: Optional[AnyUrl] = Field( + namespace: Optional[Union[str, AnyUrl]] = Field( None, description="Namespace of the measurement unit" ) + @model_validator(mode="after") + def check_measurement_unit(cls, self) -> "MeasurementUnit": + """ + Validate and convert IRI and namespace fields to AnyUrl type. + + This method is a model validator that runs after the model is initialized. + It ensures that the `iri` and `namespace` fields of the `MeasurementUnit` + are of type `AnyUrl`. If they are not, it attempts to convert them to + `AnyUrl`. + + Returns: + MeasurementUnit: The validated and potentially modified instance. + """ + + if not isinstance(self.iri, AnyUrl): + self.iri = str(self.iri) + if not isinstance(self.namespace, AnyUrl): + self.namespace = str(self.namespace) + return self + class KnowledgeItemReference(BaseModel): """Reference to a knowledge item if linked in the custom properties""" @@ -348,6 +426,39 @@ def _validate_value(cls, value: Any) -> Any: value = NumericalDataType(value) return value + def get_unit(self) -> "Dict[str, Any]": + """Get unit for the property""" + if not isinstance(self.value, NumericalDataType): + raise TypeError( + f"Cannot get unit for value {self.value} of type {type(self.value)}" + ) + return self.value.get_unit() # pylint: disable=no-member + + def convert_to( + self, + unit_symbol_or_iri: str, + decimals: "Optional[int]" = None, + use_input_iri: bool = True, + ) -> Any: + """ + Convert the data of the entry to a different unit. + + Args: + unit_symbol_or_iri (str): Symbol or IRI of the unit to convert to. + decimals (Optional[int]): Number of decimals to round the result to. Defaults to None. + use_input_iri (bool): If True, use IRI for unit comparison. Defaults to False. + + Returns: + Any: converted value of the entry + """ + if not isinstance(self.value, NumericalDataType): + raise TypeError( + f"Cannot convert value {self.value} of type {type(self.value)}" + ) + return self.value.convert_to( # pylint: disable=no-member + unit_symbol_or_iri, decimals, use_input_iri + ) + @model_validator(mode="after") @classmethod def _validate_inputs(cls, self: "Entry") -> "Entry": @@ -603,17 +714,19 @@ def __getattr__(self, key): if not key in self.model_dump() and key != "kitem": for section in self.sections: # pylint: disable=not-an-iterable + if section.name == key: + target.append(section) for entry in section.entries: if entry.label == key: target.append(entry) if len(target) == 0: raise AttributeError( - f"Custom properties model has no attribute '{key}'" + f"Custom properties model has no entry or section '{key}'" ) if len(target) > 1: raise AttributeError( - f"""Custom properties model has multiple attributes - '{key}'. Please specify section!""" + f"""Custom properties model has multiple entries or sections for + '{key}'. Please specify section via list indexing!""" ) target = target.pop() else: From e9f3f58621639925b000fd726684ff42acc7870d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Mon, 16 Dec 2024 19:07:16 +0100 Subject: [PATCH 41/66] update config --- dsms/apps/config.py | 32 ++++++++------------------------ setup.cfg | 2 +- 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/dsms/apps/config.py b/dsms/apps/config.py index a0d27a2..85aee47 100644 --- a/dsms/apps/config.py +++ b/dsms/apps/config.py @@ -18,6 +18,7 @@ _get_app_specification, ) +from dsms.knowledge.utils import print_model # isort:skip from dsms.core.logging import handler # isort:skip @@ -101,30 +102,13 @@ def __setattr__(self, name, value) -> None: ) self.session.buffers.updated.update({self.name: self}) - # def __str__(self) -> str: - # """Pretty print the app config fields""" - # fields = ", ".join( - # [ - # "{key}={value}".format( # pylint: disable=consider-using-f-string - # key=key, - # value=( - # value - # if key != "specification" - # else { - # "metadata": value.get( # pylint: disable=no-member - # "metadata" - # ) - # } - # ), - # ) - # for key, value in self.__dict__.items() - # ] - # ) - # return f"{self.__class__.__name__}({fields})" - - # def __repr__(self) -> str: - # """Pretty print the kitem Fields""" - # return str(self) + def __str__(self) -> str: + """Pretty print the kitem Fields""" + return print_model(self, "app") + + def __repr__(self) -> str: + """Pretty print the kitem Fields""" + return str(self) @field_validator("name") @classmethod diff --git a/setup.cfg b/setup.cfg index 5fbab39..382f8f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ install_requires = click>=8,<9 html5lib>=1,<2 lru-cache<1 - oyaml==1.1 + oyaml==1 pandas>=2,<3 pydantic>=2,<3 pydantic-settings From 1f60b4da63cf3c3f8bfd224e5179e4e66cfd4ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Tue, 17 Dec 2024 11:53:15 +0100 Subject: [PATCH 42/66] debug buffer context --- dsms/core/configuration.py | 24 +++++++--- dsms/knowledge/kitem.py | 44 +++++++++++++------ .../properties/custom_datatype/numerical.py | 22 +++++----- dsms/knowledge/properties/external_links.py | 28 ++++++++++-- dsms/knowledge/properties/linked_kitems.py | 2 +- dsms/knowledge/utils.py | 12 ++--- dsms/knowledge/webform.py | 4 +- 7 files changed, 90 insertions(+), 46 deletions(-) diff --git a/dsms/core/configuration.py b/dsms/core/configuration.py index 9237366..350fb2d 100644 --- a/dsms/core/configuration.py +++ b/dsms/core/configuration.py @@ -7,11 +7,21 @@ from typing import Callable, Optional, Set, Union import requests -from pydantic import AnyUrl, ConfigDict, Field, SecretStr, field_validator -from pydantic_core.core_schema import ValidationInfo -from pydantic_settings import BaseSettings, SettingsConfigDict -from .utils import get_callable +from pydantic_core.core_schema import ValidationInfo # isort: skip +from pydantic_settings import BaseSettings, SettingsConfigDict # isort: skip + + +from pydantic import ( # isort: skip + AliasChoices, + AnyUrl, + ConfigDict, + Field, + SecretStr, + field_validator, +) + +from .utils import get_callable # isort: skip MODULE_REGEX = r"^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*:[a-zA-Z_][a-zA-Z0-9_]*$" DEFAULT_UNIT_SPARQL = "dsms.knowledge.semantics.units.sparql:UnitSparqlQuery" @@ -121,7 +131,9 @@ class Configuration(BaseSettings): ) loglevel: Optional[Union[Loglevel, str]] = Field( - None, description="Set level of logging messages" + None, + description="Set level of logging messages", + alias=AliasChoices("loglevel", "log_level"), ) model_config = ConfigDict(use_enum_values=True) @@ -146,7 +158,7 @@ def validate_hide_properties(cls, val: Set) -> "Callable": from dsms import KItem for key in val: - if key not in KItem.model_fields: + if key not in KItem.model_fields: # pylint: disable=E1135 raise KeyError(f"Property `{key}` not in KItem schema") return val diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 1e87a91..ed6c554 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -231,13 +231,13 @@ def __init__(self, **kwargs: "Any") -> None: super().__init__(**kwargs) # add kitem to buffer - if not self.in_backend and self.id not in self.context.buffers.created: + if not self.in_backend and self.id not in self.session.buffers.created: logger.debug( "Marking KItem with ID `%s` as created and updated during KItem initialization.", self.id, ) - self.context.buffers.created.update({self.id: self}) - self.context.buffers.updated.update({self.id: self}) + self.session.buffers.created.update({self.id: self}) + self.session.buffers.updated.update({self.id: self}) self._set_kitem_for_properties() @@ -252,18 +252,20 @@ def __setattr__(self, name, value) -> None: self._set_kitem_for_properties() if ( - self.id not in self.context.buffers.updated + self.id not in self.session.buffers.updated and not name.startswith("_") ): logger.debug( "Setting KItem with ID `%s` as updated during KItem.__setattr__", self.id, ) - self.context.buffers.updated.update({self.id: self}) + self.session.buffers.updated.update({self.id: self}) def __str__(self) -> str: """Pretty print the kitem fields""" - return print_model(self, "kitem") + return print_model( + self, "kitem", exclude_extra=self.dsms.config.hide_properties + ) def __repr__(self) -> str: """Pretty print the kitem Fields""" @@ -573,10 +575,12 @@ def validate_custom_properties(cls, self: "KItem") -> "KItem": """A flat dictionary was provided for custom properties. Will be transformed into `KItemCustomPropertiesModel`.""" ) - + was_in_buffer = self.id in self.session.buffers.updated self.custom_properties = KItemCustomPropertiesModel( **value, kitem=self ) + if not was_in_buffer: + self.session.buffers.updated.pop(self.id) elif not isinstance( self.custom_properties, (KItemCustomPropertiesModel, type(None)) ): @@ -589,7 +593,7 @@ def validate_custom_properties(cls, self: "KItem") -> "KItem": def _set_kitem_for_properties(self) -> None: """Set kitem for CustomProperties and KProperties in order to - remain the context for the buffer if any of these properties is changed. + remain the session for the buffer if any of these properties is changed. """ for prop in self.__dict__.values(): if ( @@ -613,13 +617,13 @@ def _set_kitem_for_properties(self) -> None: @property def dsms(cls) -> "DSMS": - """DSMS context getter""" - return cls.context.dsms + """DSMS session getter""" + return cls.session.dsms @dsms.setter def dsms(cls, value: "DSMS") -> None: - """DSMS context setter""" - cls.context.dsms = value + """DSMS session setter""" + cls.session.dsms = value @property def subgraph(cls) -> Optional[Graph]: @@ -629,7 +633,7 @@ def subgraph(cls) -> Optional[Graph]: ) @property - def context(cls) -> "Session": + def session(cls) -> "Session": """Getter for Session""" from dsms import ( # isort:skip Session, @@ -641,7 +645,7 @@ def context(cls) -> "Session": def url(cls) -> str: """URL of the KItem""" return cls.access_url or urljoin( - str(cls.context.dsms.config.host_url), + str(cls.session.dsms.config.host_url), f"knowledge/{cls._get_ktype_as_str()}/{cls.slug}", ) @@ -661,3 +665,15 @@ def _get_ktype_as_str(self) -> str: else: raise TypeError(f"Datatype for KType is unknown: {type(ktype)}") return ktype + + +class KItemList(list): + """List of KItems""" + + def __str__(self): + """Pretty print the KItemList""" + return "\n".join([str(item) for item in self]) + + def __repr__(self): + """Pretty print the KItemList""" + return str(self) diff --git a/dsms/knowledge/properties/custom_datatype/numerical.py b/dsms/knowledge/properties/custom_datatype/numerical.py index 1dc0fed..dd855ca 100644 --- a/dsms/knowledge/properties/custom_datatype/numerical.py +++ b/dsms/knowledge/properties/custom_datatype/numerical.py @@ -28,7 +28,7 @@ def __new__(cls, value): def __str__(self) -> str: """Pretty print the numerical datatype""" - if self.kitem.dsms.config.display_units: + if self.kitem.dsms.config.display_units: # pylint: disable=no-member try: string = f"{self.__float__()} {self.get_unit().get('symbol')}" except Exception as error: @@ -69,32 +69,32 @@ def __repr__(self) -> str: return str(self) @property - def kitem(cls) -> "Optional[KItem]": + def kitem(self) -> "Optional[KItem]": """Context of the current kitem for this property""" - return cls._kitem + return self._kitem @kitem.setter - def kitem(cls, value: "Optional[KItem]") -> None: + def kitem(self, value: "Optional[KItem]") -> None: """Setter for current KItem context""" - cls._kitem = value + self._kitem = value @property - def name(cls) -> str: + def name(self) -> str: """Context of the name for this property""" - return cls._name + return self._name @name.setter - def name(cls, value: str) -> None: + def name(self, value: str) -> None: """Setter for the name of the property""" - cls._name = value + self._name = value def get_unit(self) -> "Dict[str, Any]": """Get unit for the property""" return get_property_unit( - self.kitem.id, + self.kitem.id, # pylint: disable=no-member self.name, is_dataframe_column=True, - autocomplete_symbol=self.kitem.dsms.config.autocomplete_units, + autocomplete_symbol=self.kitem.dsms.config.autocomplete_units, # pylint: disable=no-member ) def convert_to( diff --git a/dsms/knowledge/properties/external_links.py b/dsms/knowledge/properties/external_links.py index 60b48a7..f15720f 100644 --- a/dsms/knowledge/properties/external_links.py +++ b/dsms/knowledge/properties/external_links.py @@ -1,8 +1,8 @@ """ExternalLink property of a KItem""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Union -from pydantic import AnyUrl, Field +from pydantic import AnyUrl, Field, field_validator from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList from dsms.knowledge.utils import print_model @@ -17,12 +17,34 @@ class ExternalLink(KItemProperty): label: str = Field( ..., description="Label of the external link", max_length=50 ) - url: AnyUrl = Field(..., description="URL of the external link") + url: Union[str, AnyUrl] = Field( + ..., description="URL of the external link" + ) # OVERRIDE def __str__(self): return print_model(self, "external_link") + @field_validator("url") + @classmethod + def validate_url(cls, value: Union[str, AnyUrl]) -> AnyUrl: + """ + Validate and convert the URL to a string. + + This method is a model validator that runs after the model is initialized. + It ensures that the `url` field of the `ExternalLink` is a string. + If it is not, it attempts to convert it to a string. + + Args: + value (Union[str, AnyUrl]): The value to be validated and converted. + + Returns: + AnyUrl: The validated and potentially modified URL. + """ + if isinstance(value, AnyUrl): + value = str(value) + return value + class ExternalLinksProperty(KItemPropertyList): """KItemPropertyList for external links""" diff --git a/dsms/knowledge/properties/linked_kitems.py b/dsms/knowledge/properties/linked_kitems.py index 447f028..f27760e 100644 --- a/dsms/knowledge/properties/linked_kitems.py +++ b/dsms/knowledge/properties/linked_kitems.py @@ -136,7 +136,7 @@ class LinkedKItem(KItemProperty): _kitem = PrivateAttr(default=None) # OVERRIDE - model_config = ConfigDict(exclude={}, arbitrary_types_allowed=True) + model_config = ConfigDict(arbitrary_types_allowed=True) def fetch(self) -> "KItem": """Fetch the linked KItem""" diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 7e3b932..b880a27 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -44,11 +44,7 @@ def _is_number(value): def print_model(self, key, exclude_extra: set = set()) -> str: """Pretty print the ktype fields""" - exclude = ( - self.model_config.get("exclude", set()) - | exclude_extra - | self.dsms.config.hide_properties - ) + exclude = self.model_config.get("exclude", set()) | exclude_extra dumped = self.model_dump( exclude_none=True, exclude_unset=True, @@ -212,16 +208,14 @@ def _delete_ktype(ktype: "KType") -> None: def _get_kitem_list() -> "List[KItem]": """Get all available KItems from the remote backend.""" - from dsms.knowledge.kitem import ( # isort:skip - KItem, - ) + from dsms.knowledge.kitem import KItem, KItemList # isort:skip response = _perform_request("api/knowledge/kitems", "get") if not response.ok: raise ValueError( f"Something went wrong fetching the available kitems: {response.text}" ) - return [KItem(**kitem) for kitem in response.json()] + return KItemList(KItem(**kitem) for kitem in response.json()) def _kitem_exists(kitem: Union[Any, str, UUID]) -> bool: diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index 9d028b5..bbda050 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -178,7 +178,7 @@ def __setattr__(self, key, value) -> None: This method sets an attribute of the model and logs the operation. If the attribute being set is `kitem`, it directly assigns the value. For other attributes, it marks the associated `kitem` as updated in the - context buffers if it exists. + session buffers if it exists. Args: key (str): The name of the attribute to set. @@ -196,7 +196,7 @@ def __setattr__(self, key, value) -> None: "Setting related kitem with id `%s` as updated", self.kitem.id, ) - self.kitem.context.buffers.updated.update( + self.kitem.session.buffers.updated.update( {self.kitem.id: self.kitem} ) From 00999e4006315f2b80101836c610a383ef9d004f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Tue, 17 Dec 2024 12:10:33 +0100 Subject: [PATCH 43/66] bump dev version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 382f8f0..65666ce 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev5 +version = v2.1.0dev6 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From 38932908e25054641efee0927fe200134e4a6e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Tue, 17 Dec 2024 15:14:32 +0100 Subject: [PATCH 44/66] fix minor problems --- dsms/knowledge/kitem.py | 5 +++-- dsms/knowledge/properties/custom_datatype/numerical.py | 6 ++++-- dsms/knowledge/webform.py | 1 + setup.cfg | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index ed6c554..45c233e 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -448,11 +448,12 @@ def validate_ktype_id(cls, value: Union[str, Enum]) -> KType: from dsms import Session if isinstance(value, str): - value = Session.ktypes.get(value) - if not value: + ktype = Session.ktypes.get(value) + if not ktype: raise TypeError( f"KType for `ktype_id={value}` does not exist." ) + value = ktype return value.id diff --git a/dsms/knowledge/properties/custom_datatype/numerical.py b/dsms/knowledge/properties/custom_datatype/numerical.py index dd855ca..2309a55 100644 --- a/dsms/knowledge/properties/custom_datatype/numerical.py +++ b/dsms/knowledge/properties/custom_datatype/numerical.py @@ -45,7 +45,9 @@ def __get_validators__(cls) -> "Generator": yield cls.validate @classmethod - def validate(cls, v: "Any") -> "NumericalDataType": + def validate( + cls, v: "Any", *args, **kwargs # pylint: disable=W0613 + ) -> "NumericalDataType": """ Validate the input value as a valid NumericalDataType. @@ -59,7 +61,7 @@ def validate(cls, v: "Any") -> "NumericalDataType": TypeError: If the input value is not a float or int. """ if not isinstance(v, (float, int)): - raise TypeError(f"Expected float or int, got {type(v)}") + raise TypeError(f"Expected float or int, got {type(v)}: {v}") obj = super().__new__(cls, v) obj._kitem = None obj._name = None diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index bbda050..12f1b12 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -394,6 +394,7 @@ class Entry(BaseWebformModel): relationMapping: Optional[RelationMapping] = Field( None, description="Relation mapping of the entry" ) + required: Optional[bool] = Field(False, description="Required input") def __setattr__(self, key, value) -> None: """ diff --git a/setup.cfg b/setup.cfg index 65666ce..de51f7a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev6 +version = v2.1.0dev7 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From ae25109f3946517268b7a366151621f9fc766a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 18 Dec 2024 14:07:25 +0100 Subject: [PATCH 45/66] remove 'NumericalDatatype', debug unit converion --- dsms/knowledge/properties/__init__.py | 2 - .../properties/custom_datatype/__init__.py | 5 - .../properties/custom_datatype/numerical.py | 126 --------- dsms/knowledge/semantics/units/utils.py | 87 ++++--- dsms/knowledge/webform.py | 239 +++++++++++++++--- 5 files changed, 249 insertions(+), 210 deletions(-) delete mode 100644 dsms/knowledge/properties/custom_datatype/__init__.py delete mode 100644 dsms/knowledge/properties/custom_datatype/numerical.py diff --git a/dsms/knowledge/properties/__init__.py b/dsms/knowledge/properties/__init__.py index 13b8547..3df0c2a 100644 --- a/dsms/knowledge/properties/__init__.py +++ b/dsms/knowledge/properties/__init__.py @@ -12,7 +12,6 @@ from dsms.knowledge.properties.authors import Author, AuthorsProperty from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList from dsms.knowledge.properties.contacts import ContactInfo, ContactsProperty -from dsms.knowledge.properties.custom_datatype import NumericalDataType from dsms.knowledge.properties.dataframe import Column, DataFrameContainer from dsms.knowledge.properties.summary import Summary from dsms.knowledge.properties.user_groups import UserGroup, UserGroupsProperty @@ -60,5 +59,4 @@ "KItemProperty", "DataFrameContainer", "Column", - "NumericalDataType", ] diff --git a/dsms/knowledge/properties/custom_datatype/__init__.py b/dsms/knowledge/properties/custom_datatype/__init__.py deleted file mode 100644 index f53b7d1..0000000 --- a/dsms/knowledge/properties/custom_datatype/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Custom data type module""" - -from .numerical import NumericalDataType - -__all__ = ["NumericalDataType"] diff --git a/dsms/knowledge/properties/custom_datatype/numerical.py b/dsms/knowledge/properties/custom_datatype/numerical.py deleted file mode 100644 index 2309a55..0000000 --- a/dsms/knowledge/properties/custom_datatype/numerical.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Module for custom numerical data type""" - -import logging -from typing import TYPE_CHECKING - -from dsms.knowledge.semantics.units.utils import ( - get_conversion_factor, - get_property_unit, -) - -if TYPE_CHECKING: - from typing import Any, Dict, Generator, Optional - - from dsms import KItem - - -logger = logging.Logger(__name__) - - -class NumericalDataType(float): - """Custom Base data type for custom properties""" - - def __new__(cls, value): - obj = super().__new__(cls, value) - obj._kitem = None - obj._name = None - return obj - - def __str__(self) -> str: - """Pretty print the numerical datatype""" - if self.kitem.dsms.config.display_units: # pylint: disable=no-member - try: - string = f"{self.__float__()} {self.get_unit().get('symbol')}" - except Exception as error: - logger.debug( - "Could not fetch unit from `%i`: %i", self.name, error.args - ) - string = str(self.__float__()) - else: - string = str(self.__float__()) - return string - - @classmethod - def __get_validators__(cls) -> "Generator": - yield cls.validate - - @classmethod - def validate( - cls, v: "Any", *args, **kwargs # pylint: disable=W0613 - ) -> "NumericalDataType": - """ - Validate the input value as a valid NumericalDataType. - - Args: - v (Any): The value to be validated. - - Returns: - NumericalDataType: An instance of NumericalDataType if validation is successful. - - Raises: - TypeError: If the input value is not a float or int. - """ - if not isinstance(v, (float, int)): - raise TypeError(f"Expected float or int, got {type(v)}: {v}") - obj = super().__new__(cls, v) - obj._kitem = None - obj._name = None - return obj - - def __repr__(self) -> str: - return str(self) - - @property - def kitem(self) -> "Optional[KItem]": - """Context of the current kitem for this property""" - return self._kitem - - @kitem.setter - def kitem(self, value: "Optional[KItem]") -> None: - """Setter for current KItem context""" - self._kitem = value - - @property - def name(self) -> str: - """Context of the name for this property""" - return self._name - - @name.setter - def name(self, value: str) -> None: - """Setter for the name of the property""" - self._name = value - - def get_unit(self) -> "Dict[str, Any]": - """Get unit for the property""" - return get_property_unit( - self.kitem.id, # pylint: disable=no-member - self.name, - is_dataframe_column=True, - autocomplete_symbol=self.kitem.dsms.config.autocomplete_units, # pylint: disable=no-member - ) - - def convert_to( - self, - unit_symbol_or_iri: str, - decimals: "Optional[int]" = None, - use_input_iri: bool = True, - ) -> float: - """ - Convert the data of property to a different unit. - - Args: - unit_symbol_or_iri (str): Symbol or IRI of the unit to convert to. - decimals (Optional[int]): Number of decimals to round the result to. Defaults to None. - use_input_iri (bool): If True, use IRI for unit comparison. Defaults to False. - - Returns: - float: converted value of the property - """ - unit = self.get_unit() - if use_input_iri: - input_str = unit.get("iri") - else: - input_str = unit.get("symbol") - return self * get_conversion_factor( - input_str, unit_symbol_or_iri, decimals=decimals - ) diff --git a/dsms/knowledge/semantics/units/utils.py b/dsms/knowledge/semantics/units/utils.py index 0916da6..fdef380 100644 --- a/dsms/knowledge/semantics/units/utils.py +++ b/dsms/knowledge/semantics/units/utils.py @@ -68,56 +68,63 @@ def get_conversion_factor( def get_property_unit( kitem_id: "Union[str, UUID]", property_name: str, + measurement_unit: "Optional[Any]" = None, is_dataframe_column: bool = False, autocomplete_symbol: bool = True, ) -> "Dict[str, Any]": """ - Retrieve the unit associated with a given property of a KIitem. + Get unit for a given property name. + + This function queries the unit for a given property name with respect to the + semantics applied. If `measurement_unit` is not `None`, it will be returned + as is. If `measurement_unit` is `None`, the function will try to get the unit + from the QUDT-based semantics. If no unit is found, a `ValueError` is raised. Args: - kitem (KItem): The identifier of the KItem. - property_name (str): The name of the property of the KItem for which - the unit is to be retrieved. - is_dataframe_column (bool, optional): Indicates whether the property is an DataFrame - column or a custom property. Defaults to False. - autocomplete_symbol (bool, optional): Whether the symbol of a unit shall be - fetched automatically from the ontology when it is not given next to the - URI. + kitem_id (Union[str, UUID]): ID of the KItem. + property_name (str): Name of the property. + measurement_unit (Optional[Any], optional): Measurement unit. Defaults to None. + is_dataframe_column (bool, optional): Whether the property is a dataframe column. + Defaults to False. + autocomplete_symbol (bool, optional): Whether to autocomplete the unit symbol. + Defaults to True. Returns: - Dict[str, Any]: A dictionary with the symbol and iri of the unit associated - with the specified property. + Dict[str, Any]: The unit as a dictionary. Raises: - ValueError: If unable to retrieve the unit for the property due to any errors or if - the property does not have a unit or has more than one unit associated with it. + ValueError: If no unit is found. """ from dsms import Session - units_sparql_object = Session.dsms.config.units_sparql_object - if not issubclass(units_sparql_object, BaseUnitSparqlQuery): - raise TypeError( - f"´{units_sparql_object}´ must be a subclass of `{BaseUnitSparqlQuery}`" - ) - try: - query = units_sparql_object( - kitem_id=kitem_id, - property_name=property_name, - is_dataframe_column=is_dataframe_column, - autocomplete_symbol=autocomplete_symbol, - ) - except Exception as error: - raise ValueError( - f"Something went wrong catching the unit for property `{property_name}`." - ) from error - if len(query.results) == 0: - raise ValueError( - f"""Property `{property_name}` does not own any - unit with respect to the semantics applied.""" - ) - if len(query.results) > 1: - raise ValueError( - f"""Property `{property_name}` owns more than one - unit with respect to the semantics applied.""" - ) - return query.results.pop() + if not measurement_unit: + units_sparql_object = Session.dsms.config.units_sparql_object + if not issubclass(units_sparql_object, BaseUnitSparqlQuery): + raise TypeError( + f"´{units_sparql_object}´ must be a subclass of `{BaseUnitSparqlQuery}`" + ) + try: + query = units_sparql_object( + kitem_id=kitem_id, + property_name=property_name, + is_dataframe_column=is_dataframe_column, + autocomplete_symbol=autocomplete_symbol, + ) + except Exception as error: + raise ValueError( + f"Something went wrong catching the unit for property `{property_name}`." + ) from error + if len(query.results) == 0: + raise ValueError( + f"""Property `{property_name}` does not own any + unit with respect to the semantics applied.""" + ) + if len(query.results) > 1: + raise ValueError( + f"""Property `{property_name}` owns more than one + unit with respect to the semantics applied.""" + ) + unit = query.results.pop() + else: + unit = measurement_unit.model_dump() + return unit diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index 12f1b12..deece11 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -11,9 +11,9 @@ BaseModel, ConfigDict, Field, - field_validator, model_serializer, model_validator, + AliasChoices, ) from dsms.knowledge.utils import ( # isort:skip @@ -22,9 +22,11 @@ print_model, ) -from dsms.knowledge.properties.custom_datatype import ( # isort:skip - NumericalDataType, +from dsms.knowledge.semantics.units.utils import ( # isort:skip + get_conversion_factor, + get_property_unit, ) + from dsms.core.logging import handler # isort:skip if TYPE_CHECKING: @@ -58,6 +60,14 @@ class WebformSelectOption(BaseModel): value: Optional[Any] = Field(None, description="Value of the option") disabled: Optional[bool] = Field(False, description="Disabled option") + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "select option") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + class WebformMeasurementUnit(BaseModel): """Measurement unit""" @@ -95,6 +105,14 @@ def check_measurement_unit(cls, self) -> "MeasurementUnit": self.namespace = str(self.namespace) return self + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "webform measurement unit") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + class WebformRangeOptions(BaseModel): """Range options""" @@ -106,6 +124,14 @@ class WebformRangeOptions(BaseModel): False, description="Range value" ) + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "range options") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + class RelationMappingType(Enum): """ @@ -218,6 +244,14 @@ class RelationMapping(BaseWebformModel): description="Target class IRI if the type of relation is an object property", ) + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "relation mapping") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + @model_serializer def serialize(self) -> Dict[str, Any]: """ @@ -293,6 +327,14 @@ def serialize(self) -> Dict[str, Any]: if key not in self.model_config["exclude"] } + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "input") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + class Section(BaseWebformModel): """Section in webform""" @@ -306,6 +348,14 @@ class Section(BaseWebformModel): ) hidden: Optional[bool] = Field(False, description="Hidden section") + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "section") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + class Webform(BaseWebformModel): """User defined webform for ktype""" @@ -321,6 +371,14 @@ class Webform(BaseWebformModel): ) sections: List[Section] = Field([], description="List of sections") + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "webform") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + class MeasurementUnit(BaseWebformModel): """Measurement unit""" @@ -341,6 +399,14 @@ class MeasurementUnit(BaseWebformModel): None, description="Namespace of the measurement unit" ) + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "measurement_unit") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + @model_validator(mode="after") def check_measurement_unit(cls, self) -> "MeasurementUnit": """ @@ -368,6 +434,14 @@ class KnowledgeItemReference(BaseModel): id: str = Field(..., description="ID of the knowledge item") name: str = Field(..., description="Name of the knowledge item") + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "knowledge item reference") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + class Entry(BaseWebformModel): """ @@ -385,14 +459,21 @@ class Entry(BaseWebformModel): float, bool, List, - NumericalDataType, ] ] = Field(None, description="Value of the entry") - measurementUnit: Optional[MeasurementUnit] = Field( - None, description="Measurement unit of the entry" + measurement_unit: Optional[MeasurementUnit] = Field( + None, + description="Measurement unit of the entry", + alias=AliasChoices( + "measurementUnit", "measurement_unit", "measurementunit" + ), ) - relationMapping: Optional[RelationMapping] = Field( - None, description="Relation mapping of the entry" + relation_mapping: Optional[RelationMapping] = Field( + None, + description="Relation mapping of the entry", + alias=AliasChoices( + "relationMapping", "relation_mapping", "relationmapping" + ), ) required: Optional[bool] = Field(False, description="Required input") @@ -420,29 +501,34 @@ def __setattr__(self, key, value) -> None: super().__setattr__(key, value) - @field_validator("value") - @classmethod - def _validate_value(cls, value: Any) -> Any: - if isinstance(value, (int, float)): - value = NumericalDataType(value) - return value + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "entry") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) def get_unit(self) -> "Dict[str, Any]": """Get unit for the property""" - if not isinstance(self.value, NumericalDataType): - raise TypeError( - f"Cannot get unit for value {self.value} of type {type(self.value)}" - ) - return self.value.get_unit() # pylint: disable=no-member + if not isinstance(self.value, (float, int)): + raise ValueError("Value must be a number") + return get_property_unit( + self.kitem.id, # pylint: disable=no-member + self.label, + self.measurement_unit, + is_dataframe_column=True, + autocomplete_symbol=self.kitem.dsms.config.autocomplete_units, # pylint: disable=no-member + ) def convert_to( self, unit_symbol_or_iri: str, decimals: "Optional[int]" = None, use_input_iri: bool = True, - ) -> Any: + ) -> float: """ - Convert the data of the entry to a different unit. + Convert the data of property to a different unit. Args: unit_symbol_or_iri (str): Symbol or IRI of the unit to convert to. @@ -450,14 +536,17 @@ def convert_to( use_input_iri (bool): If True, use IRI for unit comparison. Defaults to False. Returns: - Any: converted value of the entry + float: converted value of the property """ - if not isinstance(self.value, NumericalDataType): - raise TypeError( - f"Cannot convert value {self.value} of type {type(self.value)}" - ) - return self.value.convert_to( # pylint: disable=no-member - unit_symbol_or_iri, decimals, use_input_iri + if not isinstance(self.value, (float, int)): + raise ValueError("Value must be a number") + unit = self.get_unit() + if use_input_iri: + input_str = unit.get("iri") + else: + input_str = unit.get("symbol") + return self.value * get_conversion_factor( + input_str, unit_symbol_or_iri, decimals=decimals ) @model_validator(mode="after") @@ -497,7 +586,7 @@ def _validate_inputs(cls, self: "Entry") -> "Entry": ): dtype = str elif self.type in (Widget.NUMBER, Widget.SLIDER): - dtype = NumericalDataType + dtype = float elif self.type == Widget.CHECKBOX: dtype = bool elif self.type in (Widget.SELECT, Widget.RADIO, Widget.MULTI_SELECT): @@ -537,11 +626,6 @@ def _validate_inputs(cls, self: "Entry") -> "Entry": if self.value is None and default_value is None and self.required: raise ValueError(f"Value for entry {self.label} is required") - # set name and kitem of numerical data type - if isinstance(self.value, NumericalDataType): - self.value.name = self.label - self.value.kitem = self.kitem - return self @model_serializer @@ -560,8 +644,6 @@ def serialize(self) -> Dict[str, Any]: dumped = {} for key, value in self.__dict__.items(): if key != "kitem": - if isinstance(value, NumericalDataType): - value = float(value) if key == "type": value = value.value dumped[key] = value @@ -655,7 +737,8 @@ def __getattr__(self, key) -> Any: if len(target) > 1: raise AttributeError( f"""Section with name `{self.name}` - has multiple attributes '{key}'. Please specify section!""" + has multiple attributes '{key}'. + Please specify the concrete entry via indexing !""" ) target = target.pop() @@ -663,6 +746,39 @@ def __getattr__(self, key) -> Any: target = super().__getattr__(key) return target + def __getitem__(self, key): + """ + Retrieve an entry from the section by its index. + + Args: + key (int): The index of the entry to retrieve. + + Returns: + Entry: The entry at the given index. + + Raises: + IndexError: If the index is out of range. + """ + return self.entries[key] # pylint: disable=unsubscriptable-object + + def __iter__(self): + """ + Iterate over the entries of the section. + + Yields: + Entry: The entries in the section. + """ + yield from self.entries # pylint: disable=not-an-iterable + + def __len__(self): + """ + Return the number of entries in the section. + + Returns: + int: The number of entries in the section. + """ + return len(self.entries) + @model_validator(mode="before") @classmethod def set_kitem(cls, self: Dict[str, Any]) -> Dict[str, Any]: @@ -682,6 +798,14 @@ def set_kitem(cls, self: Dict[str, Any]) -> Dict[str, Any]: entry["kitem"] = kitem return self + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "section") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) + class KItemCustomPropertiesModel(BaseWebformModel): """ @@ -776,6 +900,39 @@ def __setattr__(self, key, value) -> None: else: super().__setattr__(key, value) + def __iter__(self): + """ + Iterate over the sections in the custom properties. + + This method yields each section within the custom properties model, + allowing for iteration over all sections. + + Yields: + CustomPropertiesSection: The next section in the custom properties. + """ + yield from self.sections # pylint: disable=not-an-iterable + + def __len__(self): + """ + Return the number of sections in the custom properties. + + Returns: + int: The number of sections in the custom properties. + """ + return len(self.sections) + + def __getitem__(self, key): + """ + Retrieve a section from the custom properties by its index. + + Args: + key (int): The index of the section to retrieve. + + Returns: + CustomPropertiesSection: The section at the specified index. + """ + return self.sections[key] # pylint: disable=unsubscriptable-object + @model_validator(mode="before") @classmethod def set_kitem(cls, self: Dict[str, Any]) -> Dict[str, Any]: @@ -794,3 +951,11 @@ def set_kitem(cls, self: Dict[str, Any]) -> Dict[str, Any]: if not "kitem" in section: section["kitem"] = kitem return self + + def __str__(self) -> str: + """Pretty print the model fields""" + return print_model(self, "custom_properties") + + def __repr__(self) -> str: + """Pretty print the model fields""" + return str(self) From 69ba134372bad6cb5279ea5e3871d8d5b2f8c131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 18 Dec 2024 14:33:42 +0100 Subject: [PATCH 46/66] bump dev version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index de51f7a..2566b03 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev7 +version = v2.1.0dev8 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From 9738986cff51b0d865030543c7ea915a0c182152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 18 Dec 2024 16:18:20 +0100 Subject: [PATCH 47/66] debug type check in entry --- dsms/knowledge/webform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index deece11..d37b2cb 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -455,8 +455,8 @@ class Entry(BaseWebformModel): Union[ str, int, - KnowledgeItemReference, float, + KnowledgeItemReference, bool, List, ] @@ -586,7 +586,7 @@ def _validate_inputs(cls, self: "Entry") -> "Entry": ): dtype = str elif self.type in (Widget.NUMBER, Widget.SLIDER): - dtype = float + dtype = (int, float) elif self.type == Widget.CHECKBOX: dtype = bool elif self.type in (Widget.SELECT, Widget.RADIO, Widget.MULTI_SELECT): From 75ff8281b00c5290b58d8fc272f945407e832e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 18 Dec 2024 16:18:59 +0100 Subject: [PATCH 48/66] bump version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 2566b03..374a2ce 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev8 +version = v2.1.0dev9 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From 0a40c391b8a770feb961fc9ed5f2db5306f0385b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 18 Dec 2024 17:44:28 +0100 Subject: [PATCH 49/66] debug serialization --- dsms/knowledge/utils.py | 2 +- dsms/knowledge/webform.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index b880a27..ada5177 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -902,7 +902,7 @@ def _transform_custom_properties_schema(custom_properties: Any, webform: Any): "id": input_def.id, "label": input_def.label, "value": copy_properties.pop(input_def.label), - "measurementUnit": input_def.measurement_unit, + "measurement_unit": input_def.measurement_unit, "type": input_def.widget, **class_mapping, } diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index d37b2cb..d3c1e2f 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -490,12 +490,12 @@ def __setattr__(self, key, value) -> None: value (Any): The value to set for the attribute. """ if key == "kitem": - if self.measurementUnit: - self.measurementUnit.kitem = ( # pylint: disable=assigning-non-slot + if self.measurement_unit: + self.measurement_unit.kitem = ( # pylint: disable=assigning-non-slot value ) - if self.relationMapping: - self.relationMapping.kitem = ( # pylint: disable=assigning-non-slot + if self.relation_mapping: + self.relation_mapping.kitem = ( # pylint: disable=assigning-non-slot value ) @@ -646,6 +646,8 @@ def serialize(self) -> Dict[str, Any]: if key != "kitem": if key == "type": value = value.value + if key in ("measurement_unit", "relation_mapping"): + key = to_camel(key) dumped[key] = value return dumped From 3cc313afce6094ce376906bcb871dd6ba8a3fa1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 18 Dec 2024 19:04:24 +0100 Subject: [PATCH 50/66] update webform and custom properties aliases --- dsms/knowledge/properties/attachments.py | 2 +- dsms/knowledge/utils.py | 21 ++--- dsms/knowledge/webform.py | 104 ++++++----------------- 3 files changed, 33 insertions(+), 94 deletions(-) diff --git a/dsms/knowledge/properties/attachments.py b/dsms/knowledge/properties/attachments.py index 12494c8..93e937b 100644 --- a/dsms/knowledge/properties/attachments.py +++ b/dsms/knowledge/properties/attachments.py @@ -21,7 +21,7 @@ class Attachment(KItemProperty): ) content: Optional[Union[str, bytes]] = Field( - None, description="Content of the file" + None, description="Content of the file", exclude=True ) # OVERRIDE diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index ada5177..35b7518 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -181,6 +181,7 @@ def _update_ktype(ktype: "KType") -> Response: """Update a KType in the remote backend.""" payload = ktype.model_dump( exclude_none=True, + by_alias=True, ) logger.debug("Update KType for `%s` with body: %s", ktype.id, payload) response = _perform_request( @@ -306,15 +307,9 @@ def _update_kitem(new_kitem: "KItem", old_kitem: "Dict[str, Any]") -> Response: **differences, ) if new_kitem.custom_properties: - custom_properties = new_kitem.custom_properties.model_dump() - # # a smarted detection whether the custom properties were updated is needed - # old_properties = old_kitem.get("custom_properties") - # if isinstance(old_properties, dict): - # old_custom_properties = old_properties.get("content") - # else: - # old_custom_properties = None - # if custom_properties != old_custom_properties: - # payload.update(custom_properties={"content": custom_properties}) + custom_properties = new_kitem.custom_properties.model_dump( + by_alias=True + ) payload.update(custom_properties={"content": custom_properties}) logger.debug( "Update KItem for `%s` with payload: %s", new_kitem.id, payload @@ -957,13 +952,13 @@ def _map_data_type_to_widget(value): from dsms.knowledge.webform import Widget if isinstance(value, str): - widget = Widget.TEXT + widget = Widget.TEXT.value elif isinstance(value, (int, float)): - widget = Widget.NUMBER + widget = Widget.NUMBER.value elif isinstance(value, bool): - widget = Widget.CHECKBOX + widget = Widget.CHECKBOX.value elif isinstance(value, list): - widget = Widget.MULTI_SELECT + widget = Widget.MULTI_SELECT.value else: raise ValueError( f"Unsupported data type: {type(value)}. Value: {value}" diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index d3c1e2f..f6c9c06 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -11,8 +11,8 @@ BaseModel, ConfigDict, Field, - model_serializer, model_validator, + AliasGenerator, AliasChoices, ) @@ -147,11 +147,18 @@ class BaseWebformModel(BaseModel): """Base webform model""" model_config = ConfigDict( - alias_generator=lambda field_name: to_camel( # pylint: disable=W0108 - field_name + alias_generator=AliasGenerator( + validation_alias=lambda field_name: AliasChoices( + to_camel(field_name), field_name # pylint: disable=W0108 + ), + serialization_alias=lambda field_name: to_camel( # pylint: disable=W0108 + field_name + ), ), exclude={"kitem"}, + use_enum_values=True, ) + kitem: Optional[Any] = Field( None, description="Associated KItem instance", exclude=True, hide=True ) @@ -252,25 +259,6 @@ def __repr__(self) -> str: """Pretty print the model fields""" return str(self) - @model_serializer - def serialize(self) -> Dict[str, Any]: - """ - Serialize the Input object to a dictionary representation. - - This method transforms the Input instance into a dictionary, where the keys - are the attribute names and the values are the corresponding attribute values. - The "type" attribute is treated specially by storing its `value` instead of - the object itself. - - Returns: - Dict[str, Any]: A dictionary representation of the Input object. - """ - return { - key: (value.value if isinstance(value, Enum) else value) - for key, value in self.__dict__.items() - if key not in self.model_config["exclude"] - } - class Input(BaseWebformModel): """Input fields in the sections in webform""" @@ -308,25 +296,6 @@ class Input(BaseWebformModel): None, description="Placeholder for the input" ) - @model_serializer - def serialize(self) -> Dict[str, Any]: - """ - Serialize the Input object to a dictionary representation. - - This method transforms the Input instance into a dictionary, where the keys - are the attribute names and the values are the corresponding attribute values. - The "type" attribute is treated specially by storing its `value` instead of - the object itself. - - Returns: - Dict[str, Any]: A dictionary representation of the Input object. - """ - return { - key: (value.value if isinstance(value, Enum) else value) - for key, value in self.__dict__.items() - if key not in self.model_config["exclude"] - } - def __str__(self) -> str: """Pretty print the model fields""" return print_model(self, "input") @@ -464,16 +433,10 @@ class Entry(BaseWebformModel): measurement_unit: Optional[MeasurementUnit] = Field( None, description="Measurement unit of the entry", - alias=AliasChoices( - "measurementUnit", "measurement_unit", "measurementunit" - ), ) relation_mapping: Optional[RelationMapping] = Field( None, description="Relation mapping of the entry", - alias=AliasChoices( - "relationMapping", "relation_mapping", "relationmapping" - ), ) required: Optional[bool] = Field(False, description="Required input") @@ -579,27 +542,31 @@ def _validate_inputs(cls, self: "Entry") -> "Entry": # check if widget is mapped to a data type if self.type in ( - Widget.TEXT, - Widget.FILE, - Widget.TEXTAREA, - Widget.VOCABULARY_TERM, + Widget.TEXT.value, + Widget.FILE.value, + Widget.TEXTAREA.value, + Widget.VOCABULARY_TERM.value, ): dtype = str - elif self.type in (Widget.NUMBER, Widget.SLIDER): + elif self.type in (Widget.NUMBER.value, Widget.SLIDER.value): dtype = (int, float) - elif self.type == Widget.CHECKBOX: + elif self.type == Widget.CHECKBOX.value: dtype = bool - elif self.type in (Widget.SELECT, Widget.RADIO, Widget.MULTI_SELECT): - if self.type == Widget.MULTI_SELECT: + elif self.type in ( + Widget.SELECT.value, + Widget.RADIO.value, + Widget.MULTI_SELECT.value, + ): + if self.type == Widget.MULTI_SELECT.value: dtype = list else: dtype = str choices = [choice.value for choice in select_options] - elif self.type == Widget.KNOWLEDGE_ITEM: + elif self.type == Widget.KNOWLEDGE_ITEM.value: dtype = KnowledgeItemReference else: raise ValueError( - f"Widget type is not mapped to a data type: {self.widget}" + f"Widget type is not mapped to a data type: {self.type}" ) # check if value is set @@ -628,29 +595,6 @@ def _validate_inputs(cls, self: "Entry") -> "Entry": return self - @model_serializer - def serialize(self) -> Dict[str, Any]: - """ - Serialize the Entry object to a dictionary representation. - - This method transforms the Entry instance into a dictionary, where the keys - are the attribute names and the values are the corresponding attribute values. - The "type" attribute is treated specially by storing its `value` instead of - the object itself. - - Returns: - Dict[str, Any]: A dictionary representation of the Entry object. - """ - dumped = {} - for key, value in self.__dict__.items(): - if key != "kitem": - if key == "type": - value = value.value - if key in ("measurement_unit", "relation_mapping"): - key = to_camel(key) - dumped[key] = value - return dumped - @classmethod def _get_input_spec(cls, self: "Entry"): spec = [] From a6663469f1f2823b84f217f6253cc536be594629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 18 Dec 2024 19:04:58 +0100 Subject: [PATCH 51/66] bump version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 374a2ce..9e6ba6a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev9 +version = v2.1.0dev10 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From c2170f6eedd431f834f06bb8ca8c4996f115c2e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Wed, 18 Dec 2024 19:30:44 +0100 Subject: [PATCH 52/66] remove unneeded union --- dsms/knowledge/kitem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 45c233e..70aef5c 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -181,7 +181,7 @@ class KItem(BaseModel): [], description="User groups able to access the KItem.", ) - custom_properties: Optional[Union[Any]] = Field( + custom_properties: Optional[Any] = Field( None, description="Custom properties associated to the KItem" ) From 7e834918a3d93503a11fb63ba5d082d4c8fb740c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Thu, 19 Dec 2024 16:15:36 +0100 Subject: [PATCH 53/66] move function to sectionize metadata to sdk --- dsms/knowledge/utils.py | 41 ++++++++++++++++++++++++++++++++++++--- dsms/knowledge/webform.py | 10 +++++----- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 35b7518..ea3301d 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -935,11 +935,11 @@ def _make_misc_section(custom_properties: dict): If the ktype_id is not found, return the custom_properties dictionary as is, wrapped in a section named "Misc". """ - section = {"id": id_generator(), "name": "Misc", "entries": []} + section = {"id": generate_id(), "name": "Misc", "entries": []} for key, value in custom_properties.items(): section["entries"].append( { - "id": id_generator(), + "id": generate_id(), "label": key, "value": value, "type": _map_data_type_to_widget(value), @@ -966,7 +966,42 @@ def _map_data_type_to_widget(value): return widget -def id_generator(prefix: str = "id") -> str: +def sectionize_metadata(metadata: List[Dict[str, Any]]) -> dict: + """ + Convert a list of dictionaries representing metadata + entries into a DSMS schema dict. + + The input should be a list of dictionaries, + where each dictionary represents a metadata entry. + The output is a dictionary in the DSMS schema, + with a single section named "General", + containing the given metadata entries. + + If the input is empty, the function will + return an empty dictionary. + + :param metadata: The metadata list to convert. + :return: A dictionary in the DSMS schema. + """ + if metadata: + for metadatum in metadata: + metadatum["id"] = generate_id() + metadata = { + "sections": [ + { + "id": generate_id(), + "name": "General", + "entries": metadata, + } + ] + } + else: + metadata = {} + + return metadata + + +def generate_id(prefix: str = "id") -> str: # Generate a unique part using time and random characters """ Generates a unique id using a combination of the current time and 6 random characters. diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index f6c9c06..5250fb9 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -18,7 +18,7 @@ from dsms.knowledge.utils import ( # isort:skip _map_data_type_to_widget, - id_generator, + generate_id, print_model, ) @@ -264,7 +264,7 @@ class Input(BaseWebformModel): """Input fields in the sections in webform""" id: Optional[str] = Field( - default_factory=id_generator, description="ID of the input" + default_factory=generate_id, description="ID of the input" ) label: Optional[str] = Field(None, description="Label of the input") widget: Optional[Widget] = Field(None, description="Widget of the input") @@ -309,7 +309,7 @@ class Section(BaseWebformModel): """Section in webform""" id: Optional[str] = Field( - default_factory=id_generator, description="ID of the section" + default_factory=generate_id, description="ID of the section" ) name: Optional[str] = Field(None, description="Name of the section") inputs: List[Input] = Field( @@ -417,7 +417,7 @@ class Entry(BaseWebformModel): Entry in a custom properties section """ - id: str = Field(default_factory=id_generator) + id: str = Field(default_factory=generate_id) type: Optional[Widget] = Field(None, description="Type of the entry") label: str = Field(..., description="Label of the entry") value: Optional[ @@ -611,7 +611,7 @@ class CustomPropertiesSection(BaseWebformModel): Section for custom properties """ - id: Optional[str] = Field(default_factory=id_generator) + id: Optional[str] = Field(default_factory=generate_id) name: str = Field(..., description="Name of the section") entries: List[Entry] = Field([], description="Entries of the section") From 8871d71ca7a8c0c823faebab5d057c12295039ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCschelberger?= Date: Thu, 19 Dec 2024 16:16:22 +0100 Subject: [PATCH 54/66] bump version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9e6ba6a..a0c7a3c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v2.1.0dev10 +version = v2.1.0dev11 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown From 2c28db7699fbc6a589726e8c7783e2c9b94fe279 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Wed, 29 Jan 2025 09:49:48 +0100 Subject: [PATCH 55/66] kitem export to hdf5 --- dsms/knowledge/kitem.py | 20 ++++ dsms/knowledge/kitem_wrapper.py | 171 ++++++++++++++++++++++++++++++++ dsms/knowledge/ktype.py | 15 +++ 3 files changed, 206 insertions(+) create mode 100644 dsms/knowledge/kitem_wrapper.py diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 70aef5c..5a5f6f0 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -74,6 +74,12 @@ logger.addHandler(handler) logger.propagate = False +class Format(Enum): + """Data formats""" + + JSON = "json" + YAML = "yaml" + HDF5 = "hdf5" class KItem(BaseModel): """ @@ -667,6 +673,20 @@ def _get_ktype_as_str(self) -> str: raise TypeError(f"Datatype for KType is unknown: {type(ktype)}") return ktype + def export(self, format: Format) -> Any: + """Export kitems to different formats""" + + if format == Format.HDF5: + from dsms.knowledge.kitem_wrapper import to_hdf5 + return to_hdf5(self) + + elif format == Format.JSON: + # need to implement + return + + elif format == Format.YAML: + # need to implement + return class KItemList(list): """List of KItems""" diff --git a/dsms/knowledge/kitem_wrapper.py b/dsms/knowledge/kitem_wrapper.py new file mode 100644 index 0000000..82f019d --- /dev/null +++ b/dsms/knowledge/kitem_wrapper.py @@ -0,0 +1,171 @@ +import io +import h5py +import numpy as np + +def to_hdf5(kItem) -> io.BytesIO: + """Export KItem to HDF5""" + + data_bytes = io.BytesIO() + # with tempfile.NamedTemporaryFile(delete=False) as temp_file: + with h5py.File(data_bytes, 'w') as hdf: + + # Store top-level attributes + keys = ['name', 'id', 'ktype_id', 'in_backend', 'slug', 'avatar_exists', 'created_at', 'updated_at', 'rdf_exists', 'context_id', 'access_url'] + for key in keys: + value = getattr(kItem, key) + create_dataset(key, value, hdf) + + # Store the summary + summary = getattr(kItem, 'summary') + if summary is not None: + if summary.text is not None: + hdf.create_dataset('summary', data = summary.text) + + # Store dataframe + dataframe = getattr(kItem, 'dataframe') + if dataframe is not None: + value = dataframe.to_df().to_json() + hdf.create_dataset('dataframe', data = value) + + # Store the avatar in binary + avatar = getattr(kItem, 'avatar') + if avatar is not None: + # Get the image + image = avatar.download() + + # Create a BytesIO object and save the image to it + image_bytes = io.BytesIO() + image.save(image_bytes, format='PNG') + + # Get the bytes value + value = image_bytes.getvalue() + img_arr = np.frombuffer(value, dtype=np.uint8) + hdf.create_dataset('avatar', data=img_arr, dtype=img_arr.dtype) + + # Store the subgraph after serialization + subgraph = getattr(kItem, 'subgraph') + if subgraph is not None: + value = subgraph.serialize() + hdf.create_dataset('subgraph', data = value) + + # Store annotations + annotations_group = hdf.create_group('annotations') + for i, annotation in enumerate(getattr(kItem, 'annotations')): + annotation_group = annotations_group.create_group(f'annotation_{i}') + for key, value in annotation: + create_dataset(key, value, annotation_group) + + # Store attachments + attachments_group = hdf.create_group('attachments') + for i, attachment in enumerate(getattr(kItem,'attachments')): + attachment_group = attachments_group.create_group(f'attachment_{i}') + for key, value in attachment: + if key == 'content': + value = attachment.download().encode() + binary_data = np.frombuffer(value, dtype='uint8') + attachment_group.create_dataset(key, data=binary_data, dtype=binary_data.dtype) + else: + create_dataset(key, value, attachment_group) + + # Store linked_kitems + linked_kitems_group = hdf.create_group('linked_kitems') + for i, linked_kitem in enumerate(getattr(kItem,'linked_kitems')): + linked_kitem_group = linked_kitems_group.create_group(f'linked_kitem_{i}') + for key in ['id', 'name', 'slug', 'ktype_id']: + value = getattr(linked_kitem, key) + create_dataset(key, value, linked_kitem_group) + + # Store affiliations + affiliations_group = hdf.create_group('affiliations') + for i, affiliation in enumerate(getattr(kItem,'affiliations')): + affiliation_group = affiliations_group.create_group(f'affiliation_{i}') + for key, value in affiliation: + create_dataset(key, value, affiliation_group) + + # Store authors + authors_group = hdf.create_group('authors') + for i, author in enumerate(getattr(kItem,'authors')): + author_group = authors_group.create_group(f'author_{i}') + for key, value in author: + create_dataset(key, value, author_group) + + # Store contacts + contacts_group = hdf.create_group('contacts') + for i, contact in enumerate(getattr(kItem,'contacts')): + contact_group = contacts_group.create_group(f'contact_{i}') + for key, value in contact: + create_dataset(key, value, contact_group) + + # Store external links + external_links_group = hdf.create_group('external_links') + for i, external_link in enumerate(getattr(kItem,'external_links')): + external_link_group = external_links_group.create_group(f'external_link_{i}') + for key, value in external_link: + create_dataset(key, value, external_link_group) + + # Store kitem_apps + kitem_apps_group = hdf.create_group('kitem_apps') + for i, app in enumerate(getattr(kItem,'kitem_apps')): + app_group = kitem_apps_group.create_group(f'app_{i}') + for key, value in app: + if key == 'additional_properties': + for prop_key, prop_value in value: + app_group.create_dataset(f'additional_properties/{prop_key}', data=prop_value) + else: + create_dataset(key, value, app_group) + + # Store user groups + user_groups_group = hdf.create_group('user_groups') + for i, user_group in enumerate(getattr(kItem,'user_groups')): + user_group_group = user_groups_group.create_group(f'user_group_{i}') + for key, value in user_group: + create_dataset(key, value, user_group_group) + + # Store custom_properties + from dsms.knowledge.webform import KItemCustomPropertiesModel + custom_properties_group = hdf.create_group('custom_properties') + for item in kItem: + if 'custom_properties' in item: + break + for custom_property in item: + + if isinstance(custom_property, KItemCustomPropertiesModel): + sections_group = custom_properties_group.create_group('sections') + for i, section in enumerate(custom_property): + section_group = sections_group.create_group(f'section_{i}') + section_group.create_dataset('id', data=section.id) + section_group.create_dataset('name', data=section.name) + entries_group = section_group.create_group('entries') + + for j, entry in enumerate(section): + entry_group = entries_group.create_group(f'entry_{j}') + + for key, value in entry: + if key == 'kitem': + continue + if key == 'measurement_unit': + measurement_unit_group = entry_group.create_group('measurement_unit') + for key_, value_ in value: + if key_ == 'kitem': + continue + measurement_unit_group.create_dataset(key_, data=str(value_)) + elif key == 'relation_mapping': + relation_mapping_group = entry_group.create_group('relation_mapping') + for key_, value_ in value: + if key_ == 'kitem': + continue + relation_mapping_group.create_dataset(key_, data=str(value_)) + else: + create_dataset(key, value, entry_group) + + return data_bytes + + +def create_dataset(key, value, group): + """Create dataset depending on the type of the data""" + + basic_types = (int, float, str, bool, list, tuple, dict, set) + if isinstance(value, basic_types): + group.create_dataset(key, data=value) + else: + group.create_dataset(key, data=str(value)) diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index ba9300e..8a20759 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, Field, model_serializer from dsms.core.logging import handler +from dsms.knowledge.kitem import Format from dsms.knowledge.utils import _ktype_exists, _refresh_ktype, print_ktype from dsms.knowledge.webform import Webform @@ -143,3 +144,17 @@ def serialize(self): ) for key, value in self.__dict__.items() } + + def export(self, format: Format) -> Any: + """Export ktypes to different formats""" + + if format == Format.HDF5: + return + + elif format == Format.JSON: + # need to implement + return + + elif format == Format.YAML: + # need to implement + return From aefd5166fa94c09aa114052d956fedf75e42dfa6 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Wed, 29 Jan 2025 10:02:02 +0100 Subject: [PATCH 56/66] kitem export --- dsms/knowledge/kitem.py | 15 +++++++++++++++ dsms/knowledge/ktype.py | 20 ++++++++++---------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index d307e5e..b2dbe36 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -669,3 +669,18 @@ def is_a(self, to_be_compared: KType) -> bool: def refresh(self) -> None: """Refresh the KItem""" _refresh_kitem(self) + + def export(self, format: Format) -> Any: + """Export kitems to different formats""" + + if format == Format.HDF5: + from dsms.knowledge.kitem_wrapper import to_hdf5 + return to_hdf5(self) + + elif format == Format.JSON: + # need to implement + return + + elif format == Format.YAML: + # need to implement + return \ No newline at end of file diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 0c34570..0777c2e 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -138,16 +138,16 @@ def serialize(self): for key, value in self.__dict__.items() } - def export(self, format: Format) -> Any: - """Export ktypes to different formats""" + # def export(self, format: Format) -> Any: + # """Export ktypes to different formats""" - if format == Format.HDF5: - return + # if format == Format.HDF5: + # return - elif format == Format.JSON: - # need to implement - return + # elif format == Format.JSON: + # # need to implement + # return - elif format == Format.YAML: - # need to implement - return + # elif format == Format.YAML: + # # need to implement + # return From bbb17762ce0bf771915c4bdc2b8b85033a911aa0 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Wed, 29 Jan 2025 10:26:28 +0100 Subject: [PATCH 57/66] fixed merge issues --- dsms/core/dsms.py | 9 --------- dsms/knowledge/kitem.py | 12 ++---------- dsms/knowledge/properties/base.py | 7 +------ dsms/knowledge/webform.py | 1 + 4 files changed, 4 insertions(+), 25 deletions(-) diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index d7e9a43..a257164 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -162,15 +162,6 @@ def ktypes(self, value: "Enum") -> None: """ self._ktypes = value - @ktypes.setter - def ktypes(self, value: "Enum") -> None: - """Setter for the ktypes property of the DSMS instance. - - Args: - value: the Enum object to be set as the ktypes property. - """ - self._ktypes = value - @property def config(self) -> Configuration: """Property returning the DSMS Configuration""" diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index b2dbe36..52f639c 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -205,15 +205,7 @@ class KItem(BaseModel): avatar: Optional[Union[Avatar, Dict]] = Field( default_factory=Avatar, description="KItem avatar interface" ) - - context_id: Optional[Union[UUID, str]] = Field( - None, description="Context ID of the KItem" - ) - - access_url: Optional[str] = Field( - None, description="Access URL of the KItem", exclude=True - ) - + model_config = ConfigDict( validate_assignment=True, validate_default=True, @@ -479,7 +471,7 @@ def validate_ktype( value = Session.ktypes.get(ktype_id) if not value: raise TypeError( - f"KType for `ktype_id={value}` does not exist." + f"KType for `ktype_id={ktype_id}` does not exist." ) if not hasattr(value, "id"): raise TypeError( diff --git a/dsms/knowledge/properties/base.py b/dsms/knowledge/properties/base.py index 2bb82ef..82b5820 100644 --- a/dsms/knowledge/properties/base.py +++ b/dsms/knowledge/properties/base.py @@ -79,12 +79,7 @@ def kitem(self) -> "KItem": def dsms(self) -> "KItem": """DSMS instance related to the KItemProperty""" return self.kitem.dsms - - @property - def dsms(cls) -> "KItem": - """DSMS instance related to the KItemProperty""" - return cls.kitem.dsms - + @kitem.setter def kitem(self, item: "KItem") -> None: """Set KItem related to the KItemProperty""" diff --git a/dsms/knowledge/webform.py b/dsms/knowledge/webform.py index 354cdb7..85f4df0 100644 --- a/dsms/knowledge/webform.py +++ b/dsms/knowledge/webform.py @@ -35,6 +35,7 @@ if TYPE_CHECKING: from dsms.knowledge.ktype import KType + logger = logging.getLogger(__name__) logger.addHandler(handler) logger.propagate = False From d5fbf7cf55b28156497efb189f09711539d86619 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Wed, 29 Jan 2025 10:28:04 +0100 Subject: [PATCH 58/66] fixed merge issues --- dsms/knowledge/kitem.py | 2 +- dsms/knowledge/properties/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 52f639c..7c8530f 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -205,7 +205,7 @@ class KItem(BaseModel): avatar: Optional[Union[Avatar, Dict]] = Field( default_factory=Avatar, description="KItem avatar interface" ) - + model_config = ConfigDict( validate_assignment=True, validate_default=True, diff --git a/dsms/knowledge/properties/base.py b/dsms/knowledge/properties/base.py index 82b5820..d1e557f 100644 --- a/dsms/knowledge/properties/base.py +++ b/dsms/knowledge/properties/base.py @@ -79,7 +79,7 @@ def kitem(self) -> "KItem": def dsms(self) -> "KItem": """DSMS instance related to the KItemProperty""" return self.kitem.dsms - + @kitem.setter def kitem(self, item: "KItem") -> None: """Set KItem related to the KItemProperty""" From cc04d7d2fd76cad3946013ba2a86b35e114eab1f Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Tue, 11 Feb 2025 13:38:06 +0100 Subject: [PATCH 59/66] Export function for ktype --- dsms/knowledge/kitem_wrapper.py | 24 +++++------- dsms/knowledge/ktype.py | 29 +++++++++----- dsms/knowledge/ktype_wrapper.py | 67 +++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 dsms/knowledge/ktype_wrapper.py diff --git a/dsms/knowledge/kitem_wrapper.py b/dsms/knowledge/kitem_wrapper.py index 82f019d..1fb1019 100644 --- a/dsms/knowledge/kitem_wrapper.py +++ b/dsms/knowledge/kitem_wrapper.py @@ -139,22 +139,16 @@ def to_hdf5(kItem) -> io.BytesIO: for j, entry in enumerate(section): entry_group = entries_group.create_group(f'entry_{j}') - - for key, value in entry: - if key == 'kitem': + entry_keys = ['measurement_unit', 'relation_mapping'] + for entry_key, entry_value in entry: + if entry_key == 'kitem': continue - if key == 'measurement_unit': - measurement_unit_group = entry_group.create_group('measurement_unit') - for key_, value_ in value: - if key_ == 'kitem': - continue - measurement_unit_group.create_dataset(key_, data=str(value_)) - elif key == 'relation_mapping': - relation_mapping_group = entry_group.create_group('relation_mapping') - for key_, value_ in value: - if key_ == 'kitem': - continue - relation_mapping_group.create_dataset(key_, data=str(value_)) + elif entry_key in entry_keys and entry_value is not None: + group = entry_group.create_group(entry_key) + for key_, value_ in entry_value: + if key_ == 'kitem': + continue + create_dataset(key_, value_, group) else: create_dataset(key, value, entry_group) diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 0777c2e..bb54a32 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -4,6 +4,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Optional, Union from uuid import UUID +from enum import Enum from pydantic import BaseModel, Field, model_serializer @@ -19,6 +20,12 @@ logger.addHandler(handler) logger.propagate = False +class Format(Enum): + """Data formats""" + + JSON = "json" + YAML = "yaml" + HDF5 = "hdf5" class KType(BaseModel): """Knowledge type of the knowledge item.""" @@ -138,16 +145,18 @@ def serialize(self): for key, value in self.__dict__.items() } - # def export(self, format: Format) -> Any: - # """Export ktypes to different formats""" + def export(self, format: Format) -> Any: + """Export ktypes to different formats""" - # if format == Format.HDF5: - # return + if format == Format.HDF5: + from dsms.knowledge.ktype_wrapper import to_hdf5 + return to_hdf5(self) + - # elif format == Format.JSON: - # # need to implement - # return + elif format == Format.JSON: + # need to implement + return - # elif format == Format.YAML: - # # need to implement - # return + elif format == Format.YAML: + # need to implement + return diff --git a/dsms/knowledge/ktype_wrapper.py b/dsms/knowledge/ktype_wrapper.py new file mode 100644 index 0000000..1e9b9ed --- /dev/null +++ b/dsms/knowledge/ktype_wrapper.py @@ -0,0 +1,67 @@ +import io +import h5py +import numpy as np + +def to_hdf5(ktype) -> io.BytesIO: + + data_bytes = io.BytesIO() + with h5py.File(data_bytes, 'w') as hdf: + + # Store top-level attributes + keys = ['id', 'name', 'created_at', 'updated_at'] + for key in keys: + value = getattr(ktype, key) + create_dataset(key, value, hdf) + + # Store the Webform + webform = getattr(ktype, 'webform') + webform_group = hdf.create_group('webform') + if webform is not None: + sections_group = webform_group.create_group('sections') + section_keys = ['id', 'name', 'hidden'] + input_keys = ['measurement_unit', 'relation_mapping', 'relation_mapping_extra', 'range_options'] + for webform_key, webform_value in webform: + if webform_key == 'kitem': + continue + elif webform_key == 'sections': + for i, section in enumerate(webform_value): + section_group = sections_group.create_group(f'section_{i}') + for section_key in section_keys: + section_value = getattr(section, section_key) + create_dataset(section_key, section_value, section_group) + + inputs_group = section_group.create_group('inputs') + + for j, input in enumerate(section.inputs): + input_group = inputs_group.create_group(f'input_{j}') + for input_key, input_value in input: + if input_key == 'kitem': + continue + elif input_key == 'select_options': + select_options_group = input_group.create_group('select_options') + for k, select_option in enumerate(input_value): + select_option_group = select_options_group.create_group(f'option_{k}') + for option_key, option_value in select_option: + create_dataset(option_key, option_value, select_option_group) + elif input_key in input_keys and input_value is not None: + group = input_group.create_group(input_key) + for key_, value_ in input_value: + if key_ == 'kitem': + continue + create_dataset(key_, value_, group) + else: + create_dataset(input_key, input_value, input_group) + + else: + create_dataset(webform_key, webform_value, webform_group) + + return data_bytes + +def create_dataset(key, value, group): + """Create dataset depending on the type of the data""" + + basic_types = (int, float, str, bool, list, tuple, dict, set) + if isinstance(value, basic_types): + group.create_dataset(key, data=value) + else: + group.create_dataset(key, data=str(value)) \ No newline at end of file From 6070f8a8e9458e6e00c0b53df617cf38a9ea25b5 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Thu, 20 Mar 2025 11:40:27 +0100 Subject: [PATCH 60/66] Import kitems/ktypes implemented --- dsms/knowledge/data_format.py | 10 ++ dsms/knowledge/kitem.py | 54 +++++--- dsms/knowledge/kitem_wrapper.py | 165 ------------------------ dsms/knowledge/knowledge_wrapper.py | 192 ++++++++++++++++++++++++++++ dsms/knowledge/ktype.py | 58 ++++++--- dsms/knowledge/ktype_wrapper.py | 67 ---------- 6 files changed, 275 insertions(+), 271 deletions(-) create mode 100644 dsms/knowledge/data_format.py delete mode 100644 dsms/knowledge/kitem_wrapper.py create mode 100644 dsms/knowledge/knowledge_wrapper.py delete mode 100644 dsms/knowledge/ktype_wrapper.py diff --git a/dsms/knowledge/data_format.py b/dsms/knowledge/data_format.py new file mode 100644 index 0000000..2fe392c --- /dev/null +++ b/dsms/knowledge/data_format.py @@ -0,0 +1,10 @@ +"""Data Formats""" + +from enum import Enum + +class DataFormat(Enum): + """Data formats""" + + JSON = "json" + YAML = "yaml" + HDF5 = "hdf5" \ No newline at end of file diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index 7c8530f..d2323c8 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -1,5 +1,6 @@ """Knowledge Item implementation of the DSMS""" +import json import logging import warnings from datetime import datetime @@ -9,6 +10,7 @@ from uuid import UUID, uuid4 import pandas as pd +import yaml from rdflib import Graph from pydantic import ( # isort:skip @@ -48,6 +50,8 @@ UserGroupsProperty, ) +from dsms.knowledge.data_format import DataFormat # isort:skip + from dsms.knowledge.ktype import KType # isort:skip from dsms.knowledge.utils import ( # isort:skip @@ -74,12 +78,6 @@ logger.addHandler(handler) logger.propagate = False -class Format(Enum): - """Data formats""" - - JSON = "json" - YAML = "yaml" - HDF5 = "hdf5" class KItem(BaseModel): """ @@ -662,17 +660,37 @@ def refresh(self) -> None: """Refresh the KItem""" _refresh_kitem(self) - def export(self, format: Format) -> Any: + def export(self, data_format: DataFormat) -> Any: """Export kitems to different formats""" - if format == Format.HDF5: - from dsms.knowledge.kitem_wrapper import to_hdf5 - return to_hdf5(self) - - elif format == Format.JSON: - # need to implement - return - - elif format == Format.YAML: - # need to implement - return \ No newline at end of file + if data_format == DataFormat.HDF5: + from dsms.knowledge.knowledge_wrapper import data_to_dict, dict_to_hdf5 + + return dict_to_hdf5(data_to_dict(self)) + + if data_format == DataFormat.JSON: + from dsms.knowledge.knowledge_wrapper import data_to_dict + + return json.dumps(data_to_dict(self)) + + if data_format == DataFormat.YAML: + from dsms.knowledge.knowledge_wrapper import data_to_dict + + return yaml.dump(data_to_dict(self), default_flow_style=False) + + raise ValueError(f"Unsupported data format: {data_format}") + + def import_kitem(data, data_format: DataFormat) -> Any: + """Import objects in different formats to KItem""" + + if data_format == DataFormat.HDF5: + from dsms.knowledge.knowledge_wrapper import hdf5_to_dict + + return hdf5_to_dict(data) + + if data_format == DataFormat.JSON: + return json.load(data) + if data_format == DataFormat.YAML: + return yaml.safe_load(data) + + raise ValueError(f"Unsupported data format: {data_format}") diff --git a/dsms/knowledge/kitem_wrapper.py b/dsms/knowledge/kitem_wrapper.py deleted file mode 100644 index 1fb1019..0000000 --- a/dsms/knowledge/kitem_wrapper.py +++ /dev/null @@ -1,165 +0,0 @@ -import io -import h5py -import numpy as np - -def to_hdf5(kItem) -> io.BytesIO: - """Export KItem to HDF5""" - - data_bytes = io.BytesIO() - # with tempfile.NamedTemporaryFile(delete=False) as temp_file: - with h5py.File(data_bytes, 'w') as hdf: - - # Store top-level attributes - keys = ['name', 'id', 'ktype_id', 'in_backend', 'slug', 'avatar_exists', 'created_at', 'updated_at', 'rdf_exists', 'context_id', 'access_url'] - for key in keys: - value = getattr(kItem, key) - create_dataset(key, value, hdf) - - # Store the summary - summary = getattr(kItem, 'summary') - if summary is not None: - if summary.text is not None: - hdf.create_dataset('summary', data = summary.text) - - # Store dataframe - dataframe = getattr(kItem, 'dataframe') - if dataframe is not None: - value = dataframe.to_df().to_json() - hdf.create_dataset('dataframe', data = value) - - # Store the avatar in binary - avatar = getattr(kItem, 'avatar') - if avatar is not None: - # Get the image - image = avatar.download() - - # Create a BytesIO object and save the image to it - image_bytes = io.BytesIO() - image.save(image_bytes, format='PNG') - - # Get the bytes value - value = image_bytes.getvalue() - img_arr = np.frombuffer(value, dtype=np.uint8) - hdf.create_dataset('avatar', data=img_arr, dtype=img_arr.dtype) - - # Store the subgraph after serialization - subgraph = getattr(kItem, 'subgraph') - if subgraph is not None: - value = subgraph.serialize() - hdf.create_dataset('subgraph', data = value) - - # Store annotations - annotations_group = hdf.create_group('annotations') - for i, annotation in enumerate(getattr(kItem, 'annotations')): - annotation_group = annotations_group.create_group(f'annotation_{i}') - for key, value in annotation: - create_dataset(key, value, annotation_group) - - # Store attachments - attachments_group = hdf.create_group('attachments') - for i, attachment in enumerate(getattr(kItem,'attachments')): - attachment_group = attachments_group.create_group(f'attachment_{i}') - for key, value in attachment: - if key == 'content': - value = attachment.download().encode() - binary_data = np.frombuffer(value, dtype='uint8') - attachment_group.create_dataset(key, data=binary_data, dtype=binary_data.dtype) - else: - create_dataset(key, value, attachment_group) - - # Store linked_kitems - linked_kitems_group = hdf.create_group('linked_kitems') - for i, linked_kitem in enumerate(getattr(kItem,'linked_kitems')): - linked_kitem_group = linked_kitems_group.create_group(f'linked_kitem_{i}') - for key in ['id', 'name', 'slug', 'ktype_id']: - value = getattr(linked_kitem, key) - create_dataset(key, value, linked_kitem_group) - - # Store affiliations - affiliations_group = hdf.create_group('affiliations') - for i, affiliation in enumerate(getattr(kItem,'affiliations')): - affiliation_group = affiliations_group.create_group(f'affiliation_{i}') - for key, value in affiliation: - create_dataset(key, value, affiliation_group) - - # Store authors - authors_group = hdf.create_group('authors') - for i, author in enumerate(getattr(kItem,'authors')): - author_group = authors_group.create_group(f'author_{i}') - for key, value in author: - create_dataset(key, value, author_group) - - # Store contacts - contacts_group = hdf.create_group('contacts') - for i, contact in enumerate(getattr(kItem,'contacts')): - contact_group = contacts_group.create_group(f'contact_{i}') - for key, value in contact: - create_dataset(key, value, contact_group) - - # Store external links - external_links_group = hdf.create_group('external_links') - for i, external_link in enumerate(getattr(kItem,'external_links')): - external_link_group = external_links_group.create_group(f'external_link_{i}') - for key, value in external_link: - create_dataset(key, value, external_link_group) - - # Store kitem_apps - kitem_apps_group = hdf.create_group('kitem_apps') - for i, app in enumerate(getattr(kItem,'kitem_apps')): - app_group = kitem_apps_group.create_group(f'app_{i}') - for key, value in app: - if key == 'additional_properties': - for prop_key, prop_value in value: - app_group.create_dataset(f'additional_properties/{prop_key}', data=prop_value) - else: - create_dataset(key, value, app_group) - - # Store user groups - user_groups_group = hdf.create_group('user_groups') - for i, user_group in enumerate(getattr(kItem,'user_groups')): - user_group_group = user_groups_group.create_group(f'user_group_{i}') - for key, value in user_group: - create_dataset(key, value, user_group_group) - - # Store custom_properties - from dsms.knowledge.webform import KItemCustomPropertiesModel - custom_properties_group = hdf.create_group('custom_properties') - for item in kItem: - if 'custom_properties' in item: - break - for custom_property in item: - - if isinstance(custom_property, KItemCustomPropertiesModel): - sections_group = custom_properties_group.create_group('sections') - for i, section in enumerate(custom_property): - section_group = sections_group.create_group(f'section_{i}') - section_group.create_dataset('id', data=section.id) - section_group.create_dataset('name', data=section.name) - entries_group = section_group.create_group('entries') - - for j, entry in enumerate(section): - entry_group = entries_group.create_group(f'entry_{j}') - entry_keys = ['measurement_unit', 'relation_mapping'] - for entry_key, entry_value in entry: - if entry_key == 'kitem': - continue - elif entry_key in entry_keys and entry_value is not None: - group = entry_group.create_group(entry_key) - for key_, value_ in entry_value: - if key_ == 'kitem': - continue - create_dataset(key_, value_, group) - else: - create_dataset(key, value, entry_group) - - return data_bytes - - -def create_dataset(key, value, group): - """Create dataset depending on the type of the data""" - - basic_types = (int, float, str, bool, list, tuple, dict, set) - if isinstance(value, basic_types): - group.create_dataset(key, data=value) - else: - group.create_dataset(key, data=str(value)) diff --git a/dsms/knowledge/knowledge_wrapper.py b/dsms/knowledge/knowledge_wrapper.py new file mode 100644 index 0000000..1788ac6 --- /dev/null +++ b/dsms/knowledge/knowledge_wrapper.py @@ -0,0 +1,192 @@ +import io +import h5py +import numpy as np +import base64 +import re +from typing import Any +from dsms.knowledge.kitem import KItem +import numpy as np +from pydantic import BaseModel + + +def data_to_dict(data) -> Any: + """Convert data to python dictionary""" + + data_dict = {} + def handle_value(key, value): + """Handles the values under different scenarios""" + + if not isinstance(value, (int, float, str, bytes, bool, type(None))) and hasattr(value, "__dict__"): + return data_to_dict(value) + elif isinstance(value, list): + return [handle_value(key, v) for v in value] + elif isinstance(value, dict): + return {k: handle_value(k, v) for k, v in value.items()} + elif key == 'id': + return str(value) + elif key == 'summary': + summary = getattr(data, 'summary') + return summary.text + elif key == 'dataframe': + dataframe = getattr(data,'dataframe') + if dataframe == None: + return None + return dataframe.to_df().to_json() + elif key == 'file': + # Get the image + avatar = getattr(data,'avatar') + if avatar == None: + return None + image = avatar.download() + + # Create a BytesIO object and save the image to it + image_bytes = io.BytesIO() + image.save(image_bytes, format='PNG') + image_bytes.seek(0) + + # Get the bytes value + value = image_bytes.getvalue() + return base64.b64encode(value).decode("utf-8") + elif key == 'subgraph' and value is not None: + return value.serialize() + elif key == 'content': + content = data.download().encode() + return content.decode("utf-8") if content else None + elif isinstance(value, BaseModel): + return {k: handle_value(v) for k, v in value.model_dump().items()} + if isinstance(value, io.BytesIO): + return base64.b64encode(value.getvalue()).decode("utf-8") + else: + return str(value) + + for k, v in data.model_dump().items() : + if k == 'attachments': + for attachment in (getattr(data,'attachments')): + data_dict.setdefault("attachments", []).append(handle_value(k, attachment)) + continue + elif k == 'linked_kitems': + for linked_kitem in (getattr(data,'linked_kitems')): + item = {} + for key in ['id', 'name', 'slug', 'ktype_id']: + value = getattr(linked_kitem, key) + item[key] = str(value) + data_dict.setdefault("linked_kitems", []).append(item) + continue + data_dict[k] = handle_value(k, v) + + return data_dict + +def dict_to_hdf5(dict_data): + + byte_data = io.BytesIO() + + # Create an HDF5 file in memory + with h5py.File(byte_data, 'w') as f: + # Recursively add dictionary contents + def add_to_hdf5(data, group): + for key, value in data.items(): + if isinstance(value, dict): + # Handle nested dictionaries recursively + subgroup = group.create_group(key) + add_to_hdf5(value, subgroup) + elif isinstance(value, list): + # Handle lists, check if the list contains dictionaries + subgroup = group.create_group(key) + for idx, item in enumerate(value): + if isinstance(item, dict): + item_group = subgroup.create_group(f"item_{idx}") + add_to_hdf5(item, item_group) + else: + subgroup.create_dataset(f"item_{idx}", data=item) + elif value is not None: + group.create_dataset(key, data=value) + else: + group.create_dataset(key, data="") + + # Start adding data to the root group + add_to_hdf5(dict_data, f) + + # Get the bytes data from the memory buffer + byte_data.seek(0) + return byte_data.read() + + +def hdf5_to_dict(hdf5_file: io.BytesIO) -> dict: + """Convert an HDF5 file back into a Python dictionary.""" + + def decode_if_bytes(value): + """Decode bytes to string if needed.""" + + if isinstance(value, bytes): + return value.decode("utf-8") + elif isinstance(value, np.ndarray) and value.dtype.type is np.bytes_: + return [elem.decode("utf-8") for elem in value.tolist()] + return value + + def convert_numpy(obj): + """Recursively convert numpy data types in dictionaries or lists to native Python types.""" + + if isinstance(obj, np.generic): + return obj.item() + elif isinstance(obj, dict): + return {key: convert_numpy(value) for key, value in obj.items()} + elif isinstance(obj, list): + return [convert_numpy(item) for item in obj] + elif isinstance(obj, tuple): + return tuple(convert_numpy(item) for item in obj) + elif isinstance(obj, set): + return {convert_numpy(item) for item in obj} + return obj + + def read_group(group): + """Recursively read HDF5 groups into a dictionary.""" + + data_dict = {} + grouped_items = {} + + # Read attributes + for key, value in group.attrs.items(): + data_dict[key] = decode_if_bytes(value) + + # Read datasets + for key, dataset in group.items(): + if isinstance(dataset, h5py.Dataset): + data = dataset[()] + + # Convert binary data back to original format if needed + if isinstance(data, np.ndarray) and data.dtype == np.uint8: + try: + value = data.tobytes().decode() # Convert binary to string + except UnicodeDecodeError: + value = data.tobytes() # Keep as raw bytes + + elif isinstance(data, np.ndarray): + value = decode_if_bytes(data.tolist()) # Convert numpy arrays to lists + + else: + value = decode_if_bytes(data) + + elif isinstance(dataset, h5py.Group): + value = read_group(dataset) # Recursively read subgroups + + # If key matches 'item_X', group into a list + if re.match(r'item_\d+', key): + parent_key = dataset.parent.name.split('/')[-1] # Get the parent key + if parent_key not in grouped_items: + grouped_items[parent_key] = [] + grouped_items[parent_key].append(value) + else: + data_dict[key] = value + + # Merge grouped items back into the main dictionary + data_dict.update(grouped_items) + + return convert_numpy(data_dict) + + # Open the HDF5 file and start reading + with h5py.File(hdf5_file, 'r') as hdf: + return read_group(hdf) + + + + diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index bb54a32..5425fba 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -1,14 +1,16 @@ """KItem types""" +import json import logging from datetime import datetime from typing import TYPE_CHECKING, Any, Optional, Union from uuid import UUID -from enum import Enum +import yaml from pydantic import BaseModel, Field, model_serializer from dsms.core.logging import handler +from dsms.knowledge.data_format import DataFormat from dsms.knowledge.utils import _ktype_exists, _refresh_ktype, print_ktype from dsms.knowledge.webform import Webform @@ -20,12 +22,6 @@ logger.addHandler(handler) logger.propagate = False -class Format(Enum): - """Data formats""" - - JSON = "json" - YAML = "yaml" - HDF5 = "hdf5" class KType(BaseModel): """Knowledge type of the knowledge item.""" @@ -144,19 +140,39 @@ def serialize(self): ) for key, value in self.__dict__.items() } - - def export(self, format: Format) -> Any: + + def export(self, data_format: DataFormat) -> Any: """Export ktypes to different formats""" - if format == Format.HDF5: - from dsms.knowledge.ktype_wrapper import to_hdf5 - return to_hdf5(self) - - - elif format == Format.JSON: - # need to implement - return - - elif format == Format.YAML: - # need to implement - return + if data_format == DataFormat.HDF5: + from dsms.knowledge.knowledge_wrapper import data_to_dict, dict_to_hdf5 + + return dict_to_hdf5(data_to_dict(self)) + + if data_format == DataFormat.JSON: + from dsms.knowledge.knowledge_wrapper import data_to_dict + + return json.dumps(data_to_dict(self)) + + if data_format == DataFormat.YAML: + from dsms.knowledge.knowledge_wrapper import data_to_dict + + return yaml.dump(data_to_dict(self), default_flow_style=False) + + raise ValueError(f"Unsupported data format: {data_format}") + + def import_ktype(data, data_format: DataFormat) -> Any: + """Import objects in different formats to KType""" + + if data_format == DataFormat.HDF5: + from dsms.knowledge.knowledge_wrapper import hdf5_to_dict + + return hdf5_to_dict(data) + + if data_format == DataFormat.JSON: + return json.load(data) + + if data_format == DataFormat.YAML: + return yaml.safe_load(data) + + raise ValueError(f"Unsupported data format: {data_format}") diff --git a/dsms/knowledge/ktype_wrapper.py b/dsms/knowledge/ktype_wrapper.py deleted file mode 100644 index 1e9b9ed..0000000 --- a/dsms/knowledge/ktype_wrapper.py +++ /dev/null @@ -1,67 +0,0 @@ -import io -import h5py -import numpy as np - -def to_hdf5(ktype) -> io.BytesIO: - - data_bytes = io.BytesIO() - with h5py.File(data_bytes, 'w') as hdf: - - # Store top-level attributes - keys = ['id', 'name', 'created_at', 'updated_at'] - for key in keys: - value = getattr(ktype, key) - create_dataset(key, value, hdf) - - # Store the Webform - webform = getattr(ktype, 'webform') - webform_group = hdf.create_group('webform') - if webform is not None: - sections_group = webform_group.create_group('sections') - section_keys = ['id', 'name', 'hidden'] - input_keys = ['measurement_unit', 'relation_mapping', 'relation_mapping_extra', 'range_options'] - for webform_key, webform_value in webform: - if webform_key == 'kitem': - continue - elif webform_key == 'sections': - for i, section in enumerate(webform_value): - section_group = sections_group.create_group(f'section_{i}') - for section_key in section_keys: - section_value = getattr(section, section_key) - create_dataset(section_key, section_value, section_group) - - inputs_group = section_group.create_group('inputs') - - for j, input in enumerate(section.inputs): - input_group = inputs_group.create_group(f'input_{j}') - for input_key, input_value in input: - if input_key == 'kitem': - continue - elif input_key == 'select_options': - select_options_group = input_group.create_group('select_options') - for k, select_option in enumerate(input_value): - select_option_group = select_options_group.create_group(f'option_{k}') - for option_key, option_value in select_option: - create_dataset(option_key, option_value, select_option_group) - elif input_key in input_keys and input_value is not None: - group = input_group.create_group(input_key) - for key_, value_ in input_value: - if key_ == 'kitem': - continue - create_dataset(key_, value_, group) - else: - create_dataset(input_key, input_value, input_group) - - else: - create_dataset(webform_key, webform_value, webform_group) - - return data_bytes - -def create_dataset(key, value, group): - """Create dataset depending on the type of the data""" - - basic_types = (int, float, str, bool, list, tuple, dict, set) - if isinstance(value, basic_types): - group.create_dataset(key, data=value) - else: - group.create_dataset(key, data=str(value)) \ No newline at end of file From 95feed310f7e01f8f45659998b80a0af0624f43a Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Thu, 20 Mar 2025 12:31:27 +0100 Subject: [PATCH 61/66] Pylint error fixes --- dsms/knowledge/data_format.py | 5 +- dsms/knowledge/knowledge_wrapper.py | 201 +++++++++++++++------------- 2 files changed, 114 insertions(+), 92 deletions(-) diff --git a/dsms/knowledge/data_format.py b/dsms/knowledge/data_format.py index 2fe392c..6f1960a 100644 --- a/dsms/knowledge/data_format.py +++ b/dsms/knowledge/data_format.py @@ -2,9 +2,10 @@ from enum import Enum + class DataFormat(Enum): """Data formats""" - + JSON = "json" YAML = "yaml" - HDF5 = "hdf5" \ No newline at end of file + HDF5 = "hdf5" diff --git a/dsms/knowledge/knowledge_wrapper.py b/dsms/knowledge/knowledge_wrapper.py index 1788ac6..191e593 100644 --- a/dsms/knowledge/knowledge_wrapper.py +++ b/dsms/knowledge/knowledge_wrapper.py @@ -1,87 +1,106 @@ -import io -import h5py -import numpy as np +"""Wrapper for data conversion to and from different data formats""" + import base64 +import io import re from typing import Any -from dsms.knowledge.kitem import KItem + +import h5py import numpy as np from pydantic import BaseModel def data_to_dict(data) -> Any: - """Convert data to python dictionary""" + """Convert data to python dictionary""" - data_dict = {} - def handle_value(key, value): - """Handles the values under different scenarios""" + data_dict = {} - if not isinstance(value, (int, float, str, bytes, bool, type(None))) and hasattr(value, "__dict__"): - return data_to_dict(value) - elif isinstance(value, list): - return [handle_value(key, v) for v in value] - elif isinstance(value, dict): - return {k: handle_value(k, v) for k, v in value.items()} - elif key == 'id': - return str(value) - elif key == 'summary': - summary = getattr(data, 'summary') - return summary.text - elif key == 'dataframe': - dataframe = getattr(data,'dataframe') - if dataframe == None: - return None - return dataframe.to_df().to_json() - elif key == 'file': - # Get the image - avatar = getattr(data,'avatar') - if avatar == None: - return None + def handle_value(key, value): + """Handles the values under different scenarios""" + + result = None # Default value for result + + # Handle special cases based on 'key' and 'value' + if not isinstance( + value, (int, float, str, bytes, bool, type(None)) + ) and hasattr(value, "__dict__"): + result = data_to_dict(value) + + elif key == "id": + result = str(value) + + elif key == "summary": + summary = getattr(data, "summary", None) + result = summary.text if summary else None + + elif key == "dataframe": + dataframe = getattr(data, "dataframe", None) + if dataframe: + result = dataframe.to_df().to_json() + + elif key == "file": + avatar = getattr(data, "avatar", None) + if avatar: image = avatar.download() - - # Create a BytesIO object and save the image to it image_bytes = io.BytesIO() - image.save(image_bytes, format='PNG') + image.save(image_bytes, format="PNG") image_bytes.seek(0) - - # Get the bytes value - value = image_bytes.getvalue() - return base64.b64encode(value).decode("utf-8") - elif key == 'subgraph' and value is not None: - return value.serialize() - elif key == 'content': - content = data.download().encode() - return content.decode("utf-8") if content else None + result = base64.b64encode(image_bytes.getvalue()).decode( + "utf-8" + ) + + elif key == "subgraph" and value is not None: + result = value.serialize() + + elif key == "content": + content = data.download().encode("utf-8") + bytes_io = io.BytesIO(content) if content else None + result = base64.b64encode(bytes_io.getvalue()).decode("utf-8") + + # Process the value for other cases (lists, dicts, models, etc.) + if result is None: + if isinstance(value, (int, float, str, bytes, bool, type(None))): + result = str(value) + elif isinstance(value, list): + result = [handle_value(key, v) for v in value] + elif isinstance(value, dict): + result = {k: handle_value(k, v) for k, v in value.items()} elif isinstance(value, BaseModel): - return {k: handle_value(v) for k, v in value.model_dump().items()} - if isinstance(value, io.BytesIO): - return base64.b64encode(value.getvalue()).decode("utf-8") - else: - return str(value) - - for k, v in data.model_dump().items() : - if k == 'attachments': - for attachment in (getattr(data,'attachments')): - data_dict.setdefault("attachments", []).append(handle_value(k, attachment)) - continue - elif k == 'linked_kitems': - for linked_kitem in (getattr(data,'linked_kitems')): - item = {} - for key in ['id', 'name', 'slug', 'ktype_id']: - value = getattr(linked_kitem, key) - item[key] = str(value) - data_dict.setdefault("linked_kitems", []).append(item) - continue - data_dict[k] = handle_value(k, v) - - return data_dict - -def dict_to_hdf5(dict_data): + result = { + k: handle_value(k, v) + for k, v in value.model_dump().items() + } + elif isinstance(value, io.BytesIO): + result = base64.b64encode(value.getvalue()).decode("utf-8") + + return result + + for k, v in data.model_dump().items(): + if k == "attachments": + for attachment in getattr(data, "attachments"): + data_dict.setdefault("attachments", []).append( + handle_value(k, attachment) + ) + continue + if k == "linked_kitems": + for linked_kitem in getattr(data, "linked_kitems"): + item = {} + for key in ["id", "name", "slug", "ktype_id"]: + value = getattr(linked_kitem, key) + item[key] = str(value) + data_dict.setdefault("linked_kitems", []).append(item) + continue + data_dict[k] = handle_value(k, v) + + return data_dict + +def dict_to_hdf5(dict_data): + """Converts data from a dictionary to HDF5""" byte_data = io.BytesIO() - + # Create an HDF5 file in memory - with h5py.File(byte_data, 'w') as f: + with h5py.File(byte_data, "w") as f: # Recursively add dictionary contents def add_to_hdf5(data, group): for key, value in data.items(): @@ -102,10 +121,10 @@ def add_to_hdf5(data, group): group.create_dataset(key, data=value) else: group.create_dataset(key, data="") - + # Start adding data to the root group add_to_hdf5(dict_data, f) - + # Get the bytes data from the memory buffer byte_data.seek(0) return byte_data.read() @@ -119,31 +138,31 @@ def decode_if_bytes(value): if isinstance(value, bytes): return value.decode("utf-8") - elif isinstance(value, np.ndarray) and value.dtype.type is np.bytes_: + if isinstance(value, np.ndarray) and value.dtype.type is np.bytes_: return [elem.decode("utf-8") for elem in value.tolist()] return value - + def convert_numpy(obj): """Recursively convert numpy data types in dictionaries or lists to native Python types.""" if isinstance(obj, np.generic): return obj.item() - elif isinstance(obj, dict): + if isinstance(obj, dict): return {key: convert_numpy(value) for key, value in obj.items()} - elif isinstance(obj, list): + if isinstance(obj, list): return [convert_numpy(item) for item in obj] - elif isinstance(obj, tuple): + if isinstance(obj, tuple): return tuple(convert_numpy(item) for item in obj) - elif isinstance(obj, set): + if isinstance(obj, set): return {convert_numpy(item) for item in obj} return obj def read_group(group): """Recursively read HDF5 groups into a dictionary.""" - + data_dict = {} grouped_items = {} - + # Read attributes for key, value in group.attrs.items(): data_dict[key] = decode_if_bytes(value) @@ -152,17 +171,21 @@ def read_group(group): for key, dataset in group.items(): if isinstance(dataset, h5py.Dataset): data = dataset[()] - + # Convert binary data back to original format if needed if isinstance(data, np.ndarray) and data.dtype == np.uint8: try: - value = data.tobytes().decode() # Convert binary to string + value = ( + data.tobytes().decode() + ) # Convert binary to string except UnicodeDecodeError: value = data.tobytes() # Keep as raw bytes - + elif isinstance(data, np.ndarray): - value = decode_if_bytes(data.tolist()) # Convert numpy arrays to lists - + value = decode_if_bytes( + data.tolist() + ) # Convert numpy arrays to lists + else: value = decode_if_bytes(data) @@ -170,8 +193,10 @@ def read_group(group): value = read_group(dataset) # Recursively read subgroups # If key matches 'item_X', group into a list - if re.match(r'item_\d+', key): - parent_key = dataset.parent.name.split('/')[-1] # Get the parent key + if re.match(r"item_\d+", key): + parent_key = dataset.parent.name.split("/")[ + -1 + ] # Get the parent key if parent_key not in grouped_items: grouped_items[parent_key] = [] grouped_items[parent_key].append(value) @@ -182,11 +207,7 @@ def read_group(group): data_dict.update(grouped_items) return convert_numpy(data_dict) - + # Open the HDF5 file and start reading - with h5py.File(hdf5_file, 'r') as hdf: + with h5py.File(hdf5_file, "r") as hdf: return read_group(hdf) - - - - From 8f134feb46176050c229c74fa9927cb4462b35a5 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Thu, 20 Mar 2025 12:33:16 +0100 Subject: [PATCH 62/66] Pylint fixes --- dsms/knowledge/knowledge_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsms/knowledge/knowledge_wrapper.py b/dsms/knowledge/knowledge_wrapper.py index 191e593..4fe3a71 100644 --- a/dsms/knowledge/knowledge_wrapper.py +++ b/dsms/knowledge/knowledge_wrapper.py @@ -55,7 +55,7 @@ def handle_value(key, value): elif key == "content": content = data.download().encode("utf-8") bytes_io = io.BytesIO(content) if content else None - result = base64.b64encode(bytes_io.getvalue()).decode("utf-8") + result = base64.b64encode(bytes_io.getvalue()).decode("utf-8") # Process the value for other cases (lists, dicts, models, etc.) if result is None: From 1f1aec6d9024b5d40d5c4856460d38d32d9f2c80 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Thu, 20 Mar 2025 12:44:37 +0100 Subject: [PATCH 63/66] Updated setup requirements --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 7a3ef24..2e7f80b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,6 +21,7 @@ packages = find: install_requires = PyYAML>=6,<7 click>=8,<9 + h5py>=3,<4 html5lib>=1,<2 lru-cache<1 oyaml==1 From 0563bdcd4185dbffe83ea5ef2b102710b68f4e05 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Thu, 20 Mar 2025 15:05:21 +0100 Subject: [PATCH 64/66] pre-commit errors fixed --- dsms/knowledge/kitem.py | 5 ++++- dsms/knowledge/ktype.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index d2323c8..50567ff 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -664,7 +664,10 @@ def export(self, data_format: DataFormat) -> Any: """Export kitems to different formats""" if data_format == DataFormat.HDF5: - from dsms.knowledge.knowledge_wrapper import data_to_dict, dict_to_hdf5 + from dsms.knowledge.knowledge_wrapper import ( # isort:skip + data_to_dict, + dict_to_hdf5, + ) return dict_to_hdf5(data_to_dict(self)) diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 5425fba..c8dbb46 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -145,7 +145,10 @@ def export(self, data_format: DataFormat) -> Any: """Export ktypes to different formats""" if data_format == DataFormat.HDF5: - from dsms.knowledge.knowledge_wrapper import data_to_dict, dict_to_hdf5 + from dsms.knowledge.knowledge_wrapper import ( # isort:skip + data_to_dict, + dict_to_hdf5, + ) return dict_to_hdf5(data_to_dict(self)) From d09b49f184bd5efde7b77d9bc658d9b17d761a92 Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Tue, 25 Mar 2025 16:02:59 +0100 Subject: [PATCH 65/66] Duplicate key error fixed for hdf5 --- dsms/knowledge/knowledge_wrapper.py | 63 +++++++++-------------------- 1 file changed, 20 insertions(+), 43 deletions(-) diff --git a/dsms/knowledge/knowledge_wrapper.py b/dsms/knowledge/knowledge_wrapper.py index 4fe3a71..39e2ecf 100644 --- a/dsms/knowledge/knowledge_wrapper.py +++ b/dsms/knowledge/knowledge_wrapper.py @@ -122,92 +122,69 @@ def add_to_hdf5(data, group): else: group.create_dataset(key, data="") - # Start adding data to the root group + # Add data to the root group add_to_hdf5(dict_data, f) # Get the bytes data from the memory buffer byte_data.seek(0) return byte_data.read() - def hdf5_to_dict(hdf5_file: io.BytesIO) -> dict: - """Convert an HDF5 file back into a Python dictionary.""" + """Convert an HDF5 file into a Python dictionary.""" def decode_if_bytes(value): """Decode bytes to string if needed.""" - if isinstance(value, bytes): return value.decode("utf-8") - if isinstance(value, np.ndarray) and value.dtype.type is np.bytes_: + elif isinstance(value, np.ndarray) and value.dtype.type is np.bytes_: return [elem.decode("utf-8") for elem in value.tolist()] return value def convert_numpy(obj): - """Recursively convert numpy data types in dictionaries or lists to native Python types.""" - + """Convert numpy data types to native Python types.""" if isinstance(obj, np.generic): return obj.item() - if isinstance(obj, dict): + elif isinstance(obj, dict): return {key: convert_numpy(value) for key, value in obj.items()} - if isinstance(obj, list): + elif isinstance(obj, list): return [convert_numpy(item) for item in obj] - if isinstance(obj, tuple): - return tuple(convert_numpy(item) for item in obj) - if isinstance(obj, set): - return {convert_numpy(item) for item in obj} return obj def read_group(group): - """Recursively read HDF5 groups into a dictionary.""" - + """Recursively read HDF5 groups, grouping 'item_X' keys into lists efficiently.""" data_dict = {} - grouped_items = {} + grouped_items = [] - # Read attributes for key, value in group.attrs.items(): data_dict[key] = decode_if_bytes(value) - # Read datasets for key, dataset in group.items(): if isinstance(dataset, h5py.Dataset): data = dataset[()] - - # Convert binary data back to original format if needed if isinstance(data, np.ndarray) and data.dtype == np.uint8: try: - value = ( - data.tobytes().decode() - ) # Convert binary to string + value = data.tobytes().decode() except UnicodeDecodeError: - value = data.tobytes() # Keep as raw bytes - + value = data.tobytes() elif isinstance(data, np.ndarray): - value = decode_if_bytes( - data.tolist() - ) # Convert numpy arrays to lists - + value = decode_if_bytes(data.tolist()) else: value = decode_if_bytes(data) elif isinstance(dataset, h5py.Group): - value = read_group(dataset) # Recursively read subgroups - - # If key matches 'item_X', group into a list - if re.match(r"item_\d+", key): - parent_key = dataset.parent.name.split("/")[ - -1 - ] # Get the parent key - if parent_key not in grouped_items: - grouped_items[parent_key] = [] - grouped_items[parent_key].append(value) + value = read_group(dataset) + + if key.startswith("item_") and key[5:].isdigit(): + grouped_items.append(value) else: data_dict[key] = value - # Merge grouped items back into the main dictionary - data_dict.update(grouped_items) + # If there are grouped items, store them correctly + if grouped_items: + return grouped_items return convert_numpy(data_dict) - # Open the HDF5 file and start reading - with h5py.File(hdf5_file, "r") as hdf: + with h5py.File(hdf5_file, 'r') as hdf: return read_group(hdf) + From 4befbde4817631cfa6c8d83ff22d9194f95630df Mon Sep 17 00:00:00 2001 From: Arjun Gopalakrishnan Date: Tue, 25 Mar 2025 16:06:32 +0100 Subject: [PATCH 66/66] Pre-commit hook errors fixed --- dsms/knowledge/knowledge_wrapper.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dsms/knowledge/knowledge_wrapper.py b/dsms/knowledge/knowledge_wrapper.py index 39e2ecf..f782ce8 100644 --- a/dsms/knowledge/knowledge_wrapper.py +++ b/dsms/knowledge/knowledge_wrapper.py @@ -2,7 +2,6 @@ import base64 import io -import re from typing import Any import h5py @@ -129,6 +128,7 @@ def add_to_hdf5(data, group): byte_data.seek(0) return byte_data.read() + def hdf5_to_dict(hdf5_file: io.BytesIO) -> dict: """Convert an HDF5 file into a Python dictionary.""" @@ -136,7 +136,7 @@ def decode_if_bytes(value): """Decode bytes to string if needed.""" if isinstance(value, bytes): return value.decode("utf-8") - elif isinstance(value, np.ndarray) and value.dtype.type is np.bytes_: + if isinstance(value, np.ndarray) and value.dtype.type is np.bytes_: return [elem.decode("utf-8") for elem in value.tolist()] return value @@ -144,9 +144,9 @@ def convert_numpy(obj): """Convert numpy data types to native Python types.""" if isinstance(obj, np.generic): return obj.item() - elif isinstance(obj, dict): + if isinstance(obj, dict): return {key: convert_numpy(value) for key, value in obj.items()} - elif isinstance(obj, list): + if isinstance(obj, list): return [convert_numpy(item) for item in obj] return obj @@ -185,6 +185,5 @@ def read_group(group): return convert_numpy(data_dict) - with h5py.File(hdf5_file, 'r') as hdf: + with h5py.File(hdf5_file, "r") as hdf: return read_group(hdf) -