From ac6d100fcbb74dcee89ed67ff9a7252769d3075e Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Fri, 20 Jun 2025 10:22:16 -0700 Subject: [PATCH 1/6] Draft implementation of Blob Storage --- .../microsoft/agents/blob/__init__.py | 3 + .../microsoft/agents/blob/blob_storage.py | 180 +++++++++++ .../microsoft-agents-blob/pyproject.toml | 20 ++ .../microsoft-agents-blob/tests/__init__.py | 0 .../tests/blob_storage_test.py | 250 ++++++++++++++++ .../tests/storage_base_test.py | 281 ++++++++++++++++++ 6 files changed, 734 insertions(+) create mode 100644 libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/__init__.py create mode 100644 libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py create mode 100644 libraries/Storage/microsoft-agents-blob/pyproject.toml create mode 100644 libraries/Storage/microsoft-agents-blob/tests/__init__.py create mode 100644 libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py create mode 100644 libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py diff --git a/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/__init__.py b/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/__init__.py new file mode 100644 index 00000000..dec949a6 --- /dev/null +++ b/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/__init__.py @@ -0,0 +1,3 @@ +from .blob_storage import BlobStorage, BlobStorageSettings + +__all__ = ["BlobStorage", "BlobStorageSettings"] diff --git a/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py b/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py new file mode 100644 index 00000000..9e278a35 --- /dev/null +++ b/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py @@ -0,0 +1,180 @@ +# based on +# https://github.com/microsoft/botbuilder-python/blob/main/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py + +from typing import TypeVar +import json + +from microsoft.agents.storage._type_aliases import JSON + +from azure.core import MatchConditions +from azure.core.exceptions import ( + HttpResponseError, + ResourceExistsError, + ResourceNotFoundError, +) +from azure.storage.blob.aio import BlobServiceClient + +from microsoft.agents.storage import Storage, StoreItem + +StoreItemT = TypeVar("StoreItemT", bound=StoreItem) + +class BlobStorageSettings: + + def __init__( + self, + container_name: str, + account_name: str = "", + account_key: str = "", + connection_string: str = "" + ): + + self.container_name = container_name + self.account_name = account_name + self.account_key = account_key + self.connection_string = connection_string + +# New Azure Blob SDK only allows connection strings, but our SDK allows key+name. +# This is here for backwards compatibility. +def convert_account_name_and_key_to_connection_string(settings: BlobStorageSettings): + if not settings.account_name or not settings.account_key: + raise Exception( + "account_name and account_key are both required for BlobStorageSettings if not using a connections string." + ) + return ( + f"DefaultEndpointsProtocol=https;AccountName={settings.account_name};" + f"AccountKey={settings.account_key};EndpointSuffix=core.windows.net" + ) + +class BlobStorage(Storage): + + def __init__(self, settings: BlobStorageSettings): + + if not settings.container_name: + raise Exception("Container name is required.") + + if settings.connection_string: + blob_service_client = BlobServiceClient.from_connection_string( + settings.connection_string + ) + else: + blob_service_client = BlobServiceClient.from_connection_string( + convert_account_name_and_key_to_connection_string(settings) + ) + + self.__container_client = blob_service_client.get_container_client( + settings.container_name + ) + + self.__initialized = False + + async def _initialize(self): + + if self.__initialized is False: + # This should only happen once - assuming this is a singleton. + # ContainerClient.exists() method is available in an unreleased version of the SDK. Until then, we use: + try: + await self.__container_client.create_container() + except ResourceExistsError: + pass + self.__initialized = True + return self.__initialized + + async def read( + self, + keys: list[str], + *, + target_cls: StoreItemT = None, + **kwargs + ) -> dict[str, StoreItemT]: + """Retrieve entities from the configured blob container. + + :param keys: An array of entity keys. + :type keys: dict[str, StoreItem] + :return dict: + """ + if not keys: + raise Exception("Keys are required when reading") + + await self._initialize() + + result: dict[str, StoreItem] = {} + + for key in keys: + + blob_client = self.__container_client.get_blob_client(key) + + try: + blob = await blob_client.download_blob() + except HttpResponseError as error: + if error.status_code == 404: + continue + + item_json = json.loads(await blob.content_as_text()) + + item_json["e_tag"] = blob.properties.etag.replace('""', "") + + if not target_cls: + result[key] = item_json + else: + try: + result[key] = target_cls.from_json_to_store_item(item_json) + except AttributeError as error: + raise TypeError( + f"BlobStorage.read(): could not deserialize blob item into {target_cls} class. Error: {error}" + ) + + return result + + async def write(self, changes: dict[str, StoreItem]): + """Stores a new entity in the configured blob container. + + :param changes: The changes to write to storage. + :type changes: dict[str, StoreItem] + :return: + """ + + if changes is None: + raise ValueError("BlobStorage.write(): changes cannot be None") + + for key, item in changes.items(): + + item_json = item.store_item_to_json() + if item_json is None: + raise ValueError("BlobStorage.write(): StoreItem serialization cannot return None") + + item_str = json.dumps(item_json) + + blob_reference = self.__container_client.get_blob_client(key) + + e_tag = None if item_json is None else item_json.get("e_tag", None) + e_tag = None if e_tag == "*" else e_tag + + if e_tag == "": + raise Exception("blob_storage.write(): etag missing") + + if e_tag: + await blob_reference.upload_blob( + item_str, match_condition=MatchConditions.IfNotModified, etag=e_tag + ) + else: + await blob_reference.upload_blob(item_str, overwrite=True) + + async def delete(self, keys: list[str]): + """Deletes entity blobs from the configured container. + + :param keys: An array of entity keys. + :type keys: list[str] + """ + + if keys is None: + raise Exception("BlobStorage.delete: keys parameter can't be null") + + await self._initialize() + + for key in keys: + blob_client = self.__container_client.get_blob_client(key) + try: + await blob_client.delete_blob() + # We can't delete what's already gone. + except ResourceNotFoundError: + pass \ No newline at end of file diff --git a/libraries/Storage/microsoft-agents-blob/pyproject.toml b/libraries/Storage/microsoft-agents-blob/pyproject.toml new file mode 100644 index 00000000..71bf7367 --- /dev/null +++ b/libraries/Storage/microsoft-agents-blob/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "microsoft-agents-blob" +version = "0.0.0a1" +description = "A blob storage library for Microsoft Agents" +authors = [{name = "Microsoft Corporation"}] +requires-python = ">=3.9" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dependencies = [ +] + +[project.urls] +"Homepage" = "https://github.com/microsoft/microsoft-agents-protocol" diff --git a/libraries/Storage/microsoft-agents-blob/tests/__init__.py b/libraries/Storage/microsoft-agents-blob/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py b/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py new file mode 100644 index 00000000..8962ea7a --- /dev/null +++ b/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py @@ -0,0 +1,250 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# based on https://github.com/microsoft/botbuilder-python/blob/main/libraries/botbuilder-azure/tests/test_blob_storage.py + +import pytest +import pytest_asyncio + +from azure.core.exceptions import ResourceNotFoundError +from azure.storage.blob.aio import BlobServiceClient + +from microsoft.agents.storage import StoreItem +from microsoft.agents.storage._type_aliases import JSON +from microsoft.agents.blob import BlobStorage, BlobStorageSettings + +from .storage_base_test import StorageBaseTests + +EMULATOR_RUNNING = True + +# constructs an emulated blob storage instance +@pytest_asyncio.fixture +async def blob_storage(): + + # setup + + connection_string = ("AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq" + + "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=" + + "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" + + "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;") + + blob_service_client = BlobServiceClient.from_connection_string(connection_string) + + container_name = "test" + container_client = blob_service_client.get_container_client(container_name) + + try: + await container_client.delete_container() + except ResourceNotFoundError: + pass + + await container_client.create_container() + + blob_storage_settings = BlobStorageSettings( + account_name="", account_key="", + container_name=container_name, + connection_string=connection_string + ) + + storage = BlobStorage(blob_storage_settings) + + yield storage + + # teardown + await container_client.delete_container() + + +class SimpleStoreItem(StoreItem): + def __init__(self, counter=1, e_tag="*"): + super(SimpleStoreItem, self).__init__() + + self.counter = counter + self.e_tag = e_tag + + def store_item_to_json(self) -> JSON: + return { + "counter": self.counter, + "e_tag": self.e_tag, + } + + @staticmethod + def from_json_to_store_item(json_data: JSON) -> "StoreItem": + return SimpleStoreItem(json_data["counter"], json_data["e_tag"]) + + +class TestBlobStorageConstructor: + + @pytest.mark.asyncio + async def test_blob_storage_init_should_error_without_blob_config(self): + try: + BlobStorage(BlobStorageSettings()) # pylint: disable=no-value-for-parameter + except Exception as error: + assert error + + +class TestBlobStorageBaseTests: + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_return_empty_object_when_reading_unknown_key(self, blob_storage): + + test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key( + blob_storage + ) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_handle_null_keys_when_reading(self, blob_storage): + + test_ran = await StorageBaseTests.handle_null_keys_when_reading(blob_storage) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_handle_null_keys_when_writing(self, blob_storage): + + test_ran = await StorageBaseTests.handle_null_keys_when_writing(blob_storage) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_does_not_raise_when_writing_no_items(self, blob_storage): + + test_ran = await StorageBaseTests.does_not_raise_when_writing_no_items( + blob_storage + ) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_create_object(self, blob_storage): + + test_ran = await StorageBaseTests.create_object(blob_storage) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_handle_crazy_keys(self, blob_storage): + + test_ran = await StorageBaseTests.handle_crazy_keys(blob_storage) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_update_object(self, blob_storage): + + test_ran = await StorageBaseTests.update_object(blob_storage) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_delete_object(self, blob_storage): + + test_ran = await StorageBaseTests.delete_object(blob_storage) + + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_perform_batch_operations(self, blob_storage): + + test_ran = await StorageBaseTests.perform_batch_operations(blob_storage) + + assert test_ran + + +class TestBlobStorage: + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_read_update_should_return_new_etag(self, blob_storage): + await blob_storage.write({"test": SimpleStoreItem(counter=1)}) + data_result = await blob_storage.read(["test"], target_cls=SimpleStoreItem) + data_result["test"].counter = 2 + await blob_storage.write(data_result) + data_updated = await blob_storage.read(["test"], target_cls=SimpleStoreItem) + assert data_updated["test"].counter == 2 + assert data_updated["test"].e_tag != data_result["test"].e_tag + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_write_should_overwrite_when_new_e_tag_is_an_asterisk( + self, blob_storage + ): + await blob_storage.write({"user": SimpleStoreItem()}) + + await blob_storage.write({"user": SimpleStoreItem(counter=10, e_tag="*")}) + data = await blob_storage.read(["user"], target_cls=SimpleStoreItem) + assert data["user"].counter == 10 + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_should_delete_according_cached_data(self, blob_storage): + await blob_storage.write({"test": SimpleStoreItem()}) + try: + await blob_storage.delete(["test"]) + except Exception as error: + raise error + else: + data = await blob_storage.read(["test"], target_cls=SimpleStoreItem) + + assert isinstance(data, dict) + assert not data.keys() + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_should_delete_multiple_values_when_given_multiple_valid_keys( + self, blob_storage + ): + await blob_storage.write({"test": SimpleStoreItem(), "test2": SimpleStoreItem(2)}) + + await blob_storage.delete(["test", "test2"]) + data = await blob_storage.read(["test", "test2"]) + assert not data.keys() + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_should_delete_values_when_given_multiple_valid_keys_and_ignore_other_data( + self, blob_storage + ): + await blob_storage.write( + { + "test": SimpleStoreItem(), + "test2": SimpleStoreItem(counter=2), + "test3": SimpleStoreItem(counter=3), + } + ) + + await blob_storage.delete(["test", "test2"]) + data = await blob_storage.read(["test", "test2", "test3"]) + assert len(data.keys()) == 1 + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_invalid_key_should_do_nothing_and_not_affect_cached_data( + self, blob_storage + ): + await blob_storage.write({"test": SimpleStoreItem()}) + + await blob_storage.delete(["foo"]) + data = await blob_storage.read(["test"]) + assert len(data.keys()) == 1 + data = await blob_storage.read(["foo"]) + assert not data.keys() + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_invalid_keys_should_do_nothing_and_not_affect_cached_data( + self, blob_storage + ): + await blob_storage.write({"test": SimpleStoreItem()}) + + await blob_storage.delete(["foo", "bar"]) + data = await blob_storage.read(["test"]) + assert len(data.keys()) == 1 \ No newline at end of file diff --git a/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py b/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py new file mode 100644 index 00000000..0bd23c92 --- /dev/null +++ b/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py @@ -0,0 +1,281 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" +Base tests that all storage providers should implement in their own tests. +They handle the storage-based assertions, internally. + +All tests return true if assertions pass to indicate that the code ran to completion, passing internal assertions. +Therefore, all tests using theses static tests should strictly check that the method returns true. + +Note: Python cannot have dicts with properties with a None value like other SDKs can have properties with null values. + Because of this, StoreItem tests have "e_tag: *" where the tests in the other SDKs do not. + This has also caused us to comment out some parts of these tests where we assert that "e_tag" + is None for the same reason. A null e_tag should work just like a * e_tag when writing, + as far as the storage adapters are concerened, so this shouldn't cause issues. + + +:Example: + async def test_handle_null_keys_when_reading(self): + await reset() + + test_ran = await StorageBaseTests.handle_null_keys_when_reading(get_storage()) + + assert test_ran +""" +from copy import deepcopy + +import pytest + +from microsoft.agents.storage import MemoryStorage, StoreItem +from microsoft.agents.storage._type_aliases import JSON + +class DictStoreItem(StoreItem): + def __init__(self, data): + super().__init__() + + self.data = deepcopy(data) + + def store_item_to_json(self) -> JSON: + return deepcopy(self.data) + + @staticmethod + def from_json_to_store_item(json_data: JSON) -> "StoreItem": + return DictStoreItem(json_data) + + +class StorageBaseTests: + # pylint: disable=pointless-string-statement + @staticmethod + async def return_empty_object_when_reading_unknown_key(storage) -> bool: + result = await storage.read(["unknown"]) + + assert result is not None + assert len(result) == 0 + + return True + + @staticmethod + async def handle_null_keys_when_reading(storage) -> bool: + if isinstance(storage, (MemoryStorage)): + result = await storage.read(None) + assert len(result.keys()) == 0 + # Catch-all + else: + with pytest.raises(Exception) as err: + await storage.read(None) + assert err.value.args[0] == "Keys are required when reading" + + return True + + @staticmethod + async def handle_null_keys_when_writing(storage) -> bool: + with pytest.raises(Exception) as err: + await storage.write(None) + # assert err.value.args[0] == "Changes are required when writing" + + return True + + @staticmethod + async def does_not_raise_when_writing_no_items(storage) -> bool: + # noinspection PyBroadException + try: + await storage.write(dict()) + except: + pytest.fail("Should not raise") + + return True + + @staticmethod + async def create_object(storage) -> bool: + store_items = { + "createPoco": DictStoreItem({"id": 1}), + "createPocoStoreItem": DictStoreItem({"id": 2, "e_tag": "*"}), + } + + await storage.write(store_items) + + read_store_items = await storage.read(store_items.keys()) + + assert store_items["createPoco"].data["id"] == read_store_items["createPoco"]["id"] + assert ( + store_items["createPocoStoreItem"].data["id"] + == read_store_items["createPocoStoreItem"]["id"] + ) + + # If decided to validate e_tag integrity again, uncomment this code + # assert read_store_items["createPoco"]["e_tag"] is not None + assert read_store_items["createPocoStoreItem"]["e_tag"] is not None + + return True + + @staticmethod + async def handle_crazy_keys(storage) -> bool: + key = '!@#$%^&*()_+??><":QASD~`' + store_item = DictStoreItem({"id": 1}) + store_items = {key: store_item} + + await storage.write(store_items) + + read_store_items = await storage.read(store_items.keys()) + + assert read_store_items[key] is not None + assert read_store_items[key]["id"] == 1 + + return True + + @staticmethod + async def update_object(storage) -> bool: + original_store_items = { + "pocoItem": DictStoreItem({"id": 1, "count": 1}), + "pocoStoreItem": DictStoreItem({"id": 1, "count": 1, "e_tag": "*"}), + } + + # 1st write should work + await storage.write(original_store_items) + + loaded_store_items = await storage.read(["pocoItem", "pocoStoreItem"]) + + update_poco_item = loaded_store_items["pocoItem"] + update_poco_item["e_tag"] = None + update_poco_store_item = loaded_store_items["pocoStoreItem"] + assert update_poco_store_item["e_tag"] is not None + + # 2nd write should work + update_poco_item["count"] += 1 + update_poco_store_item["count"] += 1 + + await storage.write({ key: DictStoreItem(value) for key, value in loaded_store_items.items() }) + + reloaded_store_items = await storage.read(loaded_store_items.keys()) + + reloaded_update_poco_item = reloaded_store_items["pocoItem"] + reloaded_update_poco_store_item = reloaded_store_items["pocoStoreItem"] + + assert reloaded_update_poco_item["count"] == 2 + assert reloaded_update_poco_store_item["count"] == 2 + + # Write with old e_tag should succeed for non-storeItem + update_poco_item["count"] = 123 + await storage.write({"pocoItem": DictStoreItem(update_poco_item)}) + + # Write with old eTag should FAIL for storeItem + update_poco_store_item["count"] = 123 + + """ + This assert exists in the other SDKs but can't in python, currently + due to using "e_tag: *" above (see comment near the top of this file for details). + + with pytest.raises(Exception) as err: + await storage.write({"pocoStoreItem": update_poco_store_item}) + assert err.value is not None + """ + + reloaded_store_items2 = await storage.read(["pocoItem", "pocoStoreItem"]) + + reloaded_poco_item2 = reloaded_store_items2["pocoItem"] + reloaded_poco_item2["e_tag"] = None + reloaded_poco_store_item2 = reloaded_store_items2["pocoStoreItem"] + + assert reloaded_poco_item2["count"] == 123 + assert reloaded_poco_store_item2["count"] == 2 + + # write with wildcard etag should work + reloaded_poco_item2["count"] = 100 + reloaded_poco_store_item2["count"] = 100 + reloaded_poco_store_item2["e_tag"] = "*" + + wildcard_etag_dict = { + "pocoItem": reloaded_poco_item2, + "pocoStoreItem": reloaded_poco_store_item2, + } + + await storage.write({ key: DictStoreItem(value) for key, value in wildcard_etag_dict.items() }) + + reloaded_store_items3 = await storage.read(["pocoItem", "pocoStoreItem"]) + + assert reloaded_store_items3["pocoItem"]["count"] == 100 + assert reloaded_store_items3["pocoStoreItem"]["count"] == 100 + + # Write with empty etag should not work + reloaded_store_items4 = await storage.read(["pocoStoreItem"]) + reloaded_store_item4 = reloaded_store_items4["pocoStoreItem"] + + assert reloaded_store_item4 is not None + + reloaded_store_item4["e_tag"] = "" + dict2 = {"pocoStoreItem": DictStoreItem(reloaded_store_item4)} + + with pytest.raises(Exception) as err: + await storage.write(dict2) + assert err.value is not None + + final_store_items = await storage.read(["pocoItem", "pocoStoreItem"]) + assert final_store_items["pocoItem"]["count"] == 100 + assert final_store_items["pocoStoreItem"]["count"] == 100 + + return True + + @staticmethod + async def delete_object(storage) -> bool: + store_items = {"delete1": DictStoreItem({"id": 1, "count": 1, "e_tag": "*"})} + + await storage.write(store_items) + + read_store_items = await storage.read(["delete1"]) + + assert read_store_items["delete1"]["e_tag"] + assert read_store_items["delete1"]["count"] == 1 + + await storage.delete(["delete1"]) + + reloaded_store_items = await storage.read(["delete1"]) + + assert reloaded_store_items.get("delete1", None) is None + + return True + + @staticmethod + async def delete_unknown_object(storage) -> bool: + # noinspection PyBroadException + try: + await storage.delete(["unknown_key"]) + except: + pytest.fail("Should not raise") + + return True + + @staticmethod + async def perform_batch_operations(storage) -> bool: + await storage.write( + { + "batch1": DictStoreItem({"count": 10}), + "batch2": DictStoreItem({"count": 20}), + "batch3": DictStoreItem({"count": 30}), + } + ) + + result = await storage.read(["batch1", "batch2", "batch3"]) + + assert result.get("batch1", None) is not None + assert result.get("batch2", None) is not None + assert result.get("batch3", None) is not None + assert result["batch1"]["count"] == 10 + assert result["batch2"]["count"] == 20 + assert result["batch3"]["count"] == 30 + """ + If decided to validate e_tag integrity aagain, uncomment this code + assert result["batch1"].get("e_tag", None) is not None + assert result["batch2"].get("e_tag", None) is not None + assert result["batch3"].get("e_tag", None) is not None + """ + + await storage.delete(["batch1", "batch2", "batch3"]) + + result = await storage.read(["batch1", "batch2", "batch3"]) + + assert result.get("batch1", None) is None + assert result.get("batch2", None) is None + assert result.get("batch3", None) is None + + return True \ No newline at end of file From 309fb49716d77ba26e282721555da4169483cd19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodrigo=20Brand=C3=A3o?= Date: Thu, 26 Jun 2025 13:47:55 -0700 Subject: [PATCH 2/6] Adding blob-backed storage --- .../microsoft/agents/blob/blob_storage.py | 131 ++++++------- .../microsoft-agents-blob/pyproject.toml | 3 + .../tests/blob_storage_test.py | 85 ++++----- .../tests/storage_base_test.py | 176 +++++------------- 4 files changed, 154 insertions(+), 241 deletions(-) diff --git a/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py b/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py index 9e278a35..7bca95a6 100644 --- a/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py +++ b/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py @@ -2,18 +2,22 @@ # https://github.com/microsoft/botbuilder-python/blob/main/libraries/botbuilder-azure/botbuilder/azure/blob_storage.py from typing import TypeVar +from io import BytesIO import json -from microsoft.agents.storage._type_aliases import JSON - from azure.core import MatchConditions from azure.core.exceptions import ( HttpResponseError, ResourceExistsError, ResourceNotFoundError, ) -from azure.storage.blob.aio import BlobServiceClient +from azure.storage.blob.aio import ( + ContainerClient, + BlobServiceClient, + BlobClient, +) +from microsoft.agents.storage._type_aliases import JSON from microsoft.agents.storage import Storage, StoreItem StoreItemT = TypeVar("StoreItemT", bound=StoreItem) @@ -27,17 +31,15 @@ def __init__( account_key: str = "", connection_string: str = "" ): - self.container_name = container_name self.account_name = account_name self.account_key = account_key self.connection_string = connection_string -# New Azure Blob SDK only allows connection strings, but our SDK allows key+name. -# This is here for backwards compatibility. + def convert_account_name_and_key_to_connection_string(settings: BlobStorageSettings): if not settings.account_name or not settings.account_key: - raise Exception( + raise ValueError( "account_name and account_key are both required for BlobStorageSettings if not using a connections string." ) return ( @@ -48,36 +50,34 @@ def convert_account_name_and_key_to_connection_string(settings: BlobStorageSetti class BlobStorage(Storage): def __init__(self, settings: BlobStorageSettings): - if not settings.container_name: - raise Exception("Container name is required.") - - if settings.connection_string: - blob_service_client = BlobServiceClient.from_connection_string( - settings.connection_string - ) - else: - blob_service_client = BlobServiceClient.from_connection_string( - convert_account_name_and_key_to_connection_string(settings) - ) - - self.__container_client = blob_service_client.get_container_client( - settings.container_name - ) + raise ValueError("BlobStorage: Container name is required.") + + connection_string: str = settings.connection_string + if not connection_string: + # New Azure Blob SDK only allows connection strings, but our SDK allows key+name. + # This is here for backwards compatibility. + connection_string = convert_account_name_and_key_to_connection_string(settings) - self.__initialized = False + blob_service_client: BlobServiceClient = BlobServiceClient.from_connection_string(connection_string) - async def _initialize(self): + self._container_client: ContainerClient = blob_service_client.get_container_client( + settings.container_name + ) + self._initialized: bool = False - if self.__initialized is False: + async def _initialize_container(self): + """Initializes the storage container""" + if self._initialized is False: # This should only happen once - assuming this is a singleton. # ContainerClient.exists() method is available in an unreleased version of the SDK. Until then, we use: try: - await self.__container_client.create_container() + await self._container_client.create_container() except ResourceExistsError: pass - self.__initialized = True - return self.__initialized + self._initialized = True + + return self._initialized async def read( self, @@ -90,38 +90,37 @@ async def read( :param keys: An array of entity keys. :type keys: dict[str, StoreItem] + :param target_cls: The StoreItem class to deserialize retrieved values into. + :type target_cls: StoreItem :return dict: """ if not keys: - raise Exception("Keys are required when reading") + raise ValueError("BlobStorage.read(): Keys are required when reading.") + if not target_cls: + raise ValueError("BlobStorage.read(): target_cls cannot be None.") - await self._initialize() + await self._initialize_container() result: dict[str, StoreItem] = {} - for key in keys: - blob_client = self.__container_client.get_blob_client(key) - try: - blob = await blob_client.download_blob() + item_rep: str = await (await self._container_client.download_blob(blob=key)).readall() + item_JSON: JSON = json.loads(item_rep) except HttpResponseError as error: if error.status_code == 404: continue - - item_json = json.loads(await blob.content_as_text()) - - item_json["e_tag"] = blob.properties.etag.replace('""', "") - - if not target_cls: - result[key] = item_json - else: - try: - result[key] = target_cls.from_json_to_store_item(item_json) - except AttributeError as error: - raise TypeError( - f"BlobStorage.read(): could not deserialize blob item into {target_cls} class. Error: {error}" + else: + raise HttpResponseError( + f"BlobStorage.read(): Error reading blob '{key}': {error}" ) + + try: + result[key] = target_cls.from_json_to_store_item(item_JSON) + except AttributeError as error: + raise TypeError( + f"BlobStorage.read(): could not deserialize blob item into {target_cls} class. Error: {error}" + ) return result @@ -132,32 +131,20 @@ async def write(self, changes: dict[str, StoreItem]): :type changes: dict[str, StoreItem] :return: """ - - if changes is None: - raise ValueError("BlobStorage.write(): changes cannot be None") + if not changes: + raise ValueError("BlobStorage.write(): changes cannot be None nor empty") + + await self._initialize_container() for key, item in changes.items(): - item_json = item.store_item_to_json() - if item_json is None: + item_JSON: JSON = item.store_item_to_json() + if item_JSON is None: raise ValueError("BlobStorage.write(): StoreItem serialization cannot return None") - - item_str = json.dumps(item_json) + item_rep_bytes = json.dumps(item_JSON).encode("utf-8") - blob_reference = self.__container_client.get_blob_client(key) - - e_tag = None if item_json is None else item_json.get("e_tag", None) - e_tag = None if e_tag == "*" else e_tag - - if e_tag == "": - raise Exception("blob_storage.write(): etag missing") - - if e_tag: - await blob_reference.upload_blob( - item_str, match_condition=MatchConditions.IfNotModified, etag=e_tag - ) - else: - await blob_reference.upload_blob(item_str, overwrite=True) + # providing the length parameter may improve performance + await self._container_client.upload_blob(name=key, data=BytesIO(item_rep_bytes), overwrite=True, length=len(item_rep_bytes)) async def delete(self, keys: list[str]): """Deletes entity blobs from the configured container. @@ -165,16 +152,14 @@ async def delete(self, keys: list[str]): :param keys: An array of entity keys. :type keys: list[str] """ - if keys is None: - raise Exception("BlobStorage.delete: keys parameter can't be null") + raise ValueError("BlobStorage.delete(): keys parameter can't be null") - await self._initialize() + await self._initialize_container() for key in keys: - blob_client = self.__container_client.get_blob_client(key) try: - await blob_client.delete_blob() + await self._container_client.delete_blob(blob=key) # We can't delete what's already gone. except ResourceNotFoundError: pass \ No newline at end of file diff --git a/libraries/Storage/microsoft-agents-blob/pyproject.toml b/libraries/Storage/microsoft-agents-blob/pyproject.toml index 71bf7367..7fd16cb6 100644 --- a/libraries/Storage/microsoft-agents-blob/pyproject.toml +++ b/libraries/Storage/microsoft-agents-blob/pyproject.toml @@ -14,6 +14,9 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ + "microsoft.agents.storage", + "azure-core", + "azure-storage-blob", ] [project.urls] diff --git a/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py b/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py index 8962ea7a..0508eb01 100644 --- a/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py +++ b/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py @@ -15,14 +15,15 @@ from .storage_base_test import StorageBaseTests -EMULATOR_RUNNING = True +EMULATOR_RUNNING = False # constructs an emulated blob storage instance @pytest_asyncio.fixture async def blob_storage(): # setup - + + # Default Azure Storage Emulator connection string connection_string = ("AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq" + "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=" + "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" @@ -33,11 +34,11 @@ async def blob_storage(): container_name = "test" container_client = blob_service_client.get_container_client(container_name) + # reset state of test container try: await container_client.delete_container() except ResourceNotFoundError: pass - await container_client.create_container() blob_storage_settings = BlobStorageSettings( @@ -55,25 +56,32 @@ async def blob_storage(): class SimpleStoreItem(StoreItem): - def __init__(self, counter=1, e_tag="*"): - super(SimpleStoreItem, self).__init__() + def __init__(self, counter: int = 1, value: str = "*"): self.counter = counter - self.e_tag = e_tag + self.value = value def store_item_to_json(self) -> JSON: return { "counter": self.counter, - "e_tag": self.e_tag, + "value": self.value, } @staticmethod def from_json_to_store_item(json_data: JSON) -> "StoreItem": - return SimpleStoreItem(json_data["counter"], json_data["e_tag"]) + return SimpleStoreItem(json_data["counter"], json_data["value"]) class TestBlobStorageConstructor: + @pytest.mark.asyncio + async def test_blob_storage_init_should_error_without_container_name(self): + settings = BlobStorageSettings("") + with pytest.raises(Exception) as err: + BlobStorage(settings) + + assert err.value.args[0] == "BlobStorage: Container name is required." + @pytest.mark.asyncio async def test_blob_storage_init_should_error_without_blob_config(self): try: @@ -81,107 +89,104 @@ async def test_blob_storage_init_should_error_without_blob_config(self): except Exception as error: assert error + @pytest.mark.asyncio + async def test_blob_storage_init_should_error_with_insufficient_settings(self): + settings_0 = BlobStorageSettings("norway", account_name="some_account_name") + settings_1 = BlobStorageSettings("sweden", account_key="some_account_key") + with pytest.raises(Exception) as err: + BlobStorage(settings_0) + with pytest.raises(Exception) as err: + BlobStorage(settings_1) + + @pytest.mark.asyncio + async def test_blob_storage_init_from_account_key_and_name(self): + settings = BlobStorageSettings("norway", account_name="some_account_name", account_key="some_account_key") + BlobStorage(settings) + class TestBlobStorageBaseTests: @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_return_empty_object_when_reading_unknown_key(self, blob_storage): - test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key( blob_storage ) - assert test_ran @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_handle_null_keys_when_reading(self, blob_storage): - test_ran = await StorageBaseTests.handle_null_keys_when_reading(blob_storage) - assert test_ran @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_handle_null_keys_when_writing(self, blob_storage): - test_ran = await StorageBaseTests.handle_null_keys_when_writing(blob_storage) - assert test_ran @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio - async def test_does_not_raise_when_writing_no_items(self, blob_storage): - - test_ran = await StorageBaseTests.does_not_raise_when_writing_no_items( + async def does_raise_when_writing_no_items(self, blob_storage): + test_ran = await StorageBaseTests.does_raise_when_writing_no_items( blob_storage ) - assert test_ran @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_create_object(self, blob_storage): - test_ran = await StorageBaseTests.create_object(blob_storage) - assert test_ran @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_handle_crazy_keys(self, blob_storage): - test_ran = await StorageBaseTests.handle_crazy_keys(blob_storage) - assert test_ran @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_update_object(self, blob_storage): - test_ran = await StorageBaseTests.update_object(blob_storage) - assert test_ran @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_delete_object(self, blob_storage): - test_ran = await StorageBaseTests.delete_object(blob_storage) - assert test_ran @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def test_perform_batch_operations(self, blob_storage): - test_ran = await StorageBaseTests.perform_batch_operations(blob_storage) - assert test_ran class TestBlobStorage: + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio - async def test_blob_storage_read_update_should_return_new_etag(self, blob_storage): + async def test_blob_storage_read_update_same_data(self, blob_storage): await blob_storage.write({"test": SimpleStoreItem(counter=1)}) data_result = await blob_storage.read(["test"], target_cls=SimpleStoreItem) data_result["test"].counter = 2 await blob_storage.write(data_result) data_updated = await blob_storage.read(["test"], target_cls=SimpleStoreItem) assert data_updated["test"].counter == 2 - assert data_updated["test"].e_tag != data_result["test"].e_tag + assert data_updated["test"].value == data_result["test"].value @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio - async def test_blob_storage_write_should_overwrite_when_new_e_tag_is_an_asterisk( + async def test_blob_storage_write_should_overwrite( self, blob_storage ): await blob_storage.write({"user": SimpleStoreItem()}) - - await blob_storage.write({"user": SimpleStoreItem(counter=10, e_tag="*")}) + await blob_storage.write({"user": SimpleStoreItem(counter=10, value="*")}) data = await blob_storage.read(["user"], target_cls=SimpleStoreItem) assert data["user"].counter == 10 + assert data["user"].value == "*" @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio @@ -203,9 +208,8 @@ async def test_blob_storage_delete_should_delete_multiple_values_when_given_mult self, blob_storage ): await blob_storage.write({"test": SimpleStoreItem(), "test2": SimpleStoreItem(2)}) - await blob_storage.delete(["test", "test2"]) - data = await blob_storage.read(["test", "test2"]) + data = await blob_storage.read(["test", "test2"], target_cls=SimpleStoreItem) assert not data.keys() @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @@ -220,9 +224,8 @@ async def test_blob_storage_delete_should_delete_values_when_given_multiple_vali "test3": SimpleStoreItem(counter=3), } ) - await blob_storage.delete(["test", "test2"]) - data = await blob_storage.read(["test", "test2", "test3"]) + data = await blob_storage.read(["test", "test2", "test3"], target_cls=SimpleStoreItem) assert len(data.keys()) == 1 @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @@ -231,11 +234,10 @@ async def test_blob_storage_delete_invalid_key_should_do_nothing_and_not_affect_ self, blob_storage ): await blob_storage.write({"test": SimpleStoreItem()}) - await blob_storage.delete(["foo"]) - data = await blob_storage.read(["test"]) + data = await blob_storage.read(["test"], target_cls=SimpleStoreItem) assert len(data.keys()) == 1 - data = await blob_storage.read(["foo"]) + data = await blob_storage.read(["foo"], target_cls=SimpleStoreItem) assert not data.keys() @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @@ -244,7 +246,6 @@ async def test_blob_storage_delete_invalid_keys_should_do_nothing_and_not_affect self, blob_storage ): await blob_storage.write({"test": SimpleStoreItem()}) - await blob_storage.delete(["foo", "bar"]) - data = await blob_storage.read(["test"]) + data = await blob_storage.read(["test"], target_cls=SimpleStoreItem) assert len(data.keys()) == 1 \ No newline at end of file diff --git a/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py b/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py index 0bd23c92..aeddd026 100644 --- a/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py +++ b/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py @@ -2,19 +2,14 @@ # Licensed under the MIT License. """ +Adapted from https://github.com/microsoft/botbuilder-python/blob/main/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py + Base tests that all storage providers should implement in their own tests. They handle the storage-based assertions, internally. All tests return true if assertions pass to indicate that the code ran to completion, passing internal assertions. Therefore, all tests using theses static tests should strictly check that the method returns true. -Note: Python cannot have dicts with properties with a None value like other SDKs can have properties with null values. - Because of this, StoreItem tests have "e_tag: *" where the tests in the other SDKs do not. - This has also caused us to comment out some parts of these tests where we assert that "e_tag" - is None for the same reason. A null e_tag should work just like a * e_tag when writing, - as far as the storage adapters are concerened, so this shouldn't cause issues. - - :Example: async def test_handle_null_keys_when_reading(self): await reset() @@ -23,32 +18,32 @@ async def test_handle_null_keys_when_reading(self): assert test_ran """ -from copy import deepcopy import pytest from microsoft.agents.storage import MemoryStorage, StoreItem from microsoft.agents.storage._type_aliases import JSON -class DictStoreItem(StoreItem): - def __init__(self, data): - super().__init__() - self.data = deepcopy(data) +class MockStoreItem(StoreItem): + + def __init__(self, data: JSON = None): + self.data = data or {} def store_item_to_json(self) -> JSON: - return deepcopy(self.data) + return self.data @staticmethod - def from_json_to_store_item(json_data: JSON) -> "StoreItem": - return DictStoreItem(json_data) + def from_json_to_store_item(json_data: JSON) -> "MockStoreItem": + return MockStoreItem(json_data) class StorageBaseTests: + # pylint: disable=pointless-string-statement @staticmethod async def return_empty_object_when_reading_unknown_key(storage) -> bool: - result = await storage.read(["unknown"]) + result = await storage.read(["unknown"], target_cls=MockStoreItem) assert result is not None assert len(result) == 0 @@ -58,13 +53,12 @@ async def return_empty_object_when_reading_unknown_key(storage) -> bool: @staticmethod async def handle_null_keys_when_reading(storage) -> bool: if isinstance(storage, (MemoryStorage)): - result = await storage.read(None) + result = await storage.read(None, target_cls=MockStoreItem) assert len(result.keys()) == 0 # Catch-all else: with pytest.raises(Exception) as err: - await storage.read(None) - assert err.value.args[0] == "Keys are required when reading" + await storage.read(None, target_cls=MockStoreItem) return True @@ -77,159 +71,95 @@ async def handle_null_keys_when_writing(storage) -> bool: return True @staticmethod - async def does_not_raise_when_writing_no_items(storage) -> bool: + async def does_raise_when_writing_no_items(storage) -> bool: # noinspection PyBroadException - try: + with pytest.raises(Exception) as err: await storage.write(dict()) - except: - pytest.fail("Should not raise") - return True @staticmethod async def create_object(storage) -> bool: store_items = { - "createPoco": DictStoreItem({"id": 1}), - "createPocoStoreItem": DictStoreItem({"id": 2, "e_tag": "*"}), + "createPoco": MockStoreItem({"id": 1}), + "createPocoStoreItem": MockStoreItem({"id": 2, "value": "*"}), } await storage.write(store_items) - read_store_items = await storage.read(store_items.keys()) + read_store_items = await storage.read(store_items.keys(), target_cls=MockStoreItem) - assert store_items["createPoco"].data["id"] == read_store_items["createPoco"]["id"] + assert store_items["createPoco"].data["id"] == read_store_items["createPoco"].data["id"] assert ( - store_items["createPocoStoreItem"].data["id"] - == read_store_items["createPocoStoreItem"]["id"] + store_items["createPocoStoreItem"].data["id"] == read_store_items["createPocoStoreItem"].data["id"] ) - - # If decided to validate e_tag integrity again, uncomment this code - # assert read_store_items["createPoco"]["e_tag"] is not None - assert read_store_items["createPocoStoreItem"]["e_tag"] is not None + assert read_store_items["createPocoStoreItem"].data["value"] == "*" return True @staticmethod async def handle_crazy_keys(storage) -> bool: key = '!@#$%^&*()_+??><":QASD~`' - store_item = DictStoreItem({"id": 1}) + store_item = MockStoreItem({"id": 1}) store_items = {key: store_item} await storage.write(store_items) - read_store_items = await storage.read(store_items.keys()) + read_store_items = await storage.read(store_items.keys(), target_cls=MockStoreItem) assert read_store_items[key] is not None - assert read_store_items[key]["id"] == 1 + assert read_store_items[key].data["id"] == 1 return True @staticmethod async def update_object(storage) -> bool: original_store_items = { - "pocoItem": DictStoreItem({"id": 1, "count": 1}), - "pocoStoreItem": DictStoreItem({"id": 1, "count": 1, "e_tag": "*"}), + "pocoItem": MockStoreItem({"id": 1, "count": 1}), + "pocoStoreItem": MockStoreItem({"id": 1, "count": 1, "value": "*"}), } # 1st write should work await storage.write(original_store_items) - loaded_store_items = await storage.read(["pocoItem", "pocoStoreItem"]) + loaded_store_items = await storage.read(["pocoItem", "pocoStoreItem"], target_cls=MockStoreItem) update_poco_item = loaded_store_items["pocoItem"] - update_poco_item["e_tag"] = None + update_poco_item.data["value"] = None update_poco_store_item = loaded_store_items["pocoStoreItem"] - assert update_poco_store_item["e_tag"] is not None + assert update_poco_store_item.data["value"] == "*" # 2nd write should work - update_poco_item["count"] += 1 - update_poco_store_item["count"] += 1 + update_poco_item.data["count"] += 1 + update_poco_store_item.data["count"] += 1 - await storage.write({ key: DictStoreItem(value) for key, value in loaded_store_items.items() }) + await storage.write({ key: MockStoreItem(value.data) for key, value in loaded_store_items.items() }) - reloaded_store_items = await storage.read(loaded_store_items.keys()) + reloaded_store_items = await storage.read(loaded_store_items.keys(), target_cls=MockStoreItem) reloaded_update_poco_item = reloaded_store_items["pocoItem"] reloaded_update_poco_store_item = reloaded_store_items["pocoStoreItem"] - assert reloaded_update_poco_item["count"] == 2 - assert reloaded_update_poco_store_item["count"] == 2 - - # Write with old e_tag should succeed for non-storeItem - update_poco_item["count"] = 123 - await storage.write({"pocoItem": DictStoreItem(update_poco_item)}) - - # Write with old eTag should FAIL for storeItem - update_poco_store_item["count"] = 123 - - """ - This assert exists in the other SDKs but can't in python, currently - due to using "e_tag: *" above (see comment near the top of this file for details). - - with pytest.raises(Exception) as err: - await storage.write({"pocoStoreItem": update_poco_store_item}) - assert err.value is not None - """ - - reloaded_store_items2 = await storage.read(["pocoItem", "pocoStoreItem"]) - - reloaded_poco_item2 = reloaded_store_items2["pocoItem"] - reloaded_poco_item2["e_tag"] = None - reloaded_poco_store_item2 = reloaded_store_items2["pocoStoreItem"] - - assert reloaded_poco_item2["count"] == 123 - assert reloaded_poco_store_item2["count"] == 2 - - # write with wildcard etag should work - reloaded_poco_item2["count"] = 100 - reloaded_poco_store_item2["count"] = 100 - reloaded_poco_store_item2["e_tag"] = "*" - - wildcard_etag_dict = { - "pocoItem": reloaded_poco_item2, - "pocoStoreItem": reloaded_poco_store_item2, - } - - await storage.write({ key: DictStoreItem(value) for key, value in wildcard_etag_dict.items() }) - - reloaded_store_items3 = await storage.read(["pocoItem", "pocoStoreItem"]) - - assert reloaded_store_items3["pocoItem"]["count"] == 100 - assert reloaded_store_items3["pocoStoreItem"]["count"] == 100 - - # Write with empty etag should not work - reloaded_store_items4 = await storage.read(["pocoStoreItem"]) - reloaded_store_item4 = reloaded_store_items4["pocoStoreItem"] - - assert reloaded_store_item4 is not None - - reloaded_store_item4["e_tag"] = "" - dict2 = {"pocoStoreItem": DictStoreItem(reloaded_store_item4)} - - with pytest.raises(Exception) as err: - await storage.write(dict2) - assert err.value is not None - - final_store_items = await storage.read(["pocoItem", "pocoStoreItem"]) - assert final_store_items["pocoItem"]["count"] == 100 - assert final_store_items["pocoStoreItem"]["count"] == 100 + assert reloaded_update_poco_item.data["count"] == 2 + assert reloaded_update_poco_store_item.data["count"] == 2 + assert reloaded_update_poco_item.data["value"] is None + assert reloaded_update_poco_store_item.data["value"] == "*" return True @staticmethod async def delete_object(storage) -> bool: - store_items = {"delete1": DictStoreItem({"id": 1, "count": 1, "e_tag": "*"})} + store_items = {"delete1": MockStoreItem({"id": 1, "count": 1, "value": "*"})} await storage.write(store_items) - read_store_items = await storage.read(["delete1"]) + read_store_items = await storage.read(["delete1"], target_cls=MockStoreItem) - assert read_store_items["delete1"]["e_tag"] - assert read_store_items["delete1"]["count"] == 1 + assert read_store_items["delete1"].data["value"] + assert read_store_items["delete1"].data["count"] == 1 await storage.delete(["delete1"]) - reloaded_store_items = await storage.read(["delete1"]) + reloaded_store_items = await storage.read(["delete1"], target_cls=MockStoreItem) assert reloaded_store_items.get("delete1", None) is None @@ -249,30 +179,24 @@ async def delete_unknown_object(storage) -> bool: async def perform_batch_operations(storage) -> bool: await storage.write( { - "batch1": DictStoreItem({"count": 10}), - "batch2": DictStoreItem({"count": 20}), - "batch3": DictStoreItem({"count": 30}), + "batch1": MockStoreItem({"count": 10}), + "batch2": MockStoreItem({"count": 20}), + "batch3": MockStoreItem({"count": 30}), } ) - result = await storage.read(["batch1", "batch2", "batch3"]) + result = await storage.read(["batch1", "batch2", "batch3"], target_cls=MockStoreItem) assert result.get("batch1", None) is not None assert result.get("batch2", None) is not None assert result.get("batch3", None) is not None - assert result["batch1"]["count"] == 10 - assert result["batch2"]["count"] == 20 - assert result["batch3"]["count"] == 30 - """ - If decided to validate e_tag integrity aagain, uncomment this code - assert result["batch1"].get("e_tag", None) is not None - assert result["batch2"].get("e_tag", None) is not None - assert result["batch3"].get("e_tag", None) is not None - """ + assert result["batch1"].data["count"] == 10 + assert result["batch2"].data["count"] == 20 + assert result["batch3"].data["count"] == 30 await storage.delete(["batch1", "batch2", "batch3"]) - result = await storage.read(["batch1", "batch2", "batch3"]) + result = await storage.read(["batch1", "batch2", "batch3"], target_cls=MockStoreItem) assert result.get("batch1", None) is None assert result.get("batch2", None) is None From ec764d3f9d93443ba5942bbb80ebbbefdbd88e93 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 26 Jun 2025 14:07:53 -0700 Subject: [PATCH 3/6] Reformatted code and removed unused imports --- .../microsoft-agents-blob/microsoft/agents/blob/blob_storage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py b/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py index 7bca95a6..89f4749f 100644 --- a/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py +++ b/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py @@ -5,7 +5,6 @@ from io import BytesIO import json -from azure.core import MatchConditions from azure.core.exceptions import ( HttpResponseError, ResourceExistsError, @@ -14,7 +13,6 @@ from azure.storage.blob.aio import ( ContainerClient, BlobServiceClient, - BlobClient, ) from microsoft.agents.storage._type_aliases import JSON From 6829ac3c21c759f45003923ca098e95b5d44b722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodrigo=20Brand=C3=A3o?= Date: Thu, 26 Jun 2025 14:37:50 -0700 Subject: [PATCH 4/6] Updated ci pipelines to include microsoft-agents-blob dependency --- .azdo/ci-pr.yaml | 1 + .github/workflows/python-package.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.azdo/ci-pr.yaml b/.azdo/ci-pr.yaml index b8439344..64742e94 100644 --- a/.azdo/ci-pr.yaml +++ b/.azdo/ci-pr.yaml @@ -57,6 +57,7 @@ steps: python -m pip install ./dist/microsoft_agents_copilotstudio_client*.whl python -m pip install ./dist/microsoft_agents_hosting_aiohttp*.whl python -m pip install ./dist/microsoft_agents_storage*.whl + python -m pip install ./dist/microsoft_agents_blob*.whl displayName: 'Install wheels' - script: | diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 31ee8efd..5e0b070f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -63,6 +63,7 @@ jobs: python -m pip install ./dist/microsoft_agents_copilotstudio_client*.whl python -m pip install ./dist/microsoft_agents_hosting_aiohttp*.whl python -m pip install ./dist/microsoft_agents_storage*.whl + python -m pip install ./dist/microsoft_agents_blob*.whl - name: Test with pytest run: | pytest From 64ed5683dbbf2ba1bcc9aa7e000d45a770ce79a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodrigo=20Brand=C3=A3o?= Date: Thu, 26 Jun 2025 14:42:48 -0700 Subject: [PATCH 5/6] Reformatting --- .../microsoft/agents/blob/blob_storage.py | 51 +++++++++++-------- .../tests/blob_storage_test.py | 48 +++++++++-------- .../tests/storage_base_test.py | 41 +++++++++++---- 3 files changed, 90 insertions(+), 50 deletions(-) diff --git a/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py b/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py index 89f4749f..1925b7e2 100644 --- a/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py +++ b/libraries/Storage/microsoft-agents-blob/microsoft/agents/blob/blob_storage.py @@ -20,6 +20,7 @@ StoreItemT = TypeVar("StoreItemT", bound=StoreItem) + class BlobStorageSettings: def __init__( @@ -27,7 +28,7 @@ def __init__( container_name: str, account_name: str = "", account_key: str = "", - connection_string: str = "" + connection_string: str = "", ): self.container_name = container_name self.account_name = account_name @@ -45,22 +46,27 @@ def convert_account_name_and_key_to_connection_string(settings: BlobStorageSetti f"AccountKey={settings.account_key};EndpointSuffix=core.windows.net" ) + class BlobStorage(Storage): def __init__(self, settings: BlobStorageSettings): if not settings.container_name: raise ValueError("BlobStorage: Container name is required.") - + connection_string: str = settings.connection_string if not connection_string: # New Azure Blob SDK only allows connection strings, but our SDK allows key+name. # This is here for backwards compatibility. - connection_string = convert_account_name_and_key_to_connection_string(settings) + connection_string = convert_account_name_and_key_to_connection_string( + settings + ) - blob_service_client: BlobServiceClient = BlobServiceClient.from_connection_string(connection_string) + blob_service_client: BlobServiceClient = ( + BlobServiceClient.from_connection_string(connection_string) + ) - self._container_client: ContainerClient = blob_service_client.get_container_client( - settings.container_name + self._container_client: ContainerClient = ( + blob_service_client.get_container_client(settings.container_name) ) self._initialized: bool = False @@ -78,12 +84,8 @@ async def _initialize_container(self): return self._initialized async def read( - self, - keys: list[str], - *, - target_cls: StoreItemT = None, - **kwargs - ) -> dict[str, StoreItemT]: + self, keys: list[str], *, target_cls: StoreItemT = None, **kwargs + ) -> dict[str, StoreItemT]: """Retrieve entities from the configured blob container. :param keys: An array of entity keys. @@ -103,7 +105,9 @@ async def read( for key in keys: try: - item_rep: str = await (await self._container_client.download_blob(blob=key)).readall() + item_rep: str = await ( + await self._container_client.download_blob(blob=key) + ).readall() item_JSON: JSON = json.loads(item_rep) except HttpResponseError as error: if error.status_code == 404: @@ -119,7 +123,7 @@ async def read( raise TypeError( f"BlobStorage.read(): could not deserialize blob item into {target_cls} class. Error: {error}" ) - + return result async def write(self, changes: dict[str, StoreItem]): @@ -131,18 +135,25 @@ async def write(self, changes: dict[str, StoreItem]): """ if not changes: raise ValueError("BlobStorage.write(): changes cannot be None nor empty") - + await self._initialize_container() - + for key, item in changes.items(): item_JSON: JSON = item.store_item_to_json() if item_JSON is None: - raise ValueError("BlobStorage.write(): StoreItem serialization cannot return None") + raise ValueError( + "BlobStorage.write(): StoreItem serialization cannot return None" + ) item_rep_bytes = json.dumps(item_JSON).encode("utf-8") - + # providing the length parameter may improve performance - await self._container_client.upload_blob(name=key, data=BytesIO(item_rep_bytes), overwrite=True, length=len(item_rep_bytes)) + await self._container_client.upload_blob( + name=key, + data=BytesIO(item_rep_bytes), + overwrite=True, + length=len(item_rep_bytes), + ) async def delete(self, keys: list[str]): """Deletes entity blobs from the configured container. @@ -160,4 +171,4 @@ async def delete(self, keys: list[str]): await self._container_client.delete_blob(blob=key) # We can't delete what's already gone. except ResourceNotFoundError: - pass \ No newline at end of file + pass diff --git a/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py b/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py index 0508eb01..03074c0b 100644 --- a/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py +++ b/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py @@ -17,17 +17,20 @@ EMULATOR_RUNNING = False + # constructs an emulated blob storage instance @pytest_asyncio.fixture async def blob_storage(): - + # setup - + # Default Azure Storage Emulator connection string - connection_string = ("AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq" - + "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=" - + "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" - + "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;") + connection_string = ( + "AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq" + + "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=" + + "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" + + "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + ) blob_service_client = BlobServiceClient.from_connection_string(connection_string) @@ -42,9 +45,10 @@ async def blob_storage(): await container_client.create_container() blob_storage_settings = BlobStorageSettings( - account_name="", account_key="", + account_name="", + account_key="", container_name=container_name, - connection_string=connection_string + connection_string=connection_string, ) storage = BlobStorage(blob_storage_settings) @@ -79,7 +83,7 @@ async def test_blob_storage_init_should_error_without_container_name(self): settings = BlobStorageSettings("") with pytest.raises(Exception) as err: BlobStorage(settings) - + assert err.value.args[0] == "BlobStorage: Container name is required." @pytest.mark.asyncio @@ -100,7 +104,9 @@ async def test_blob_storage_init_should_error_with_insufficient_settings(self): @pytest.mark.asyncio async def test_blob_storage_init_from_account_key_and_name(self): - settings = BlobStorageSettings("norway", account_name="some_account_name", account_key="some_account_key") + settings = BlobStorageSettings( + "norway", account_name="some_account_name", account_key="some_account_key" + ) BlobStorage(settings) @@ -128,9 +134,7 @@ async def test_handle_null_keys_when_writing(self, blob_storage): @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio async def does_raise_when_writing_no_items(self, blob_storage): - test_ran = await StorageBaseTests.does_raise_when_writing_no_items( - blob_storage - ) + test_ran = await StorageBaseTests.does_raise_when_writing_no_items(blob_storage) assert test_ran @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @@ -179,9 +183,7 @@ async def test_blob_storage_read_update_same_data(self, blob_storage): @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio - async def test_blob_storage_write_should_overwrite( - self, blob_storage - ): + async def test_blob_storage_write_should_overwrite(self, blob_storage): await blob_storage.write({"user": SimpleStoreItem()}) await blob_storage.write({"user": SimpleStoreItem(counter=10, value="*")}) data = await blob_storage.read(["user"], target_cls=SimpleStoreItem) @@ -190,7 +192,9 @@ async def test_blob_storage_write_should_overwrite( @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @pytest.mark.asyncio - async def test_blob_storage_delete_should_delete_according_cached_data(self, blob_storage): + async def test_blob_storage_delete_should_delete_according_cached_data( + self, blob_storage + ): await blob_storage.write({"test": SimpleStoreItem()}) try: await blob_storage.delete(["test"]) @@ -207,7 +211,9 @@ async def test_blob_storage_delete_should_delete_according_cached_data(self, blo async def test_blob_storage_delete_should_delete_multiple_values_when_given_multiple_valid_keys( self, blob_storage ): - await blob_storage.write({"test": SimpleStoreItem(), "test2": SimpleStoreItem(2)}) + await blob_storage.write( + {"test": SimpleStoreItem(), "test2": SimpleStoreItem(2)} + ) await blob_storage.delete(["test", "test2"]) data = await blob_storage.read(["test", "test2"], target_cls=SimpleStoreItem) assert not data.keys() @@ -225,7 +231,9 @@ async def test_blob_storage_delete_should_delete_values_when_given_multiple_vali } ) await blob_storage.delete(["test", "test2"]) - data = await blob_storage.read(["test", "test2", "test3"], target_cls=SimpleStoreItem) + data = await blob_storage.read( + ["test", "test2", "test3"], target_cls=SimpleStoreItem + ) assert len(data.keys()) == 1 @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") @@ -248,4 +256,4 @@ async def test_blob_storage_delete_invalid_keys_should_do_nothing_and_not_affect await blob_storage.write({"test": SimpleStoreItem()}) await blob_storage.delete(["foo", "bar"]) data = await blob_storage.read(["test"], target_cls=SimpleStoreItem) - assert len(data.keys()) == 1 \ No newline at end of file + assert len(data.keys()) == 1 diff --git a/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py b/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py index aeddd026..4ea2227c 100644 --- a/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py +++ b/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py @@ -86,11 +86,17 @@ async def create_object(storage) -> bool: await storage.write(store_items) - read_store_items = await storage.read(store_items.keys(), target_cls=MockStoreItem) + read_store_items = await storage.read( + store_items.keys(), target_cls=MockStoreItem + ) - assert store_items["createPoco"].data["id"] == read_store_items["createPoco"].data["id"] assert ( - store_items["createPocoStoreItem"].data["id"] == read_store_items["createPocoStoreItem"].data["id"] + store_items["createPoco"].data["id"] + == read_store_items["createPoco"].data["id"] + ) + assert ( + store_items["createPocoStoreItem"].data["id"] + == read_store_items["createPocoStoreItem"].data["id"] ) assert read_store_items["createPocoStoreItem"].data["value"] == "*" @@ -104,7 +110,9 @@ async def handle_crazy_keys(storage) -> bool: await storage.write(store_items) - read_store_items = await storage.read(store_items.keys(), target_cls=MockStoreItem) + read_store_items = await storage.read( + store_items.keys(), target_cls=MockStoreItem + ) assert read_store_items[key] is not None assert read_store_items[key].data["id"] == 1 @@ -121,7 +129,9 @@ async def update_object(storage) -> bool: # 1st write should work await storage.write(original_store_items) - loaded_store_items = await storage.read(["pocoItem", "pocoStoreItem"], target_cls=MockStoreItem) + loaded_store_items = await storage.read( + ["pocoItem", "pocoStoreItem"], target_cls=MockStoreItem + ) update_poco_item = loaded_store_items["pocoItem"] update_poco_item.data["value"] = None @@ -132,9 +142,16 @@ async def update_object(storage) -> bool: update_poco_item.data["count"] += 1 update_poco_store_item.data["count"] += 1 - await storage.write({ key: MockStoreItem(value.data) for key, value in loaded_store_items.items() }) + await storage.write( + { + key: MockStoreItem(value.data) + for key, value in loaded_store_items.items() + } + ) - reloaded_store_items = await storage.read(loaded_store_items.keys(), target_cls=MockStoreItem) + reloaded_store_items = await storage.read( + loaded_store_items.keys(), target_cls=MockStoreItem + ) reloaded_update_poco_item = reloaded_store_items["pocoItem"] reloaded_update_poco_store_item = reloaded_store_items["pocoStoreItem"] @@ -185,7 +202,9 @@ async def perform_batch_operations(storage) -> bool: } ) - result = await storage.read(["batch1", "batch2", "batch3"], target_cls=MockStoreItem) + result = await storage.read( + ["batch1", "batch2", "batch3"], target_cls=MockStoreItem + ) assert result.get("batch1", None) is not None assert result.get("batch2", None) is not None @@ -196,10 +215,12 @@ async def perform_batch_operations(storage) -> bool: await storage.delete(["batch1", "batch2", "batch3"]) - result = await storage.read(["batch1", "batch2", "batch3"], target_cls=MockStoreItem) + result = await storage.read( + ["batch1", "batch2", "batch3"], target_cls=MockStoreItem + ) assert result.get("batch1", None) is None assert result.get("batch2", None) is None assert result.get("batch3", None) is None - return True \ No newline at end of file + return True From 022440a2018e10a8113ca45a2a0b077972cf8f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodrigo=20Brand=C3=A3o?= Date: Thu, 26 Jun 2025 15:46:39 -0700 Subject: [PATCH 6/6] Implemented temporary fix to ensure Blob storage unit tests run --- .../microsoft-agents-blob/tests/__init__.py | 0 .../tests/storage_base_test.py | 226 ------ ...b_storage_test.py => test_blob_storage.py} | 747 ++++++++++++------ 3 files changed, 488 insertions(+), 485 deletions(-) delete mode 100644 libraries/Storage/microsoft-agents-blob/tests/__init__.py delete mode 100644 libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py rename libraries/Storage/microsoft-agents-blob/tests/{blob_storage_test.py => test_blob_storage.py} (58%) diff --git a/libraries/Storage/microsoft-agents-blob/tests/__init__.py b/libraries/Storage/microsoft-agents-blob/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py b/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py deleted file mode 100644 index 4ea2227c..00000000 --- a/libraries/Storage/microsoft-agents-blob/tests/storage_base_test.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -""" -Adapted from https://github.com/microsoft/botbuilder-python/blob/main/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py - -Base tests that all storage providers should implement in their own tests. -They handle the storage-based assertions, internally. - -All tests return true if assertions pass to indicate that the code ran to completion, passing internal assertions. -Therefore, all tests using theses static tests should strictly check that the method returns true. - -:Example: - async def test_handle_null_keys_when_reading(self): - await reset() - - test_ran = await StorageBaseTests.handle_null_keys_when_reading(get_storage()) - - assert test_ran -""" - -import pytest - -from microsoft.agents.storage import MemoryStorage, StoreItem -from microsoft.agents.storage._type_aliases import JSON - - -class MockStoreItem(StoreItem): - - def __init__(self, data: JSON = None): - self.data = data or {} - - def store_item_to_json(self) -> JSON: - return self.data - - @staticmethod - def from_json_to_store_item(json_data: JSON) -> "MockStoreItem": - return MockStoreItem(json_data) - - -class StorageBaseTests: - - # pylint: disable=pointless-string-statement - @staticmethod - async def return_empty_object_when_reading_unknown_key(storage) -> bool: - result = await storage.read(["unknown"], target_cls=MockStoreItem) - - assert result is not None - assert len(result) == 0 - - return True - - @staticmethod - async def handle_null_keys_when_reading(storage) -> bool: - if isinstance(storage, (MemoryStorage)): - result = await storage.read(None, target_cls=MockStoreItem) - assert len(result.keys()) == 0 - # Catch-all - else: - with pytest.raises(Exception) as err: - await storage.read(None, target_cls=MockStoreItem) - - return True - - @staticmethod - async def handle_null_keys_when_writing(storage) -> bool: - with pytest.raises(Exception) as err: - await storage.write(None) - # assert err.value.args[0] == "Changes are required when writing" - - return True - - @staticmethod - async def does_raise_when_writing_no_items(storage) -> bool: - # noinspection PyBroadException - with pytest.raises(Exception) as err: - await storage.write(dict()) - return True - - @staticmethod - async def create_object(storage) -> bool: - store_items = { - "createPoco": MockStoreItem({"id": 1}), - "createPocoStoreItem": MockStoreItem({"id": 2, "value": "*"}), - } - - await storage.write(store_items) - - read_store_items = await storage.read( - store_items.keys(), target_cls=MockStoreItem - ) - - assert ( - store_items["createPoco"].data["id"] - == read_store_items["createPoco"].data["id"] - ) - assert ( - store_items["createPocoStoreItem"].data["id"] - == read_store_items["createPocoStoreItem"].data["id"] - ) - assert read_store_items["createPocoStoreItem"].data["value"] == "*" - - return True - - @staticmethod - async def handle_crazy_keys(storage) -> bool: - key = '!@#$%^&*()_+??><":QASD~`' - store_item = MockStoreItem({"id": 1}) - store_items = {key: store_item} - - await storage.write(store_items) - - read_store_items = await storage.read( - store_items.keys(), target_cls=MockStoreItem - ) - - assert read_store_items[key] is not None - assert read_store_items[key].data["id"] == 1 - - return True - - @staticmethod - async def update_object(storage) -> bool: - original_store_items = { - "pocoItem": MockStoreItem({"id": 1, "count": 1}), - "pocoStoreItem": MockStoreItem({"id": 1, "count": 1, "value": "*"}), - } - - # 1st write should work - await storage.write(original_store_items) - - loaded_store_items = await storage.read( - ["pocoItem", "pocoStoreItem"], target_cls=MockStoreItem - ) - - update_poco_item = loaded_store_items["pocoItem"] - update_poco_item.data["value"] = None - update_poco_store_item = loaded_store_items["pocoStoreItem"] - assert update_poco_store_item.data["value"] == "*" - - # 2nd write should work - update_poco_item.data["count"] += 1 - update_poco_store_item.data["count"] += 1 - - await storage.write( - { - key: MockStoreItem(value.data) - for key, value in loaded_store_items.items() - } - ) - - reloaded_store_items = await storage.read( - loaded_store_items.keys(), target_cls=MockStoreItem - ) - - reloaded_update_poco_item = reloaded_store_items["pocoItem"] - reloaded_update_poco_store_item = reloaded_store_items["pocoStoreItem"] - - assert reloaded_update_poco_item.data["count"] == 2 - assert reloaded_update_poco_store_item.data["count"] == 2 - assert reloaded_update_poco_item.data["value"] is None - assert reloaded_update_poco_store_item.data["value"] == "*" - - return True - - @staticmethod - async def delete_object(storage) -> bool: - store_items = {"delete1": MockStoreItem({"id": 1, "count": 1, "value": "*"})} - - await storage.write(store_items) - - read_store_items = await storage.read(["delete1"], target_cls=MockStoreItem) - - assert read_store_items["delete1"].data["value"] - assert read_store_items["delete1"].data["count"] == 1 - - await storage.delete(["delete1"]) - - reloaded_store_items = await storage.read(["delete1"], target_cls=MockStoreItem) - - assert reloaded_store_items.get("delete1", None) is None - - return True - - @staticmethod - async def delete_unknown_object(storage) -> bool: - # noinspection PyBroadException - try: - await storage.delete(["unknown_key"]) - except: - pytest.fail("Should not raise") - - return True - - @staticmethod - async def perform_batch_operations(storage) -> bool: - await storage.write( - { - "batch1": MockStoreItem({"count": 10}), - "batch2": MockStoreItem({"count": 20}), - "batch3": MockStoreItem({"count": 30}), - } - ) - - result = await storage.read( - ["batch1", "batch2", "batch3"], target_cls=MockStoreItem - ) - - assert result.get("batch1", None) is not None - assert result.get("batch2", None) is not None - assert result.get("batch3", None) is not None - assert result["batch1"].data["count"] == 10 - assert result["batch2"].data["count"] == 20 - assert result["batch3"].data["count"] == 30 - - await storage.delete(["batch1", "batch2", "batch3"]) - - result = await storage.read( - ["batch1", "batch2", "batch3"], target_cls=MockStoreItem - ) - - assert result.get("batch1", None) is None - assert result.get("batch2", None) is None - assert result.get("batch3", None) is None - - return True diff --git a/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py b/libraries/Storage/microsoft-agents-blob/tests/test_blob_storage.py similarity index 58% rename from libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py rename to libraries/Storage/microsoft-agents-blob/tests/test_blob_storage.py index 03074c0b..e68a31a9 100644 --- a/libraries/Storage/microsoft-agents-blob/tests/blob_storage_test.py +++ b/libraries/Storage/microsoft-agents-blob/tests/test_blob_storage.py @@ -1,259 +1,488 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# based on https://github.com/microsoft/botbuilder-python/blob/main/libraries/botbuilder-azure/tests/test_blob_storage.py - -import pytest -import pytest_asyncio - -from azure.core.exceptions import ResourceNotFoundError -from azure.storage.blob.aio import BlobServiceClient - -from microsoft.agents.storage import StoreItem -from microsoft.agents.storage._type_aliases import JSON -from microsoft.agents.blob import BlobStorage, BlobStorageSettings - -from .storage_base_test import StorageBaseTests - -EMULATOR_RUNNING = False - - -# constructs an emulated blob storage instance -@pytest_asyncio.fixture -async def blob_storage(): - - # setup - - # Default Azure Storage Emulator connection string - connection_string = ( - "AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq" - + "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=" - + "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" - + "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" - ) - - blob_service_client = BlobServiceClient.from_connection_string(connection_string) - - container_name = "test" - container_client = blob_service_client.get_container_client(container_name) - - # reset state of test container - try: - await container_client.delete_container() - except ResourceNotFoundError: - pass - await container_client.create_container() - - blob_storage_settings = BlobStorageSettings( - account_name="", - account_key="", - container_name=container_name, - connection_string=connection_string, - ) - - storage = BlobStorage(blob_storage_settings) - - yield storage - - # teardown - await container_client.delete_container() - - -class SimpleStoreItem(StoreItem): - - def __init__(self, counter: int = 1, value: str = "*"): - self.counter = counter - self.value = value - - def store_item_to_json(self) -> JSON: - return { - "counter": self.counter, - "value": self.value, - } - - @staticmethod - def from_json_to_store_item(json_data: JSON) -> "StoreItem": - return SimpleStoreItem(json_data["counter"], json_data["value"]) - - -class TestBlobStorageConstructor: - - @pytest.mark.asyncio - async def test_blob_storage_init_should_error_without_container_name(self): - settings = BlobStorageSettings("") - with pytest.raises(Exception) as err: - BlobStorage(settings) - - assert err.value.args[0] == "BlobStorage: Container name is required." - - @pytest.mark.asyncio - async def test_blob_storage_init_should_error_without_blob_config(self): - try: - BlobStorage(BlobStorageSettings()) # pylint: disable=no-value-for-parameter - except Exception as error: - assert error - - @pytest.mark.asyncio - async def test_blob_storage_init_should_error_with_insufficient_settings(self): - settings_0 = BlobStorageSettings("norway", account_name="some_account_name") - settings_1 = BlobStorageSettings("sweden", account_key="some_account_key") - with pytest.raises(Exception) as err: - BlobStorage(settings_0) - with pytest.raises(Exception) as err: - BlobStorage(settings_1) - - @pytest.mark.asyncio - async def test_blob_storage_init_from_account_key_and_name(self): - settings = BlobStorageSettings( - "norway", account_name="some_account_name", account_key="some_account_key" - ) - BlobStorage(settings) - - -class TestBlobStorageBaseTests: - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_return_empty_object_when_reading_unknown_key(self, blob_storage): - test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key( - blob_storage - ) - assert test_ran - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_handle_null_keys_when_reading(self, blob_storage): - test_ran = await StorageBaseTests.handle_null_keys_when_reading(blob_storage) - assert test_ran - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_handle_null_keys_when_writing(self, blob_storage): - test_ran = await StorageBaseTests.handle_null_keys_when_writing(blob_storage) - assert test_ran - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def does_raise_when_writing_no_items(self, blob_storage): - test_ran = await StorageBaseTests.does_raise_when_writing_no_items(blob_storage) - assert test_ran - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_create_object(self, blob_storage): - test_ran = await StorageBaseTests.create_object(blob_storage) - assert test_ran - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_handle_crazy_keys(self, blob_storage): - test_ran = await StorageBaseTests.handle_crazy_keys(blob_storage) - assert test_ran - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_update_object(self, blob_storage): - test_ran = await StorageBaseTests.update_object(blob_storage) - assert test_ran - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_delete_object(self, blob_storage): - test_ran = await StorageBaseTests.delete_object(blob_storage) - assert test_ran - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_perform_batch_operations(self, blob_storage): - test_ran = await StorageBaseTests.perform_batch_operations(blob_storage) - assert test_ran - - -class TestBlobStorage: - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_blob_storage_read_update_same_data(self, blob_storage): - await blob_storage.write({"test": SimpleStoreItem(counter=1)}) - data_result = await blob_storage.read(["test"], target_cls=SimpleStoreItem) - data_result["test"].counter = 2 - await blob_storage.write(data_result) - data_updated = await blob_storage.read(["test"], target_cls=SimpleStoreItem) - assert data_updated["test"].counter == 2 - assert data_updated["test"].value == data_result["test"].value - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_blob_storage_write_should_overwrite(self, blob_storage): - await blob_storage.write({"user": SimpleStoreItem()}) - await blob_storage.write({"user": SimpleStoreItem(counter=10, value="*")}) - data = await blob_storage.read(["user"], target_cls=SimpleStoreItem) - assert data["user"].counter == 10 - assert data["user"].value == "*" - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_blob_storage_delete_should_delete_according_cached_data( - self, blob_storage - ): - await blob_storage.write({"test": SimpleStoreItem()}) - try: - await blob_storage.delete(["test"]) - except Exception as error: - raise error - else: - data = await blob_storage.read(["test"], target_cls=SimpleStoreItem) - - assert isinstance(data, dict) - assert not data.keys() - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_blob_storage_delete_should_delete_multiple_values_when_given_multiple_valid_keys( - self, blob_storage - ): - await blob_storage.write( - {"test": SimpleStoreItem(), "test2": SimpleStoreItem(2)} - ) - await blob_storage.delete(["test", "test2"]) - data = await blob_storage.read(["test", "test2"], target_cls=SimpleStoreItem) - assert not data.keys() - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_blob_storage_delete_should_delete_values_when_given_multiple_valid_keys_and_ignore_other_data( - self, blob_storage - ): - await blob_storage.write( - { - "test": SimpleStoreItem(), - "test2": SimpleStoreItem(counter=2), - "test3": SimpleStoreItem(counter=3), - } - ) - await blob_storage.delete(["test", "test2"]) - data = await blob_storage.read( - ["test", "test2", "test3"], target_cls=SimpleStoreItem - ) - assert len(data.keys()) == 1 - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_blob_storage_delete_invalid_key_should_do_nothing_and_not_affect_cached_data( - self, blob_storage - ): - await blob_storage.write({"test": SimpleStoreItem()}) - await blob_storage.delete(["foo"]) - data = await blob_storage.read(["test"], target_cls=SimpleStoreItem) - assert len(data.keys()) == 1 - data = await blob_storage.read(["foo"], target_cls=SimpleStoreItem) - assert not data.keys() - - @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") - @pytest.mark.asyncio - async def test_blob_storage_delete_invalid_keys_should_do_nothing_and_not_affect_cached_data( - self, blob_storage - ): - await blob_storage.write({"test": SimpleStoreItem()}) - await blob_storage.delete(["foo", "bar"]) - data = await blob_storage.read(["test"], target_cls=SimpleStoreItem) - assert len(data.keys()) == 1 +# temporary fix for pytest import issue. There are two separate scripts here + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +""" +Adapted from https://github.com/microsoft/botbuilder-python/blob/main/libraries/botbuilder-testing/botbuilder/testing/storage_base_tests.py + +Base tests that all storage providers should implement in their own tests. +They handle the storage-based assertions, internally. + +All tests return true if assertions pass to indicate that the code ran to completion, passing internal assertions. +Therefore, all tests using theses static tests should strictly check that the method returns true. + +:Example: + async def test_handle_null_keys_when_reading(self): + await reset() + + test_ran = await StorageBaseTests.handle_null_keys_when_reading(get_storage()) + + assert test_ran +""" + +import pytest + +from microsoft.agents.storage import MemoryStorage, StoreItem +from microsoft.agents.storage._type_aliases import JSON + + +class MockStoreItem(StoreItem): + + def __init__(self, data: JSON = None): + self.data = data or {} + + def store_item_to_json(self) -> JSON: + return self.data + + @staticmethod + def from_json_to_store_item(json_data: JSON) -> "MockStoreItem": + return MockStoreItem(json_data) + + +class StorageBaseTests: + + # pylint: disable=pointless-string-statement + @staticmethod + async def return_empty_object_when_reading_unknown_key(storage) -> bool: + result = await storage.read(["unknown"], target_cls=MockStoreItem) + + assert result is not None + assert len(result) == 0 + + return True + + @staticmethod + async def handle_null_keys_when_reading(storage) -> bool: + if isinstance(storage, (MemoryStorage)): + result = await storage.read(None, target_cls=MockStoreItem) + assert len(result.keys()) == 0 + # Catch-all + else: + with pytest.raises(Exception) as err: + await storage.read(None, target_cls=MockStoreItem) + + return True + + @staticmethod + async def handle_null_keys_when_writing(storage) -> bool: + with pytest.raises(Exception) as err: + await storage.write(None) + # assert err.value.args[0] == "Changes are required when writing" + + return True + + @staticmethod + async def does_raise_when_writing_no_items(storage) -> bool: + # noinspection PyBroadException + with pytest.raises(Exception) as err: + await storage.write(dict()) + return True + + @staticmethod + async def create_object(storage) -> bool: + store_items = { + "createPoco": MockStoreItem({"id": 1}), + "createPocoStoreItem": MockStoreItem({"id": 2, "value": "*"}), + } + + await storage.write(store_items) + + read_store_items = await storage.read( + store_items.keys(), target_cls=MockStoreItem + ) + + assert ( + store_items["createPoco"].data["id"] + == read_store_items["createPoco"].data["id"] + ) + assert ( + store_items["createPocoStoreItem"].data["id"] + == read_store_items["createPocoStoreItem"].data["id"] + ) + assert read_store_items["createPocoStoreItem"].data["value"] == "*" + + return True + + @staticmethod + async def handle_crazy_keys(storage) -> bool: + key = '!@#$%^&*()_+??><":QASD~`' + store_item = MockStoreItem({"id": 1}) + store_items = {key: store_item} + + await storage.write(store_items) + + read_store_items = await storage.read( + store_items.keys(), target_cls=MockStoreItem + ) + + assert read_store_items[key] is not None + assert read_store_items[key].data["id"] == 1 + + return True + + @staticmethod + async def update_object(storage) -> bool: + original_store_items = { + "pocoItem": MockStoreItem({"id": 1, "count": 1}), + "pocoStoreItem": MockStoreItem({"id": 1, "count": 1, "value": "*"}), + } + + # 1st write should work + await storage.write(original_store_items) + + loaded_store_items = await storage.read( + ["pocoItem", "pocoStoreItem"], target_cls=MockStoreItem + ) + + update_poco_item = loaded_store_items["pocoItem"] + update_poco_item.data["value"] = None + update_poco_store_item = loaded_store_items["pocoStoreItem"] + assert update_poco_store_item.data["value"] == "*" + + # 2nd write should work + update_poco_item.data["count"] += 1 + update_poco_store_item.data["count"] += 1 + + await storage.write( + { + key: MockStoreItem(value.data) + for key, value in loaded_store_items.items() + } + ) + + reloaded_store_items = await storage.read( + loaded_store_items.keys(), target_cls=MockStoreItem + ) + + reloaded_update_poco_item = reloaded_store_items["pocoItem"] + reloaded_update_poco_store_item = reloaded_store_items["pocoStoreItem"] + + assert reloaded_update_poco_item.data["count"] == 2 + assert reloaded_update_poco_store_item.data["count"] == 2 + assert reloaded_update_poco_item.data["value"] is None + assert reloaded_update_poco_store_item.data["value"] == "*" + + return True + + @staticmethod + async def delete_object(storage) -> bool: + store_items = {"delete1": MockStoreItem({"id": 1, "count": 1, "value": "*"})} + + await storage.write(store_items) + + read_store_items = await storage.read(["delete1"], target_cls=MockStoreItem) + + assert read_store_items["delete1"].data["value"] + assert read_store_items["delete1"].data["count"] == 1 + + await storage.delete(["delete1"]) + + reloaded_store_items = await storage.read(["delete1"], target_cls=MockStoreItem) + + assert reloaded_store_items.get("delete1", None) is None + + return True + + @staticmethod + async def delete_unknown_object(storage) -> bool: + # noinspection PyBroadException + try: + await storage.delete(["unknown_key"]) + except: + pytest.fail("Should not raise") + + return True + + @staticmethod + async def perform_batch_operations(storage) -> bool: + await storage.write( + { + "batch1": MockStoreItem({"count": 10}), + "batch2": MockStoreItem({"count": 20}), + "batch3": MockStoreItem({"count": 30}), + } + ) + + result = await storage.read( + ["batch1", "batch2", "batch3"], target_cls=MockStoreItem + ) + + assert result.get("batch1", None) is not None + assert result.get("batch2", None) is not None + assert result.get("batch3", None) is not None + assert result["batch1"].data["count"] == 10 + assert result["batch2"].data["count"] == 20 + assert result["batch3"].data["count"] == 30 + + await storage.delete(["batch1", "batch2", "batch3"]) + + result = await storage.read( + ["batch1", "batch2", "batch3"], target_cls=MockStoreItem + ) + + assert result.get("batch1", None) is None + assert result.get("batch2", None) is None + assert result.get("batch3", None) is None + + return True + + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# based on https://github.com/microsoft/botbuilder-python/blob/main/libraries/botbuilder-azure/tests/test_blob_storage.py + +import pytest +import pytest_asyncio + +from azure.core.exceptions import ResourceNotFoundError +from azure.storage.blob.aio import BlobServiceClient + +from microsoft.agents.storage import StoreItem +from microsoft.agents.storage._type_aliases import JSON +from microsoft.agents.blob import BlobStorage, BlobStorageSettings + + +EMULATOR_RUNNING = False + + +# constructs an emulated blob storage instance +@pytest_asyncio.fixture +async def blob_storage(): + + # setup + + # Default Azure Storage Emulator connection string + connection_string = ( + "AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq" + + "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=" + + "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" + + "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + ) + + blob_service_client = BlobServiceClient.from_connection_string(connection_string) + + container_name = "test" + container_client = blob_service_client.get_container_client(container_name) + + # reset state of test container + try: + await container_client.delete_container() + except ResourceNotFoundError: + pass + await container_client.create_container() + + blob_storage_settings = BlobStorageSettings( + account_name="", + account_key="", + container_name=container_name, + connection_string=connection_string, + ) + + storage = BlobStorage(blob_storage_settings) + + yield storage + + # teardown + await container_client.delete_container() + + +class SimpleStoreItem(StoreItem): + + def __init__(self, counter: int = 1, value: str = "*"): + self.counter = counter + self.value = value + + def store_item_to_json(self) -> JSON: + return { + "counter": self.counter, + "value": self.value, + } + + @staticmethod + def from_json_to_store_item(json_data: JSON) -> "StoreItem": + return SimpleStoreItem(json_data["counter"], json_data["value"]) + + +class TestBlobStorageConstructor: + + @pytest.mark.asyncio + async def test_blob_storage_init_should_error_without_container_name(self): + settings = BlobStorageSettings("") + with pytest.raises(Exception) as err: + BlobStorage(settings) + + assert err.value.args[0] == "BlobStorage: Container name is required." + + @pytest.mark.asyncio + async def test_blob_storage_init_should_error_without_blob_config(self): + try: + BlobStorage(BlobStorageSettings()) # pylint: disable=no-value-for-parameter + except Exception as error: + assert error + + @pytest.mark.asyncio + async def test_blob_storage_init_should_error_with_insufficient_settings(self): + settings_0 = BlobStorageSettings("norway", account_name="some_account_name") + settings_1 = BlobStorageSettings("sweden", account_key="some_account_key") + with pytest.raises(Exception) as err: + BlobStorage(settings_0) + with pytest.raises(Exception) as err: + BlobStorage(settings_1) + + @pytest.mark.asyncio + async def test_blob_storage_init_from_account_key_and_name(self): + settings = BlobStorageSettings( + "norway", account_name="some_account_name", account_key="some_account_key" + ) + BlobStorage(settings) + + +class TestBlobStorageBaseTests: + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_return_empty_object_when_reading_unknown_key(self, blob_storage): + test_ran = await StorageBaseTests.return_empty_object_when_reading_unknown_key( + blob_storage + ) + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_handle_null_keys_when_reading(self, blob_storage): + test_ran = await StorageBaseTests.handle_null_keys_when_reading(blob_storage) + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_handle_null_keys_when_writing(self, blob_storage): + test_ran = await StorageBaseTests.handle_null_keys_when_writing(blob_storage) + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def does_raise_when_writing_no_items(self, blob_storage): + test_ran = await StorageBaseTests.does_raise_when_writing_no_items(blob_storage) + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_create_object(self, blob_storage): + test_ran = await StorageBaseTests.create_object(blob_storage) + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_handle_crazy_keys(self, blob_storage): + test_ran = await StorageBaseTests.handle_crazy_keys(blob_storage) + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_update_object(self, blob_storage): + test_ran = await StorageBaseTests.update_object(blob_storage) + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_delete_object(self, blob_storage): + test_ran = await StorageBaseTests.delete_object(blob_storage) + assert test_ran + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_perform_batch_operations(self, blob_storage): + test_ran = await StorageBaseTests.perform_batch_operations(blob_storage) + assert test_ran + + +class TestBlobStorage: + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_read_update_same_data(self, blob_storage): + await blob_storage.write({"test": SimpleStoreItem(counter=1)}) + data_result = await blob_storage.read(["test"], target_cls=SimpleStoreItem) + data_result["test"].counter = 2 + await blob_storage.write(data_result) + data_updated = await blob_storage.read(["test"], target_cls=SimpleStoreItem) + assert data_updated["test"].counter == 2 + assert data_updated["test"].value == data_result["test"].value + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_write_should_overwrite(self, blob_storage): + await blob_storage.write({"user": SimpleStoreItem()}) + await blob_storage.write({"user": SimpleStoreItem(counter=10, value="*")}) + data = await blob_storage.read(["user"], target_cls=SimpleStoreItem) + assert data["user"].counter == 10 + assert data["user"].value == "*" + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_should_delete_according_cached_data( + self, blob_storage + ): + await blob_storage.write({"test": SimpleStoreItem()}) + try: + await blob_storage.delete(["test"]) + except Exception as error: + raise error + else: + data = await blob_storage.read(["test"], target_cls=SimpleStoreItem) + + assert isinstance(data, dict) + assert not data.keys() + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_should_delete_multiple_values_when_given_multiple_valid_keys( + self, blob_storage + ): + await blob_storage.write( + {"test": SimpleStoreItem(), "test2": SimpleStoreItem(2)} + ) + await blob_storage.delete(["test", "test2"]) + data = await blob_storage.read(["test", "test2"], target_cls=SimpleStoreItem) + assert not data.keys() + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_should_delete_values_when_given_multiple_valid_keys_and_ignore_other_data( + self, blob_storage + ): + await blob_storage.write( + { + "test": SimpleStoreItem(), + "test2": SimpleStoreItem(counter=2), + "test3": SimpleStoreItem(counter=3), + } + ) + await blob_storage.delete(["test", "test2"]) + data = await blob_storage.read( + ["test", "test2", "test3"], target_cls=SimpleStoreItem + ) + assert len(data.keys()) == 1 + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_invalid_key_should_do_nothing_and_not_affect_cached_data( + self, blob_storage + ): + await blob_storage.write({"test": SimpleStoreItem()}) + await blob_storage.delete(["foo"]) + data = await blob_storage.read(["test"], target_cls=SimpleStoreItem) + assert len(data.keys()) == 1 + data = await blob_storage.read(["foo"], target_cls=SimpleStoreItem) + assert not data.keys() + + @pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.") + @pytest.mark.asyncio + async def test_blob_storage_delete_invalid_keys_should_do_nothing_and_not_affect_cached_data( + self, blob_storage + ): + await blob_storage.write({"test": SimpleStoreItem()}) + await blob_storage.delete(["foo", "bar"]) + data = await blob_storage.read(["test"], target_cls=SimpleStoreItem) + assert len(data.keys()) == 1