diff --git a/.github/workflows/ci_checks.yml b/.github/workflows/ci_checks.yml index 4488a9a..f3cf642 100644 --- a/.github/workflows/ci_checks.yml +++ b/.github/workflows/ci_checks.yml @@ -73,4 +73,24 @@ jobs: uv sync --extra dev - name: Run unit tests run: | - uv run python -m pytest tests/ -v + uv run pytest -v + + integration_test: + name: Integration Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.13" + - name: Set up uv + uses: astral-sh/setup-uv@v1 + - name: Install dependencies + run: | + uv sync --extra dev + - name: Run unit tests + run: | + uv run pytest -v --integration-tests diff --git a/cadence/_internal/rpc/yarpc.py b/cadence/_internal/rpc/yarpc.py index 266df13..1c2cdbb 100644 --- a/cadence/_internal/rpc/yarpc.py +++ b/cadence/_internal/rpc/yarpc.py @@ -44,7 +44,8 @@ def _replace_details(self, client_call_details: ClientCallDetails) -> ClientCall return _ClientCallDetails( method=client_call_details.method, - timeout=client_call_details.timeout, + # YARPC seems to require a TTL value + timeout=client_call_details.timeout or 60.0, metadata=metadata, credentials=client_call_details.credentials, wait_for_ready=client_call_details.wait_for_ready, diff --git a/cadence/client.py b/cadence/client.py index 3f085dd..3a0c5e7 100644 --- a/cadence/client.py +++ b/cadence/client.py @@ -6,6 +6,7 @@ from cadence._internal.rpc.error import CadenceErrorInterceptor from cadence._internal.rpc.yarpc import YarpcMetadataInterceptor +from cadence.api.v1.service_domain_pb2_grpc import DomainAPIStub from cadence.api.v1.service_worker_pb2_grpc import WorkerAPIStub from grpc.aio import Channel, ClientInterceptor, secure_channel, insecure_channel from cadence.data_converter import DataConverter, DefaultDataConverter @@ -39,6 +40,7 @@ def __init__(self, **kwargs: Unpack[ClientOptions]) -> None: self._options = _validate_and_copy_defaults(ClientOptions(**kwargs)) self._channel = _create_channel(self._options) self._worker_stub = WorkerAPIStub(self._channel) + self._domain_stub = DomainAPIStub(self._channel) @property def data_converter(self) -> DataConverter: @@ -52,13 +54,26 @@ def domain(self) -> str: def identity(self) -> str: return self._options["identity"] + @property + def domain_stub(self) -> DomainAPIStub: + return self._domain_stub + @property def worker_stub(self) -> WorkerAPIStub: return self._worker_stub + async def ready(self) -> None: + await self._channel.channel_ready() + async def close(self) -> None: await self._channel.close() + async def __aenter__(self) -> 'Client': + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + await self.close() + def _validate_and_copy_defaults(options: ClientOptions) -> ClientOptions: if "target" not in options: raise ValueError("target must be specified") diff --git a/cadence/sample/client_example.py b/cadence/sample/client_example.py index ece4346..152f7a8 100644 --- a/cadence/sample/client_example.py +++ b/cadence/sample/client_example.py @@ -6,9 +6,9 @@ async def main(): - client = Client(target="localhost:7833", domain="foo") - worker = Worker(client, "task_list", Registry()) - await worker.run() + async with Client(target="localhost:7833", domain="foo") as client: + worker = Worker(client, "task_list", Registry()) + await worker.run() if __name__ == '__main__': asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 9b3a27e..b860554 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dev = [ "flake8>=6.0.0", "mypy>=1.0.0", "pre-commit>=3.0.0", + "pytest-docker>=3.2.3", ] docs = [ "sphinx>=6.0.0", @@ -138,14 +139,14 @@ ignore_missing_imports = true [tool.pytest.ini_options] minversion = "7.0" -addopts = "-ra -q --strict-markers --strict-config" +addopts = "-ra -q --strict-markers --strict-config --import-mode=importlib" +asyncio_mode = "auto" testpaths = ["tests"] python_files = ["test_*.py", "*_test.py"] python_classes = ["Test*"] python_functions = ["test_*"] markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", - "integration: marks tests as integration tests", "unit: marks tests as unit tests", "asyncio: marks tests as async tests", ] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..5899674 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,8 @@ + + +ENABLE_INTEGRATION_TESTS = "--integration-tests" + +# Need to define the option in the root conftest.py file +def pytest_addoption(parser): + parser.addoption(ENABLE_INTEGRATION_TESTS, action="store_true", + help="enables running integration tests, which rely on docker and docker-compose") \ No newline at end of file diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py new file mode 100644 index 0000000..0833e24 --- /dev/null +++ b/tests/integration_tests/conftest.py @@ -0,0 +1,47 @@ +import asyncio +import os +from datetime import timedelta + +import pytest + +from google.protobuf.duration import from_timedelta +from pytest_docker import Services + +from cadence.api.v1.service_domain_pb2 import RegisterDomainRequest +from cadence.client import ClientOptions +from tests.conftest import ENABLE_INTEGRATION_TESTS +from tests.integration_tests.helper import CadenceHelper, DOMAIN_NAME + +# Run tests in this directory and lower only if integration tests are enabled +def pytest_runtest_setup(item): + if not item.config.getoption(ENABLE_INTEGRATION_TESTS): + pytest.skip(f"{ENABLE_INTEGRATION_TESTS} not enabled") + +@pytest.fixture(scope="session") +def docker_compose_file(pytestconfig): + return os.path.join(str(pytestconfig.rootdir), "tests", "integration_tests", "docker-compose.yml") + +@pytest.fixture(scope="session") +def client_options(docker_ip: str, docker_services: Services) -> ClientOptions: + return ClientOptions( + domain=DOMAIN_NAME, + target=f'{docker_ip}:{docker_services.port_for("cadence", 7833)}', + ) + +# We can't pass around Client objects between tests/fixtures without changing our pytest-asyncio version +# to ensure that they use the same event loop. +# Instead, we can wait for the server to be ready, create the common domain, and then provide a helper capable +# of creating additional clients within each test as needed +@pytest.fixture(scope="session") +async def helper(client_options: ClientOptions) -> CadenceHelper: + helper = CadenceHelper(client_options) + async with helper.client() as client: + # It takes around a minute for the Cadence server to start up with Cassandra + async with asyncio.timeout(120): + await client.ready() + + await client.domain_stub.RegisterDomain(RegisterDomainRequest( + name=DOMAIN_NAME, + workflow_execution_retention_period=from_timedelta(timedelta(days=1)), + )) + return CadenceHelper(client_options) diff --git a/tests/integration_tests/docker-compose.yml b/tests/integration_tests/docker-compose.yml new file mode 100644 index 0000000..1130578 --- /dev/null +++ b/tests/integration_tests/docker-compose.yml @@ -0,0 +1,47 @@ +version: "3.5" + +services: + cassandra: + image: cassandra:4.1.3 + ports: + - "9042:9042" + networks: + services-network: + aliases: + - cassandra + + statsd: + image: hopsoft/graphite-statsd + ports: + - "8080:80" + - "2003:2003" + - "8125:8125" + - "8126:8126" + networks: + services-network: + aliases: + - statsd + + cadence: + image: ubercadence/server:master-auto-setup + ports: + - "7933:7933" + - "7833:7833" + - "7934:7934" + - "7935:7935" + - "7939:7939" + environment: + - "CASSANDRA_SEEDS=cassandra" + - "STATSD_ENDPOINT=statsd:8125" + - "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml" + depends_on: + - cassandra + - statsd + networks: + services-network: + aliases: + - cadence +networks: + services-network: + name: services-network + driver: bridge \ No newline at end of file diff --git a/tests/integration_tests/helper.py b/tests/integration_tests/helper.py new file mode 100644 index 0000000..06fb5a0 --- /dev/null +++ b/tests/integration_tests/helper.py @@ -0,0 +1,11 @@ +from cadence.client import ClientOptions, Client + +DOMAIN_NAME = "test-domain" + + +class CadenceHelper: + def __init__(self, options: ClientOptions): + self.options = options + + def client(self): + return Client(**self.options) \ No newline at end of file diff --git a/tests/integration_tests/test_client.py b/tests/integration_tests/test_client.py new file mode 100644 index 0000000..7d5c03d --- /dev/null +++ b/tests/integration_tests/test_client.py @@ -0,0 +1,18 @@ +import pytest + +from cadence.api.v1.service_domain_pb2 import DescribeDomainRequest, DescribeDomainResponse +from cadence.error import EntityNotExistsError +from tests.integration_tests.helper import CadenceHelper, DOMAIN_NAME + + +@pytest.mark.usefixtures("helper") +async def test_domain_exists(helper: CadenceHelper): + async with helper.client() as client: + response: DescribeDomainResponse = await client.domain_stub.DescribeDomain(DescribeDomainRequest(name=DOMAIN_NAME)) + assert response.domain.name == DOMAIN_NAME + +@pytest.mark.usefixtures("helper") +async def test_domain_not_exists(helper: CadenceHelper): + with pytest.raises(EntityNotExistsError): + async with helper.client() as client: + await client.domain_stub.DescribeDomain(DescribeDomainRequest(name="unknown-domain")) diff --git a/uv.lock b/uv.lock index 90d9b36..ca5750c 100644 --- a/uv.lock +++ b/uv.lock @@ -170,6 +170,7 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "pytest-docker" }, ] docs = [ { name = "myst-parser" }, @@ -198,6 +199,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4.1" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.21.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" }, + { name = "pytest-docker", marker = "extra == 'dev'", specifier = ">=3.2.3" }, { name = "requests", marker = "extra == 'examples'", specifier = ">=2.28.0" }, { name = "sphinx", marker = "extra == 'docs'", specifier = ">=6.0.0" }, { name = "sphinx-rtd-theme", marker = "extra == 'docs'", specifier = ">=1.2.0" }, @@ -1107,6 +1109,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/16/4ea354101abb1287856baa4af2732be351c7bee728065aed451b678153fd/pytest_cov-6.2.1-py3-none-any.whl", hash = "sha256:f5bc4c23f42f1cdd23c70b1dab1bbaef4fc505ba950d53e0081d0730dd7e86d5", size = 24644, upload-time = "2025-06-12T10:47:45.932Z" }, ] +[[package]] +name = "pytest-docker" +version = "3.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/75/285187953062ebe38108e77a7919c75e157943fa3513371c88e27d3df7b2/pytest_docker-3.2.3.tar.gz", hash = "sha256:26a1c711d99ef01e86e7c9c007f69641552c1554df4fccb065b35581cca24206", size = 13452, upload-time = "2025-07-04T07:46:17.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/c7/e057e0d1de611ce1bbb26cccf07ddf56eb30a6f6a03aa512a09dac356e03/pytest_docker-3.2.3-py3-none-any.whl", hash = "sha256:f973c35e6f2b674c8fc87e8b3354b02c15866a21994c0841a338c240a05de1eb", size = 8585, upload-time = "2025-07-04T07:46:16.439Z" }, +] + [[package]] name = "pyyaml" version = "6.0.2"