Add test path that hits a more traditional local test database instead of Docker (#3)

Here, add a testing path that hits a more traditional local test
database URL instead of requiring that the tests go through Docker, and
making the suite a little more like other River projects and able to
share a test database with them (`river_test` being a common name
convention between them all).

I've also changed the test code to rollback test transactions after each
test case so as to not leave any artifacts in the database that might
affect other test suites.

I've left the Docker path in as well, and it can be activated by sending
the `RIVER_USE_DOCKER` env var when running tests:

    RIVER_USE_DOCKER=true rye test

I also added code so that the tests get type checked as well as the main
code, which I found helped detect bugs, and being a Python beginner,
help better understand what exactly various fixture functions were
supposed to be returning. I didn't type everything, mostly just common
fixture functions. To make this possible I had to make a few changes
like having the main package declare itself as typed by exporting a
`py.typed` file, and also making `tests` a module with `__init__.py` so
MyPy could resolve common utilities in it like `SimpleArgs`.

Author: brandur

.github/workflows/ci.yaml

@@ -11,7 +11,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout
+        uses: actions/checkout@v4
 
       - name: Install the latest version of rye
         uses: eifinger/setup-rye@v3
@@ -32,4 +33,4 @@ jobs:
         run: rye build
 
       - name: Test
-        run: rye test
+        run: RIVER_USE_DOCKER=true rye test

Makefile

@@ -12,4 +12,4 @@ test:
 
 .PHONY: typecheck
 typecheck:
-	rye run mypy -p src.riverqueue
+	rye run mypy -p riverqueue -p tests

docs/development.md

@@ -22,10 +22,32 @@ $ rye sync
 
 ## Run tests
 
+Create a test database and migrate with River's CLI:
+
+```shell
+$ go install github.com/riverqueue/river/cmd/river
+$ createdb river_test
+$ river migrate-up --database-url "postgres://localhost/river_test"
+```
+
+Run all tests:
+
 ```shell
 $ rye test
 ```
 
+_Or_, using a Docker test Postgres container instead of a test database:
+
+```shell
+RIVER_USE_DOCKER=true rye test
+```
+
+Run a specific test (or without `-k` option for all tests in a single file):
+
+```shell
+rye test -- tests/driver/sqlalchemy/sqlalchemy_driver_test.py -k test_insert_with_unique_opts_by_queue
+```
+
 ## Run lint
 
 ```shell
@@ -44,4 +66,4 @@ $ make typecheck
 $ rye fmt
 ```
 
-Rye uses [Ruff](https://github.com/astral-sh/ruff) under the hood for code formatting.
+Rye uses [Ruff](https://github.com/astral-sh/ruff) under the hood for code formatting.

mypy.ini

@@ -1,4 +1,8 @@
 [mypy]
 
-[mypy-riverqueue.sqlcdb]
-ignore_errors = True
+# testcontainers is typed, but it doesn't correctly declare itself as such.
+# Hopefully it can be fixed one day:
+#
+# https://github.com/testcontainers/testcontainers-python/issues/305
+[mypy-testcontainers.*]
+ignore_missing_imports = True

pyproject.toml

@@ -28,4 +28,4 @@ dev-dependencies = [
 allow-direct-references = true
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/riverqueue"]
+packages = ["src/riverqueue"]

src/riverqueue/py.typed

@@ -1,5 +1,3 @@
-import json
-from dataclasses import dataclass
 from datetime import datetime, timezone
 from unittest.mock import MagicMock, patch
 
@@ -8,6 +6,8 @@
 from riverqueue.client import Client
 from riverqueue.models import InsertOpts, UniqueOpts
 
+from tests.simple_args import SimpleArgs
+
 
 @pytest.fixture
 def mock_driver():
@@ -19,15 +19,6 @@ def client(mock_driver):
     return Client(mock_driver)
 
 
-@dataclass
-class SimpleArgs:
-    kind: str = "simple"
-
-    @staticmethod
-    def to_json() -> str:
-        return json.dumps({"job_num": 1})
-
-
 @patch("datetime.datetime")
 def test_insert_with_proto_args(mock_datetime, client, mock_driver):
     mock_datetime.now.return_value = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc)

tests/__init__.py

@@ -1,13 +1,30 @@
 import os
+from typing import Iterator
 
 import pytest
 from testcontainers.postgres import PostgresContainer
 import sqlalchemy
-from sqlalchemy import text
+from sqlalchemy import Engine, text
 
 
 @pytest.fixture(scope="session")
-def engine():
+def engine() -> Iterator[Engine]:
+    if os.getenv("RIVER_USE_DOCKER"):
+        yield from engine_with_docker()
+    else:
+        yield from engine_with_database_url()
+
+
+def engine_with_database_url() -> Iterator[Engine]:
+    database_url = os.getenv("TEST_DATABASE_URL", "postgres://localhost/river_test")
+
+    # sqlalchemy removed support for postgres:// for reasons beyond comprehension
+    database_url = database_url.replace("postgres://", "postgresql://")
+
+    yield sqlalchemy.create_engine(database_url)
+
+
+def engine_with_docker() -> Iterator[Engine]:
     with PostgresContainer("postgres:16") as postgres:
         engine = sqlalchemy.create_engine(postgres.get_connection_url())
         with engine.connect() as conn:

tests/client_test.py

@@ -1,24 +1,27 @@
 from datetime import datetime, timezone
+from typing import Iterator
 from unittest.mock import patch
 
 import pytest
 
 from riverqueue.client import Client
 from riverqueue.models import InsertOpts, UniqueOpts
 from riverqueue.drivers.sqlalchemy.sqlalchemy_driver import SqlAlchemyDriver
-from client_test import SimpleArgs
-from sqlalchemy import text
+from sqlalchemy import Engine, text
+
+from tests.simple_args import SimpleArgs
 
 
 @pytest.fixture
-def driver(engine):
+def driver(engine: Engine) -> Iterator[SqlAlchemyDriver]:
     with engine.begin() as conn:
         conn.execute(text("SET search_path TO public"))
         yield SqlAlchemyDriver(conn)
+        conn.rollback()
 
 
 @pytest.fixture
-def client(driver):
+def client(driver: SqlAlchemyDriver) -> Client:
     return Client(driver)