From f3bba73f69bb80f48342b2d81b57e54f99eee721 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 00:24:20 +0000 Subject: [PATCH 01/16] feat: Add Task processor - Port `flagsmith/flagsmith-task-processor` - Port `waitfordb` management command - Introduce PostgreSQL for tests - Bump Poetry from 2.0.1 to 2.1.1 --- poetry.lock | 81 +++---------------- pyproject.toml | 2 +- .../migrations/helpers/__init__.py | 9 +++ .../migrations/helpers/postgres_helpers.py | 41 ++++++++++ 4 files changed, 63 insertions(+), 70 deletions(-) create mode 100644 src/task_processor/migrations/helpers/__init__.py create mode 100644 src/task_processor/migrations/helpers/postgres_helpers.py diff --git a/poetry.lock b/poetry.lock index 9bcaa48..f157d0f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -775,80 +775,23 @@ files = [ twisted = ["twisted"] [[package]] -name = "psycopg2-binary" +name = "psycopg2" version = "2.9.10" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:3e9c76f0ac6f92ecfc79516a8034a544926430f7b080ec5a0537bca389ee0906"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ad26b467a405c798aaa1458ba09d7e2b6e5f96b1ce0ac15d82fd9f95dc38a92"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:270934a475a0e4b6925b5f804e3809dd5f90f8613621d062848dd82f9cd62007"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48b338f08d93e7be4ab2b5f1dbe69dc5e9ef07170fe1f86514422076d9c010d0"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4152f8f76d2023aac16285576a9ecd2b11a9895373a1f10fd9db54b3ff06b4"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:32581b3020c72d7a421009ee1c6bf4a131ef5f0a968fab2e2de0c9d2bb4577f1"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2ce3e21dc3437b1d960521eca599d57408a695a0d3c26797ea0f72e834c7ffe5"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e984839e75e0b60cfe75e351db53d6db750b00de45644c5d1f7ee5d1f34a1ce5"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c4745a90b78e51d9ba06e2088a2fe0c693ae19cc8cb051ccda44e8df8a6eb53"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-win32.whl", hash = "sha256:e5720a5d25e3b99cd0dc5c8a440570469ff82659bb09431c1439b92caf184d3b"}, - {file = "psycopg2_binary-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:3c18f74eb4386bf35e92ab2354a12c17e5eb4d9798e4c0ad3a00783eae7cd9f1"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-win32.whl", hash = "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392"}, - {file = "psycopg2_binary-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-win32.whl", hash = "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64"}, - {file = "psycopg2_binary-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0"}, - {file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d"}, - {file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb"}, - {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7"}, - {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d"}, - {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73"}, - {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673"}, - {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f"}, - {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909"}, - {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1"}, - {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567"}, - {file = "psycopg2_binary-2.9.10-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4"}, - {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8"}, - {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864"}, - {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:056470c3dc57904bbf63d6f534988bafc4e970ffd50f6271fc4ee7daad9498a5"}, - {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aa0e31fa4bb82578f3a6c74a73c273367727de397a7a0f07bd83cbea696baa"}, - {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8de718c0e1c4b982a54b41779667242bc630b2197948405b7bd8ce16bcecac92"}, - {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5c370b1e4975df846b0277b4deba86419ca77dbc25047f535b0bb03d1a544d44"}, - {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ffe8ed017e4ed70f68b7b371d84b7d4a790368db9203dfc2d222febd3a9c8863"}, - {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:8aecc5e80c63f7459a1a2ab2c64df952051df196294d9f739933a9f6687e86b3"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:7a813c8bdbaaaab1f078014b9b0b13f5de757e2b5d9be6403639b298a04d218b"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d00924255d7fc916ef66e4bf22f354a940c67179ad3fd7067d7a0a9c84d2fbfc"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7559bce4b505762d737172556a4e6ea8a9998ecac1e39b5233465093e8cee697"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8b58f0a96e7a1e341fc894f62c1177a7c83febebb5ff9123b579418fdc8a481"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b269105e59ac96aba877c1707c600ae55711d9dcd3fc4b5012e4af68e30c648"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:79625966e176dc97ddabc142351e0409e28acf4660b88d1cf6adb876d20c490d"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8aabf1c1a04584c168984ac678a668094d831f152859d06e055288fa515e4d30"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:19721ac03892001ee8fdd11507e6a2e01f4e37014def96379411ca99d78aeb2c"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7f5d859928e635fa3ce3477704acee0f667b3a3d3e4bb109f2b18d4005f38287"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-win32.whl", hash = "sha256:3216ccf953b3f267691c90c6fe742e45d890d8272326b4a8b20850a03d05b7b8"}, - {file = "psycopg2_binary-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:30e34c4e97964805f715206c7b789d54a78b70f3ff19fbe590104b71c45600e5"}, + {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, + {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, + {file = "psycopg2-2.9.10-cp311-cp311-win32.whl", hash = "sha256:47c4f9875125344f4c2b870e41b6aad585901318068acd01de93f3677a6522c2"}, + {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, + {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, + {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, + {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, + {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, + {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, + {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, ] [[package]] @@ -1571,4 +1514,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.1" python-versions = ">=3.11,<4.0" -content-hash = "cbbf795f761c8166b098abbd6d3ef7ace6c1315fac0598a60253e9358b49332a" +content-hash = "3005f742451d90ecf950ebfbae482b06ea44a5ea062823722017d07ed4ae7959" diff --git a/pyproject.toml b/pyproject.toml index 55c5512..031215a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,8 @@ dependencies = [ "flagsmith-flag-engine", "gunicorn (>=19.1)", "prometheus-client (>=0.0.16)", + "psycopg2 (>=2,<3)", "simplejson (>=3,<4)", - "psycopg2-binary (>=2.9,<3)", ] authors = [ { name = "Matthew Elwell" }, diff --git a/src/task_processor/migrations/helpers/__init__.py b/src/task_processor/migrations/helpers/__init__.py new file mode 100644 index 0000000..4fb432d --- /dev/null +++ b/src/task_processor/migrations/helpers/__init__.py @@ -0,0 +1,9 @@ +""" +Note: django doesn't support adding submodules to the migrations module directory +that don't include a Migration class. As such, I've defined this helpers submodule +and simplified the imports by defining the __all__ attribute. +""" + +from task_processor.migrations.helpers.postgres_helpers import PostgresOnlyRunSQL + +__all__ = ["PostgresOnlyRunSQL"] diff --git a/src/task_processor/migrations/helpers/postgres_helpers.py b/src/task_processor/migrations/helpers/postgres_helpers.py new file mode 100644 index 0000000..f3de3aa --- /dev/null +++ b/src/task_processor/migrations/helpers/postgres_helpers.py @@ -0,0 +1,41 @@ +from contextlib import suppress + +from django.db import migrations +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.state import ProjectState + + +class PostgresOnlyRunSQL(migrations.RunSQL): + @classmethod + def from_sql_file( + cls, + file_path: str, + reverse_sql: str = "", + ) -> "PostgresOnlyRunSQL": + with open(file_path) as forward_sql: + with suppress(FileNotFoundError): + with open(reverse_sql) as reverse_sql_file: + reverse_sql = reverse_sql_file.read() + return cls(forward_sql.read(), reverse_sql=reverse_sql) + + def database_forwards( + self, + app_label: str, + schema_editor: BaseDatabaseSchemaEditor, + from_state: ProjectState, + to_state: ProjectState, + ) -> None: + if schema_editor.connection.vendor != "postgresql": + return + super().database_forwards(app_label, schema_editor, from_state, to_state) + + def database_backwards( + self, + app_label: str, + schema_editor: BaseDatabaseSchemaEditor, + from_state: ProjectState, + to_state: ProjectState, + ) -> None: + if schema_editor.connection.vendor != "postgresql": + return + super().database_backwards(app_label, schema_editor, from_state, to_state) From c581a117ec6904cb5cbed23f40d73b61c80e19a6 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 13:05:21 +0000 Subject: [PATCH 02/16] make sure the entrypoint works --- src/common/core/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/common/core/utils.py b/src/common/core/utils.py index 4b785b4..2330e0a 100644 --- a/src/common/core/utils.py +++ b/src/common/core/utils.py @@ -1,4 +1,5 @@ import json +import logging import pathlib from functools import lru_cache from typing import NotRequired, TypedDict @@ -11,6 +12,8 @@ UNKNOWN = "unknown" VERSIONS_INFO_FILE_LOCATION = ".versions.json" +logger = logging.getLogger(__name__) + class SelfHostedData(TypedDict): has_users: bool From 2ac0a0b122ae66081ecf768a628df5ad9b456212 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 16:06:54 +0000 Subject: [PATCH 03/16] remove unused logger --- src/common/core/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/common/core/utils.py b/src/common/core/utils.py index 2330e0a..4b785b4 100644 --- a/src/common/core/utils.py +++ b/src/common/core/utils.py @@ -1,5 +1,4 @@ import json -import logging import pathlib from functools import lru_cache from typing import NotRequired, TypedDict @@ -12,8 +11,6 @@ UNKNOWN = "unknown" VERSIONS_INFO_FILE_LOCATION = ".versions.json" -logger = logging.getLogger(__name__) - class SelfHostedData(TypedDict): has_users: bool From 6d76515afb25143c0989cfcf730b7f794f3079d8 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 17:09:47 +0000 Subject: [PATCH 04/16] improve code organisation --- .../migrations/helpers/__init__.py | 9 ---- .../migrations/helpers/postgres_helpers.py | 41 ------------------- 2 files changed, 50 deletions(-) delete mode 100644 src/task_processor/migrations/helpers/__init__.py delete mode 100644 src/task_processor/migrations/helpers/postgres_helpers.py diff --git a/src/task_processor/migrations/helpers/__init__.py b/src/task_processor/migrations/helpers/__init__.py deleted file mode 100644 index 4fb432d..0000000 --- a/src/task_processor/migrations/helpers/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Note: django doesn't support adding submodules to the migrations module directory -that don't include a Migration class. As such, I've defined this helpers submodule -and simplified the imports by defining the __all__ attribute. -""" - -from task_processor.migrations.helpers.postgres_helpers import PostgresOnlyRunSQL - -__all__ = ["PostgresOnlyRunSQL"] diff --git a/src/task_processor/migrations/helpers/postgres_helpers.py b/src/task_processor/migrations/helpers/postgres_helpers.py deleted file mode 100644 index f3de3aa..0000000 --- a/src/task_processor/migrations/helpers/postgres_helpers.py +++ /dev/null @@ -1,41 +0,0 @@ -from contextlib import suppress - -from django.db import migrations -from django.db.backends.base.schema import BaseDatabaseSchemaEditor -from django.db.migrations.state import ProjectState - - -class PostgresOnlyRunSQL(migrations.RunSQL): - @classmethod - def from_sql_file( - cls, - file_path: str, - reverse_sql: str = "", - ) -> "PostgresOnlyRunSQL": - with open(file_path) as forward_sql: - with suppress(FileNotFoundError): - with open(reverse_sql) as reverse_sql_file: - reverse_sql = reverse_sql_file.read() - return cls(forward_sql.read(), reverse_sql=reverse_sql) - - def database_forwards( - self, - app_label: str, - schema_editor: BaseDatabaseSchemaEditor, - from_state: ProjectState, - to_state: ProjectState, - ) -> None: - if schema_editor.connection.vendor != "postgresql": - return - super().database_forwards(app_label, schema_editor, from_state, to_state) - - def database_backwards( - self, - app_label: str, - schema_editor: BaseDatabaseSchemaEditor, - from_state: ProjectState, - to_state: ProjectState, - ) -> None: - if schema_editor.connection.vendor != "postgresql": - return - super().database_backwards(app_label, schema_editor, from_state, to_state) From 49e534c60212cefe01cbb59e936a6db8318e0535 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 13:42:32 +0000 Subject: [PATCH 05/16] add initial metrics --- src/task_processor/metrics.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/task_processor/metrics.py diff --git a/src/task_processor/metrics.py b/src/task_processor/metrics.py new file mode 100644 index 0000000..afe96ef --- /dev/null +++ b/src/task_processor/metrics.py @@ -0,0 +1,19 @@ +import prometheus_client + +from common.prometheus import Histogram + +task_processor_task_duration_seconds = Histogram( + "task_processor_task_duration_seconds", + "Task processor task duration in seconds", + ["task_identifier"], +) +task_processor_queued_tasks_total = prometheus_client.Counter( + "task_processor_queued_tasks_total", + "Total number of queued tasks", + ["task_identifier"], +) +task_processor_finished_tasks_total = prometheus_client.Counter( + "task_processor_finished_tasks_total", + "Total number of finished tasks", + ["task_identifier", "status"], +) From cd768859cd5f4b7505c0b33445e9caf0719d0b32 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 14:20:10 +0000 Subject: [PATCH 06/16] prevent exposing task processor netrics from api --- settings/dev.py | 1 + src/task_processor/metrics.py | 29 ++++++++++++++++------------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/settings/dev.py b/settings/dev.py index cdd1655..bca15dc 100644 --- a/settings/dev.py +++ b/settings/dev.py @@ -46,6 +46,7 @@ TASK_DELETE_RETENTION_DAYS = 15 TASK_DELETE_RUN_EVERY = timedelta(days=1) TASK_DELETE_RUN_TIME = time(5, 0, 0) +TASK_PROCESSOR_MODE = True TASK_RUN_METHOD = TaskRunMethod.TASK_PROCESSOR # Avoid models.W042 warnings diff --git a/src/task_processor/metrics.py b/src/task_processor/metrics.py index afe96ef..35e7662 100644 --- a/src/task_processor/metrics.py +++ b/src/task_processor/metrics.py @@ -1,19 +1,22 @@ import prometheus_client +from django.conf import settings from common.prometheus import Histogram -task_processor_task_duration_seconds = Histogram( - "task_processor_task_duration_seconds", - "Task processor task duration in seconds", +task_processor_enqueued_tasks_total = prometheus_client.Counter( + "task_processor_enqueued_tasks_total", + "Total number of enqueued tasks", ["task_identifier"], ) -task_processor_queued_tasks_total = prometheus_client.Counter( - "task_processor_queued_tasks_total", - "Total number of queued tasks", - ["task_identifier"], -) -task_processor_finished_tasks_total = prometheus_client.Counter( - "task_processor_finished_tasks_total", - "Total number of finished tasks", - ["task_identifier", "status"], -) + +if settings.TASK_PROCESSOR_MODE: + task_processor_finished_tasks_total = prometheus_client.Counter( + "task_processor_finished_tasks_total", + "Total number of finished tasks", + ["task_identifier", "result"], + ) + task_processor_task_duration_seconds = Histogram( + "task_processor_task_duration_seconds", + "Task processor task duration in seconds", + ["task_identifier", "result"], + ) From 02833ed14bd409854b1f399cfdd57b03f95644e7 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 14:20:34 +0000 Subject: [PATCH 07/16] implement `task_processor_task_duration_seconds`, `task_processor_finished_tasks_total` --- src/task_processor/processor.py | 42 ++++++++++--- .../test_unit_task_processor_processor.py | 60 +++++++++++++++++++ 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/src/task_processor/processor.py b/src/task_processor/processor.py index 78aa820..9e9545d 100644 --- a/src/task_processor/processor.py +++ b/src/task_processor/processor.py @@ -2,10 +2,13 @@ import traceback import typing from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack from datetime import timedelta +from django.conf import settings from django.utils import timezone +from task_processor import metrics from task_processor.models import ( AbstractBaseTask, RecurringTask, @@ -101,10 +104,21 @@ def run_recurring_tasks() -> list[RecurringTaskRun]: def _run_task( task: T, ) -> typing.Tuple[T, AnyTaskRun]: + assert settings.TASK_PROCESSOR_MODE, ( + "Attempt to run tasks in a non-task-processor environment" + ) + + ctx = ExitStack() + timer = metrics.task_processor_task_duration_seconds.time() + ctx.enter_context(timer) + + task_identifier = task.task_identifier + logger.debug( - f"Running task {task.task_identifier} id={task.pk} args={task.args} kwargs={task.kwargs}" + f"Running task {task_identifier} id={task.pk} args={task.args} kwargs={task.kwargs}" ) task_run: AnyTaskRun = task.task_runs.model(started_at=timezone.now(), task=task) # type: ignore[attr-defined] + result: str try: with ThreadPoolExecutor(max_workers=1) as executor: @@ -112,27 +126,41 @@ def _run_task( timeout = task.timeout.total_seconds() if task.timeout else None future.result(timeout=timeout) # Wait for completion or timeout - task_run.result = TaskResult.SUCCESS.value + task_run.result = result = TaskResult.SUCCESS.value task_run.finished_at = timezone.now() task.mark_success() - logger.debug(f"Task {task.task_identifier} id={task.pk} completed") + + logger.debug(f"Task {task_identifier} id={task.pk} completed") except Exception as e: # For errors that don't include a default message (e.g., TimeoutError), # fall back to using repr. err_msg = str(e) or repr(e) + task.mark_failure() + + task_run.result = result = TaskResult.FAILURE.value + task_run.error_details = str(traceback.format_exc()) + logger.error( "Failed to execute task '%s', with id %d. Exception: %s", - task.task_identifier, + task_identifier, task.pk, err_msg, exc_info=True, ) - task.mark_failure() + result_label_value = result.lower() - task_run.result = TaskResult.FAILURE.value - task_run.error_details = str(traceback.format_exc()) + timer.labels( + task_identifier=task_identifier, + result=result_label_value, + ) # type: ignore[no-untyped-call] + ctx.close() + + metrics.task_processor_finished_tasks_total.labels( + task_identifier=task_identifier, + result=result_label_value, + ).inc() return task, task_run diff --git a/tests/unit/task_processor/test_unit_task_processor_processor.py b/tests/unit/task_processor/test_unit_task_processor_processor.py index 4edbce5..22bad55 100644 --- a/tests/unit/task_processor/test_unit_task_processor_processor.py +++ b/tests/unit/task_processor/test_unit_task_processor_processor.py @@ -12,6 +12,7 @@ from pytest import MonkeyPatch from pytest_mock import MockerFixture +from common.test_tools import AssertMetricFixture from task_processor.decorators import ( TaskHandler, register_recurring_task, @@ -578,6 +579,65 @@ def test_run_task_runs_tasks_in_correct_priority( assert task_runs_3[0].task == task_2 +@pytest.mark.django_db(transaction=True) +def test_run_tasks__expected_metrics( + dummy_task: TaskHandler[[str, str]], + raise_exception_task: TaskHandler[[str]], + assert_metric: AssertMetricFixture, + mocker: MockerFixture, +) -> None: + # Given + dummy_task_identifier = dummy_task.task_identifier + raise_exception_task_identifier = raise_exception_task.task_identifier + Task.create( + dummy_task_identifier, + scheduled_for=timezone.now(), + args=("arg1", "arg2"), + ).save() + Task.create( + raise_exception_task_identifier, + scheduled_for=timezone.now(), + args=("arg1",), + ).save() + + # When + run_tasks(2) + + # Then + assert_metric( + name="task_processor_finished_tasks_total", + value=1.0, + labels={ + "task_identifier": dummy_task_identifier, + "result": "success", + }, + ) + assert_metric( + name="task_processor_finished_tasks_total", + value=1.0, + labels={ + "task_identifier": raise_exception_task_identifier, + "result": "failure", + }, + ) + assert_metric( + name="task_processor_task_duration_seconds", + value=mocker.ANY, + labels={ + "task_identifier": dummy_task_identifier, + "result": "success", + }, + ) + assert_metric( + name="task_processor_task_duration_seconds", + value=mocker.ANY, + labels={ + "task_identifier": raise_exception_task_identifier, + "result": "success", + }, + ) + + @pytest.mark.django_db(transaction=True) def test_run_tasks_skips_locked_tasks( dummy_task: TaskHandler[[str, str]], From 4938a978076d153bc9e61f34eb63971724121491 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 14:25:19 +0000 Subject: [PATCH 08/16] implement `task_processor_enqueued_tasks_total` --- src/task_processor/decorators.py | 14 +++++++---- .../test_unit_task_processor_decorators.py | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/task_processor/decorators.py b/src/task_processor/decorators.py index 8b74078..1b75ede 100644 --- a/src/task_processor/decorators.py +++ b/src/task_processor/decorators.py @@ -8,7 +8,7 @@ from django.db.transaction import on_commit from django.utils import timezone -from task_processor import task_registry +from task_processor import metrics, task_registry from task_processor.exceptions import InvalidArgumentsError, TaskQueueFullError from task_processor.models import RecurringTask, Task, TaskPriority from task_processor.task_run_method import TaskRunMethod @@ -69,7 +69,8 @@ def delay( args: tuple[typing.Any, ...] = (), kwargs: dict[str, typing.Any] | None = None, ) -> Task | None: - logger.debug("Request to run task '%s' asynchronously.", self.task_identifier) + task_identifier = self.task_identifier + logger.debug("Request to run task '%s' asynchronously.", task_identifier) kwargs = kwargs or {} @@ -84,13 +85,16 @@ def delay( _validate_inputs(*args, **kwargs) self.unwrapped(*args, **kwargs) elif settings.TASK_RUN_METHOD == TaskRunMethod.SEPARATE_THREAD: - logger.debug("Running task '%s' in separate thread", self.task_identifier) + logger.debug("Running task '%s' in separate thread", task_identifier) self.run_in_thread(args=args, kwargs=kwargs) else: - logger.debug("Creating task for function '%s'...", self.task_identifier) + logger.debug("Creating task for function '%s'...", task_identifier) + metrics.task_processor_enqueued_tasks_total.labels( + task_identifier=task_identifier + ).inc() try: task = Task.create( - task_identifier=self.task_identifier, + task_identifier=task_identifier, scheduled_for=delay_until or timezone.now(), priority=self.priority, queue_size=self.queue_size, diff --git a/tests/unit/task_processor/test_unit_task_processor_decorators.py b/tests/unit/task_processor/test_unit_task_processor_decorators.py index 3609515..93e843b 100644 --- a/tests/unit/task_processor/test_unit_task_processor_decorators.py +++ b/tests/unit/task_processor/test_unit_task_processor_decorators.py @@ -8,6 +8,7 @@ from pytest_django.fixtures import SettingsWrapper from pytest_mock import MockerFixture +from common.test_tools import AssertMetricFixture from task_processor.decorators import ( register_recurring_task, register_task_handler, @@ -208,6 +209,29 @@ def my_function(*args: typing.Any, **kwargs: typing.Any) -> None: assert task is None +def test_delay__expected_metrics( + settings: SettingsWrapper, + db: None, + assert_metric: AssertMetricFixture, +) -> None: + # Given + settings.TASK_RUN_METHOD = TaskRunMethod.TASK_PROCESSOR + + @register_task_handler(queue_size=1) + def my_function(*args: typing.Any, **kwargs: typing.Any) -> None: + pass + + # When + my_function.delay() + + # Then + assert_metric( + name="task_processor_enqueued_tasks_total", + value=1.0, + labels={"task_identifier": "test_unit_task_processor_decorators.my_function"}, + ) + + def test_can_create_task_with_priority(settings: SettingsWrapper, db: None) -> None: # Given settings.TASK_RUN_METHOD = TaskRunMethod.TASK_PROCESSOR From e66a5ccc5b92b6495bf61d1a41954cef667c5d37 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 14:28:20 +0000 Subject: [PATCH 09/16] update docs --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 30a668c..c5dac6c 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,17 @@ Flagsmith uses Prometheus to track performance metrics. The following default metrics are exposed: +##### Common metrics + - `flagsmith_build_info`: Has the labels `version` and `ci_commit_sha`. - `http_server_request_duration_seconds`: Histogram labeled with `method`, `route`, and `response_status`. - `http_server_requests_total`: Counter labeled with `method`, `route`, and `response_status`. +- `task_processor_enqueued_tasks_total`: Counter labeled with `task_identifier`. + +##### Task Processor metrics + +- `task_processor_finished_tasks_total`: Counter labeled with `task_identifier` and `result` (`"success"`, `"failure"`). +- `task_processor_task_duration_seconds`: Histogram labeled with `task_identifier` and `result` (`"success"`, `"failure"`). ##### Guidelines From 54d9189dc502ab01f8ddd7d44822d7841e2de029 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 14:36:07 +0000 Subject: [PATCH 10/16] correct assertion --- tests/unit/task_processor/test_unit_task_processor_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/task_processor/test_unit_task_processor_processor.py b/tests/unit/task_processor/test_unit_task_processor_processor.py index 22bad55..ea97224 100644 --- a/tests/unit/task_processor/test_unit_task_processor_processor.py +++ b/tests/unit/task_processor/test_unit_task_processor_processor.py @@ -633,7 +633,7 @@ def test_run_tasks__expected_metrics( value=mocker.ANY, labels={ "task_identifier": raise_exception_task_identifier, - "result": "success", + "result": "failure", }, ) From 5bb0efe3f117f5bfb1ce9ae4d33bdf078fa4d890 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 17:04:42 +0000 Subject: [PATCH 11/16] test for `TASK_PROCESSOR_MODE` - consistent usage of `settings.TASK_PROCESSOR_MODE` - test for failure running tasks when `TASK_PROCESSOR_MODE` is `False` - add `task_processor_mode` pytest marker - consistent usage of `django_db` marker --- settings/dev.py | 2 +- src/common/prometheus/utils.py | 24 +++++ src/task_processor/decorators.py | 3 +- tests/unit/task_processor/conftest.py | 18 +++- .../test_unit_task_processor_decorators.py | 20 ++-- .../test_unit_task_processor_health.py | 3 +- .../test_unit_task_processor_monitoring.py | 4 +- .../test_unit_task_processor_processor.py | 99 ++++++++++--------- .../test_unit_task_processor_tasks.py | 19 ++-- .../test_unit_task_processor_threads.py | 2 +- 10 files changed, 118 insertions(+), 76 deletions(-) diff --git a/settings/dev.py b/settings/dev.py index bca15dc..efde469 100644 --- a/settings/dev.py +++ b/settings/dev.py @@ -46,7 +46,7 @@ TASK_DELETE_RETENTION_DAYS = 15 TASK_DELETE_RUN_EVERY = timedelta(days=1) TASK_DELETE_RUN_TIME = time(5, 0, 0) -TASK_PROCESSOR_MODE = True +TASK_PROCESSOR_MODE = False TASK_RUN_METHOD = TaskRunMethod.TASK_PROCESSOR # Avoid models.W042 warnings diff --git a/src/common/prometheus/utils.py b/src/common/prometheus/utils.py index 2258561..8b7cd15 100644 --- a/src/common/prometheus/utils.py +++ b/src/common/prometheus/utils.py @@ -1,3 +1,4 @@ +import importlib import typing import prometheus_client @@ -16,3 +17,26 @@ def get_registry() -> prometheus_client.CollectorRegistry: registry = prometheus_client.CollectorRegistry() MultiProcessCollector(registry) # type: ignore[no-untyped-call] return registry + + +def reload_metrics(*metric_module_names: str) -> None: + """ + Clear the registry of all collectors from the given modules + and reload the modules to register the collectors again. + + Used in tests to reset the state of the metrics module + when needed. + """ + + for module_name in metric_module_names: + metrics_module = importlib.import_module(module_name) + + _collectors = metrics_module.__dict__.values() + + for collector in [ + *(registry := prometheus_client.REGISTRY)._collector_to_names + ]: + if collector in _collectors: + registry.unregister(collector) + + importlib.reload(metrics_module) diff --git a/src/task_processor/decorators.py b/src/task_processor/decorators.py index 1b75ede..40c2b5f 100644 --- a/src/task_processor/decorators.py +++ b/src/task_processor/decorators.py @@ -1,5 +1,4 @@ import logging -import os import typing from datetime import datetime, time, timedelta from threading import Thread @@ -178,7 +177,7 @@ def register_recurring_task( first_run_time: time | None = None, timeout: timedelta | None = timedelta(minutes=30), ) -> typing.Callable[[TaskCallable[TaskParameters]], TaskCallable[TaskParameters]]: - if not os.environ.get("RUN_BY_PROCESSOR"): + if not settings.TASK_PROCESSOR_MODE: # Do not register recurring tasks if not invoked by task processor return lambda f: f diff --git a/tests/unit/task_processor/conftest.py b/tests/unit/task_processor/conftest.py index 23769b2..6b4a827 100644 --- a/tests/unit/task_processor/conftest.py +++ b/tests/unit/task_processor/conftest.py @@ -2,13 +2,25 @@ import typing import pytest +from pytest_django.fixtures import SettingsWrapper +from common.prometheus.utils import reload_metrics from task_processor.task_registry import RegisteredTask -@pytest.fixture -def run_by_processor(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("RUN_BY_PROCESSOR", "True") +@pytest.fixture() +def task_processor_mode(settings: SettingsWrapper) -> None: + settings.TASK_PROCESSOR_MODE = True + # The setting is supposed to be set before the metrics module is imported, + # so reload it + reload_metrics("task_processor.metrics") + + +@pytest.fixture(autouse=True) +def task_processor_mode_marked(request: pytest.FixtureRequest) -> None: + for marker in request.node.iter_markers(): + if marker.name == "task_processor_mode": + request.getfixturevalue("task_processor_mode") class GetTaskProcessorCaplog(typing.Protocol): diff --git a/tests/unit/task_processor/test_unit_task_processor_decorators.py b/tests/unit/task_processor/test_unit_task_processor_decorators.py index 93e843b..a84b89a 100644 --- a/tests/unit/task_processor/test_unit_task_processor_decorators.py +++ b/tests/unit/task_processor/test_unit_task_processor_decorators.py @@ -99,10 +99,10 @@ def my_function(*args: typing.Any, **kwargs: typing.Any) -> None: ) +@pytest.mark.django_db +@pytest.mark.task_processor_mode def test_register_recurring_task( mocker: MockerFixture, - db: None, - run_by_processor: None, ) -> None: # Given mock = mocker.Mock() @@ -128,10 +128,8 @@ def test_register_recurring_task( assert task.callable is mock -def test_register_recurring_task_does_nothing_if_not_run_by_processor( - mocker: MockerFixture, - db: None, -) -> None: +@pytest.mark.django_db +def test_register_recurring_task_does_nothing_if_not_run_by_processor() -> None: # Given task_kwargs = {"first_arg": "foo", "second_arg": "bar"} @@ -187,9 +185,8 @@ class NonSerializableObj: my_function.delay(args=(NonSerializableObj(),)) -def test_delay_returns_none_if_task_queue_is_full( - settings: SettingsWrapper, db: None -) -> None: +@pytest.mark.django_db +def test_delay_returns_none_if_task_queue_is_full(settings: SettingsWrapper) -> None: # Given settings.TASK_RUN_METHOD = TaskRunMethod.TASK_PROCESSOR @@ -209,9 +206,9 @@ def my_function(*args: typing.Any, **kwargs: typing.Any) -> None: assert task is None +@pytest.mark.django_db def test_delay__expected_metrics( settings: SettingsWrapper, - db: None, assert_metric: AssertMetricFixture, ) -> None: # Given @@ -232,7 +229,8 @@ def my_function(*args: typing.Any, **kwargs: typing.Any) -> None: ) -def test_can_create_task_with_priority(settings: SettingsWrapper, db: None) -> None: +@pytest.mark.django_db +def test_can_create_task_with_priority(settings: SettingsWrapper) -> None: # Given settings.TASK_RUN_METHOD = TaskRunMethod.TASK_PROCESSOR diff --git a/tests/unit/task_processor/test_unit_task_processor_health.py b/tests/unit/task_processor/test_unit_task_processor_health.py index 21a76bf..86de292 100644 --- a/tests/unit/task_processor/test_unit_task_processor_health.py +++ b/tests/unit/task_processor/test_unit_task_processor_health.py @@ -1,3 +1,4 @@ +import pytest from pytest_django.fixtures import SettingsWrapper from pytest_mock import MockerFixture @@ -25,8 +26,8 @@ def test_is_processor_healthy_returns_false_if_task_not_processed( assert result is False +@pytest.mark.django_db def test_is_processor_healthy_returns_true_if_task_processed( - db: None, settings: SettingsWrapper, ) -> None: # Given diff --git a/tests/unit/task_processor/test_unit_task_processor_monitoring.py b/tests/unit/task_processor/test_unit_task_processor_monitoring.py index ffe27f7..eae67a2 100644 --- a/tests/unit/task_processor/test_unit_task_processor_monitoring.py +++ b/tests/unit/task_processor/test_unit_task_processor_monitoring.py @@ -1,12 +1,14 @@ from datetime import timedelta +import pytest from django.utils import timezone from task_processor.models import Task from task_processor.monitoring import get_num_waiting_tasks -def test_get_num_waiting_tasks(db: None) -> None: +@pytest.mark.django_db +def test_get_num_waiting_tasks() -> None: # Given now = timezone.now() diff --git a/tests/unit/task_processor/test_unit_task_processor_processor.py b/tests/unit/task_processor/test_unit_task_processor_processor.py index ea97224..ee02335 100644 --- a/tests/unit/task_processor/test_unit_task_processor_processor.py +++ b/tests/unit/task_processor/test_unit_task_processor_processor.py @@ -9,7 +9,6 @@ from django.core.cache import cache from django.utils import timezone from freezegun import freeze_time -from pytest import MonkeyPatch from pytest_mock import MockerFixture from common.test_tools import AssertMetricFixture @@ -79,6 +78,7 @@ def _sleep_task(seconds: int) -> None: return _sleep_task +@pytest.mark.task_processor_mode def test_run_task_runs_task_and_creates_task_run_object_when_success( dummy_task: TaskHandler[[str, str]], ) -> None: @@ -106,6 +106,7 @@ def test_run_task_runs_task_and_creates_task_run_object_when_success( assert task.completed +@pytest.mark.task_processor_mode def test_run_task_kills_task_after_timeout( sleep_task: TaskHandler[[int]], get_task_processor_caplog: "GetTaskProcessorCaplog", @@ -144,14 +145,13 @@ def test_run_task_kills_task_after_timeout( ) +@pytest.mark.django_db +@pytest.mark.task_processor_mode def test_run_recurring_task_kills_task_after_timeout( - db: None, - monkeypatch: MonkeyPatch, get_task_processor_caplog: "GetTaskProcessorCaplog", ) -> None: # Given caplog = get_task_processor_caplog(logging.ERROR) - monkeypatch.setenv("RUN_BY_PROCESSOR", "True") @register_recurring_task( run_every=timedelta(seconds=1), timeout=timedelta(microseconds=1) @@ -187,13 +187,12 @@ def _dummy_recurring_task() -> None: ) -def test_run_recurring_tasks_runs_task_and_creates_recurring_task_run_object_when_success( - db: None, - monkeypatch: MonkeyPatch, -) -> None: +@pytest.mark.django_db +@pytest.mark.task_processor_mode +def test_run_recurring_tasks_runs_task_and_creates_recurring_task_run_object_when_success() -> ( + None +): # Given - monkeypatch.setenv("RUN_BY_PROCESSOR", "True") - @register_recurring_task(run_every=timedelta(seconds=1)) def _dummy_recurring_task() -> None: cache.set(DEFAULT_CACHE_KEY, DEFAULT_CACHE_VALUE) @@ -217,13 +216,10 @@ def _dummy_recurring_task() -> None: assert task_run.error_details is None -def test_run_recurring_tasks_runs_locked_task_after_tiemout( - db: None, - monkeypatch: MonkeyPatch, -) -> None: +@pytest.mark.django_db +@pytest.mark.task_processor_mode +def test_run_recurring_tasks_runs_locked_task_after_tiemout() -> None: # Given - monkeypatch.setenv("RUN_BY_PROCESSOR", "True") - @register_recurring_task(run_every=timedelta(hours=1)) def _dummy_recurring_task() -> None: cache.set(DEFAULT_CACHE_KEY, DEFAULT_CACHE_VALUE) @@ -258,10 +254,8 @@ def _dummy_recurring_task() -> None: @pytest.mark.django_db(transaction=True) -def test_run_recurring_tasks_multiple_runs( - db: None, - run_by_processor: None, -) -> None: +@pytest.mark.task_processor_mode +def test_run_recurring_tasks_multiple_runs() -> None: # Given @register_recurring_task(run_every=timedelta(milliseconds=200)) def _dummy_recurring_task() -> None: @@ -305,10 +299,8 @@ def _dummy_recurring_task() -> None: @pytest.mark.django_db(transaction=True) -def test_run_recurring_tasks_loops_over_all_tasks( - db: None, - run_by_processor: None, -) -> None: +@pytest.mark.task_processor_mode +def test_run_recurring_tasks_loops_over_all_tasks() -> None: # Given, Three recurring tasks @register_recurring_task(run_every=timedelta(milliseconds=200)) def _dummy_recurring_task_1() -> None: @@ -364,15 +356,10 @@ def _dummy_recurring_task() -> None: assert RecurringTaskRun.objects.filter(task=task).count() == 1 -def test_run_recurring_tasks_does_nothing_if_unregistered_task_is_new( - db: None, - run_by_processor: None, - caplog: pytest.LogCaptureFixture, -) -> None: +@pytest.mark.django_db +@pytest.mark.task_processor_mode +def test_run_recurring_tasks_does_nothing_if_unregistered_task_is_new() -> None: # Given - task_processor_logger = logging.getLogger("task_processor") - task_processor_logger.propagate = True - task_identifier = "test_unit_task_processor_processor._a_task" @register_recurring_task(run_every=timedelta(milliseconds=100)) @@ -394,11 +381,9 @@ def _a_task() -> None: assert RecurringTask.objects.filter(task_identifier=task_identifier).exists() -def test_run_recurring_tasks_deletes_the_task_if_unregistered_task_is_old( - db: None, - run_by_processor: None, - mocker: MockerFixture, -) -> None: +@pytest.mark.django_db +@pytest.mark.task_processor_mode +def test_run_recurring_tasks_deletes_the_task_if_unregistered_task_is_old() -> None: # Given task_processor_logger = logging.getLogger("task_processor") task_processor_logger.propagate = True @@ -426,6 +411,7 @@ def _a_task() -> None: ) +@pytest.mark.task_processor_mode def test_run_task_runs_task_and_creates_task_run_object_when_failure( raise_exception_task: TaskHandler[[str]], get_task_processor_caplog: "GetTaskProcessorCaplog", @@ -471,6 +457,7 @@ def test_run_task_runs_task_and_creates_task_run_object_when_failure( ] +@pytest.mark.task_processor_mode def test_run_task_runs_failed_task_again( raise_exception_task: TaskHandler[[str]], ) -> None: @@ -502,10 +489,11 @@ def test_run_task_runs_failed_task_again( assert task.is_locked is False -def test_run_recurring_task_runs_task_and_creates_recurring_task_run_object_when_failure( - db: None, - run_by_processor: None, -) -> None: +@pytest.mark.django_db +@pytest.mark.task_processor_mode +def test_run_recurring_task_runs_task_and_creates_recurring_task_run_object_when_failure() -> ( + None +): # Given task_identifier = "test_unit_task_processor_processor._raise_exception" @@ -529,7 +517,8 @@ def _raise_exception(organisation_name: str) -> None: assert task_run.error_details is not None -def test_run_task_does_nothing_if_no_tasks(db: None) -> None: +@pytest.mark.django_db +def test_run_task_does_nothing_if_no_tasks() -> None: # Given - no tasks # When result = run_tasks() @@ -539,6 +528,7 @@ def test_run_task_does_nothing_if_no_tasks(db: None) -> None: @pytest.mark.django_db(transaction=True) +@pytest.mark.task_processor_mode def test_run_task_runs_tasks_in_correct_priority( dummy_task: TaskHandler[[str, str]], ) -> None: @@ -579,7 +569,23 @@ def test_run_task_runs_tasks_in_correct_priority( assert task_runs_3[0].task == task_2 +def test_run_tasks__fails_if_not_in_task_processor_mode( + dummy_task: TaskHandler[[str, str]], +) -> None: + # Given + Task.create( + dummy_task.task_identifier, + scheduled_for=timezone.now(), + args=("arg1", "arg2"), + ).save() + + # When + with pytest.raises(AssertionError): + run_tasks() + + @pytest.mark.django_db(transaction=True) +@pytest.mark.task_processor_mode def test_run_tasks__expected_metrics( dummy_task: TaskHandler[[str, str]], raise_exception_task: TaskHandler[[str]], @@ -639,6 +645,7 @@ def test_run_tasks__expected_metrics( @pytest.mark.django_db(transaction=True) +@pytest.mark.task_processor_mode def test_run_tasks_skips_locked_tasks( dummy_task: TaskHandler[[str, str]], sleep_task: TaskHandler[[int]], @@ -680,6 +687,7 @@ def test_run_tasks_skips_locked_tasks( task_runner_thread.join() +@pytest.mark.task_processor_mode def test_run_more_than_one_task(dummy_task: TaskHandler[[str, str]]) -> None: # Given num_tasks = 5 @@ -713,10 +721,9 @@ def test_run_more_than_one_task(dummy_task: TaskHandler[[str, str]]) -> None: assert task.completed -def test_recurring_tasks_are_unlocked_if_picked_up_but_not_executed( - db: None, - run_by_processor: None, -) -> None: +@pytest.mark.django_db +@pytest.mark.task_processor_mode +def test_recurring_tasks_are_unlocked_if_picked_up_but_not_executed() -> None: # Given @register_recurring_task(run_every=timedelta(days=1)) def my_task() -> None: diff --git a/tests/unit/task_processor/test_unit_task_processor_tasks.py b/tests/unit/task_processor/test_unit_task_processor_tasks.py index cfd8e19..99df3e6 100644 --- a/tests/unit/task_processor/test_unit_task_processor_tasks.py +++ b/tests/unit/task_processor/test_unit_task_processor_tasks.py @@ -1,5 +1,6 @@ from datetime import timedelta +import pytest from django.utils import timezone from pytest_django.fixtures import DjangoAssertNumQueries, SettingsWrapper @@ -16,7 +17,8 @@ sixty_days_ago = now - timedelta(days=60) -def test_clean_up_old_tasks_does_nothing_when_no_tasks(db: None) -> None: +@pytest.mark.django_db +def test_clean_up_old_tasks_does_nothing_when_no_tasks() -> None: # Given assert Task.objects.count() == 0 @@ -38,10 +40,10 @@ def test_clean_up_old_recurring_task_runs_does_nothing_when_no_runs(db: None) -> assert RecurringTaskRun.objects.count() == 0 +@pytest.mark.django_db def test_clean_up_old_tasks( settings: SettingsWrapper, django_assert_num_queries: DjangoAssertNumQueries, - db: None, ) -> None: # Given settings.TASK_DELETE_RETENTION_DAYS = 2 @@ -89,10 +91,10 @@ def test_clean_up_old_tasks( ] +@pytest.mark.django_db def test_clean_up_old_recurring_task_runs( settings: SettingsWrapper, django_assert_num_queries: DjangoAssertNumQueries, - db: None, ) -> None: # Given settings.RECURRING_TASK_RUN_RETENTION_DAYS = 2 @@ -125,11 +127,8 @@ def test_clean_up_old_recurring_task_runs( assert list(RecurringTaskRun.objects.all()) == [task_in_retention_period] -def test_clean_up_old_tasks_include_failed_tasks( - settings: SettingsWrapper, - django_assert_num_queries: DjangoAssertNumQueries, - db: None, -) -> None: +@pytest.mark.django_db +def test_clean_up_old_tasks_include_failed_tasks(settings: SettingsWrapper) -> None: # Given settings.TASK_DELETE_RETENTION_DAYS = 2 settings.TASK_DELETE_INCLUDE_FAILED_TASKS = True @@ -146,10 +145,10 @@ def test_clean_up_old_tasks_include_failed_tasks( assert not Task.objects.exists() +@pytest.mark.django_db def test_clean_up_old_tasks_does_not_run_if_disabled( settings: SettingsWrapper, django_assert_num_queries: DjangoAssertNumQueries, - db: None, ) -> None: # Given settings.ENABLE_CLEAN_UP_OLD_TASKS = False @@ -166,10 +165,10 @@ def test_clean_up_old_tasks_does_not_run_if_disabled( assert Task.objects.filter(id=task.id).exists() +@pytest.mark.django_db def test_clean_up_old_recurring_task_runs_does_not_run_if_disabled( settings: SettingsWrapper, django_assert_num_queries: DjangoAssertNumQueries, - db: None, ) -> None: # Given settings.RECURRING_TASK_RUN_RETENTION_DAYS = 2 diff --git a/tests/unit/task_processor/test_unit_task_processor_threads.py b/tests/unit/task_processor/test_unit_task_processor_threads.py index 3647d50..7702b49 100644 --- a/tests/unit/task_processor/test_unit_task_processor_threads.py +++ b/tests/unit/task_processor/test_unit_task_processor_threads.py @@ -17,8 +17,8 @@ "exception_class, exception_message", [(DatabaseError, "Database error"), (Exception, "Generic error")], ) +@pytest.mark.django_db def test_task_runner_is_resilient_to_errors( - db: None, mocker: MockerFixture, get_task_processor_caplog: "GetTaskProcessorCaplog", exception_class: Type[Exception], From 6b3f074a68353d3db30ea5e081767d0c20be1ee6 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Thu, 27 Mar 2025 19:11:03 +0000 Subject: [PATCH 12/16] fix bad merge --- poetry.lock | 81 ++++++++++++++++++++++++++++++++++++++++++-------- pyproject.toml | 2 +- 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/poetry.lock b/poetry.lock index f157d0f..4dae194 100644 --- a/poetry.lock +++ b/poetry.lock @@ -775,23 +775,80 @@ files = [ twisted = ["twisted"] [[package]] -name = "psycopg2" +name = "psycopg2-binary" version = "2.9.10" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, - {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, - {file = "psycopg2-2.9.10-cp311-cp311-win32.whl", hash = "sha256:47c4f9875125344f4c2b870e41b6aad585901318068acd01de93f3677a6522c2"}, - {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, - {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, - {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, - {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, - {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, - {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, - {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, + {file = "psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:3e9c76f0ac6f92ecfc79516a8034a544926430f7b080ec5a0537bca389ee0906"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ad26b467a405c798aaa1458ba09d7e2b6e5f96b1ce0ac15d82fd9f95dc38a92"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:270934a475a0e4b6925b5f804e3809dd5f90f8613621d062848dd82f9cd62007"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48b338f08d93e7be4ab2b5f1dbe69dc5e9ef07170fe1f86514422076d9c010d0"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4152f8f76d2023aac16285576a9ecd2b11a9895373a1f10fd9db54b3ff06b4"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:32581b3020c72d7a421009ee1c6bf4a131ef5f0a968fab2e2de0c9d2bb4577f1"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2ce3e21dc3437b1d960521eca599d57408a695a0d3c26797ea0f72e834c7ffe5"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e984839e75e0b60cfe75e351db53d6db750b00de45644c5d1f7ee5d1f34a1ce5"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c4745a90b78e51d9ba06e2088a2fe0c693ae19cc8cb051ccda44e8df8a6eb53"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-win32.whl", hash = "sha256:e5720a5d25e3b99cd0dc5c8a440570469ff82659bb09431c1439b92caf184d3b"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:3c18f74eb4386bf35e92ab2354a12c17e5eb4d9798e4c0ad3a00783eae7cd9f1"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-win32.whl", hash = "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-win32.whl", hash = "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:056470c3dc57904bbf63d6f534988bafc4e970ffd50f6271fc4ee7daad9498a5"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aa0e31fa4bb82578f3a6c74a73c273367727de397a7a0f07bd83cbea696baa"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8de718c0e1c4b982a54b41779667242bc630b2197948405b7bd8ce16bcecac92"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5c370b1e4975df846b0277b4deba86419ca77dbc25047f535b0bb03d1a544d44"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ffe8ed017e4ed70f68b7b371d84b7d4a790368db9203dfc2d222febd3a9c8863"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:8aecc5e80c63f7459a1a2ab2c64df952051df196294d9f739933a9f6687e86b3"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:7a813c8bdbaaaab1f078014b9b0b13f5de757e2b5d9be6403639b298a04d218b"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d00924255d7fc916ef66e4bf22f354a940c67179ad3fd7067d7a0a9c84d2fbfc"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7559bce4b505762d737172556a4e6ea8a9998ecac1e39b5233465093e8cee697"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8b58f0a96e7a1e341fc894f62c1177a7c83febebb5ff9123b579418fdc8a481"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b269105e59ac96aba877c1707c600ae55711d9dcd3fc4b5012e4af68e30c648"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:79625966e176dc97ddabc142351e0409e28acf4660b88d1cf6adb876d20c490d"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8aabf1c1a04584c168984ac678a668094d831f152859d06e055288fa515e4d30"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:19721ac03892001ee8fdd11507e6a2e01f4e37014def96379411ca99d78aeb2c"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7f5d859928e635fa3ce3477704acee0f667b3a3d3e4bb109f2b18d4005f38287"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-win32.whl", hash = "sha256:3216ccf953b3f267691c90c6fe742e45d890d8272326b4a8b20850a03d05b7b8"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:30e34c4e97964805f715206c7b789d54a78b70f3ff19fbe590104b71c45600e5"}, ] [[package]] @@ -1514,4 +1571,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.1" python-versions = ">=3.11,<4.0" -content-hash = "3005f742451d90ecf950ebfbae482b06ea44a5ea062823722017d07ed4ae7959" +content-hash = "947b4e34fce30980f39cfc2ca1090bcbb4a89a5725647c9ce83df15719d2119d" diff --git a/pyproject.toml b/pyproject.toml index 031215a..88e6f8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "flagsmith-flag-engine", "gunicorn (>=19.1)", "prometheus-client (>=0.0.16)", - "psycopg2 (>=2,<3)", + "psycopg2-binary (>=2.9,<3)", "simplejson (>=3,<4)", ] authors = [ From 60cf548e03d9fd54d258aad336681594cca89be2 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Fri, 28 Mar 2025 10:13:11 +0000 Subject: [PATCH 13/16] improvement --- src/common/prometheus/utils.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/common/prometheus/utils.py b/src/common/prometheus/utils.py index 8b7cd15..c1793c5 100644 --- a/src/common/prometheus/utils.py +++ b/src/common/prometheus/utils.py @@ -1,13 +1,10 @@ import importlib -import typing import prometheus_client from django.conf import settings from prometheus_client.metrics import MetricWrapperBase from prometheus_client.multiprocess import MultiProcessCollector -T = typing.TypeVar("T", bound=MetricWrapperBase) - class Histogram(prometheus_client.Histogram): DEFAULT_BUCKETS = settings.PROMETHEUS_HISTOGRAM_BUCKETS @@ -28,15 +25,14 @@ def reload_metrics(*metric_module_names: str) -> None: when needed. """ + registry = prometheus_client.REGISTRY + for module_name in metric_module_names: metrics_module = importlib.import_module(module_name) - _collectors = metrics_module.__dict__.values() - - for collector in [ - *(registry := prometheus_client.REGISTRY)._collector_to_names - ]: - if collector in _collectors: - registry.unregister(collector) + for module_attr in vars(metrics_module).values(): + if isinstance(module_attr, MetricWrapperBase): + # Unregister the collector from the registry + registry.unregister(module_attr) importlib.reload(metrics_module) From 722d9e5f2d41c7cecd228e9959d937aac9ac45e3 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Fri, 28 Mar 2025 17:42:50 +0000 Subject: [PATCH 14/16] less code --- tests/unit/task_processor/conftest.py | 26 ------------------- .../test_unit_task_processor_decorators.py | 13 ++++------ .../test_unit_task_processor_processor.py | 17 +++--------- .../test_unit_task_processor_threads.py | 9 +------ 4 files changed, 10 insertions(+), 55 deletions(-) diff --git a/tests/unit/task_processor/conftest.py b/tests/unit/task_processor/conftest.py index 6b4a827..4d4a07a 100644 --- a/tests/unit/task_processor/conftest.py +++ b/tests/unit/task_processor/conftest.py @@ -23,32 +23,6 @@ def task_processor_mode_marked(request: pytest.FixtureRequest) -> None: request.getfixturevalue("task_processor_mode") -class GetTaskProcessorCaplog(typing.Protocol): - def __call__( - self, log_level: str | int = logging.INFO - ) -> pytest.LogCaptureFixture: ... - - -@pytest.fixture -def get_task_processor_caplog( - caplog: pytest.LogCaptureFixture, -) -> GetTaskProcessorCaplog: - # caplog doesn't allow you to capture logging outputs from loggers that don't - # propagate to root. Quick hack here to get the task_processor logger to - # propagate. - # TODO: look into using loguru. - - def _inner(log_level: str | int = logging.INFO) -> pytest.LogCaptureFixture: - task_processor_logger = logging.getLogger("task_processor") - task_processor_logger.propagate = True - # Assume required level for the logger. - task_processor_logger.setLevel(log_level) - caplog.set_level(log_level) - return caplog - - return _inner - - @pytest.fixture(autouse=True) def task_registry() -> typing.Generator[dict[str, RegisteredTask], None, None]: from task_processor.task_registry import registered_tasks diff --git a/tests/unit/task_processor/test_unit_task_processor_decorators.py b/tests/unit/task_processor/test_unit_task_processor_decorators.py index a84b89a..4f0c443 100644 --- a/tests/unit/task_processor/test_unit_task_processor_decorators.py +++ b/tests/unit/task_processor/test_unit_task_processor_decorators.py @@ -1,4 +1,5 @@ import json +import logging import typing from datetime import timedelta from unittest.mock import MagicMock @@ -18,10 +19,6 @@ from task_processor.task_registry import get_task, initialise from task_processor.task_run_method import TaskRunMethod -if typing.TYPE_CHECKING: - # This import breaks private-package-test workflow in core - from tests.unit.task_processor.conftest import GetTaskProcessorCaplog - @pytest.fixture def mock_thread_class( @@ -36,12 +33,12 @@ def mock_thread_class( @pytest.mark.django_db def test_register_task_handler_run_in_thread__transaction_commit__true__default( - get_task_processor_caplog: "GetTaskProcessorCaplog", + caplog: pytest.LogCaptureFixture, mock_thread_class: MagicMock, django_capture_on_commit_callbacks: DjangoCaptureOnCommitCallbacks, ) -> None: # Given - caplog = get_task_processor_caplog() + caplog.set_level(logging.DEBUG) @register_task_handler() def my_function(*args: str, **kwargs: str) -> None: @@ -69,11 +66,11 @@ def my_function(*args: str, **kwargs: str) -> None: def test_register_task_handler_run_in_thread__transaction_commit__false( - get_task_processor_caplog: "GetTaskProcessorCaplog", + caplog: pytest.LogCaptureFixture, mock_thread_class: MagicMock, ) -> None: # Given - caplog = get_task_processor_caplog() + caplog.set_level(logging.DEBUG) @register_task_handler(transaction_on_commit=False) def my_function(*args: typing.Any, **kwargs: typing.Any) -> None: diff --git a/tests/unit/task_processor/test_unit_task_processor_processor.py b/tests/unit/task_processor/test_unit_task_processor_processor.py index ee02335..6aa324f 100644 --- a/tests/unit/task_processor/test_unit_task_processor_processor.py +++ b/tests/unit/task_processor/test_unit_task_processor_processor.py @@ -32,11 +32,6 @@ ) from task_processor.task_registry import initialise, registered_tasks -if typing.TYPE_CHECKING: - # This import breaks private-package-test workflow in core - from tests.unit.task_processor.conftest import GetTaskProcessorCaplog - - DEFAULT_CACHE_KEY = "foo" DEFAULT_CACHE_VALUE = "bar" @@ -109,10 +104,9 @@ def test_run_task_runs_task_and_creates_task_run_object_when_success( @pytest.mark.task_processor_mode def test_run_task_kills_task_after_timeout( sleep_task: TaskHandler[[int]], - get_task_processor_caplog: "GetTaskProcessorCaplog", + caplog: pytest.LogCaptureFixture, ) -> None: # Given - caplog = get_task_processor_caplog(logging.ERROR) task = Task.create( sleep_task.task_identifier, scheduled_for=timezone.now(), @@ -148,11 +142,9 @@ def test_run_task_kills_task_after_timeout( @pytest.mark.django_db @pytest.mark.task_processor_mode def test_run_recurring_task_kills_task_after_timeout( - get_task_processor_caplog: "GetTaskProcessorCaplog", + caplog: pytest.LogCaptureFixture, ) -> None: # Given - caplog = get_task_processor_caplog(logging.ERROR) - @register_recurring_task( run_every=timedelta(seconds=1), timeout=timedelta(microseconds=1) ) @@ -414,11 +406,10 @@ def _a_task() -> None: @pytest.mark.task_processor_mode def test_run_task_runs_task_and_creates_task_run_object_when_failure( raise_exception_task: TaskHandler[[str]], - get_task_processor_caplog: "GetTaskProcessorCaplog", + caplog: pytest.LogCaptureFixture, ) -> None: # Given - caplog = get_task_processor_caplog(logging.DEBUG) - + caplog.set_level(logging.DEBUG) msg = "Error!" task = Task.create( raise_exception_task.task_identifier, args=(msg,), scheduled_for=timezone.now() diff --git a/tests/unit/task_processor/test_unit_task_processor_threads.py b/tests/unit/task_processor/test_unit_task_processor_threads.py index 7702b49..a2f5812 100644 --- a/tests/unit/task_processor/test_unit_task_processor_threads.py +++ b/tests/unit/task_processor/test_unit_task_processor_threads.py @@ -1,5 +1,4 @@ import logging -import typing from typing import Type import pytest @@ -8,10 +7,6 @@ from task_processor.threads import TaskRunner -if typing.TYPE_CHECKING: - # This import breaks private-package-test workflow in core - from tests.unit.task_processor.conftest import GetTaskProcessorCaplog - @pytest.mark.parametrize( "exception_class, exception_message", @@ -20,13 +15,11 @@ @pytest.mark.django_db def test_task_runner_is_resilient_to_errors( mocker: MockerFixture, - get_task_processor_caplog: "GetTaskProcessorCaplog", + caplog: pytest.LogCaptureFixture, exception_class: Type[Exception], exception_message: str, ) -> None: # Given - caplog = get_task_processor_caplog(logging.DEBUG) - task_runner = TaskRunner() mocker.patch( "task_processor.threads.run_tasks", From 80943c0d9426cd07a8075cf9c0148bde8d39897a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Mar 2025 17:44:12 +0000 Subject: [PATCH 15/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/task_processor/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/task_processor/conftest.py b/tests/unit/task_processor/conftest.py index 4d4a07a..d601660 100644 --- a/tests/unit/task_processor/conftest.py +++ b/tests/unit/task_processor/conftest.py @@ -1,4 +1,3 @@ -import logging import typing import pytest From 1acd7c326533fd8c865d0ec94914593006e949d3 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Fri, 4 Apr 2025 10:17:44 +0100 Subject: [PATCH 16/16] fix fixture --- .../task_processor/test_unit_task_processor_processor.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/task_processor/test_unit_task_processor_processor.py b/tests/unit/task_processor/test_unit_task_processor_processor.py index 6aa324f..492f782 100644 --- a/tests/unit/task_processor/test_unit_task_processor_processor.py +++ b/tests/unit/task_processor/test_unit_task_processor_processor.py @@ -321,10 +321,11 @@ def _dummy_recurring_task_3() -> None: assert RecurringTaskRun.objects.filter(task=task).count() == 1 -def test_run_recurring_tasks_only_executes_tasks_after_interval_set_by_run_every( - db: None, - run_by_processor: None, -) -> None: +@pytest.mark.django_db +@pytest.mark.task_processor_mode +def test_run_recurring_tasks_only_executes_tasks_after_interval_set_by_run_every() -> ( + None +): # Given @register_recurring_task(run_every=timedelta(milliseconds=200)) def _dummy_recurring_task() -> None: