From a45278b1aba12d8f7fb72c43427df7d3b78b2b68 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Fri, 7 Nov 2025 18:30:41 +0000 Subject: [PATCH 01/23] os.chmod to force permissions --- src/common/core/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/common/core/main.py b/src/common/core/main.py index 47cd10e..0066a6b 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -50,6 +50,11 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: ) shutil.rmtree(prometheus_multiproc_dir_name, ignore_errors=True) os.makedirs(prometheus_multiproc_dir_name, exist_ok=True) + + # Force expected permissions (see here: + # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-in-python) + os.chmod(prometheus_multiproc_dir_name, 0o777) + logger.info( "Re-created %s for Prometheus multi-process mode", prometheus_multiproc_dir_name, From cc03e9a493d7f0a548e68239c1f50f3eab00964c Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Fri, 7 Nov 2025 18:46:45 +0000 Subject: [PATCH 02/23] Clear all files and dirs inside of multiproc dir instead of the dir itself --- src/common/core/main.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index 0066a6b..da7299b 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -48,12 +48,18 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: "PROMETHEUS_MULTIPROC_DIR", DEFAULT_PROMETHEUS_MULTIPROC_DIR, ) - shutil.rmtree(prometheus_multiproc_dir_name, ignore_errors=True) - os.makedirs(prometheus_multiproc_dir_name, exist_ok=True) - # Force expected permissions (see here: - # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-in-python) - os.chmod(prometheus_multiproc_dir_name, 0o777) + # Clear all files and directories inside the multiproc dir, + # but not the dir itself. + for filename in os.listdir(prometheus_multiproc_dir_name): + file_path = os.path.join(prometheus_multiproc_dir_name, filename) + try: + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) # delete file or symlink + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + except Exception as e: + print(f"Failed to delete {file_path}. Reason: {e}") logger.info( "Re-created %s for Prometheus multi-process mode", From 2fbb7e43b921b8834b89eb89ccb4acdd87556888 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Fri, 7 Nov 2025 18:51:05 +0000 Subject: [PATCH 03/23] Only clear if it exists --- src/common/core/main.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index da7299b..6685945 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -49,17 +49,18 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: DEFAULT_PROMETHEUS_MULTIPROC_DIR, ) - # Clear all files and directories inside the multiproc dir, - # but not the dir itself. - for filename in os.listdir(prometheus_multiproc_dir_name): - file_path = os.path.join(prometheus_multiproc_dir_name, filename) - try: - if os.path.isfile(file_path) or os.path.islink(file_path): - os.unlink(file_path) # delete file or symlink - elif os.path.isdir(file_path): - shutil.rmtree(file_path) - except Exception as e: - print(f"Failed to delete {file_path}. Reason: {e}") + if os.path.exists(prometheus_multiproc_dir_name): + # Clear all files and directories inside the multiproc dir, + # but not the dir itself. + for filename in os.listdir(prometheus_multiproc_dir_name): + file_path = os.path.join(prometheus_multiproc_dir_name, filename) + try: + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) # delete file or symlink + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + except Exception as e: + print(f"Failed to delete {file_path}. Reason: {e}") logger.info( "Re-created %s for Prometheus multi-process mode", From 74ca0e3cfcc9d1ed738ba93815c08442ba52d638 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Fri, 7 Nov 2025 19:01:05 +0000 Subject: [PATCH 04/23] Only clear subdirs --- src/common/core/main.py | 14 ++------------ src/task_processor/utils.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index 6685945..5687885 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -1,7 +1,6 @@ import contextlib import logging import os -import shutil import sys import typing @@ -11,6 +10,7 @@ from common.core.cli import healthcheck from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR +from task_processor.utils import clear_subdirs logger = logging.getLogger(__name__) @@ -50,17 +50,7 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: ) if os.path.exists(prometheus_multiproc_dir_name): - # Clear all files and directories inside the multiproc dir, - # but not the dir itself. - for filename in os.listdir(prometheus_multiproc_dir_name): - file_path = os.path.join(prometheus_multiproc_dir_name, filename) - try: - if os.path.isfile(file_path) or os.path.islink(file_path): - os.unlink(file_path) # delete file or symlink - elif os.path.isdir(file_path): - shutil.rmtree(file_path) - except Exception as e: - print(f"Failed to delete {file_path}. Reason: {e}") + clear_subdirs(prometheus_multiproc_dir_name) logger.info( "Re-created %s for Prometheus multi-process mode", diff --git a/src/task_processor/utils.py b/src/task_processor/utils.py index f526ff5..a8d2d6c 100644 --- a/src/task_processor/utils.py +++ b/src/task_processor/utils.py @@ -1,6 +1,8 @@ import argparse import inspect import logging +import os +import shutil from contextlib import contextmanager from typing import Any, Generator @@ -69,3 +71,13 @@ def start_task_processor( yield coordinator finally: coordinator.stop() + + +def clear_subdirs(dir_path: str) -> None: + for filename in os.listdir(dir_path): + file_path = os.path.join(dir_path, filename) + try: + if os.path.isdir(file_path): + shutil.rmtree(file_path) + except Exception as e: + logger.info(f"Failed to delete {file_path}. Reason: {e}") From f3cadf428cc06e9326251343a2a0a6cfa77fa966 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Fri, 7 Nov 2025 19:08:49 +0000 Subject: [PATCH 05/23] Fix circular import --- src/common/core/main.py | 14 ++++++++++++-- src/task_processor/utils.py | 12 ------------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index 5687885..7da3bd0 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -1,6 +1,7 @@ import contextlib import logging import os +import shutil import sys import typing @@ -10,7 +11,6 @@ from common.core.cli import healthcheck from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR -from task_processor.utils import clear_subdirs logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: ) if os.path.exists(prometheus_multiproc_dir_name): - clear_subdirs(prometheus_multiproc_dir_name) + _clear_subdirs(prometheus_multiproc_dir_name) logger.info( "Re-created %s for Prometheus multi-process mode", @@ -104,3 +104,13 @@ def main(argv: list[str] = sys.argv) -> None: with ensure_cli_env(): # Run own commands and Django execute_from_command_line(argv) + + +def _clear_subdirs(dir_path: str) -> None: + for filename in os.listdir(dir_path): + file_path = os.path.join(dir_path, filename) + try: + if os.path.isdir(file_path): + shutil.rmtree(file_path) + except Exception as e: + logger.info(f"Failed to delete {file_path}. Reason: {e}") diff --git a/src/task_processor/utils.py b/src/task_processor/utils.py index a8d2d6c..f526ff5 100644 --- a/src/task_processor/utils.py +++ b/src/task_processor/utils.py @@ -1,8 +1,6 @@ import argparse import inspect import logging -import os -import shutil from contextlib import contextmanager from typing import Any, Generator @@ -71,13 +69,3 @@ def start_task_processor( yield coordinator finally: coordinator.stop() - - -def clear_subdirs(dir_path: str) -> None: - for filename in os.listdir(dir_path): - file_path = os.path.join(dir_path, filename) - try: - if os.path.isdir(file_path): - shutil.rmtree(file_path) - except Exception as e: - logger.info(f"Failed to delete {file_path}. Reason: {e}") From 709b05ebd12cf67680b4fcc81a27368f2d47a1de Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Fri, 7 Nov 2025 19:21:28 +0000 Subject: [PATCH 06/23] Only remove directories beginning with --- src/common/core/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index 7da3bd0..42aeff1 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -50,7 +50,7 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: ) if os.path.exists(prometheus_multiproc_dir_name): - _clear_subdirs(prometheus_multiproc_dir_name) + _clear_temporary_directories(prometheus_multiproc_dir_name) logger.info( "Re-created %s for Prometheus multi-process mode", @@ -106,11 +106,11 @@ def main(argv: list[str] = sys.argv) -> None: execute_from_command_line(argv) -def _clear_subdirs(dir_path: str) -> None: +def _clear_temporary_directories(dir_path: str) -> None: for filename in os.listdir(dir_path): file_path = os.path.join(dir_path, filename) try: - if os.path.isdir(file_path): + if os.path.isdir(file_path) and filename.startswith("tmp"): shutil.rmtree(file_path) except Exception as e: logger.info(f"Failed to delete {file_path}. Reason: {e}") From a2ef2a5962c6e690480da7cf603bd4932dd2778e Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 10:07:28 +0000 Subject: [PATCH 07/23] Try setting mode to 0o777 manually on creating dir --- src/common/core/main.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index 42aeff1..e11737d 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -48,10 +48,8 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: "PROMETHEUS_MULTIPROC_DIR", DEFAULT_PROMETHEUS_MULTIPROC_DIR, ) - - if os.path.exists(prometheus_multiproc_dir_name): - _clear_temporary_directories(prometheus_multiproc_dir_name) - + shutil.rmtree(prometheus_multiproc_dir_name, ignore_errors=True) + os.makedirs(prometheus_multiproc_dir_name, exist_ok=True, mode=0o777) logger.info( "Re-created %s for Prometheus multi-process mode", prometheus_multiproc_dir_name, @@ -104,13 +102,3 @@ def main(argv: list[str] = sys.argv) -> None: with ensure_cli_env(): # Run own commands and Django execute_from_command_line(argv) - - -def _clear_temporary_directories(dir_path: str) -> None: - for filename in os.listdir(dir_path): - file_path = os.path.join(dir_path, filename) - try: - if os.path.isdir(file_path) and filename.startswith("tmp"): - shutil.rmtree(file_path) - except Exception as e: - logger.info(f"Failed to delete {file_path}. Reason: {e}") From 4a34e083f1796b2747820c64d94469a32df7a2f0 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 11:40:57 +0000 Subject: [PATCH 08/23] Try replicating the permissions used by mkdtemp --- src/common/core/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index e11737d..88f5b51 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -49,7 +49,7 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: DEFAULT_PROMETHEUS_MULTIPROC_DIR, ) shutil.rmtree(prometheus_multiproc_dir_name, ignore_errors=True) - os.makedirs(prometheus_multiproc_dir_name, exist_ok=True, mode=0o777) + os.makedirs(prometheus_multiproc_dir_name, exist_ok=True, mode=0o700) logger.info( "Re-created %s for Prometheus multi-process mode", prometheus_multiproc_dir_name, From c5e983447837966818c97fbc0b5e4cfe1c2e136d Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 11:55:03 +0000 Subject: [PATCH 09/23] Use `gettempdir` --- src/common/core/constants.py | 2 +- src/common/core/main.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/common/core/constants.py b/src/common/core/constants.py index d4d74ea..b55a6ca 100644 --- a/src/common/core/constants.py +++ b/src/common/core/constants.py @@ -1 +1 @@ -DEFAULT_PROMETHEUS_MULTIPROC_DIR = "/tmp/flagsmith-prometheus" +DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME = "flagsmith-prometheus" diff --git a/src/common/core/main.py b/src/common/core/main.py index 88f5b51..122bc64 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -4,13 +4,14 @@ import shutil import sys import typing +from tempfile import gettempdir from django.core.management import ( execute_from_command_line as django_execute_from_command_line, ) from common.core.cli import healthcheck -from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR +from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME logger = logging.getLogger(__name__) @@ -46,7 +47,7 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: # Set up Prometheus' multiprocess mode prometheus_multiproc_dir_name = os.environ.setdefault( "PROMETHEUS_MULTIPROC_DIR", - DEFAULT_PROMETHEUS_MULTIPROC_DIR, + os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), ) shutil.rmtree(prometheus_multiproc_dir_name, ignore_errors=True) os.makedirs(prometheus_multiproc_dir_name, exist_ok=True, mode=0o700) From 14022154e4ab8348966694240e06c27f2bc91f3d Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 15:07:19 +0000 Subject: [PATCH 10/23] Remove all prometheus setup from python --- src/common/core/main.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index 122bc64..db3a2cd 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -1,17 +1,14 @@ import contextlib import logging import os -import shutil import sys import typing -from tempfile import gettempdir from django.core.management import ( execute_from_command_line as django_execute_from_command_line, ) from common.core.cli import healthcheck -from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME logger = logging.getLogger(__name__) @@ -44,18 +41,6 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: # without resorting to it being set outside of the application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.dev") - # Set up Prometheus' multiprocess mode - prometheus_multiproc_dir_name = os.environ.setdefault( - "PROMETHEUS_MULTIPROC_DIR", - os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), - ) - shutil.rmtree(prometheus_multiproc_dir_name, ignore_errors=True) - os.makedirs(prometheus_multiproc_dir_name, exist_ok=True, mode=0o700) - logger.info( - "Re-created %s for Prometheus multi-process mode", - prometheus_multiproc_dir_name, - ) - if "docgen" in sys.argv: os.environ["DOCGEN_MODE"] = "true" From a127cc218c533ea2ae891f42c26eb38f028832a0 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 15:36:20 +0000 Subject: [PATCH 11/23] Blindly add code from gpt :grimacing: --- src/common/core/main.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/common/core/main.py b/src/common/core/main.py index db3a2cd..3cfca7b 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -1,8 +1,12 @@ import contextlib import logging import os +import shutil +import stat import sys import typing +from pathlib import Path +from tempfile import gettempdir from django.core.management import ( execute_from_command_line as django_execute_from_command_line, @@ -33,6 +37,9 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: # TODO @khvn26 Move logging setup to here + # Prometheus multiproc support + _prepare_prom_multiproc_dir() + # Currently we don't install Flagsmith modules as a package, so we need to add # $CWD to the Python path to be able to import them sys.path.append(os.getcwd()) @@ -88,3 +95,35 @@ def main(argv: list[str] = sys.argv) -> None: with ensure_cli_env(): # Run own commands and Django execute_from_command_line(argv) + + +def _prepare_prom_multiproc_dir() -> str: + # Use env if already provided (e.g. via docker-compose), else default under /tmp + prom_dir = Path( + os.environ.get( + "PROMETHEUS_MULTIPROC_DIR", + os.path.join(gettempdir(), "flagsmith-prometheus"), + ) + ) + + # Best-effort: if dir exists, make everything writable so rmtree can't fail on perms + if prom_dir.exists(): + for p in prom_dir.rglob("*"): + try: + p.chmod(p.stat().st_mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) + except Exception: + pass + shutil.rmtree(prom_dir, ignore_errors=True) + + prom_dir.mkdir(parents=True, exist_ok=True) + + # Ensure the directory is writable by whoever will run the app + # Option A (most robust across Docker uid differences): world-writable tmp-style + try: + prom_dir.chmod(0o777) + except Exception: + pass + + os.environ["PROMETHEUS_MULTIPROC_DIR"] = str(prom_dir) + logger.info("Prepared Prometheus multiprocess dir at %s", prom_dir) + return str(prom_dir) From a64b30bf69deafab62dbb12bd178e01935e172fd Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 18:28:15 +0000 Subject: [PATCH 12/23] Tidy up gpt code... --- src/common/core/main.py | 39 ++------------------------------------- src/common/core/utils.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index 3cfca7b..eaba236 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -1,18 +1,15 @@ import contextlib import logging import os -import shutil -import stat import sys import typing -from pathlib import Path -from tempfile import gettempdir from django.core.management import ( execute_from_command_line as django_execute_from_command_line, ) from common.core.cli import healthcheck +from common.core.utils import prepare_prom_multiproc_dir logger = logging.getLogger(__name__) @@ -38,7 +35,7 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: # TODO @khvn26 Move logging setup to here # Prometheus multiproc support - _prepare_prom_multiproc_dir() + prepare_prom_multiproc_dir() # Currently we don't install Flagsmith modules as a package, so we need to add # $CWD to the Python path to be able to import them @@ -95,35 +92,3 @@ def main(argv: list[str] = sys.argv) -> None: with ensure_cli_env(): # Run own commands and Django execute_from_command_line(argv) - - -def _prepare_prom_multiproc_dir() -> str: - # Use env if already provided (e.g. via docker-compose), else default under /tmp - prom_dir = Path( - os.environ.get( - "PROMETHEUS_MULTIPROC_DIR", - os.path.join(gettempdir(), "flagsmith-prometheus"), - ) - ) - - # Best-effort: if dir exists, make everything writable so rmtree can't fail on perms - if prom_dir.exists(): - for p in prom_dir.rglob("*"): - try: - p.chmod(p.stat().st_mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) - except Exception: - pass - shutil.rmtree(prom_dir, ignore_errors=True) - - prom_dir.mkdir(parents=True, exist_ok=True) - - # Ensure the directory is writable by whoever will run the app - # Option A (most robust across Docker uid differences): world-writable tmp-style - try: - prom_dir.chmod(0o777) - except Exception: - pass - - os.environ["PROMETHEUS_MULTIPROC_DIR"] = str(prom_dir) - logger.info("Prepared Prometheus multiprocess dir at %s", prom_dir) - return str(prom_dir) diff --git a/src/common/core/utils.py b/src/common/core/utils.py index cf747ab..ef56cfc 100644 --- a/src/common/core/utils.py +++ b/src/common/core/utils.py @@ -1,9 +1,12 @@ import json import logging +import os import pathlib import random +import shutil from functools import lru_cache from itertools import cycle +from tempfile import gettempdir from typing import ( TYPE_CHECKING, Iterator, @@ -20,6 +23,7 @@ from django.db.utils import OperationalError from common.core import ReplicaReadStrategy +from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME if TYPE_CHECKING: from django.contrib.auth.models import AbstractBaseUser @@ -198,3 +202,30 @@ def using_database_replica( return manager return manager.db_manager(chosen_replica) + + +def prepare_prom_multiproc_dir() -> None: + prom_dir = pathlib.Path( + os.environ.setdefault( + "PROMETHEUS_MULTIPROC_DIR", + os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), + ) + ) + + if prom_dir.exists(): + for p in prom_dir.rglob("*"): + try: + # Ensure that the cleanup doesn't silently fail on + # files and subdirs created by other users. + p.chmod(0o777) + except Exception: + pass + + shutil.rmtree(prom_dir, ignore_errors=True) + + prom_dir.mkdir(parents=True, exist_ok=True) + + # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in + # lesser permissions for other users. This step ensures the directory is writable for + # all users. + prom_dir.chmod(0o777) From 786e4904d885b1af7e04856bd141673e5dc77805 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 18:31:55 +0000 Subject: [PATCH 13/23] Move to gunicorn conf.py --- src/common/core/main.py | 4 ---- src/common/core/utils.py | 31 ------------------------------- src/common/gunicorn/conf.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index eaba236..db3a2cd 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -9,7 +9,6 @@ ) from common.core.cli import healthcheck -from common.core.utils import prepare_prom_multiproc_dir logger = logging.getLogger(__name__) @@ -34,9 +33,6 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: # TODO @khvn26 Move logging setup to here - # Prometheus multiproc support - prepare_prom_multiproc_dir() - # Currently we don't install Flagsmith modules as a package, so we need to add # $CWD to the Python path to be able to import them sys.path.append(os.getcwd()) diff --git a/src/common/core/utils.py b/src/common/core/utils.py index ef56cfc..cf747ab 100644 --- a/src/common/core/utils.py +++ b/src/common/core/utils.py @@ -1,12 +1,9 @@ import json import logging -import os import pathlib import random -import shutil from functools import lru_cache from itertools import cycle -from tempfile import gettempdir from typing import ( TYPE_CHECKING, Iterator, @@ -23,7 +20,6 @@ from django.db.utils import OperationalError from common.core import ReplicaReadStrategy -from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME if TYPE_CHECKING: from django.contrib.auth.models import AbstractBaseUser @@ -202,30 +198,3 @@ def using_database_replica( return manager return manager.db_manager(chosen_replica) - - -def prepare_prom_multiproc_dir() -> None: - prom_dir = pathlib.Path( - os.environ.setdefault( - "PROMETHEUS_MULTIPROC_DIR", - os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), - ) - ) - - if prom_dir.exists(): - for p in prom_dir.rglob("*"): - try: - # Ensure that the cleanup doesn't silently fail on - # files and subdirs created by other users. - p.chmod(0o777) - except Exception: - pass - - shutil.rmtree(prom_dir, ignore_errors=True) - - prom_dir.mkdir(parents=True, exist_ok=True) - - # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in - # lesser permissions for other users. This step ensures the directory is writable for - # all users. - prom_dir.chmod(0o777) diff --git a/src/common/gunicorn/conf.py b/src/common/gunicorn/conf.py index 8dc3350..fbad2fa 100644 --- a/src/common/gunicorn/conf.py +++ b/src/common/gunicorn/conf.py @@ -4,10 +4,16 @@ It is used to correctly support Prometheus metrics in a multi-process environment. """ +import os +import pathlib +import shutil import typing +from tempfile import gettempdir from prometheus_client.multiprocess import mark_process_dead +from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME + if typing.TYPE_CHECKING: # pragma: no cover from gunicorn.arbiter import Arbiter # type: ignore[import-untyped] from gunicorn.workers.base import Worker # type: ignore[import-untyped] @@ -16,3 +22,31 @@ def worker_exit(server: "Arbiter", worker: "Worker") -> None: """Detach the process Prometheus metrics collector when a worker exits.""" mark_process_dead(worker.pid) # type: ignore[no-untyped-call] + + +def on_starting(server: "Arbiter") -> None: + """Prepare multiprocessing directory for prometheus metrics collector.""" + prom_dir = pathlib.Path( + os.environ.setdefault( + "PROMETHEUS_MULTIPROC_DIR", + os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), + ) + ) + + if prom_dir.exists(): + for p in prom_dir.rglob("*"): + try: + # Ensure that the cleanup doesn't silently fail on + # files and subdirs created by other users. + p.chmod(0o777) + except Exception: + pass + + shutil.rmtree(prom_dir, ignore_errors=True) + + prom_dir.mkdir(parents=True, exist_ok=True) + + # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in + # lesser permissions for other users. This step ensures the directory is writable for + # all users. + prom_dir.chmod(0o777) From 5b88f8108d90751bf47b947aef815be7d57cbdfa Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 18:43:33 +0000 Subject: [PATCH 14/23] move back to main.py and refactor into prometheus.utils --- src/common/core/main.py | 4 ++++ src/common/gunicorn/conf.py | 34 ---------------------------------- src/common/prometheus/utils.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index db3a2cd..300b053 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -9,6 +9,7 @@ ) from common.core.cli import healthcheck +from common.prometheus.utils import prepare_prom_multiproc_dir logger = logging.getLogger(__name__) @@ -33,6 +34,9 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: # TODO @khvn26 Move logging setup to here + # Prometheus multiproc support + prepare_prom_multiproc_dir() + # Currently we don't install Flagsmith modules as a package, so we need to add # $CWD to the Python path to be able to import them sys.path.append(os.getcwd()) diff --git a/src/common/gunicorn/conf.py b/src/common/gunicorn/conf.py index fbad2fa..8dc3350 100644 --- a/src/common/gunicorn/conf.py +++ b/src/common/gunicorn/conf.py @@ -4,16 +4,10 @@ It is used to correctly support Prometheus metrics in a multi-process environment. """ -import os -import pathlib -import shutil import typing -from tempfile import gettempdir from prometheus_client.multiprocess import mark_process_dead -from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME - if typing.TYPE_CHECKING: # pragma: no cover from gunicorn.arbiter import Arbiter # type: ignore[import-untyped] from gunicorn.workers.base import Worker # type: ignore[import-untyped] @@ -22,31 +16,3 @@ def worker_exit(server: "Arbiter", worker: "Worker") -> None: """Detach the process Prometheus metrics collector when a worker exits.""" mark_process_dead(worker.pid) # type: ignore[no-untyped-call] - - -def on_starting(server: "Arbiter") -> None: - """Prepare multiprocessing directory for prometheus metrics collector.""" - prom_dir = pathlib.Path( - os.environ.setdefault( - "PROMETHEUS_MULTIPROC_DIR", - os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), - ) - ) - - if prom_dir.exists(): - for p in prom_dir.rglob("*"): - try: - # Ensure that the cleanup doesn't silently fail on - # files and subdirs created by other users. - p.chmod(0o777) - except Exception: - pass - - shutil.rmtree(prom_dir, ignore_errors=True) - - prom_dir.mkdir(parents=True, exist_ok=True) - - # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in - # lesser permissions for other users. This step ensures the directory is writable for - # all users. - prom_dir.chmod(0o777) diff --git a/src/common/prometheus/utils.py b/src/common/prometheus/utils.py index c1793c5..99479d4 100644 --- a/src/common/prometheus/utils.py +++ b/src/common/prometheus/utils.py @@ -1,10 +1,16 @@ import importlib +import os +import pathlib +import shutil +from tempfile import gettempdir import prometheus_client from django.conf import settings from prometheus_client.metrics import MetricWrapperBase from prometheus_client.multiprocess import MultiProcessCollector +from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME + class Histogram(prometheus_client.Histogram): DEFAULT_BUCKETS = settings.PROMETHEUS_HISTOGRAM_BUCKETS @@ -36,3 +42,30 @@ def reload_metrics(*metric_module_names: str) -> None: registry.unregister(module_attr) importlib.reload(metrics_module) + + +def prepare_prom_multiproc_dir() -> None: + prom_dir = pathlib.Path( + os.environ.setdefault( + "PROMETHEUS_MULTIPROC_DIR", + os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), + ) + ) + + if prom_dir.exists(): + for p in prom_dir.rglob("*"): + try: + # Ensure that the cleanup doesn't silently fail on + # files and subdirs created by other users. + p.chmod(0o777) + except Exception: + pass + + shutil.rmtree(prom_dir, ignore_errors=True) + + prom_dir.mkdir(parents=True, exist_ok=True) + + # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in + # lesser permissions for other users. This step ensures the directory is writable for + # all users. + prom_dir.chmod(0o777) From 4bead799c138f00e100a741052585d3702988ba1 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 18:44:37 +0000 Subject: [PATCH 15/23] Add pragma no cover --- src/common/prometheus/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/prometheus/utils.py b/src/common/prometheus/utils.py index 99479d4..03bb92d 100644 --- a/src/common/prometheus/utils.py +++ b/src/common/prometheus/utils.py @@ -58,7 +58,7 @@ def prepare_prom_multiproc_dir() -> None: # Ensure that the cleanup doesn't silently fail on # files and subdirs created by other users. p.chmod(0o777) - except Exception: + except Exception: # pragma: no cover pass shutil.rmtree(prom_dir, ignore_errors=True) From 0653e2c53aa72a0157fc2c5e437c03b563c24131 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 19:16:30 +0000 Subject: [PATCH 16/23] Fix issue with settings being accessed at import time --- src/common/core/main.py | 2 +- src/common/prometheus/__init__.py | 16 ++++++++++-- src/common/prometheus/multiprocessing.py | 33 ++++++++++++++++++++++++ src/common/prometheus/utils.py | 33 ------------------------ 4 files changed, 48 insertions(+), 36 deletions(-) create mode 100644 src/common/prometheus/multiprocessing.py diff --git a/src/common/core/main.py b/src/common/core/main.py index 300b053..62774fd 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -9,7 +9,7 @@ ) from common.core.cli import healthcheck -from common.prometheus.utils import prepare_prom_multiproc_dir +from common.prometheus.multiprocessing import prepare_prom_multiproc_dir logger = logging.getLogger(__name__) diff --git a/src/common/prometheus/__init__.py b/src/common/prometheus/__init__.py index c055032..5c6c0b4 100644 --- a/src/common/prometheus/__init__.py +++ b/src/common/prometheus/__init__.py @@ -1,3 +1,15 @@ -from common.prometheus.utils import Histogram +from typing import Any -__all__ = ("Histogram",) +_utils = ("Histogram",) + + +def __getattr__(name: str) -> Any: + """ + Since utils imports settings, we lazy load any objects that we want to import to + prevent Django's settings-at-import-time trap + """ + if name in _utils: + from common.prometheus import utils + + return getattr(utils, name) + raise AttributeError(name) diff --git a/src/common/prometheus/multiprocessing.py b/src/common/prometheus/multiprocessing.py new file mode 100644 index 0000000..42154f7 --- /dev/null +++ b/src/common/prometheus/multiprocessing.py @@ -0,0 +1,33 @@ +import os +import pathlib +import shutil +from tempfile import gettempdir + +from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME + + +def prepare_prom_multiproc_dir() -> None: + prom_dir = pathlib.Path( + os.environ.setdefault( + "PROMETHEUS_MULTIPROC_DIR", + os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), + ) + ) + + if prom_dir.exists(): + for p in prom_dir.rglob("*"): + try: + # Ensure that the cleanup doesn't silently fail on + # files and subdirs created by other users. + p.chmod(0o777) + except Exception: # pragma: no cover + pass + + shutil.rmtree(prom_dir, ignore_errors=True) + + prom_dir.mkdir(parents=True, exist_ok=True) + + # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in + # lesser permissions for other users. This step ensures the directory is writable for + # all users. + prom_dir.chmod(0o777) diff --git a/src/common/prometheus/utils.py b/src/common/prometheus/utils.py index 03bb92d..c1793c5 100644 --- a/src/common/prometheus/utils.py +++ b/src/common/prometheus/utils.py @@ -1,16 +1,10 @@ import importlib -import os -import pathlib -import shutil -from tempfile import gettempdir import prometheus_client from django.conf import settings from prometheus_client.metrics import MetricWrapperBase from prometheus_client.multiprocess import MultiProcessCollector -from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME - class Histogram(prometheus_client.Histogram): DEFAULT_BUCKETS = settings.PROMETHEUS_HISTOGRAM_BUCKETS @@ -42,30 +36,3 @@ def reload_metrics(*metric_module_names: str) -> None: registry.unregister(module_attr) importlib.reload(metrics_module) - - -def prepare_prom_multiproc_dir() -> None: - prom_dir = pathlib.Path( - os.environ.setdefault( - "PROMETHEUS_MULTIPROC_DIR", - os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), - ) - ) - - if prom_dir.exists(): - for p in prom_dir.rglob("*"): - try: - # Ensure that the cleanup doesn't silently fail on - # files and subdirs created by other users. - p.chmod(0o777) - except Exception: # pragma: no cover - pass - - shutil.rmtree(prom_dir, ignore_errors=True) - - prom_dir.mkdir(parents=True, exist_ok=True) - - # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in - # lesser permissions for other users. This step ensures the directory is writable for - # all users. - prom_dir.chmod(0o777) From 85233a1be758c77b03f481382e1e5e5891bbc7ea Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 19:18:04 +0000 Subject: [PATCH 17/23] Remove type ignore --- src/task_processor/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/task_processor/processor.py b/src/task_processor/processor.py index daab3ae..159b20c 100644 --- a/src/task_processor/processor.py +++ b/src/task_processor/processor.py @@ -194,7 +194,7 @@ def _run_task( "result": result.lower(), } - timer.labels(**labels) # type: ignore[no-untyped-call] + timer.labels(**labels) ctx.close() metrics.flagsmith_task_processor_finished_tasks_total.labels(**labels).inc() From 1aae629de018cd9c4b21d0b51399748c931c5f1d Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 19:21:23 +0000 Subject: [PATCH 18/23] Improve docstring --- src/common/prometheus/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/common/prometheus/__init__.py b/src/common/prometheus/__init__.py index 5c6c0b4..fc52dd3 100644 --- a/src/common/prometheus/__init__.py +++ b/src/common/prometheus/__init__.py @@ -5,8 +5,9 @@ def __getattr__(name: str) -> Any: """ - Since utils imports settings, we lazy load any objects that we want to import to - prevent Django's settings-at-import-time trap + Since utils imports django.conf.settings, we lazy load any objects that + we want to import to prevent django.core.exceptions.ImproperlyConfigured + due to settings not being configured. """ if name in _utils: from common.prometheus import utils From 38c55dde706139e60d38b9d6e6cb09623fcdca15 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 19:35:09 +0000 Subject: [PATCH 19/23] Revert all logic to `core` --- src/common/core/main.py | 22 ++++++++++++++-- src/common/core/utils.py | 16 ++++++++++++ src/common/prometheus/__init__.py | 17 ++---------- src/common/prometheus/multiprocessing.py | 33 ------------------------ src/task_processor/processor.py | 2 +- 5 files changed, 39 insertions(+), 51 deletions(-) delete mode 100644 src/common/prometheus/multiprocessing.py diff --git a/src/common/core/main.py b/src/common/core/main.py index 62774fd..ceb3e52 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -1,15 +1,18 @@ import contextlib import logging import os +import pathlib import sys import typing +from tempfile import gettempdir from django.core.management import ( execute_from_command_line as django_execute_from_command_line, ) from common.core.cli import healthcheck -from common.prometheus.multiprocessing import prepare_prom_multiproc_dir +from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME +from common.core.utils import clear_directory logger = logging.getLogger(__name__) @@ -35,7 +38,22 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: # TODO @khvn26 Move logging setup to here # Prometheus multiproc support - prepare_prom_multiproc_dir() + prom_dir = pathlib.Path( + os.environ.setdefault( + "PROMETHEUS_MULTIPROC_DIR", + os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), + ) + ) + + if prom_dir.exists(): + clear_directory(prom_dir) + + prom_dir.mkdir(parents=True, exist_ok=True) + + # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in + # lesser permissions for other users. This step ensures the directory is writable for + # all users. + prom_dir.chmod(0o777) # Currently we don't install Flagsmith modules as a package, so we need to add # $CWD to the Python path to be able to import them diff --git a/src/common/core/utils.py b/src/common/core/utils.py index cf747ab..5a23fca 100644 --- a/src/common/core/utils.py +++ b/src/common/core/utils.py @@ -2,6 +2,7 @@ import logging import pathlib import random +import shutil from functools import lru_cache from itertools import cycle from typing import ( @@ -198,3 +199,18 @@ def using_database_replica( return manager return manager.db_manager(chosen_replica) + + +def clear_directory(dir_: pathlib.Path) -> None: + """ + Safely clear a directory including all subdirectories and files. + """ + for p in dir_.rglob("*"): + try: + # Ensure that the cleanup doesn't silently fail on + # files and subdirs created by other users. + p.chmod(0o777) + except Exception: # pragma: no cover + pass + + shutil.rmtree(dir_, ignore_errors=True) diff --git a/src/common/prometheus/__init__.py b/src/common/prometheus/__init__.py index fc52dd3..c055032 100644 --- a/src/common/prometheus/__init__.py +++ b/src/common/prometheus/__init__.py @@ -1,16 +1,3 @@ -from typing import Any +from common.prometheus.utils import Histogram -_utils = ("Histogram",) - - -def __getattr__(name: str) -> Any: - """ - Since utils imports django.conf.settings, we lazy load any objects that - we want to import to prevent django.core.exceptions.ImproperlyConfigured - due to settings not being configured. - """ - if name in _utils: - from common.prometheus import utils - - return getattr(utils, name) - raise AttributeError(name) +__all__ = ("Histogram",) diff --git a/src/common/prometheus/multiprocessing.py b/src/common/prometheus/multiprocessing.py deleted file mode 100644 index 42154f7..0000000 --- a/src/common/prometheus/multiprocessing.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import pathlib -import shutil -from tempfile import gettempdir - -from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME - - -def prepare_prom_multiproc_dir() -> None: - prom_dir = pathlib.Path( - os.environ.setdefault( - "PROMETHEUS_MULTIPROC_DIR", - os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), - ) - ) - - if prom_dir.exists(): - for p in prom_dir.rglob("*"): - try: - # Ensure that the cleanup doesn't silently fail on - # files and subdirs created by other users. - p.chmod(0o777) - except Exception: # pragma: no cover - pass - - shutil.rmtree(prom_dir, ignore_errors=True) - - prom_dir.mkdir(parents=True, exist_ok=True) - - # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in - # lesser permissions for other users. This step ensures the directory is writable for - # all users. - prom_dir.chmod(0o777) diff --git a/src/task_processor/processor.py b/src/task_processor/processor.py index 159b20c..daab3ae 100644 --- a/src/task_processor/processor.py +++ b/src/task_processor/processor.py @@ -194,7 +194,7 @@ def _run_task( "result": result.lower(), } - timer.labels(**labels) + timer.labels(**labels) # type: ignore[no-untyped-call] ctx.close() metrics.flagsmith_task_processor_finished_tasks_total.labels(**labels).inc() From 4af7e3be768e65449ce00756d33ece12d5771724 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 19:48:09 +0000 Subject: [PATCH 20/23] Remove `chmod` --- src/common/core/main.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index ceb3e52..0e2e418 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -50,11 +50,6 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: prom_dir.mkdir(parents=True, exist_ok=True) - # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in - # lesser permissions for other users. This step ensures the directory is writable for - # all users. - prom_dir.chmod(0o777) - # Currently we don't install Flagsmith modules as a package, so we need to add # $CWD to the Python path to be able to import them sys.path.append(os.getcwd()) From 48209d93a3b9edbab1a3b549171aeef4f9b10014 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 19:59:01 +0000 Subject: [PATCH 21/23] Use try/except --- src/common/core/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/common/core/main.py b/src/common/core/main.py index 0e2e418..010f993 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -50,6 +50,14 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: prom_dir.mkdir(parents=True, exist_ok=True) + # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in + # lesser permissions for other users. This step ensures the directory is writable for + # all users. + try: + prom_dir.chmod(0o777) + except Exception: + pass + # Currently we don't install Flagsmith modules as a package, so we need to add # $CWD to the Python path to be able to import them sys.path.append(os.getcwd()) From 3ec55b18e53bb184689274cd8cb94228ba216a47 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Mon, 10 Nov 2025 20:08:53 +0000 Subject: [PATCH 22/23] Tidy up --- src/common/core/main.py | 25 ++++++------------------- src/common/core/utils.py | 21 +++++++++++++++++---- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/common/core/main.py b/src/common/core/main.py index 010f993..20149e3 100644 --- a/src/common/core/main.py +++ b/src/common/core/main.py @@ -1,7 +1,6 @@ import contextlib import logging import os -import pathlib import sys import typing from tempfile import gettempdir @@ -12,7 +11,7 @@ from common.core.cli import healthcheck from common.core.constants import DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME -from common.core.utils import clear_directory +from common.core.utils import clear_directory, make_writable_directory logger = logging.getLogger(__name__) @@ -38,25 +37,13 @@ def ensure_cli_env() -> typing.Generator[None, None, None]: # TODO @khvn26 Move logging setup to here # Prometheus multiproc support - prom_dir = pathlib.Path( - os.environ.setdefault( - "PROMETHEUS_MULTIPROC_DIR", - os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), - ) + prom_dir = os.environ.setdefault( + "PROMETHEUS_MULTIPROC_DIR", + os.path.join(gettempdir(), DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME), ) - - if prom_dir.exists(): + if os.path.exists(prom_dir): clear_directory(prom_dir) - - prom_dir.mkdir(parents=True, exist_ok=True) - - # While `mkdir` sets mode=0o777 by default, this can be affected by umask resulting in - # lesser permissions for other users. This step ensures the directory is writable for - # all users. - try: - prom_dir.chmod(0o777) - except Exception: - pass + make_writable_directory(prom_dir) # Currently we don't install Flagsmith modules as a package, so we need to add # $CWD to the Python path to be able to import them diff --git a/src/common/core/utils.py b/src/common/core/utils.py index 5a23fca..5bef106 100644 --- a/src/common/core/utils.py +++ b/src/common/core/utils.py @@ -1,5 +1,6 @@ import json import logging +import os import pathlib import random import shutil @@ -201,16 +202,28 @@ def using_database_replica( return manager.db_manager(chosen_replica) -def clear_directory(dir_: pathlib.Path) -> None: +def clear_directory(directory_path: str) -> None: """ Safely clear a directory including all subdirectories and files. """ - for p in dir_.rglob("*"): + for p in pathlib.Path(directory_path).rglob("*"): try: # Ensure that the cleanup doesn't silently fail on # files and subdirs created by other users. p.chmod(0o777) - except Exception: # pragma: no cover + except (PermissionError, FileNotFoundError): # pragma: no cover pass - shutil.rmtree(dir_, ignore_errors=True) + shutil.rmtree(directory_path, ignore_errors=True) + + +def make_writable_directory(directory_path: str) -> None: + os.makedirs(directory_path, exist_ok=True) + + try: + # While `mkdir` sets mode=0o777 by default, this can be affected by umask + # resulting in lesser permissions for other users. This step ensures the + # directory is writable for all users. + os.chmod(directory_path, 0o777) + except PermissionError: + pass From 051d617026e779435baf2c89e6b8d677be0e9575 Mon Sep 17 00:00:00 2001 From: Matthew Elwell Date: Tue, 11 Nov 2025 08:29:47 +0000 Subject: [PATCH 23/23] Add pragma no cover --- src/common/core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/core/utils.py b/src/common/core/utils.py index 5bef106..c73f4d3 100644 --- a/src/common/core/utils.py +++ b/src/common/core/utils.py @@ -225,5 +225,5 @@ def make_writable_directory(directory_path: str) -> None: # resulting in lesser permissions for other users. This step ensures the # directory is writable for all users. os.chmod(directory_path, 0o777) - except PermissionError: + except PermissionError: # pragma: no cover pass