From 6a29529422228b54a080d0d2e355779f1a7bac1e Mon Sep 17 00:00:00 2001 From: domi41 Date: Thu, 29 May 2025 22:32:24 +0200 Subject: [PATCH 1/2] config: review the setup with docker compose - dynamized some hardcoded values - added some explanations - commented out restic for now - addded a core profile - support for spaces in various env vars - ignore some more files and document - active reading in the README --- .env | 19 +++++- .gitignore | 21 +++++-- README.md | 39 ++++++------ docker-compose.yml | 151 ++++++++++++++++++++++++++------------------- 4 files changed, 139 insertions(+), 91 deletions(-) diff --git a/.env b/.env index aa4b636..6f0c30b 100644 --- a/.env +++ b/.env @@ -1,20 +1,37 @@ +# Environment configuration file. +# Copy this file to `.env.local` and fill the latter with your own configuration. +# Best not edit this file directly. See README.md + +# You can put pretty much any string in here but it can't be empty. +# It is used to generate the web tokens. Make sure this stays secret. SECRET= +# Configure the postgres database. +# The database password may not be empty. DB_NAME=mj DB_HOST=mj_db DB_PORT=5433 DB_USER=mj DB_PASS= -DOMAIN= +# Put your online domain name here, such as "mieuxvoter.fr" for example. +DOMAIN=localhost + +# Various admin reports may be sent at this email address. EMAIL= + +# Eg: "Europe/Paris" TIMEZONE= +# User Id of the owner of the uploaded images via imgpush. +PUID=1000 + # This is used by restic's backup SMTP_HOST= SMTP_USER= SMTP_PASS= +# Restic is a backup engine on AWS, disabled for now (see docker compose file). RESTIC_REPOSITORY=s3:s3.amazonaws.com/your-bucket RESTIC_PASSWORD= RESTIC_SEND_MAIL= diff --git a/.gitignore b/.gitignore index 4a4765c..09271d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,22 @@ -__pycache__/ +# Temporary files *.swp *.mo .#* -.ipynb_checkpoints/ -.env.local + backup -.venv/ + +# Python +__pycache__/ +.mypy_cache +.ipynb_checkpoints/ +/.venv/ + +# Local (secret) configuration +/.env.local + +# Databases (dumps?) test.db main.db -.mypy_cache + +# IDE +/.idea/ diff --git a/README.md b/README.md index 9d1f638..0b5f8f2 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ -# API for Mieux Voter +# Majority Judgment App Server in Python -This API allows you to create elections, vote and obtain results with [majority judgment](https://en.wikipedia.org/wiki/Majority_judgment). +> This is the REST API Backend for the Mieux Voter Online WebApp. + +You can use our instance of this at [api.mieuxvoter.fr/](https://api.mieuxvoter.fr/). -You can use our server at [api.mieuxvoter.fr/](api.mieuxvoter.fr/). +This API allows you to create elections, vote and obtain results with [majority judgment](https://en.wikipedia.org/wiki/Majority_judgment). Since our API relies on OpenAPI, documentation is automatically generated and it is available at [api.mieuxvoter.fr/redoc](api.mieuxvoter.fr/redoc) or [api.mieuxvoter.fr/docs](api.mieuxvoter.fr/docs). @@ -11,32 +13,33 @@ Since our API relies on OpenAPI, documentation is automatically generated and it Copy the `.env` into `.env.local` with your own settings. -Then launch the dockers with: +Then launch the docker services with: -`docker compose --profile all --env-file .env.local up -d` + docker compose --profile all --env-file .env.local up --detach -Note that you can use the `profile` called `dashboard` if you only need Metabase, `image` if you only need to store images, or `backup` for restic. +Note that you can use the `profile` called: +- `core` if you only need the backend and database, +- `dashboard` if you only need Metabase, +- `image` if you only need to store images, +- or `backup` for restic. -You certainly want to apply databases migrations with: +You certainly want to apply the database migrations with: -`docker/migrate.sh` + ./docker/migrate.sh ## Run the tests -`docker/test.sh` + ./docker/test.sh ## Local development 1. Install [postgresql](https://www.postgresql.org/download/). - 2. Install python 3.11. - 3. Create a new virtual environment and activate it: - ```bash venv .venv source .venv/bin/activate @@ -60,9 +63,9 @@ POSTGRES_USER POSTGRES_PASSWORD ``` -(In docker, DB_* variables are injected to POSTGRES_* variables) +> (In docker, `DB_*` variables are injected to `POSTGRES_*` variables) -:warning: If your using launch.json on vscode, .env create a conflict. You need to remove it. +:warning: If you're using `launch.json` on vscode, `.env` creates a conflict. You need to remove it. 6. Start the server: @@ -70,13 +73,9 @@ POSTGRES_PASSWORD uvicorn app.main:app --reload --env-file .env.local ``` -7. Visit the generated documentation: - -``` -http://127.0.0.1:8000/redoc -``` +7. Visit the generated documentation at http://127.0.0.1:8000/redoc -If you need to alter the database, you can create new migrations using [alembic](https://alembic.sqlalchemy.org/en/latest/index.html). +> If you need to alter the database, you can create new migrations using [alembic](https://alembic.sqlalchemy.org/en/latest/index.html). ## TODO diff --git a/docker-compose.yml b/docker-compose.yml index fe5693a..3c94f75 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,53 +1,63 @@ services: mj_db: + profiles: + - core + - all + # https://hub.docker.com/_/postgres image: postgres:15.1 restart: unless-stopped - hostname: mj_db + hostname: ${DB_HOST:-mj_db} healthcheck: - start_period: 10s - test: ["CMD", "pg_isready", "-U", "postgres"] - interval: 30s + start_period: 15s + test: ["CMD", "pg_isready", "--user", "${DB_USER:-mj}"] + interval: 1m + timeout: 10s + retries: 3 environment: - - POSTGRES_USER=${DB_USER:-mj} - - POSTGRES_PASSWORD=$DB_PASS - - POSTGRES_DB=${DB_NAME:-mj} - - TZ=${TIMEZONE:-Europe/Paris} + POSTGRES_DB: "${DB_NAME:-mj}" + POSTGRES_USER: "${DB_USER:-mj}" + POSTGRES_PASSWORD: "${DB_PASS}" + TZ: "${TIMEZONE:-Europe/Paris}" networks: - lan ports: - - 5432:5432 + - ${DB_PORT:-5433}:5432 volumes: - db:/var/lib/mysql mj_api: + profiles: + - core + - all + #image: majority-judgment/api-python:latest build: context: . dockerfile: docker/Dockerfile - image: majority-judgment/api-python:latest restart: unless-stopped # TODO remove reload command: uvicorn app.main:app --host 0.0.0.0 --port 8877 --proxy-headers --env-file ${ENV_FILE:-.env.local} --reload healthcheck: start_period: 30s test: ['CMD-SHELL', 'curl localhost:8877/liveness -s -f -o /dev/null || exit 1'] - interval: 30s - retries: 5 + interval: 10m + timeout: 10s + retries: 3 depends_on: mj_db: - condition: service_healthy + condition: service_healthy volumes: - .:/code networks: - lan - traefik_network environment: - POSTGRES_USER: ${DB_USER:-mj} - POSTGRES_PASSWORD: $DB_PASS - POSTGRES_DB: ${DB_NAME:-mj} - POSTGRES_HOST: mj_db - TZ: ${TIMEZONE:-Europe/Paris} + POSTGRES_HOST: ${DB_HOST:-mj_db} + POSTGRES_USER: "${DB_USER:-mj}" + POSTGRES_PASSWORD: "${DB_PASS}" + POSTGRES_DB: "${DB_NAME:-mj}" + TZ: "${TIMEZONE:-Europe/Paris}" ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-["*"]} - SECRET: $SECRET + SECRET: "${SECRET}" ports: - 8877:8877 labels: @@ -59,33 +69,36 @@ services: - "traefik.http.routers.mj.tls=true" - "traefik.http.routers.mj.tls.certresolver=leresolver" - mj_restic: - profiles: - - backup - - all - depends_on: - - mj_db - image: restic - networks: - - lan - build: - context: ./docker/restic - dockerfile: Dockerfile - args: - RESTIC_INIT_ARGS: $RESTIC_INIT_ARGS - RESTIC_PASSWORD: $RESTIC_PASSWORD - restart: unless-stopped - volumes: - - db:/data/db - - imgpush:/data/images - environment: - - TZ=${TIMEZONE:-Europe/Paris} - - RESTIC_REPOSITORY=$RESTIC_REPOSITORY - - BACKUP_CRON=${RESTIC_BACKUP_CRON:-0 0 * * *} - - RESTIC_FORGET_ARGS=--prune --keep-last 1 --keep-daily 1 - - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - - MAILX_ARGS=-r "${RESTIC_SEND_MAIL}" -s "Result of the last restic backup run" -S smtp="${SMTP_HOST}:${SMTP_PORT:-587}" -S smtp-use-starttls -S smtp-auth=login -S smtp-auth-user="${SMTP_USER}" -S smtp-auth-password="${SMTP_PASS}" "${RESTIC_DEST_MAIL}" + # Disabled because on our server this service fails repeatedly: + # > Fatal: create repository at s3:s3.amazonaws.com/mieuxvoter-app failed: client.BucketExists: Access Denied. +# mj_restic: +# profiles: +# - backup +# - all +# depends_on: +# mj_db: +# condition: service_healthy +# image: restic +# networks: +# - lan +# build: +# context: ./docker/restic +# dockerfile: Dockerfile +# args: +# RESTIC_INIT_ARGS: $RESTIC_INIT_ARGS +# RESTIC_PASSWORD: $RESTIC_PASSWORD +# restart: unless-stopped +# volumes: +# - db:/data/db +# - imgpush:/data/images +# environment: +# - TZ=${TIMEZONE:-Europe/Paris} +# - RESTIC_REPOSITORY=$RESTIC_REPOSITORY +# - BACKUP_CRON=${RESTIC_BACKUP_CRON:-0 0 * * *} +# - RESTIC_FORGET_ARGS=--prune --keep-last 1 --keep-daily 1 +# - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID +# - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY +# - MAILX_ARGS=-r "${RESTIC_SEND_MAIL}" -s "Result of the last restic backup run" -S smtp="${SMTP_HOST}:${SMTP_PORT:-587}" -S smtp-use-starttls -S smtp-auth=login -S smtp-auth-user="${SMTP_USER}" -S smtp-auth-password="${SMTP_PASS}" "${RESTIC_DEST_MAIL}" mj_imgpush: profiles: @@ -94,8 +107,8 @@ services: image: hauxir/imgpush:latest restart: unless-stopped environment: - PUID: $PUID - PGID: $PUID + PUID: ${PUID:-1000} + PGID: ${PUID:-1000} TZ: ${TIMEZONE:-Europe/Paris} IMAGES_DIR: /images MAX_SIZE_MB: 16 @@ -106,9 +119,11 @@ services: VALID_SIZES: ${VALID_SIZES:-"[100,200,300]"} NAME_STRATEGY: "uuidv4" healthcheck: - start_period: 0s + start_period: 15s test: ['CMD-SHELL', 'curl localhost:5000/liveness -s -f -o /dev/null || exit 1'] - interval: 30s + interval: 10m + timeout: 10s + retries: 3 networks: - lan - traefik_network @@ -124,23 +139,24 @@ services: - "traefik.http.routers.imgpush.tls.certresolver=leresolver" mj_metabase: - image: metabase/metabase - restart: unless-stopped profiles: - dashboard - all + image: metabase/metabase + restart: unless-stopped depends_on: - - mj_db + mj_db: + condition: service_healthy networks: - lan - traefik_network environment: MB_DB_TYPE: postgres - MB_DB_DBNAME: ${DB_NAME:-mj} - MB_DB_PORT: ${DB_PORT:-5432} - MB_DB_USER: ${DB_USER:-mj} - MB_DB_PASS: $DB_PASS - MB_DB_HOST: mj_db + MB_DB_DBNAME: "${DB_NAME:-mj}" + MB_DB_HOST: ${DB_HOST:-mj_db} + MB_DB_PORT: ${DB_PORT:-5433} + MB_DB_USER: "${DB_USER:-mj}" + MB_DB_PASS: "${DB_PASS}" TZ: ${TIMEZONE:-Europe/Paris} labels: - "traefik.enable=true" @@ -152,17 +168,22 @@ services: - "traefik.http.routers.metabase.tls.certresolver=leresolver" mj_pgadmin: + profiles: + - admin + - all image: dpage/pgadmin4 + restart: unless-stopped + depends_on: + mj_db: + condition: service_healthy environment: PGADMIN_DEFAULT_EMAIL: ${EMAIL:-pgadmin4@pgadmin.org} PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} PGADMIN_CONFIG_SERVER_MODE: 'False' - profiles: - - admin - - all - healthcheck: - test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:5050"] - interval: 1m30s + healthcheck: + start_period: 15s + test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:${PGADMIN_PORT:-5050}"] + interval: 10m timeout: 10s retries: 3 volumes: @@ -172,7 +193,6 @@ services: networks: - lan - traefik_network - restart: unless-stopped labels: - "traefik.enable=true" - "traefik.docker.network=traefik_network" @@ -190,6 +210,7 @@ volumes: redis_data: driver: local + networks: lan: traefik_network: From 001304439d9b9ab04a5343adff594a45b111a8cc Mon Sep 17 00:00:00 2001 From: domi41 Date: Fri, 30 May 2025 09:29:32 +0200 Subject: [PATCH 2/2] config: reduce the healthcheck interval --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3c94f75..84eae73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,7 +39,7 @@ services: healthcheck: start_period: 30s test: ['CMD-SHELL', 'curl localhost:8877/liveness -s -f -o /dev/null || exit 1'] - interval: 10m + interval: 1m timeout: 10s retries: 3 depends_on: