From de20335ea1b7b3131e318d2577808ec2344bcb12 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Fri, 31 Oct 2025 15:53:10 +0100 Subject: [PATCH 1/3] chore: add profiles to docker compose to use the local hydra database --- .circleci/config.yml | 4 ++-- README.md | 29 ++++++++++++++++------------- docker-compose.yml | 24 ++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2578db9..4ffe169 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,9 +38,9 @@ jobs: steps: - checkout - run: - name: Start containers + name: Start test containers command: | - docker compose up -d + docker compose --profile test up -d - run: name: Wait for PostgreSQL to be ready command: | diff --git a/README.md b/README.md index 18355a1..411e229 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,11 @@ The production API is deployed on data.gouv.fr infrastructure at [`https://tabul 1. **Start the Infrastructure** - Start this project via `docker compose`: + Start the test CSV database and test PostgREST container: ```shell - docker compose up + docker compose --profile test up -d ``` - - This starts PostgREST container and PostgreSQL container with fake test data. You can access the raw PostgREST API on http://localhost:8080. + The `--profile test` flag tells Docker Compose to start the PostgREST and PostgreSQL services for the test CSV database. This starts PostgREST on port 8080, connecting to the test CSV database. You can access the raw PostgREST API on http://localhost:8080. 2. **Launch the main API proxy** @@ -53,27 +52,31 @@ The production API is deployed on data.gouv.fr infrastructure at [`https://tabul To use the API with a real database served by [Hydra](https://github.com/datagouv/hydra) instead of the fake test database: -1. **Configure the PostgREST endpoint** to point to your Hydra database: +1. **Start the real Hydra CSV database locally:** + + First, you need to have Hydra CSV database running locally. See the [Hydra repository](https://github.com/datagouv/hydra) for instructions on how to set it up. Make sure the Hydra CSV database is accessible on `localhost:5434`. +2. **Start PostgREST pointing to your local Hydra database:** ```shell - export PGREST_ENDPOINT="http://your-hydra-postgrest:8080" + docker compose --profile hydra up -d ``` + The `--profile hydra` flag tells Docker Compose to start the PostgREST service configured for the real Hydra CSV database (instead of the test services). This starts PostgREST on port 8080, connecting to your local Hydra CSV database. - Or create a `config.toml` file: - ```toml - PGREST_ENDPOINT = "http://your-hydra-postgrest:8080" +3. **Configure the API to use it:** + ```shell + export PGREST_ENDPOINT="http://localhost:8080" ``` -2. **Start only the API services** (skip the fake database): +4. **Start the API services:** ```shell uv sync uv run adev runserver -p8005 api_tabular/tabular/app.py uv run adev runserver -p8006 api_tabular/metrics/app.py ``` -3. **Use real resource IDs** from your Hydra database instead of the test IDs. +5. **Use real resource IDs** from your Hydra database instead of the test IDs. -**Note:** Make sure your Hydra PostgREST instance is accessible and the database schema matches the expected structure (tables in the `csvapi` schema). +**Note:** Make sure your Hydra CSV database is accessible and the database schema matches the expected structure. The test database uses the `csvapi` schema, while real Hydra databases typically use the `public` schema. ## ๐Ÿ“š API Documentation @@ -441,7 +444,7 @@ export CSVAPI_SETTINGS="/path/to/your/config.toml" ## ๐Ÿงช Testing -This project uses [pytest](https://pytest.org/) for testing with async support and mocking capabilities. You must have the two tests containers running for the tests to run. +This project uses [pytest](https://pytest.org/) for testing with async support and mocking capabilities. You must have the two test containers running for the tests to run (see [### ๐Ÿงช Run with a test database](#-run-with-a-test-database) for setup instructions). ### Running Tests diff --git a/docker-compose.yml b/docker-compose.yml index ebe3ea1..c79d067 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,23 @@ services: + # PostgREST for real Hydra database (used for development with real data) postgrest: + image: "postgrest/postgrest:v14.0" + environment: + # Connect to Hydra CSV database + - PGRST_DB_URI=postgres://postgres:postgres@host.docker.internal:5434/postgres + - PGRST_SERVER_PORT=8080 + - PGRST_DB_ANON_ROLE=postgres + - PGRST_DB_SCHEMA=public + - PGRST_DB_AGGREGATES_ENABLED=true + ports: + - 8080:8080 + extra_hosts: + - "host.docker.internal:host-gateway" + profiles: + - hydra + + # PostgREST for test database (used for running tests) + postgrest-test: image: "postgrest/postgrest:v14.0" environment: # connect to hydra database-csv @@ -13,6 +31,10 @@ services: - 8080:8080 depends_on: - postgres-test + profiles: + - test + + # Test database (used for running tests) postgres-test: image: "postgres:15" volumes: @@ -23,3 +45,5 @@ services: - POSTGRES_DB=csvapi - POSTGRES_USER=csvapi - POSTGRES_PASSWORD=csvapi + profiles: + - test From 93571257cd2ad759e90ba9597d57e1f2cfd574f3 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 3 Nov 2025 14:07:54 +0100 Subject: [PATCH 2/3] fix: flexible postgREST port depending on env var --- README.md | 13 ++++++++++++- docker-compose.yml | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 411e229..97e69c5 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,22 @@ To use the API with a real database served by [Hydra](https://github.com/datagou ```shell docker compose --profile hydra up -d ``` - The `--profile hydra` flag tells Docker Compose to start the PostgREST service configured for the real Hydra CSV database (instead of the test services). This starts PostgREST on port 8080, connecting to your local Hydra CSV database. + The `--profile hydra` flag tells Docker Compose to start the PostgREST service configured for the real Hydra CSV database (instead of the test services). By default, this starts PostgREST on port 8080. You can customize the port using the `PGREST_PORT` environment variable: + ```shell + # Use default port 8080 + docker compose --profile hydra up -d + + # Use custom port (e.g., 8081) + PGREST_PORT=8081 docker compose --profile hydra up -d + ``` 3. **Configure the API to use it:** ```shell + # If using default port 8080 export PGREST_ENDPOINT="http://localhost:8080" + + # If using custom port (e.g., 8081) + export PGREST_ENDPOINT="http://localhost:8081" ``` 4. **Start the API services:** diff --git a/docker-compose.yml b/docker-compose.yml index c79d067..671b5a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,12 +5,12 @@ services: environment: # Connect to Hydra CSV database - PGRST_DB_URI=postgres://postgres:postgres@host.docker.internal:5434/postgres - - PGRST_SERVER_PORT=8080 + - PGRST_SERVER_PORT=${PGREST_PORT:-8080} - PGRST_DB_ANON_ROLE=postgres - PGRST_DB_SCHEMA=public - PGRST_DB_AGGREGATES_ENABLED=true ports: - - 8080:8080 + - "${PGREST_PORT:-8080}:${PGREST_PORT:-8080}" extra_hosts: - "host.docker.internal:host-gateway" profiles: From e3023217234cc64a07bdf47a9e10303b47d818f6 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Thu, 6 Nov 2025 17:16:06 +0100 Subject: [PATCH 3/3] docs: better readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 97e69c5..c3cd49d 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ To use the API with a real database served by [Hydra](https://github.com/datagou ```shell docker compose --profile hydra up -d ``` - The `--profile hydra` flag tells Docker Compose to start the PostgREST service configured for the real Hydra CSV database (instead of the test services). By default, this starts PostgREST on port 8080. You can customize the port using the `PGREST_PORT` environment variable: + The `--profile hydra` flag tells Docker Compose to start the PostgREST service configured for a local real Hydra CSV database (instead of the test one provided by the docker compose in this repo). By default, this starts PostgREST on port 8080. You can customize the port using the `PGREST_PORT` environment variable: ```shell # Use default port 8080 docker compose --profile hydra up -d