diff --git a/nix/devShells.nix b/nix/devShells.nix index ef464b949..74c60a3d0 100644 --- a/nix/devShells.nix +++ b/nix/devShells.nix @@ -32,28 +32,28 @@ export HISTFILE=.history ''; }; + docvenv = pkgs.python3.buildEnv.override { + extraLibs = self'.packages.docs.nativeBuildInputs; + }; in { devShells = { default = pkgs.devshell.mkShell { - packages = - with pkgs; - [ - coreutils - just - nix-update - #pg_prove - shellcheck - ansible - ansible-lint - aws-vault - packer - dbmate - nushell - pythonEnv - config.treefmt.build.wrapper - ] - ++ self'.packages.docs.nativeBuildInputs; + packages = with pkgs; [ + coreutils + just + nix-update + #pg_prove + shellcheck + ansible + ansible-lint + aws-vault + packer + dbmate + nushell + pythonEnv + config.treefmt.build.wrapper + ]; devshell.startup.pre-commit.text = config.pre-commit.installationScript; commands = [ { @@ -74,6 +74,12 @@ command = "pre-commit run --all-files"; category = "check"; } + { + name = "serve-nix-doc"; + help = "Spin up a server exposing the nix documentation"; + command = "pushd $(git rev-parse --show-toplevel)/nix && ${docvenv}/bin/mkdocs serve -o"; + category = "doc"; + } { name = "watch"; help = "Watch for file changes and run all checks"; diff --git a/nix/docs/README.md b/nix/docs/README.md index 0d7b080ee..5d177b13d 100644 --- a/nix/docs/README.md +++ b/nix/docs/README.md @@ -18,14 +18,19 @@ learn how to play with `postgres` in the [build guide](./build-postgres.md). - **[Receipt Files](./receipt-files.md)** - Understanding build receipts - **[Start Client/Server](./start-client-server.md)** - Running PostgreSQL client and server - **[Docker](./docker.md)** - Docker integration and usage +- **[Docker Image Size Analyzer](./image-size-analyzer-usage.md)** - Tool to analyze the Docker image sizes - **[Use direnv](./use-direnv.md)** - Development environment with direnv - **[Pre-commit Hooks](./pre-commit-hooks.md)** - Automatic formatting and code checks before commits - **[Nix Formatter](./nix-formatter.md)** - Code formatting with treefmt +- **[Create a New pgrx Extension](./creating-pgrx-extension.md)** - How to set up a new cargo pgrx extension +- **[Updating pgrx Extensions](./updating-pgrx-extensions.md)** - How to upgrade the cargo pgrx extensions +- **[Receipt Files](./receipt-files.md)** - Understand what receipt files are ## Package Management - **[Adding New Packages](./adding-new-package.md)** - How to add new PostgreSQL extensions - **[Update Extensions](./update-extension.md)** - How to update existing extensions +- **[Update Nix Dependecies](./updating-dependencies.md)** - How to update the Nix dependencies - **[New Major PostgreSQL](./new-major-postgres.md)** - Adding support for new PostgreSQL versions - **[Nix Overlays](./nix-overlays.md)** - Understanding and using Nix overlays @@ -34,6 +39,7 @@ learn how to play with `postgres` in the [build guide](./build-postgres.md). - **[Adding Tests](./adding-tests.md)** - How to add tests for extensions - **[Migration Tests](./migration-tests.md)** - Testing database migrations - **[Testing PG Upgrade Scripts](./testing-pg-upgrade-scripts.md)** - Testing PostgreSQL upgrades +- **[Docker Image testing](./docker-testing.md)** - How to test the docker images against the pg_regress test suite. ## Reference diff --git a/nix/docs/adding-tests.md b/nix/docs/adding-tests.md index 8ab21ad9b..6d86d2fed 100644 --- a/nix/docs/adding-tests.md +++ b/nix/docs/adding-tests.md @@ -5,13 +5,13 @@ There are basically two types of tests you can add: - Migration tests. In all cases, a number of extensions may be installed into the database for -use; you can see those in both [postgresql.conf.in](../tests/postgresql.conf.in) -and [prime.sql](../tests/prime.sql) (extensions may be enabled in either place.) +use; you can see those in both `/nix/tests/postgresql.conf.in` +and `/nix/tests/prime.sql` (extensions may be enabled in either place.) ## pg\_regress tests -pg\_regress tests are in [tests/sql](./../tests/sql/) with output in [tests/expected](./../tests/expected/). -To create a new test, create a new SQL file in [tests/sql](./../tests/sql/) +pg\_regress tests are in `/nix/tests/sql/` with output in `/nix/tests/expected/`. +To create a new test, create a new SQL file in `/nix/tests/sql/`. Next, for each current major version of postgres, we run a "flake check" build one at a time. @@ -29,8 +29,8 @@ Next, review the logs to identify where the test output was written ``` postgres> CREATE EXTENSION IF NOT EXISTS index_advisor; -postgres> CREATE EXTENSION -postgres> (using postmaster on localhost, port 5432) +postgres> CREATE EXTENSION +postgres> (using postmaster on localhost, port 5432) postgres> ============== running regression test queries ============== postgres> test new_test ... diff: /nix/store/5gk419ddz7mzzwhc9j6yj5i8lkw67pdl-tests/expected/new_test.out: No such file or directory postgres> diff command failed with status 512: diff "/nix/store/5gk419ddz7mzzwhc9j6yj5i8lkw67pdl-tests/expected/new_test.out" "/nix/store/2fbrvnnr7iz6yigyf0rb0vxnyqvrgxzp-postgres-15.6-check-harness/regression_output/results/new_test.out" > "/nix/store/2fbrvnnr7iz6yigyf0rb0vxnyqvrgxzp-postgres-15.6-check-harness/regression_output/results/new_test.out.diff @@ -44,7 +44,7 @@ cp -r /nix/store/2fbrvnnr7iz6yigyf0rb0vxnyqvrgxzp-postgres-15.6-check-harness/re Then you can review the contents of `regression_output/results/new_test.out` to see if it matches what you expected. -If it does match your expectations, copy the file to [tests/expected](./../tests/expected/) and the test will pass on the next run. +If it does match your expectations, copy the file to `/nix/tests/expected/` and the test will pass on the next run. If the output does not match your expectations, update the `.sql` file, re-run with `nix flake check -L` and try again @@ -52,7 +52,7 @@ If the output does not match your expectations, update the `.sql` file ## pgTAP tests These are super easy: simply add `.sql` files to the -[tests/smoke](./../tests/smoke/) directory, then: +`/nix/tests/smoke/` directory, then: ``` nix flake check -L @@ -100,7 +100,7 @@ extension authors), step 3 isn't guaranteed, so that's what the whole idea is designed to test. To add data into the database, modify the -[data.sql](../nix/tests/migrations/data.sql) script and add whatever you want into +`/nix/tests/migrations/data.sql` script and add whatever you want into it. This script gets loaded into the old version of the database at startup, and it's expected that the new version of the database can handle it. diff --git a/nix/docs/development-workflow.md b/nix/docs/development-workflow.md index 695427abc..e300a68f8 100644 --- a/nix/docs/development-workflow.md +++ b/nix/docs/development-workflow.md @@ -20,13 +20,14 @@ nix run .#trigger-nix-build ``` This will: + - Trigger a GitHub Actions workflow - Build PostgreSQL and extensions - Run nix flake check tests (evaluation of nix code, pg_regress and migrations tests) - Cache the results in the Supabase Nix binary cache - Watch the workflow progress until completion -The workflow will run on the branch you're currently on. +The workflow will run on the branch you're currently on. If you're on a feature different branch, you'll be prompted to confirm before proceeding. @@ -46,6 +47,7 @@ aws-vault exec -- nix run .#build-test-ami orioledb-17 ``` This will: + - Build two AMI stages using Packer - Clean up temporary instances after AMI builds - Output the final AMI name (e.g., `supabase-postgres-abc123`) @@ -62,6 +64,7 @@ nix run .#run-testinfra -- --aws-vault-profile --ami-name supabas ``` This will: + - Create a Python virtual environment - Install required Python packages - Create an EC2 instance from the AMI @@ -69,6 +72,7 @@ This will: - Automatically terminate the EC2 instance when done The script handles: + - Setting up AWS credentials via aws-vault - Creating and managing the Python virtual environment - Running the tests @@ -85,6 +89,7 @@ aws-vault exec -- nix run .#cleanup-ami supabase-postgres-abc123 ``` This will: + - Deregister the AMI - Clean up any associated resources @@ -109,6 +114,7 @@ This will: ### Environment Variables The following environment variables are used: + - `AWS_VAULT`: AWS Vault profile name (default: staging) - `AWS_REGION`: AWS region (default: ap-southeast-1) - `AMI_NAME`: Name of the AMI to test @@ -116,19 +122,21 @@ The following environment variables are used: ## Best Practices 1. **Branch Management** - - Use feature branches for development - - Merge to develop for testing - - Use release branches for version-specific changes + + - Use feature branches for development + - Merge to develop for testing + - Use release branches for version-specific changes 2. **Resource Cleanup** - - Always run the cleanup step after testing - - Monitor AWS console for any lingering resources - - Use the cleanup-ami command when done with an AMI + + - Always run the cleanup step after testing + - Monitor AWS console for any lingering resources + - Use the cleanup-ami command when done with an AMI 3. **Testing** - - Run tests locally before pushing changes - - Verify AMI builds before running testinfra - - Check test output for any warnings or errors + - Run tests locally before pushing changes + - Verify AMI builds before running testinfra + - Check test output for any warnings or errors ## Additional Commands @@ -138,4 +146,4 @@ nix run .#show-commands # Update README with latest command information nix run .#update-readme -``` \ No newline at end of file +``` diff --git a/nix/docs/documentation.md b/nix/docs/documentation.md index 034b1f460..4c0dc2fb0 100644 --- a/nix/docs/documentation.md +++ b/nix/docs/documentation.md @@ -7,8 +7,7 @@ We are using markdown for documentation in the `nix/docs` directory, which is re To generate the documentation locally, you can use the following command in a development shell: ```bash -cd nix -mkdocs serve +serve-nix-doc ``` This will start a local server at `http://localhost:8000` where you can view the documentation. diff --git a/nix/docs/nix-formatter.md b/nix/docs/nix-formatter.md index 22047a4d4..03441b66f 100644 --- a/nix/docs/nix-formatter.md +++ b/nix/docs/nix-formatter.md @@ -7,20 +7,20 @@ This repository uses [treefmt](https://treefmt.com/) with [nixfmt](https://githu ### nixfmt-rfc-style - **Purpose**: Formats Nix code according to [RFC 166](https://github.com/NixOS/rfcs/blob/master/rfcs/0166-nix-formatting.md) style guidelines -- **What it does**: +- **What it does**: - - Standardizes indentation and spacing - - Formats function calls and attribute sets consistently - - Ensures consistent line breaks and alignment + - Standardizes indentation and spacing + - Formats function calls and attribute sets consistently + - Ensures consistent line breaks and alignment ### deadnix - **Purpose**: Removes unused/dead code from Nix expressions - **What it does**: - - Identifies unused variables and bindings - - Removes unused function arguments - - Cleans up dead code paths + - Identifies unused variables and bindings + - Removes unused function arguments + - Cleans up dead code paths ## Usage diff --git a/nix/docs/nix-overlays.md b/nix/docs/nix-overlays.md index 4ca7d246e..af7562cf9 100644 --- a/nix/docs/nix-overlays.md +++ b/nix/docs/nix-overlays.md @@ -8,7 +8,7 @@ Overlays are a feature of Nixpkgs that allow you to: disabled-by-default feature. First, you need to define a file for the overlay under -[overlays/](../overlays/), and then import it in `nix/overlays/default.nix`. There is an +`/nix/overlays/`, and then import it in `nix/overlays/default.nix`. There is an example pull request in [#14](https://github.com/supabase/nix-postgres/issues/14) for this; an overlay typically looks like this: diff --git a/nix/docs/receipt-files.md b/nix/docs/receipt-files.md index e97036b74..4fa5eb5f2 100644 --- a/nix/docs/receipt-files.md +++ b/nix/docs/receipt-files.md @@ -1,3 +1,5 @@ +# Receipt Files + Every time you run `nix build` on this repository to build PostgreSQL, the installation directory comes with a _receipt_ file that tells you what's inside of it. Primarily, this tells you: diff --git a/nix/docs/start-client-server.md b/nix/docs/start-client-server.md index 5f1d0d214..3bf586281 100644 --- a/nix/docs/start-client-server.md +++ b/nix/docs/start-client-server.md @@ -3,7 +3,7 @@ If you want to run a postgres server, just do this from the root of the repository: -``` +```bash nix run .#start-server 15 ``` @@ -15,7 +15,7 @@ Actually, you don't even need the repository. You can do this from arbitrary directories, if the left-hand side of the hash character (`.` in this case) is a valid "flake reference": -``` +```bash # from any arbitrary directory nix run github:supabase/postgres#start-server 15 ``` @@ -26,7 +26,7 @@ Let's say you want to use a PostgreSQL build from a specific version of the repository. You can change the syntax of the above to use _any_ version of the repository, at any time, by adding the commit hash after the repository name: -``` +```bash # use postgresql 15 build at commit nix run github:supabase/postgres/#start-server 15 ``` @@ -36,7 +36,7 @@ nix run github:supabase/postgres/#start-server 15 All of the same rules apply, but try using `start-client` on the right-hand side of the hash character, instead. For example: -``` +```bash nix run github:supabase/postgres#start-server 15 & sleep 5 nix run github:supabase/postgres#start-client 16 @@ -52,7 +52,7 @@ To start a replica you can use the `start-postgres-replica` command. First start a server and a couple of replicas: -``` +```bash $ start-postgres-server 15 5435 $ start-postgres-replica 15 5439 @@ -62,7 +62,7 @@ $ start-postgres-replica 15 5440 Now check the master server: -``` +```bash $ start-postgres-client 15 5435 ``` @@ -80,7 +80,7 @@ create table items as select x::int from generate_series(1,100) x; And a replica: -``` +```bash $ start-postgres-client 15 5439 ``` diff --git a/nix/docs/update-extension.md b/nix/docs/update-extension.md index 30f0094ef..4ba3fa61d 100644 --- a/nix/docs/update-extension.md +++ b/nix/docs/update-extension.md @@ -1,4 +1,3 @@ - # Update an existing nix extension ## Overview @@ -107,7 +106,7 @@ These extensions use `mkPgrxExtension` and require additional Rust and pgrx vers Nix build will fail and print the correct hash. Update the `hash` field in `versions.json`. - If needed, you can access the extension name by running the command `nix flake show` + If needed, you can access the extension name by running the command `nix flake show` 4. **Update `previouslyPackagedVersions`** in the extension's `default.nix` file: diff --git a/nix/docs/updating-dependencies.md b/nix/docs/updating-dependencies.md index 73c5cf79a..695d85a85 100644 --- a/nix/docs/updating-dependencies.md +++ b/nix/docs/updating-dependencies.md @@ -1,4 +1,4 @@ -# Updating Dependencies +# Update Nix Dependencies This document explains how to update various dependencies used in the nix configuration. @@ -62,7 +62,8 @@ Packer is used for creating machine images and is defined in `nix/packages/packe ## Updating Other Dependencies Similar patterns can be followed for other dependencies defined in the nix packages. Always: + 1. Check for breaking changes in changelogs 2. Update version numbers and hashes 3. Run local tests -4. Verify functionality before creating PR \ No newline at end of file +4. Verify functionality before creating PR diff --git a/nix/mkdocs.yml b/nix/mkdocs.yml index bf7c4752d..52c2be539 100644 --- a/nix/mkdocs.yml +++ b/nix/mkdocs.yml @@ -18,19 +18,25 @@ nav: - Build PostgreSQL: build-postgres.md - Start Client/Server: start-client-server.md - Docker: docker.md + - Docker Image Size Analyzer: image-size-analyzer-usage.md - Use direnv: use-direnv.md - Code Formatter: nix-formatter.md - Pre-commit Hooks: pre-commit-hooks.md - Documentation: documentation.md + - Create pgrx Extension: creating-pgrx-extension.md + - Update pgrx Extensions: updating-pgrx-extensions.md + - Receipt Files: receipt-files.md - Package Management: - Adding New Packages: adding-new-package.md - Update Extensions: update-extension.md + - Update Nix Dependencies: updating-dependencies.md - New Major PostgreSQL: new-major-postgres.md - Nix Overlays: nix-overlays.md - Testing: - Adding Tests: adding-tests.md - Migration Tests: migration-tests.md - Testing PG Upgrade Scripts: testing-pg-upgrade-scripts.md + - Test Docker Images: docker-testing.md - NixOS Tests on macOS: nixos-tests-on-macos.md - References: references.md