diff --git a/.dclintrc b/.dclintrc index 0137ea8..fe31f34 100644 --- a/.dclintrc +++ b/.dclintrc @@ -3,5 +3,6 @@ rules: services-alphabetical-order: 0 service-image-require-explicit-tag: 0 + require-quotes-in-ports: 0 quiet: false debug: true diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index b844751..9b68f43 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -11,5 +11,5 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: docker-compose-linter/dclint-github-action@18659f6a7956706cb67cf9c1ad5e55f4352cbc17 # 1.6.0 with: - path: ./docker/docker-compose.yml + path: ./docker/ recursive: true diff --git a/Makefile b/Makefile index c705fe1..67d9773 100644 --- a/Makefile +++ b/Makefile @@ -105,6 +105,14 @@ genesis: check-not-running | build $(CHAINSTATE_DIR) /usr/bin/sudo # Secondary name to boot the genesis network up-genesis: genesis +up-prom: + @echo "Starting $(PROJECT) prometheus monitoring" + docker compose -f docker/docker-compose.prom.yml --profile default -p monitoring up -d + +down-prom: + @echo "Shutting down $(PROJECT) prometheus monitoring" + docker compose -f docker/docker-compose.prom.yml --profile default -p monitoring down + # Shut down the network (chainstate and logs will be preserved) down: backup-logs current-chainstate-dir @echo "Shutting down $(PROJECT) network" diff --git a/README.md b/README.md index 2f8025d..2fc2b79 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ make log stacks-signer-1 -- -f make log stacks-signer-1 ``` +### Container management #### Pause/Unpause service To pause all services on the network ```sh @@ -88,6 +89,19 @@ Restart the stopped service make start ``` +#### Force stop the hacknet network +If the network is in a "stuck" state where the Makefile targets are not stopping the services (i.e. the `.current-chainstate-dir` file was removed while network was running), `down-force` may be used to force stop the network. + +```sh +make down-force +``` + +Additionally, `clean` target will call `down-force` *and also* delete any chainstates on disk in `./docker/chainstate/*` +```sh +make clean +``` + +### Additional Features #### Stress the CPU To simulate CPU load. Can be modified with: - `STRESS_CORES` to target how many worker threads (default will use all cores) @@ -99,6 +113,12 @@ make stress STRESS_CORES=10 STRESS_TIMEOUT=60 make stress ``` +#### Monitor chain heights +Run a script outputting the current chain heights of each miner +```sh +make monitor +``` + #### Create a chainstate snapshot - Setting the env var `PAUSE_HEIGHT` is optional to pause the chain at a specific height, else a default of Bitcoin block `999999999999` is used. - Setting the env var `MINE_INTERVAL_EPOCH3` is recommended to reach the `PAUSE_HEIGHT` more quickly to create the snapshot @@ -129,19 +149,17 @@ ex: CHAINSTATE_ARCHIVE=./docker/chainstate_new.tar.zstd make up ``` -#### Force stop the hacknet network -If the network is in a "stuck" state where the Makefile targets are not stopping the services (i.e. the `.current-chainstate-dir` file was removed while network was running), `down-force` may be used to force stop the network. - +#### Prometheus sidecar +##### Run prometheus and cadvisor +Runs a prometheus container to record data collected by `cadvisor` for tracking host/container metrics ```sh -make down-force +make up-prom ``` - -Additionally, `clean` target will call `down-force` *and also* delete any chainstates on disk in `./docker/chainstate/*` +##### Stop prometheus and cadvisor ```sh -make clean +make down-prom ``` - ## Containers - **bitcoin**: Runs a bitcoin regtest node diff --git a/docker/docker-compose.prom.yml b/docker/docker-compose.prom.yml new file mode 100644 index 0000000..2523d3d --- /dev/null +++ b/docker/docker-compose.prom.yml @@ -0,0 +1,74 @@ +x-common-vars: + - &DOCKER_NETWORK ${DOCKER_NETWORK:-stacks} + - &DOCKER_NETWORK_CIDR 10.0.0.0/24 + - &DOCKER_NETWORK_GATEWAY 10.0.0.1 + - &PROMETHEUS_IP 10.0.0.252 + - &CADVISOR_IP 10.0.0.253 + +name: monitoring +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + depends_on: + - cadvisor + configs: + - source: prometheus.yml + target: /etc/prometheus/prometheus.yml + ports: + - "0.0.0.0:9090:9090" + networks: + default: + ipv4_address: *PROMETHEUS_IP + command: + - --config.file=/etc/prometheus/prometheus.yml + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: cadvisor + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + ports: + - "0.0.0.0:8080:8080" + networks: + default: + ipv4_address: *CADVISOR_IP + command: "--enable_metrics=cpu,cpuLoad,disk,diskIO,memory" + +networks: + default: + name: *DOCKER_NETWORK + ipam: + driver: default + config: + - subnet: *DOCKER_NETWORK_CIDR + gateway: *DOCKER_NETWORK_GATEWAY + +configs: + # prometheus scraping config + prometheus.yml: + content: | + # Global config + global: + external_labels: + monitor: "hacknet" + # A scrape configuration containing Prometheus and Docker + scrape_configs: + - job_name: cadvisor + scrape_interval: 5s + static_configs: + - targets: + - cadvisor:8080 + metric_relabel_configs: + - source_labels: [ __name__ ] + regex: '^(go_|cadvisor_version_info).+' + action: drop + - job_name: stacks + scrape_interval: 5s + static_configs: + - targets: + - stacks-miner-1:9153 + - stacks-miner-2:9153 + - stacks-miner-3:9153