diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 9a919ff..0000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: CI -on: - pull_request: - branches: - - master -jobs: - build-binaries: - runs-on: [self-hosted, linux, x64] - env: - NIX_PATH: nixpkgs=channel:nixos-unstable - BASE_SHA: ${{ github.event.pull_request.base.sha }} - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - name: Fetch base commit - run: | - echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" - git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} - - name: Setup ccache - run: | - mkdir -p /data/ccache - export CCACHE_DIR=/data/ccache - export CCACHE_MAXSIZE=50G - ccache -M 50G - ccache -s - - name: Build both binaries - env: - CCACHE_DIR: /data/ccache - run: | - mkdir -p ${{ runner.temp }}/binaries/base - mkdir -p ${{ runner.temp }}/binaries/head - nix-shell --command "just build-assumeutxo-binaries $BASE_SHA $CHECKOUT_COMMIT" - cp build-base/src/bitcoind ${{ runner.temp }}/binaries/base/bitcoind - cp build-head/src/bitcoind ${{ runner.temp }}/binaries/head/bitcoind - - name: Upload binaries - uses: actions/upload-artifact@v4 - with: - name: bitcoind-binaries - path: ${{ runner.temp }}/binaries/ - assumeutxo: - needs: build-binaries - strategy: - matrix: - include: - - network: signet - timeout: 20 - utxo_path: /var/lib/bitcoin/utxo-signet-160000.dat - dbcache: 550 - - network: mainnet-default - timeout: 600 - utxo_path: /var/lib/bitcoin/utxo-840000.dat - dbcache: 550 - - network: mainnet-large - timeout: 600 - utxo_path: /var/lib/bitcoin/utxo-840000.dat - dbcache: 32000 - runs-on: [self-hosted, linux, x64] - timeout-minutes: ${{ matrix.timeout }} - env: - NIX_PATH: nixpkgs=channel:nixos-unstable - UTXO_PATH: ${{ matrix.utxo_path }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - name: Download binaries - uses: actions/download-artifact@v4 - with: - name: bitcoind-binaries - path: ${{ runner.temp }}/binaries - - name: Set binary permissions - run: | - chmod +x ${{ runner.temp }}/binaries/base/bitcoind - chmod +x ${{ runner.temp }}/binaries/head/bitcoind - - name: Fetch base commit - run: | - echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" - git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} - - name: Run AssumeUTXO ${{ matrix.network }} - env: - TMP_DATADIR: "${{ runner.temp }}/base_datadir" - BINARIES_DIR: "${{ runner.temp }}/binaries" - run: | - env - mkdir -p "$TMP_DATADIR" - CMD="nix-shell --command \"just run-assumeutxo-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $UTXO_PATH ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR\"" - echo "Running command: $CMD" - eval "$CMD" - - uses: actions/upload-artifact@v4 - with: - name: result-${{ matrix.network }} - path: "${{ runner.temp }}/results.json" - - uses: actions/upload-artifact@v4 - with: - name: pngs-${{ matrix.network }} - path: "${{ runner.temp }}/pngs/*.png" - - uses: actions/upload-artifact@v4 - with: - name: flamegraph-${{ matrix.network }} - path: "**/*-flamegraph.svg" - - name: Write GitHub and runner context files - env: - GITHUB_CONTEXT: ${{ toJSON(github) }} - RUNNER_CONTEXT: ${{ toJSON(runner) }} - run: | - mkdir contexts - echo "$GITHUB_CONTEXT" | nix-shell -p jq --command "jq 'del(.token)' > contexts/github.json" - echo "$RUNNER_CONTEXT" > contexts/runner.json - - name: Upload context metadata as artifact - uses: actions/upload-artifact@v4 - with: - name: run-metadata-${{ matrix.network }} - path: ./contexts/ diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml deleted file mode 100644 index 9d29324..0000000 --- a/.github/workflows/publish-results.yml +++ /dev/null @@ -1,298 +0,0 @@ -name: Publish Results -on: - workflow_run: - workflows: ["CI"] - types: [completed] -jobs: - build: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - permissions: - actions: read - contents: write - checks: read - env: - NETWORKS: "mainnet-default,mainnet-large,signet" - outputs: - speedups: ${{ steps.organize.outputs.speedups }} - pr-number: ${{ steps.organize.outputs.pr-number }} - steps: - - uses: actions/checkout@v4 - with: - ref: gh-pages - - name: Download artifacts - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh run download ${{ github.event.workflow_run.id }} --repo ${{ github.repository }} - - - name: Extract artifacts - run: | - for network in ${NETWORKS//,/ }; do - if [ -d "result-${network}" ]; then - mkdir -p "${network}-results" - mv "result-${network}/results.json" "${network}-results/" - fi - - if [ -d "flamegraph-${network}" ]; then - mkdir -p "${network}-flamegraph" - mv "flamegraph-${network}"/* "${network}-flamegraph/" - fi - - if [ -d "run-metadata-${network}" ]; then - mkdir -p "${network}-metadata" - mv "run-metadata-${network}"/* "${network}-metadata/" - fi - - if [ -d "pngs-${network}" ]; then - mkdir -p "${network}-plots" - mv "pngs-${network}"/*.png "${network}-plots/" - fi - done - - name: Organize results - id: organize - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const path = require('path'); - const networks = process.env.NETWORKS.split(','); - let prNumber = 'main'; - let runId; - - // First, extract metadata and get PR number - for (const network of networks) { - if (fs.existsSync(`${network}-metadata/github.json`)) { - const metadata = JSON.parse(fs.readFileSync(`${network}-metadata/github.json`, 'utf8')); - prNumber = metadata.event.pull_request?.number || prNumber; - runId = metadata.run_id; - } - } - - if (!runId) { - console.error('No valid metadata found for any network'); - process.exit(1); - } - - // Create directory structure - const resultDir = `results/pr-${prNumber}/${runId}`; - fs.mkdirSync(resultDir, { recursive: true }); - - // Now copy metadata files - for (const network of networks) { - if (fs.existsSync(`${network}-metadata/github.json`)) { - const metadataDir = `${resultDir}/${network}-metadata`; - fs.mkdirSync(metadataDir, { recursive: true }); - fs.copyFileSync(`${network}-metadata/github.json`, `${metadataDir}/github.json`); - } - } - - // Process each network's results - const combinedResults = { - results: [], - speedups: {} - }; - - for (const network of networks) { - if (fs.existsSync(`${network}-results`)) { - const networkResults = JSON.parse(fs.readFileSync(`${network}-results/results.json`, 'utf8')); - let baseMean, headMean; - - // Add network name to each result and collect means - networkResults.results.forEach(result => { - result.network = network; - combinedResults.results.push(result); - if (result.command.includes('base')) { - baseMean = result.mean; - } else if (result.command.includes('head')) { - headMean = result.mean; - } - }); - - // Calculate speedup if we have both measurements - if (baseMean && headMean) { - const speedup = baseMean > 0 ? ((baseMean - headMean) / baseMean * 100).toFixed(1) : 'N/A'; - combinedResults.speedups[network] = speedup; - } - - // Move flamegraphs - if (fs.existsSync(`${network}-flamegraph`)) { - fs.readdirSync(`${network}-flamegraph`).forEach(file => { - const sourceFile = `${network}-flamegraph/${file}`; - const targetFile = `${resultDir}/${network}-${file}`; - fs.copyFileSync(sourceFile, targetFile); - }); - } - - // Move plots - if (fs.existsSync(`${network}-plots`)) { - const targetPlotsDir = `${resultDir}/${network}-plots`; - fs.mkdirSync(targetPlotsDir, { recursive: true }); - fs.readdirSync(`${network}-plots`).forEach(plot => { - const sourcePlot = `${network}-plots/${plot}`; - const targetPlot = `${targetPlotsDir}/${plot}`; - fs.copyFileSync(sourcePlot, targetPlot); - }); - } - } - } - - // Write combined results - fs.writeFileSync(`${resultDir}/results.json`, JSON.stringify(combinedResults, null, 2)); - - // Create index.html for this run - const indexHtml = ` - - - Benchmark Results - - - -
-

Benchmark Results

-
-

PR #${prNumber} - Run ${runId}

- ${networks.map(network => ` -
-

- ${network} Results - ${combinedResults.speedups[network] ? - `(${combinedResults.speedups[network]}% speedup)` - : ''} -

-
- ${combinedResults.results - .filter(result => result.network === network) - .map(result => { - const commitShortId = result.parameters.commit.slice(0, 8); - const flameGraphFile = `${network}-${result.parameters.commit}-flamegraph.svg`; - const flameGraphPath = `${resultDir}/${network}-${result.parameters.commit}-flamegraph.svg`; - - // Query PNG files dynamically - const plotDir = `${resultDir}/${network}-plots`; - const plots = fs.existsSync(plotDir) - ? fs.readdirSync(plotDir) - .map(plot => ` - - ${plot} - - `) - .join('') - : ''; - - return ` - - - - - - - - - - - - - - - - - - - -
CommandMean (s)Std DevUser (s)System (s)
- ${result.command.replace( - /\((\w+)\)/, - (_, commit) => `(${commit.slice(0, 8)})` - )} - ${result.mean.toFixed(3)}${result.stddev?.toFixed(3) || 'N/A'}${result.user.toFixed(3)}${result.system.toFixed(3)}
- ${fs.existsSync(flameGraphPath) ? ` - - ` : ''} - ${plots} - `; - }).join('')} -
-
- `).join('')} -
-
- - `; - - fs.writeFileSync(`${resultDir}/index.html`, indexHtml); - - // Update main index.html - const prs = fs.readdirSync('results') - .filter(dir => dir.startsWith('pr-')) - .map(dir => ({ - pr: dir.replace('pr-', ''), - runs: fs.readdirSync(`results/${dir}`) - })); - - const mainIndexHtml = ` - - - Bitcoin Benchmark Results - - - -
-

Bitcoin Benchmark Results

-
-

Available Results

- -
-
- - `; - - fs.writeFileSync('index.html', mainIndexHtml); - - // Set outputs for use in PR comment - const resultUrl = `https://${context.repo.owner}.github.io/${context.repo.name}/results/pr-${prNumber}/${runId}/index.html`; - const speedupString = Object.entries(combinedResults.speedups) - .map(([network, speedup]) => `${network}: ${speedup}%`) - .join(', '); - - core.setOutput('result-url', resultUrl); - core.setOutput('speedups', speedupString); - core.setOutput('pr-number', prNumber); - return { url: resultUrl, speedups: speedupString }; - - name: Upload Pages artifact - uses: actions/upload-pages-artifact@v3 - with: - path: results - - name: Commit and push to gh-pages - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git add results/ index.html - git commit -m "Update benchmark results from run ${{ github.event.workflow_run.id }}" - git push origin gh-pages - comment-pr: - needs: build - runs-on: ubuntu-latest - permissions: - pull-requests: write - actions: read - steps: - - name: Comment on PR - if: ${{ needs.build.outputs.pr-number != 'main' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh pr comment ${{ needs.build.outputs.pr-number }} \ - --repo ${{ github.repository }} \ - --body "📊 Benchmark results for this run (${{ github.event.workflow_run.id }}) will be available at: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/results/pr-${{ needs.build.outputs.pr-number }}/${{ github.event.workflow_run.id }}/index.html after the github pages \"build and deployment\" action has completed. - 🚀 Speedups: ${{ needs.build.outputs.speedups }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ee0e232..28f3214 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,9 @@ jobs: uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.toolchain }} + - name: Install hwloc + run: | + sudo apt-get update && sudo apt-get install hwloc libhwloc-dev - name: cargo generate-lockfile run: cargo generate-lockfile - name: cargo test --locked diff --git a/justfile b/justfile deleted file mode 100644 index 6d019c7..0000000 --- a/justfile +++ /dev/null @@ -1,117 +0,0 @@ -set shell := ["bash", "-uc"] - -os := os() - -default: - just --list - -# Build base and head binaries for CI -[group('ci')] -build-assumeutxo-binaries base_commit head_commit: - #!/usr/bin/env bash - set -euxo pipefail - for build in "base:{{ base_commit }}" "head:{{ head_commit }}"; do - name="${build%%:*}" - commit="${build#*:}" - git checkout "$commit" - taskset -c 0-15 cmake -B "build-$name" \ - -DBUILD_BENCH=OFF \ - -DBUILD_CLI=OFF \ - -DBUILD_TESTS=OFF \ - -DBUILD_TX=OFF \ - -DBUILD_UTIL=OFF \ - -DENABLE_EXTERNAL_SIGNER=OFF \ - -DENABLE_WALLET=OFF \ - -DINSTALL_MAN=OFF \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer -g" - taskset -c 0-15 cmake --build "build-$name" -j {{ num_cpus() }} - done - -# Run signet assumeutxo CI workflow -[group('ci')] -run-assumeutxo-signet-ci base_commit head_commit TMP_DATADIR UTXO_PATH results_file dbcache png_dir binaries_dir: - ./bench-ci/run-assumeutxo-bench.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ UTXO_PATH }} {{ results_file }} {{ png_dir }} signet 220000 "148.251.128.115:55555" {{ dbcache }} {{ binaries_dir }} - -# Run mainnet assumeutxo CI workflow for default cache -[group('ci')] -run-assumeutxo-mainnet-default-ci base_commit head_commit TMP_DATADIR UTXO_PATH results_file dbcache png_dir binaries_dir: - ./bench-ci/run-assumeutxo-bench.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ UTXO_PATH }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} - -# Run mainnet assumeutxo CI workflow for large cache -[group('ci')] -run-assumeutxo-mainnet-large-ci base_commit head_commit TMP_DATADIR UTXO_PATH results_file dbcache png_dir binaries_dir: - ./bench-ci/run-assumeutxo-bench.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ UTXO_PATH }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} - -# Run a signet benchmark locally -[group('local')] -run-signet: - #!/usr/bin/env bash - set -euo pipefail - set -x - - # Get git HEAD and merge-base with master (as BASE) - HEAD=$(git rev-parse HEAD) - BASE=$(git merge-base HEAD master) - echo "Using BASE: $BASE" - echo "Using HEAD: $HEAD" - - # Make a random temp dir and save it as TMPDIR - TMPDIR=$(mktemp -d) - echo "Using temporary directory: $TMPDIR" - - # Create required directories - mkdir -p "$TMPDIR/datadir" - mkdir -p "$TMPDIR/png" - mkdir -p "$TMPDIR/binaries" - - # Build binaries - just build-assumeutxo-binaries "$BASE" "$HEAD" - cp build-head/src/bitcoind "$TMPDIR/binaries/bitcoind-head" - cp build-base/src/bitcoind "$TMPDIR/binaries/bitcoind-base" - - # Fetch utxo-signet-160000.dat if not exists in $CWD - if [ ! -f "./utxo-signet-160000.dat" ]; then - echo "Downloading utxo-signet-160000.dat..." - if command -v curl &> /dev/null; then - curl -L -o "./utxo-signet-160000.dat" "https://tmp.256k1.dev/utxo-signet-160000.dat" - elif command -v wget &> /dev/null; then - wget -O "./utxo-signet-160000.dat" "https://tmp.256k1.dev/utxo-signet-160000.dat" - else - echo "Error: Neither curl nor wget is available. Please install one of them." - exit 1 - fi - echo "Download complete." - else - echo "Using existing utxo-signet-160000.dat" - fi - - # Run signet CI - CI=1 just run-assumeutxo-signet-ci \ - "$BASE" \ - "$HEAD" \ - "$TMPDIR/datadir" \ - "$PWD/utxo-signet-160000.dat" \ - "$TMPDIR/result" \ - 16000 \ - "$TMPDIR/png" \ - "$TMPDIR/binaries" - - echo "Results saved in: $TMPDIR/result" - echo "PNG files saved in: $TMPDIR/png" - -# Cherry-pick commits from a bitcoin core PR onto this branch -[group('git')] -pick-pr pr_number: - #!/usr/bin/env bash - set -euxo pipefail - - if ! git remote get-url upstream 2>/dev/null | grep -q "bitcoin/bitcoin"; then - echo "Error: 'upstream' remote not found or doesn't point to bitcoin/bitcoin" - echo "Please add it with: `git remote add upstream https://github.com/bitcoin/bitcoin.git`" - exit 1 - fi - - git fetch upstream pull/{{ pr_number }}/head:bench-{{ pr_number }} && git cherry-pick $(git rev-list --reverse bench-{{ pr_number }} --not upstream/master)