diff --git a/.github/ci-groups.yml b/.github/ci-groups.yml new file mode 100644 index 000000000..d1ab4e8a2 --- /dev/null +++ b/.github/ci-groups.yml @@ -0,0 +1,33 @@ +# Test group configuration for CI +# CI will fail if any workspace crate is not assigned to a group or excluded + +groups: + core: + - dashcore + - dashcore_hashes + - dashcore-private + - dash-network + + spv: + - dash-spv + + wallet: + - key-wallet + - key-wallet-manager + + ffi: + - dash-network-ffi + - dash-spv-ffi + - key-wallet-ffi + + rpc: + - dashcore-rpc + - dashcore-rpc-json + + tools: + - dashcore-test-utils + - dash-fuzz + +# Crates intentionally not tested (with reason) +excluded: + - integration_test # Requires live Dash node diff --git a/.github/ci-no-std.yml b/.github/ci-no-std.yml new file mode 100644 index 000000000..e82d8696c --- /dev/null +++ b/.github/ci-no-std.yml @@ -0,0 +1,16 @@ +# No-std build checks +# +# Lists crates that support no-std and their test configurations. +# Each entry runs: cargo check --no-default-features --features + +dashcore_hashes: + - bare # --no-default-features only + - alloc + - alloc serde # multiple features + +dashcore-private: + - bare + - alloc + +dash-network: + - no-std diff --git a/.github/scripts/ci_config.py b/.github/scripts/ci_config.py new file mode 100755 index 000000000..64e0e8e1d --- /dev/null +++ b/.github/scripts/ci_config.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""CI configuration management script. + +Used by GitHub Actions workflows for test management. + +Subcommands: + verify-groups Check all workspace crates are assigned to test groups + run-group Run tests for all crates in a group + run-no-std Run no-std build checks +""" + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +import yaml + + +def get_workspace_metadata(): + """Get workspace metadata from cargo.""" + result = subprocess.run( + ["cargo", "metadata", "--no-deps", "--format-version", "1"], + capture_output=True, + text=True, + check=True, + ) + return json.loads(result.stdout) + + +def load_yaml(path: Path): + """Load YAML file with error handling.""" + try: + with open(path) as f: + content = yaml.safe_load(f) + return content if content is not None else {} + except FileNotFoundError: + github_error(f"Configuration file not found: {path}") + sys.exit(1) + except yaml.YAMLError as e: + github_error(f"Invalid YAML in {path}: {e}") + sys.exit(1) + + +def github_error(msg: str): + """Print GitHub Actions error annotation.""" + print(f"::error::{msg}") + + +def github_notice(msg: str): + """Print GitHub Actions notice annotation.""" + print(f"::notice::{msg}") + + +def github_group_start(name: str): + """Start a GitHub Actions log group.""" + print(f"::group::{name}", flush=True) + + +def github_group_end(): + """End a GitHub Actions log group.""" + print("::endgroup::", flush=True) + + +def github_output(name: str, value: str): + """Write a GitHub Actions output variable.""" + output_file = os.environ.get("GITHUB_OUTPUT") + if output_file: + with open(output_file, "a") as f: + f.write(f"{name}={value}\n") + + +def verify_groups(args): + """Verify all workspace crates are assigned to test groups.""" + metadata = get_workspace_metadata() + workspace_crates = {pkg["name"] for pkg in metadata["packages"]} + + config = load_yaml(args.groups_file) + groups = config.get("groups", {}) + + assigned = set() + for group_crates in groups.values(): + if group_crates: + assigned.update(group_crates) + assigned.update(config.get("excluded", []) or []) + + unassigned = workspace_crates - assigned + if unassigned: + github_error( + f"Crates not assigned to any test group: {', '.join(sorted(unassigned))}" + ) + print("\nPlease add them to a group or 'excluded' section in ci-groups.yml") + return 1 + + print(f"All {len(workspace_crates)} workspace crates are assigned to test groups") + + # Output groups for GitHub Actions matrix + github_output("groups", json.dumps(list(groups.keys()))) + + return 0 + + +def run_no_std(args): + """Run no-std build checks from ci-no-std.yml. + + Format: crate_name: [list of configs] + Each config runs: cargo check -p crate --no-default-features --features + Special: 'bare' means just --no-default-features (no features) + """ + config = load_yaml(args.no_std_file) + + failed = [] + + for crate_name, entries in config.items(): + if not entries: + continue + + for entry in entries: + if not isinstance(entry, str) or not entry.strip(): + continue + + entry_clean = entry.strip() + + # Build cargo flags + if entry_clean == "bare": + flags = ["--no-default-features"] + display_name = "bare" + elif entry_clean == "no-std": + flags = ["--no-default-features", "--features", "no-std"] + display_name = "no-std" + elif " " in entry_clean: + # Multiple features (space-separated) + features = entry_clean.replace(" ", ",") + flags = ["--no-default-features", "--features", features] + display_name = entry_clean.replace(" ", "+") + else: + # Single feature + flags = ["--no-default-features", "--features", entry_clean] + display_name = entry_clean + + github_group_start(f"{crate_name} ({display_name})") + + cmd = ["cargo", "check", "-p", crate_name] + flags + result = subprocess.run(cmd) + + github_group_end() + + if result.returncode != 0: + failed.append(f"{crate_name} ({display_name})") + github_error(f"No-std check failed: {crate_name} with {' '.join(flags)}") + + if failed: + print("\n" + "=" * 40) + print("FAILED NO-STD CHECKS:") + for f in failed: + print(f" - {f}") + print("=" * 40) + return 1 + + return 0 + + +def run_group_tests(args): + """Run tests for all crates in a group.""" + config = load_yaml(args.groups_file) + groups = config.get("groups", {}) + + if args.group not in groups: + github_error(f"Unknown group: {args.group}") + return 1 + + crates = groups[args.group] or [] + failed = [] + + for crate in crates: + # Skip dash-fuzz on Windows + if args.os == "windows-latest" and crate == "dash-fuzz": + github_notice(f"Skipping {crate} on Windows (honggfuzz not supported)") + continue + + github_group_start(f"Testing {crate}") + + cmd = ["cargo", "test", "-p", crate, "--all-features"] + result = subprocess.run(cmd) + + github_group_end() + + if result.returncode != 0: + failed.append(crate) + github_error(f"Test failed for {crate} on {args.os}") + + if failed: + print("\n" + "=" * 40) + print(f"FAILED TESTS ({args.group} on {args.os}):") + for f in failed: + print(f" - {f}") + print("=" * 40) + return 1 + + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="CI configuration management") + parser.add_argument( + "--groups-file", + type=Path, + default=Path(".github/ci-groups.yml"), + help="Path to ci-groups.yml", + ) + parser.add_argument( + "--no-std-file", + type=Path, + default=Path(".github/ci-no-std.yml"), + help="Path to ci-no-std.yml", + ) + + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser("verify-groups", help="Verify all crates assigned to groups") + subparsers.add_parser("run-no-std", help="Run no-std checks") + + run_group_parser = subparsers.add_parser("run-group", help="Run tests for a group") + run_group_parser.add_argument("group", help="Group name") + run_group_parser.add_argument("--os", default="ubuntu-latest", help="OS name") + + args = parser.parse_args() + + commands = { + "verify-groups": verify_groups, + "run-no-std": run_no_std, + "run-group": run_group_tests, + } + + return commands[args.command](args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 000000000..bfe3079bd --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,32 @@ +name: Build and Test + +on: + workflow_call: + inputs: + os: + required: true + type: string + groups: + required: true + type: string + +permissions: + contents: read + +jobs: + test: + name: ${{ matrix.group }} + runs-on: ${{ inputs.os }} + strategy: + fail-fast: false + matrix: + group: ${{ fromJson(inputs.groups) }} + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "test-${{ inputs.os }}-${{ matrix.group }}" + - run: pip install pyyaml + - name: Run tests + run: python .github/scripts/ci_config.py run-group ${{ matrix.group }} --os ${{ inputs.os }} diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 4ea1576df..ecb8426eb 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -5,7 +5,7 @@ on: push: branches: - master - - 'test-ci/**' + - 'v**-dev' pull_request: permissions: @@ -18,12 +18,11 @@ concurrency: jobs: fuzz: if: ${{ !github.event.act }} - runs-on: ubuntu-22.04-arm + runs-on: ubuntu-latest strategy: fail-fast: false matrix: fuzz_target: [ - dash_outpoint_string, dash_deserialize_amount, # TODO fix the inputs placed in fuzz/hfuzz_input/deserialize_transaction/input # dash_deserialize_transaction, @@ -33,20 +32,21 @@ jobs: # dash_deserialize_prefilled_transaction, dash_deserialize_witness, # dash_deserialize_psbt, - dash_deserialize_block, + dash_outpoint_string, dash_deserialize_script, + dash_deserialize_block, + hashes_sha1, + hashes_sha256, + hashes_sha512, hashes_json, hashes_cbor, - hashes_sha256, hashes_ripemd160, hashes_sha512_256, - hashes_sha512, - hashes_sha1, ] steps: - name: Install test dependencies run: sudo apt-get update -y && sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc libiberty-dev - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -58,7 +58,7 @@ jobs: - name: fuzz run: cd fuzz && ./fuzz.sh "${{ matrix.fuzz_target }}" - run: echo "${{ matrix.fuzz_target }}" >executed_${{ matrix.fuzz_target }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v6 with: name: executed_${{ matrix.fuzz_target }} path: executed_${{ matrix.fuzz_target }} @@ -66,11 +66,13 @@ jobs: verify-execution: if: ${{ !github.event.act }} needs: fuzz - runs-on: ubuntu-22.04-arm + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + - uses: actions/checkout@v6 + - uses: actions/download-artifact@v6 - name: Display structure of downloaded files run: ls -R - - run: find executed_* -type f -exec cat {} + | sort > executed - - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed + - name: Verify all fuzz targets were executed + run: | + find executed_* -type f -exec cat {} + | sort > executed + source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index bb237ad8f..668a46428 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -7,6 +7,10 @@ on: - 'v**-dev' pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: pre-commit: name: Pre-commit (${{ matrix.os }}) @@ -14,16 +18,15 @@ jobs: strategy: fail-fast: false matrix: - # TODO: Windows excluded - rs-x11-hash requires POSIX headers (unistd.h) - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.x" + python-version: "3.12" - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -33,7 +36,7 @@ jobs: - name: Cache cargo dependencies uses: Swatinem/rust-cache@v2 with: - shared-key: "rust-cache-${{ matrix.os }}" + shared-key: "pre-commit-rust-${{ matrix.os }}" - name: Run pre-commit uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 25474d909..fea9d9efa 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,4 +1,5 @@ on: + workflow_dispatch: push: branches: - master @@ -15,184 +16,89 @@ concurrency: cancel-in-progress: true jobs: - spv_tests: - name: SPV Components Tests - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-22.04-arm + security: + name: Security Audit + runs-on: ubuntu-latest steps: - - name: Checkout Crate - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: Cache cargo dependencies - uses: Swatinem/rust-cache@v2 - - name: Test dash-spv crate (capture log) - id: spvtest - shell: bash - run: | - set +e - set -o pipefail - cargo test -p dash-spv --all-features 2>&1 | tee spv_test.log - status=${PIPESTATUS[0]} - echo "status=$status" >> "$GITHUB_OUTPUT" - if grep -qE 'SIGSEGV|signal: 11' spv_test.log; then - echo "segv=true" >> "$GITHUB_OUTPUT" - else - echo "segv=false" >> "$GITHUB_OUTPUT" - fi - # Always continue; follow-up steps decide based on outputs - exit 0 + - uses: actions/checkout@v6 + - uses: EmbarkStudios/cargo-deny-action@v2 - - name: Upload dash-spv test log - if: always() - uses: actions/upload-artifact@v4 - with: - name: dash-spv-test-log - path: spv_test.log + msrv: + name: MSRV Check (1.89) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@1.89 + - uses: Swatinem/rust-cache@v2 + - name: Check workspace compiles on MSRV + run: cargo check --workspace --all-features - - name: Scan for segfault culprit (dash-spv) - id: segvscan - if: ${{ steps.spvtest.outputs.segv == 'true' }} - shell: bash - run: | - set +e - echo "Detected SIGSEGV in parallel run; scanning tests sequentially..." - # List all tests for the crate - mapfile -t TESTS < <(cargo test -p dash-spv --all-features -- --list | sed -n 's/^\(.*\) ... .*/\1/p') - echo "Found ${#TESTS[@]} tests" - : > segv_scan.log - SEGV_TESTS=() - # Retry isolation up to 3 times to catch flaky crashes - for attempt in 1 2 3; do - echo "=== Isolation attempt $attempt/3 ===" | tee -a segv_scan.log - FOUND_THIS_ATTEMPT=() - for t in "${TESTS[@]}"; do - echo "--- Running $t (single-threaded) ---" | tee -a segv_scan.log - # Run test in isolation, single-threaded to improve determinism - cargo test -p dash-spv --all-features -- --nocapture --test-threads=1 "$t" 2>&1 | tee -a segv_scan.log - rc=${PIPESTATUS[0]} - if [ "$rc" -ne 0 ]; then - if tail -n 200 segv_scan.log | grep -qE 'SIGSEGV|signal: 11'; then - FOUND_THIS_ATTEMPT+=("$t") - SEGV_TESTS+=("$t") - fi - fi - done - if [ ${#FOUND_THIS_ATTEMPT[@]} -gt 0 ]; then - echo "Attempt $attempt found segfaulting tests: ${FOUND_THIS_ATTEMPT[*]}" | tee -a segv_scan.log - break - else - echo "Attempt $attempt found no segfaulting tests" | tee -a segv_scan.log - fi - done - if [ ${#SEGV_TESTS[@]} -eq 0 ]; then - echo "No consistent segfaulting tests found after 3 attempts" | tee -a segv_scan.log - fi - # De-duplicate the list for output - if [ ${#SEGV_TESTS[@]} -gt 0 ]; then - mapfile -t SEGV_UNIQ < <(printf "%s\n" "${SEGV_TESTS[@]}" | awk 'NF' | sort -u) - else - SEGV_UNIQ=() - fi - printf "%s\n" "Segfaulting tests detected: ${SEGV_UNIQ[*]-none}" | tee -a segv_scan.log - # Write outputs for later steps - { - echo 'segv_tests<> "$GITHUB_OUTPUT" - # Save scan log - echo "scan_log=segv_scan.log" >> "$GITHUB_OUTPUT" + docs: + name: Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Build documentation + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --workspace --all-features --no-deps - - name: Upload segv scan log - if: ${{ steps.spvtest.outputs.segv == 'true' }} - uses: actions/upload-artifact@v4 - with: - name: dash-spv-segv-scan-log - path: segv_scan.log + verify-execution: + name: Verify Test Execution + runs-on: ubuntu-latest + outputs: + groups: ${{ steps.verify.outputs.groups }} + steps: + - uses: actions/checkout@v6 + - run: pip install pyyaml + - name: Verify all crates are assigned and get matrix + id: verify + run: python .github/scripts/ci_config.py verify-groups - - name: Comment segfault summary on PR - if: ${{ github.event_name == 'pull_request' && steps.spvtest.outputs.segv == 'true' }} - uses: actions/github-script@v7 - with: - script: | - const tests = `${{ steps.segvscan.outputs.segv_tests }}`.trim(); - const lines = tests ? tests.split(/\r?\n/).filter(Boolean) : []; - const body = lines.length - ? `⚠️ SPV tests hit a SIGSEGV. Candidates (single-threaded isolation):\n\n${lines.map(t => `- ${t}`).join('\n')}` - : `⚠️ SPV tests hit a SIGSEGV, but single-test isolation found no consistent culprit. See artifacts for logs.`; - const issue_number = context.payload.pull_request.number; - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number, - body, - }); - - name: Fail job if initial tests failed - if: ${{ steps.spvtest.outputs.status != '0' }} - run: | - echo "Initial dash-spv tests failed with status ${{ steps.spvtest.outputs.status }}" >&2 - exit 1 - - name: Test dash-spv-ffi crate - run: cargo test -p dash-spv-ffi --all-features + test-ubuntu: + name: Ubuntu + needs: verify-execution + uses: ./.github/workflows/build-and-test.yml + with: + os: ubuntu-latest + groups: ${{ needs.verify-execution.outputs.groups }} - key_wallet_tests: - name: Key Wallet Components Tests - runs-on: ubuntu-22.04-arm - steps: - - name: Checkout Crate - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: Cache cargo dependencies - uses: Swatinem/rust-cache@v2 - - name: Test key-wallet crate - run: cargo test -p key-wallet --all-features - - name: Test key-wallet-manager crate - run: cargo test -p key-wallet-manager --all-features - - name: Test key-wallet-ffi crate - run: cargo test -p key-wallet-ffi --all-features + test-ubuntu-arm: + name: Ubuntu ARM + needs: verify-execution + uses: ./.github/workflows/build-and-test.yml + with: + os: ubuntu-22.04-arm + groups: ${{ needs.verify-execution.outputs.groups }} - core_components_tests: - name: Core Components Tests - runs-on: ubuntu-22.04-arm - steps: - - name: Checkout Crate - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: Cache cargo dependencies - uses: Swatinem/rust-cache@v2 - - name: Test dashcore crate - run: cargo test -p dashcore --all-features - - name: Test dashcore_hashes crate - run: cargo test -p dashcore_hashes --all-features - - name: Test dash-network crate - run: cargo test -p dash-network --all-features - - name: Test internals crate - run: cargo test -p dashcore-private --all-features - - name: Run script-based tests - run: ./contrib/test.sh + test-macos: + name: macOS + needs: verify-execution + uses: ./.github/workflows/build-and-test.yml + with: + os: macos-latest + groups: ${{ needs.verify-execution.outputs.groups }} - rpc_tests: - name: RPC Tests - runs-on: ubuntu-22.04-arm - strategy: - matrix: - include: - - rust: stable - env: - PIN_VERSIONS: true + test-windows: + name: Windows + needs: verify-execution + uses: ./.github/workflows/build-and-test.yml + with: + os: windows-latest + groups: ${{ needs.verify-execution.outputs.groups }} + + no-std: + name: No-std Checks + runs-on: ubuntu-latest steps: - - name: Checkout Crate - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: Running test script - env: ${{ matrix.env }} - run: ./contrib/test.sh + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: pip install pyyaml + - name: Run no-std checks + run: python .github/scripts/ci_config.py run-no-std integrations_tests: name: Integration Tests @@ -203,10 +109,8 @@ jobs: rust: [stable] dashversion: ["22.1.3"] steps: - - name: Checkout Crate - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable - name: Running test script env: DASHVERSION: ${{ matrix.dashversion }} diff --git a/.github/workflows/sanitizer.yml b/.github/workflows/sanitizer.yml new file mode 100644 index 000000000..da1dec750 --- /dev/null +++ b/.github/workflows/sanitizer.yml @@ -0,0 +1,68 @@ +name: Sanitizer + +on: + workflow_dispatch: + push: + branches: + - master + - 'v**-dev' + pull_request: + paths: + - 'key-wallet-ffi/**' + - 'dash-spv-ffi/**' + - 'dash-network-ffi/**' + - 'dashcore/**' + - 'dashcore_hashes/**' + - 'key-wallet/**' + - 'dash-spv/**' + - 'dash-network/**' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + asan: + name: Address Sanitizer + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@nightly + with: + components: rust-src + + - name: Run tests with ASAN + env: + RUSTFLAGS: "-Zsanitizer=address -Cdebuginfo=2" + ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer + ASAN_OPTIONS: "symbolize=1" + LSAN_OPTIONS: "fast_unwind_on_malloc=0" + run: | + # FFI crates (C interop) + cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \ + -p key-wallet-ffi -p dash-spv-ffi -p dash-network-ffi --lib --tests + + # Core crypto crates (unsafe optimizations) + cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \ + -p dashcore -p dashcore_hashes --lib --tests + + tsan: + name: Thread Sanitizer + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@nightly + with: + components: rust-src + + - name: Run tests with TSAN + env: + RUSTFLAGS: "-Zsanitizer=thread -Cdebuginfo=2" + TSAN_OPTIONS: "second_deadlock_stack=1" + run: | + # Async crate with concurrent code + cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \ + -p dash-spv --lib --tests diff --git a/contrib/test.sh b/contrib/test.sh deleted file mode 100755 index f3fd3c6d4..000000000 --- a/contrib/test.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -set -ex - -CRATES="dash hashes internals fuzz" - -for crate in ${CRATES} -do - ( - cd "$crate" - ./contrib/test.sh - ) -done - -exit 0 diff --git a/dash/contrib/test.sh b/dash/contrib/test.sh deleted file mode 100755 index 2a19cb731..000000000 --- a/dash/contrib/test.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/bin/sh - -set -ex - -FEATURES="base64 bitcoinconsensus serde rand secp-recovery" - -if [ "$DO_COV" = true ] -then - export RUSTFLAGS="-C link-dead-code" -fi - -cargo --version -rustc --version - -# Some tests require certain toolchain types. -NIGHTLY=false -STABLE=true -if cargo --version | grep nightly; then - STABLE=false - NIGHTLY=true -fi -if cargo --version | grep beta; then - STABLE=false -fi - -# Pin dependencies as required if we are using MSRV toolchain. -if cargo --version | grep "1\.48"; then - # 1.0.157 uses syn 2.0 which requires edition 2018 - cargo update -p serde --precise 1.0.156 -fi - -# TODO disable this for now, it's broken -# We should not have any duplicate dependencies. This catches mistakes made upgrading dependencies -# in one crate and not in another (e.g. upgrade bitcoin_hashes in bitcoin but not in secp). -#duplicate_dependencies=$( -# # Only show the actual duplicated deps, not their reverse tree, then -# # whitelist the 'syn' crate which is duplicated but it's not our fault. -# cargo tree --target=all --all-features --duplicates \ -# | grep '^[0-9A-Za-z]' \ -# | grep -v 'syn' \ -# | wc -l -#) -#if [ "$duplicate_dependencies" -ne 0 ]; then -# echo "Dependency tree is broken, contains duplicates" -# cargo tree --target=all --all-features --duplicates -# exit 1 -#fi - -if [ "$DO_LINT" = true ] -then - cargo clippy --all-features --all-targets -- -D warnings - cargo clippy --example bip32 -- -D warnings - cargo clippy --example handshake --features=rand-std -- -D warnings - cargo clippy --example ecdsa-psbt --features=bitcoinconsensus -- -D warnings - cargo clippy --example taproot-psbt --features=rand-std,bitcoinconsensus -- -D warnings -fi - -echo "********* Testing std *************" -# Test without any features other than std first -cargo test --verbose --no-default-features --features="std" - -echo "********* Testing default *************" -# Then test with the default features -cargo test --verbose - -if [ "$DO_NO_STD" = true ] -then - echo "********* Testing no-std build *************" - # Build no_std, to make sure that cfg(test) doesn't hide any issues - cargo build --verbose --features="no-std" --no-default-features - - # Build std + no_std, to make sure they are not incompatible - cargo build --verbose --features="no-std" - - # Test no_std - cargo test --verbose --features="no-std" --no-default-features - - # Build all features - cargo build --verbose --features="no-std $FEATURES" --no-default-features - - # Build specific features - for feature in ${FEATURES} - do - cargo build --verbose --features="no-std $feature" --no-default-features - done - - cargo run --example bip32 7934c09359b234e076b9fa5a1abfd38e3dc2a9939745b7cc3c22a48d831d14bd - cargo run --no-default-features --features no-std --example bip32 7934c09359b234e076b9fa5a1abfd38e3dc2a9939745b7cc3c22a48d831d14bd -fi - -# Test each feature -for feature in ${FEATURES} -do - echo "********* Testing $feature *************" - cargo test --verbose --features="$feature" -done - -# TODO need to fix these examples to support dash keys and addresses -#cargo run --example ecdsa-psbt --features=bitcoinconsensus -#cargo run --example taproot-psbt --features=rand-std,bitcoinconsensus - -# Build the docs if told to (this only works with the nightly toolchain) -if [ "$DO_DOCSRS" = true ]; then - RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo +nightly doc --all-features -fi - -# Build the docs with a stable toolchain, in unison with the DO_DOCSRS command -# above this checks that we feature guarded docs imports correctly. -if [ "$DO_DOCS" = true ]; then - RUSTDOCFLAGS="-D warnings" cargo +stable doc --all-features -fi - -# Run formatter if told to. -if [ "$DO_FMT" = true ]; then - if [ "$NIGHTLY" = false ]; then - echo "DO_FMT requires a nightly toolchain (consider using RUSTUP_TOOLCHAIN)" - exit 1 - fi - rustup component add rustfmt - cargo fmt --check -fi - -# Bench if told to, only works with non-stable toolchain (nightly, beta). -if [ "$DO_BENCH" = true ] -then - if [ "$STABLE" = true ]; then - if [ -n "$RUSTUP_TOOLCHAIN" ]; then - echo "RUSTUP_TOOLCHAIN is set to a stable toolchain but DO_BENCH requires a non-stable (beta, nightly) toolchain" - else - echo "DO_BENCH requires a non-stable (beta, nightly) toolchain" - fi - exit 1 - fi - RUSTFLAGS='--cfg=bench' cargo bench -fi - -# Use as dependency if told to -if [ "$AS_DEPENDENCY" = true ] -then - cargo new dep_test 2> /dev/null # Mute warning about workspace, fixed below. - cd dep_test - echo 'dashcore = { path = "..", features = ["serde"] }\n\n' >> Cargo.toml - # Adding an empty workspace section excludes this crate from the rust-bitcoin workspace. - echo '[workspace]\n\n' >> Cargo.toml - - cargo test --verbose -fi diff --git a/deny.toml b/deny.toml new file mode 100644 index 000000000..92b2788e1 --- /dev/null +++ b/deny.toml @@ -0,0 +1,38 @@ +# cargo-deny configuration +# https://embarkstudios.github.io/cargo-deny/ + +[advisories] +version = 2 +db-path = "~/.cargo/advisory-db" +db-urls = ["https://github.com/rustsec/advisory-db"] +ignore = [ + # serde_cbor is unmaintained but only used in fuzz targets + "RUSTSEC-2021-0127", +] + +[licenses] +version = 2 +allow = [ + "MIT", + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Unlicense", + "Zlib", + "CC0-1.0", + "MPL-2.0", + "Unicode-3.0", + "MITNFA", +] +confidence-threshold = 0.8 + +[bans] +multiple-versions = "warn" +wildcards = "allow" +highlight = "all" + +[sources] +unknown-registry = "deny" +unknown-git = "warn" +allow-registry = ["https://github.com/rust-lang/crates.io-index"] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 0f52e5294..d9b47e8e7 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -1,21 +1,18 @@ [package] name = "dash-fuzz" edition = "2024" -version = { workspace = true} +version = { workspace = true } authors = ["Generated by fuzz/generate-files.sh"] license = "CC0-1.0" publish = false [profile.release] -# Honggfuzz links with -nodefaultlibs and aggressive link flags; disable LTO to avoid -# symbol visibility/GC issues with recent Rust toolchains and rlibs. lto = false codegen-units = 16 debug = 1 panic = "unwind" [lints.rust] -# Silence `cfg(fuzzing)` lint in harness test modules unexpected_cfgs = { level = "warn", check-cfg = ["cfg(fuzzing)"] } [package.metadata] @@ -31,17 +28,13 @@ serde_json = "1.0" serde_cbor = "0.11" [[bin]] -name = "dash_outpoint_string" -path = "fuzz_targets/dash/outpoint_string.rs" +name = "dash_deserialize_prefilled_transaction" +path = "fuzz_targets/dash/deserialize_prefilled_transaction.rs" [[bin]] name = "dash_deserialize_amount" path = "fuzz_targets/dash/deserialize_amount.rs" -[[bin]] -name = "dash_deserialize_transaction" -path = "fuzz_targets/dash/deserialize_transaction.rs" - [[bin]] name = "dash_deser_net_msg" path = "fuzz_targets/dash/deser_net_msg.rs" @@ -54,25 +47,41 @@ path = "fuzz_targets/dash/deserialize_address.rs" name = "dash_script_bytes_to_asm_fmt" path = "fuzz_targets/dash/script_bytes_to_asm_fmt.rs" -[[bin]] -name = "dash_deserialize_prefilled_transaction" -path = "fuzz_targets/dash/deserialize_prefilled_transaction.rs" - [[bin]] name = "dash_deserialize_witness" path = "fuzz_targets/dash/deserialize_witness.rs" +[[bin]] +name = "dash_outpoint_string" +path = "fuzz_targets/dash/outpoint_string.rs" + +[[bin]] +name = "dash_deserialize_transaction" +path = "fuzz_targets/dash/deserialize_transaction.rs" + [[bin]] name = "dash_deserialize_psbt" path = "fuzz_targets/dash/deserialize_psbt.rs" +[[bin]] +name = "dash_deserialize_script" +path = "fuzz_targets/dash/deserialize_script.rs" + [[bin]] name = "dash_deserialize_block" path = "fuzz_targets/dash/deserialize_block.rs" [[bin]] -name = "dash_deserialize_script" -path = "fuzz_targets/dash/deserialize_script.rs" +name = "hashes_sha1" +path = "fuzz_targets/hashes/sha1.rs" + +[[bin]] +name = "hashes_sha256" +path = "fuzz_targets/hashes/sha256.rs" + +[[bin]] +name = "hashes_sha512" +path = "fuzz_targets/hashes/sha512.rs" [[bin]] name = "hashes_json" @@ -82,10 +91,6 @@ path = "fuzz_targets/hashes/json.rs" name = "hashes_cbor" path = "fuzz_targets/hashes/cbor.rs" -[[bin]] -name = "hashes_sha256" -path = "fuzz_targets/hashes/sha256.rs" - [[bin]] name = "hashes_ripemd160" path = "fuzz_targets/hashes/ripemd160.rs" @@ -93,11 +98,3 @@ path = "fuzz_targets/hashes/ripemd160.rs" [[bin]] name = "hashes_sha512_256" path = "fuzz_targets/hashes/sha512_256.rs" - -[[bin]] -name = "hashes_sha512" -path = "fuzz_targets/hashes/sha512.rs" - -[[bin]] -name = "hashes_sha1" -path = "fuzz_targets/hashes/sha1.rs" diff --git a/fuzz/contrib/test.sh b/fuzz/contrib/test.sh deleted file mode 100755 index edce7845b..000000000 --- a/fuzz/contrib/test.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh - -set -ex - -FEATURES="" - -cargo --version -rustc --version - -# Make all cargo invocations verbose -export CARGO_TERM_VERBOSE=true - -# Pin dependencies as required if we are using MSRV toolchain. -if cargo --version | grep "1\.41"; then - # 1.0.157 uses syn 2.0 which requires edition 2021 - cargo update -p serde --precise 1.0.156 - # 1.0.108 uses `matches!` macro so does not work with Rust 1.41.1, bad `syn` no biscuit. - cargo update -p syn --precise 1.0.107 - # Half 1.8 uses edition 2021 features - cargo update -p half --precise 1.7.1 -fi - -if [ "$DO_LINT" = true ] -then - cargo clippy --all-features --all-targets -- -D warnings -fi - -# Defaults / sanity checks -cargo build -cargo test - -# Address Sanitizer -if [ "$DO_ASAN" = true ]; then - cargo clean - CC='clang -fsanitize=address -fno-omit-frame-pointer' \ - RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \ - ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \ - cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu - cargo clean - CC='clang -fsanitize=memory -fno-omit-frame-pointer' \ - RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \ - cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu -fi diff --git a/fuzz/fuzz-util.sh b/fuzz/fuzz-util.sh index 0afe24222..e09844207 100755 --- a/fuzz/fuzz-util.sh +++ b/fuzz/fuzz-util.sh @@ -2,13 +2,19 @@ REPO_DIR=$(git rev-parse --show-toplevel) -listTargetFiles() { +listAllTargetFiles() { pushd "$REPO_DIR/fuzz" > /dev/null || exit 1 - find fuzz_targets -type f -name "*.rs" \ - | grep -v 'deserialize_transaction\|deserialize_prefilled_transaction\|deserialize_psbt' + # List dash targets first, then hashes + find fuzz_targets/dash -type f -name "*.rs" 2>/dev/null + find fuzz_targets/hashes -type f -name "*.rs" 2>/dev/null popd > /dev/null || exit 1 } +listTargetFiles() { + # Exclude targets that don't work in CI + listAllTargetFiles | grep -v 'deserialize_transaction\|deserialize_prefilled_transaction\|deserialize_psbt' +} + targetFileToName() { echo "$1" \ | sed 's/^fuzz_targets\///' \ diff --git a/fuzz/generate-files.sh b/fuzz/generate-files.sh index 5e29c82c1..4bb1e4705 100755 --- a/fuzz/generate-files.sh +++ b/fuzz/generate-files.sh @@ -12,24 +12,34 @@ cat > "$REPO_DIR/fuzz/Cargo.toml" <> "$REPO_DIR/fuzz/Cargo.toml" < target" - name: fuzz run: cd fuzz && ./fuzz.sh "\${{ matrix.fuzz_target }}" - run: echo "\${{ matrix.fuzz_target }}" >executed_\${{ matrix.fuzz_target }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v6 with: name: executed_\${{ matrix.fuzz_target }} path: executed_\${{ matrix.fuzz_target }} @@ -91,10 +103,12 @@ $(for name in $(listTargetNames); do echo "$name,"; done) needs: fuzz runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + - uses: actions/checkout@v6 + - uses: actions/download-artifact@v6 - name: Display structure of downloaded files run: ls -R - - run: find executed_* -type f -exec cat {} + | sort > executed - - run: source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed + - name: Verify all fuzz targets were executed + run: | + find executed_* -type f -exec cat {} + | sort > executed + source ./fuzz/fuzz-util.sh && listTargetNames | sort | diff - executed EOF diff --git a/hashes/contrib/test.sh b/hashes/contrib/test.sh deleted file mode 100755 index b934976d1..000000000 --- a/hashes/contrib/test.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -FEATURES="serde serde-std std core2" - -if [ "$DO_ALLOC_TESTS" = true ]; then - FEATURES="$FEATURES alloc" -fi - -cargo --version -rustc --version - -# Work out if we are using a nightly toolchain. -NIGHTLY=false -if cargo --version | grep nightly >/dev/null; then - NIGHTLY=true -fi - -# Pin dependencies as required if we are using MSRV toolchain. -if cargo --version | grep "1\.48"; then - # 1.0.157 uses syn 2.0 which requires edition 2021 - cargo update -p serde --precise 1.0.156 -fi - -# Make all cargo invocations verbose -export CARGO_TERM_VERBOSE=true - -# Defaults / sanity checks -cargo build -cargo test - -if [ "$DO_LINT" = true ] -then - cargo clippy --all-features --all-targets -- -D warnings -fi - -if [ "$DO_FEATURE_MATRIX" = true ]; then - cargo build --no-default-features - cargo test --no-default-features - - # All features - cargo build --no-default-features --features="$FEATURES" - cargo test --no-default-features --features="$FEATURES" - # Single features - for feature in ${FEATURES} - do - cargo build --no-default-features --features="$feature" - cargo test --no-default-features --features="$feature" - # All combos of two features - for featuretwo in ${FEATURES}; do - cargo build --no-default-features --features="$feature $featuretwo" - cargo test --no-default-features --features="$feature $featuretwo" - done - done - - # Other combos - cargo test --no-default-features --features="std,schemars" -fi - -REPO_DIR=$(git rev-parse --show-toplevel) - -if [ "$DO_SCHEMARS_TESTS" = true ]; then - pushd "$REPO_DIR/hashes/extended_tests/schemars" > /dev/null - cargo test - popd > /dev/null -fi - -# Build the docs if told to (this only works with the nightly toolchain) -if [ "$DO_DOCSRS" = true ]; then - RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo +nightly doc --all-features -fi - -# Build the docs with a stable toolchain, in unison with the DO_DOCSRS command -# above this checks that we feature guarded docs imports correctly. -if [ "$DO_DOCS" = true ]; then - RUSTDOCFLAGS="-D warnings" cargo +stable doc --all-features -fi - -# Webassembly stuff -if [ "$DO_WASM" = true ]; then - clang --version && - CARGO_TARGET_DIR=wasm cargo install --force wasm-pack && - printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml && - CC=clang-9 wasm-pack build && - CC=clang-9 wasm-pack test --node; -fi - -# Address Sanitizer -if [ "$DO_ASAN" = true ]; then - cargo clean - CC='clang -fsanitize=address -fno-omit-frame-pointer' \ - RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \ - ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' \ - cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu - cargo clean - CC='clang -fsanitize=memory -fno-omit-frame-pointer' \ - RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \ - cargo test --lib --no-default-features --features="$FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu -fi - -# Run formatter if told to. -if [ "$DO_FMT" = true ]; then - if [ "$NIGHTLY" = false ]; then - echo "DO_FMT requires a nightly toolchain (consider using RUSTUP_TOOLCHAIN)" - exit 1 - fi - rustup component add rustfmt - cargo fmt --check -fi - -# Bench if told to, only works with non-stable toolchain (nightly, beta). -if [ "$DO_BENCH" = true ] -then - if [ "$NIGHTLY" = false ] - then - if [ -n "$RUSTUP_TOOLCHAIN" ] - then - echo "RUSTUP_TOOLCHAIN is set to a non-nightly toolchain but DO_BENCH requires a nightly toolchain" - else - echo "DO_BENCH requires a nightly toolchain" - fi - exit 1 - fi - RUSTFLAGS='--cfg=bench' cargo bench -fi diff --git a/internals/contrib/test.sh b/internals/contrib/test.sh deleted file mode 100755 index 13f2d3d8a..000000000 --- a/internals/contrib/test.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh - -set -ex - -FEATURES="std alloc" - -cargo --version -rustc --version - -# Work out if we are using a nightly toolchain. -NIGHTLY=false -if cargo --version | grep nightly >/dev/null; then - NIGHTLY=true -fi - -# Make all cargo invocations verbose -export CARGO_TERM_VERBOSE=true - -# Defaults / sanity checks -cargo build -cargo test - -if [ "$DO_LINT" = true ] -then - cargo clippy --all-features --all-targets -- -D warnings -fi - -if [ "$DO_FEATURE_MATRIX" = true ]; then - # No features - cargo build --no-default-features - cargo test --no-default-features - - # All features - cargo build --no-default-features --features="$FEATURES" - cargo test --no-default-features --features="$FEATURES" - - # Single features - for feature in ${FEATURES} - do - cargo build --no-default-features --features="$feature" - cargo test --no-default-features --features="$feature" - done -fi - -# Build the docs if told to (this only works with the nightly toolchain) -if [ "$DO_DOCSRS" = true ]; then - RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo +nightly doc --all-features -fi - -# Build the docs with a stable toolchain, in unison with the DO_DOCSRS command -# above this checks that we feature guarded docs imports correctly. -if [ "$DO_DOCS" = true ]; then - RUSTDOCFLAGS="-D warnings" cargo +stable doc --all-features -fi - -# Run formatter if told to. -if [ "$DO_FMT" = true ]; then - if [ "$NIGHTLY" = false ]; then - echo "DO_FMT requires a nightly toolchain (consider using RUSTUP_TOOLCHAIN)" - exit 1 - fi - rustup component add rustfmt - cargo fmt --check -fi