diff --git a/.github/actions/fetch-artifact/actions.yaml b/.github/actions/fetch-artifact/actions.yaml new file mode 100644 index 000000000000..2d9ad4c30fb4 --- /dev/null +++ b/.github/actions/fetch-artifact/actions.yaml @@ -0,0 +1,31 @@ +name: Fetch artifact (local or GH) +description: Fetch artifact from local cache if available, otherwise download + +inputs: + name: + required: true + path: + required: false + default: . + +runs: + using: "composite" + steps: + - shell: bash + run: | + CACHE_BASE="${CACHE_BASE:-/var/lib/github-runner/cache}" + CACHE_DIR="$CACHE_BASE/${{ github.run_id }}" + + if [ -d "$CACHE_DIR" ] && [ -f "$CACHE_DIR/${{ inputs.name }}" ]; then + echo "Using local cached artifact" + cp "$CACHE_DIR/${{ inputs.name }}" "${{ inputs.path }}/" + else + echo "Falling back to GitHub artifact" + echo "DOWNLOAD_FROM_GH=1" >> "$GITHUB_ENV" + fi + + - uses: actions/download-artifact@v4 + if: env.DOWNLOAD_FROM_GH == '1' + with: + name: ${{ inputs.name }} + path: ${{ inputs.path }} diff --git a/.github/scripts/setup.sh b/.github/scripts/setup.sh index 2409cdf0d543..f87cb486e1a4 100755 --- a/.github/scripts/setup.sh +++ b/.github/scripts/setup.sh @@ -3,6 +3,13 @@ set -e export DEBIAN_FRONTEND=noninteractive export RUST_VERSION=stable +# If you have your own runner, you can pre-setup and avoid this +# (also means you don't need to allow sudo!) +if [ -f "$HOME"/runner-no-setup ]; then + echo "Runner does not need setup, skipping" + exit 0 +fi + sudo useradd -ms /bin/bash tester sudo apt-get update -qq @@ -67,10 +74,6 @@ sudo chmod 0440 /etc/sudoers.d/tester curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \ -y --default-toolchain ${RUST_VERSION} -uv sync --all-extras --all-groups -# required for reckless till poetry to uv migration -uv tool install poetry - # We also need a relatively recent protobuf-compiler, at least 3.12.0, # in order to support the experimental `optional` flag. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7dda61720455..6ca1d1883eb1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,9 +11,6 @@ concurrency: cancel-in-progress: true env: - # Makes the upload-artifact work more reliably at the cost - # of a bit of compile time. - RUST_PROFILE: release SLOW_MACHINE: 1 CI_SERVER_URL: "http://35.239.136.52:3170" PYTEST_OPTS_BASE: "-vvv --junit-xml=report.xml --timeout=1800 --durations=10" @@ -22,7 +19,10 @@ env: jobs: prebuild: name: Pre-build checks - runs-on: ubuntu-24.04 + runs-on: &prefer-self-hosted > + ${{ github.actor == 'rustyrussell' + && fromJSON('["self-hosted","rusty"]') + || 'ubuntu-24.04' }} timeout-minutes: 120 if: | github.event.action != 'edited' || @@ -82,10 +82,13 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh - # We're going to check BOLT quotes, so get the latest version - git clone https://github.com/lightning/bolts.git ../${BOLTDIR} + uv sync --all-extras --all-groups + # We're going to check BOLT quotes, so get the latest version (or refresh) + git clone https://github.com/lightning/bolts.git ../${BOLTDIR} || git -C ../${BOLTDIR} pull - name: Configure run: ./configure --enable-debugbuild --enable-rust - name: Check source @@ -93,7 +96,7 @@ jobs: VALGRIND: 0 PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} run: | - uv run make check-source BASE_REF="origin/${{ github.base_ref }}" + uv run make -j $(nproc) check-source BASE_REF="origin/${{ github.base_ref }}" - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -101,14 +104,10 @@ jobs: name: pytest-results-prebuild path: report.xml if-no-files-found: ignore - - name: Check Generated Files have been updated - run: uv run make check-gen-updated - - name: Check docs - run: uv run make check-doc compile: name: Compile CLN ${{ matrix.cfg }} - runs-on: ubuntu-24.04 + runs-on: *prefer-self-hosted timeout-minutes: 30 needs: - prebuild @@ -145,8 +144,11 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh + uv sync --all-extras --all-groups - name: Build env: @@ -164,18 +166,74 @@ jobs: # Rename now so we don't clash mv testpack.tar.bz2 cln-${CFG}.tar.bz2 - - name: Check rust packages - run: cargo test --all + - name: Save build locally if cache dir exists + run: | + CACHE_BASE="/var/lib/github-runner/cache/" + if [ -d "$CACHE_BASE" ]; then + CACHE_DIR="$CACHE_BASE/${{ github.run_id }}" + echo "Using local cache directory" + mkdir -p -m g+w "$CACHE_DIR" + # Might exist from previous run: overwrite. + cp -f cln-${{ matrix.CFG }}.tar.bz2 "$CACHE_DIR/" + echo "LOCAL_ARTIFACT=1" >> "$GITHUB_ENV" + else + echo "No local cache directory found" + echo "LOCAL_ARTIFACT=0" >> "$GITHUB_ENV" + fi - uses: actions/upload-artifact@v4 + if: env.LOCAL_ARTIFACT != '1' with: name: cln-${{ matrix.CFG }}.tar.bz2 path: cln-${{ matrix.CFG }}.tar.bz2 + check-compiled-source: + runs-on: > + ${{ github.actor == 'rustyrussell' + && fromJSON('["self-hosted","rusty"]') + || 'ubuntu-24.04' }} + needs: + - compile + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} + run: | + bash -x .github/scripts/setup.sh + uv sync --all-extras --all-groups + + - name: Download build + uses: ./.github/actions/fetch-artifact + with: + name: cln-compile-gcc.tar.bz2 + + - name: Unpack pre-built CLN + env: + CFG: ${{ matrix.CFG }} + run: | + tar -xaf cln-compile-gcc.tar.bz2 + + - name: Check + run: | + uv run eatmydata make -j $(nproc) check-source-bolt check-python check-gen-updated check-doc + - name: Check rust packages + run: cargo test --all + check-units: # The unit test checks are not in the critical path (not dependent # on the integration tests), so run them with `valgrind` name: Run unit tests - runs-on: ubuntu-24.04 + runs-on: *prefer-self-hosted timeout-minutes: 60 env: BOLTDIR: bolts @@ -202,14 +260,16 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh - sudo apt-get update -qq - # We're going to check BOLT quotes, so get the latest version - git clone https://github.com/lightning/bolts.git ../${BOLTDIR} + uv sync --all-extras --all-groups + # We're going to check BOLT quotes, so get the latest version (or refresh) + git clone https://github.com/lightning/bolts.git ../${BOLTDIR} || git -C ../${BOLTDIR} pull - name: Download build - uses: actions/download-artifact@v4 + uses: ./.github/actions/fetch-artifact with: name: cln-${{ matrix.CFG }}.tar.bz2 @@ -220,7 +280,7 @@ jobs: check-fuzz: name: Run fuzz regression tests - runs-on: ubuntu-24.04 + runs-on: *prefer-self-hosted needs: - prebuild steps: @@ -236,8 +296,11 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh + uv sync --all-extras --all-groups - name: Build run: | @@ -246,7 +309,7 @@ jobs: check-downgrade: name: Check we can downgrade the node - runs-on: ubuntu-24.04 + runs-on: *prefer-self-hosted needs: - compile strategy: @@ -276,16 +339,14 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Install bitcoind env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - run: .github/scripts/install-bitcoind.sh + run: | + bash -x .github/scripts/setup.sh + uv sync --all-extras --all-groups - name: Download build - uses: actions/download-artifact@v4 + uses: ./.github/actions/fetch-artifact with: name: cln-${{ matrix.CFG }}.tar.bz2 @@ -312,7 +373,6 @@ jobs: - name: Test env: SLOW_MACHINE: 1 - PYTEST_PAR: 4 TEST_DEBUG: 1 TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -323,7 +383,7 @@ jobs: run: | env cat config.vars - uv run eatmydata pytest tests/test_downgrade.py -n ${PYTEST_PAR} ${PYTEST_OPTS} + uv run eatmydata pytest tests/test_downgrade.py -n $(($(nproc) + 1)) ${PYTEST_OPTS} - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -334,10 +394,9 @@ jobs: integration: name: Test CLN ${{ matrix.name }} - runs-on: ubuntu-24.04 + runs-on: *prefer-self-hosted timeout-minutes: 120 env: - RUST_PROFILE: release # Has to match the one in the compile step PYTEST_OPTS: -vvv --junit-xml=report.xml --timeout=1800 --durations=10 needs: - compile @@ -396,16 +455,14 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Install bitcoind env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - run: .github/scripts/install-bitcoind.sh + run: | + bash -x .github/scripts/setup.sh + uv sync --all-extras --all-groups - name: Download build - uses: actions/download-artifact@v4 + uses: ./.github/actions/fetch-artifact with: name: cln-${{ matrix.CFG }}.tar.bz2 @@ -430,7 +487,6 @@ jobs: COMPAT: 1 CFG: ${{ matrix.CFG }} SLOW_MACHINE: 1 - PYTEST_PAR: 4 TEST_DEBUG: 1 TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -438,7 +494,7 @@ jobs: run: | env cat config.vars - VALGRIND=0 uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS} + VALGRIND=0 uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS} - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -449,10 +505,9 @@ jobs: integration-valgrind: name: Valgrind Test CLN ${{ matrix.name }} - runs-on: ubuntu-24.04 + runs-on: *prefer-self-hosted timeout-minutes: 120 env: - RUST_PROFILE: release # Has to match the one in the compile step CFG: compile-gcc PYTEST_OPTS: -vvv --junit-xml=report.xml --timeout=1800 --durations=10 --test-group-random-seed=42 needs: @@ -504,16 +559,14 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | - sudo apt-get update -qq - sudo apt-get install -yyq valgrind bash -x .github/scripts/setup.sh - - - name: Install bitcoind - run: .github/scripts/install-bitcoind.sh + uv sync --all-extras --all-groups - name: Download build - uses: actions/download-artifact@v4 + uses: ./.github/actions/fetch-artifact with: name: cln-compile-gcc.tar.bz2 @@ -524,9 +577,8 @@ jobs: env: SLOW_MACHINE: 1 TEST_DEBUG: 1 - PYTEST_PAR: 2 run: | - VALGRIND=1 uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} + VALGRIND=1 uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -537,10 +589,9 @@ jobs: integration-sanitizers: name: Sanitizers Test CLN - runs-on: ubuntu-24.04 + runs-on: *prefer-self-hosted timeout-minutes: 120 env: - RUST_PROFILE: release SLOW_MACHINE: 1 TEST_DEBUG: 1 PYTEST_OPTS: -vvv --junit-xml=report.xml --timeout=1800 --durations=10 --test-group-random-seed=42 @@ -599,14 +650,14 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh - - - name: Install bitcoind - run: .github/scripts/install-bitcoind.sh + uv sync --all-extras --all-groups - name: Download build - uses: actions/download-artifact@v4 + uses: ./.github/actions/fetch-artifact with: name: cln-compile-clang-sanitizers.tar.bz2 @@ -614,10 +665,8 @@ jobs: run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2 - name: Test - env: - PYTEST_PAR: 2 run: | - uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} + uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -629,7 +678,7 @@ jobs: update-docs-examples: name: Update examples in doc schemas (disabled temporarily!) if: false - runs-on: ubuntu-24.04 + runs-on: *prefer-self-hosted timeout-minutes: 30 strategy: fail-fast: false @@ -652,14 +701,14 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh - - name: Install bitcoind - env: - TEST_NETWORK: regtest - run: .github/scripts/install-bitcoind.sh + uv sync --all-extras --all-groups + - name: Download build - uses: actions/download-artifact@v4 + uses: ./.github/actions/fetch-artifact with: name: cln-compile-gcc.tar.bz2 - name: Unpack pre-built CLN @@ -667,7 +716,7 @@ jobs: tar -xaf cln-compile-gcc.tar.bz2 - name: Test run: | - uv run eatmydata make -j $(nproc) check-doc-examples + uv run eatmydata make -j $(($(nproc) + 1)) check-doc-examples - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -678,10 +727,9 @@ jobs: min-btc-support: name: Test minimum supported BTC v${{ matrix.MIN_BTC_VERSION }} with ${{ matrix.NAME }} - runs-on: ubuntu-24.04 + runs-on: *prefer-self-hosted timeout-minutes: 120 env: - RUST_PROFILE: release # Has to match the one in the compile step PYTEST_OPTS: -vvv --junit-xml=report.xml --timeout=1800 --durations=10 needs: - compile @@ -708,8 +756,11 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Install dependencies + env: + TEST_NETWORK: ${{ matrix.TEST_NETWORK }} run: | bash -x .github/scripts/setup.sh + uv sync --all-extras --all-groups - name: Download Bitcoin Core run: wget "https://bitcoincore.org/bin/bitcoin-core-${{ matrix.MIN_BTC_VERSION }}/bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz" @@ -724,7 +775,7 @@ jobs: run: rm -rf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz" "bitcoin-${{ matrix.MIN_BTC_VERSION }}" - name: Download build - uses: actions/download-artifact@v4 + uses: ./.github/actions/fetch-artifact with: name: cln-${{ matrix.CFG }}.tar.bz2 @@ -740,7 +791,6 @@ jobs: COMPAT: 1 CFG: ${{ matrix.CFG }} SLOW_MACHINE: 1 - PYTEST_PAR: 4 TEST_DEBUG: 1 TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -748,7 +798,7 @@ jobs: run: | env cat config.vars - VALGRIND=0 uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS} + VALGRIND=0 uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS} - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -771,11 +821,12 @@ jobs: - integration-sanitizers - min-btc-support - check-downgrade + - check-compiled-source if: ${{ always() }} steps: - name: Complete env: - JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC" + JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC CHECK_COMPILED_SOURCE" INTEGRATION: ${{ needs.integration.result }} CHECK_UNITS: ${{ needs['check-units'].result }} VALGRIND: ${{ needs['integration-valgrind'].result }} @@ -783,6 +834,7 @@ jobs: DOCS: ${{ needs['update-docs-examples'].result }} BTC: ${{ needs['min-btc-support'].result }} CHECK_DOWNGRADE: ${{ needs['check-downgrade'].result }} + CHECK_COMPILED_SOURCE: ${{ needs['check-compiled-source'].result }} run: | failed="" for name in $JOB_NAMES; do diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml deleted file mode 100644 index 71ca8e9d235f..000000000000 --- a/.github/workflows/macos.yaml +++ /dev/null @@ -1,73 +0,0 @@ ---- -name: Mac OS pytest -on: - pull_request: -jobs: - smoke-test: - name: Smoke Test macOS - runs-on: macos-14 - timeout-minutes: 120 - strategy: - fail-fast: true - matrix: - bitcoind-version: ["27.1"] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Download Bitcoin ${{ matrix.bitcoind-version }} & install binaries - run: | - export BITCOIND_VERSION=${{ matrix.bitcoind-version }} - export TARGET_ARCH="arm64-apple-darwin" - - wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz - tar -xzf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz - sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin - rm -rf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz bitcoin-${BITCOIND_VERSION} - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH" - - brew install gnu-sed autoconf automake libtool protobuf openssl lowdown libsodium - - # https://github.com/grpc/grpc/issues/31737#issuecomment-1323796842 - export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 - export GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 - uv sync --all-groups - - - name: Build and install CLN - run: | - export CPATH=/opt/homebrew/include - export LIBRARY_PATH=/opt/homebrew/lib - - uv run ./configure --disable-valgrind --disable-compat - uv run make - - - name: Start bitcoind in regtest mode - run: | - bitcoind -regtest -daemon - sleep 5 - - - name: Generate initial block - run: | - bitcoin-cli -regtest createwallet default_wallet - bitcoin-cli -regtest generatetoaddress 1 $(bitcoin-cli -regtest getnewaddress) - sleep 2 - - - name: Start CLN in regtest mode - run: | - lightningd/lightningd --network=regtest --log-file=/tmp/l1.log --daemon - sleep 5 - - - name: Verify CLN is running - run: | - cli/lightning-cli --regtest getinfo diff --git a/Makefile b/Makefile index fd6c8aa7a203..7901023ed551 100644 --- a/Makefile +++ b/Makefile @@ -643,10 +643,8 @@ update-doc-examples: check-doc-examples: update-doc-examples git diff --exit-code HEAD -# For those without working cppcheck -check-source-no-cppcheck: check-makefile check-source-bolt check-whitespace check-spelling check-python check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access check-bad-sprintf - -check-source: check-source-no-cppcheck +# This should NOT compile things! +check-source: check-makefile check-whitespace check-spelling check-python-flake8 check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access check-bad-sprintf full-check: check check-source diff --git a/common/Makefile b/common/Makefile index 333e3321cb6a..adf23ed02d29 100644 --- a/common/Makefile +++ b/common/Makefile @@ -159,8 +159,8 @@ common/htlc_state_names_gen.h: common/htlc_state.h ccan/ccan/cdump/tools/cdump-e common/gossip_store.o: gossipd/gossip_store_wiregen.h -check-source-bolt: $(COMMON_SRC_NOGEN:%=bolt-check/%) $(COMMON_HEADERS:%=bolt-check/%) -check-whitespace: $(COMMON_SRC_NOGEN:%=check-whitespace/%) $(COMMON_HEADERS:%=check-whitespace/%) +check-source-bolt: $(COMMON_SRC_NOGEN:%=bolt-check/%) $(COMMON_HEADERS_NOGEN:%=bolt-check/%) +check-whitespace: $(COMMON_SRC_NOGEN:%=check-whitespace/%) $(COMMON_HEADERS_NOGEN:%=check-whitespace/%) clean: common-clean