diff --git a/.github/actions/install-deps/action.yml b/.github/actions/install-deps/action.yml deleted file mode 100644 index 0fa119037..000000000 --- a/.github/actions/install-deps/action.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: 'Install Dependencies' -description: 'Install common dependencies' - -runs: - using: 'composite' - steps: - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y curl ca-certificates gnupg ocl-icd-opencl-dev libhwloc-dev - shell: bash - - - name: Fetch all tags - run: git fetch --all - shell: bash - - - name: Sync submodules - run: git submodule sync - shell: bash - - - name: Update submodules - run: git submodule update --init - shell: bash - diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml new file mode 100644 index 000000000..316058263 --- /dev/null +++ b/.github/actions/setup-build-env/action.yml @@ -0,0 +1,64 @@ +name: 'Setup Build Environment' +description: 'Setup Go, install deps, cache/build FFI - everything needed to build' + +inputs: + go-version: + description: 'Go version to use' + required: true + +runs: + using: 'composite' + steps: + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + cache: true + cache-dependency-path: go.sum + + - name: Install system dependencies (Ubuntu only) + run: | + if command -v apt-get &> /dev/null; then + sudo apt-get update + sudo apt-get install -y curl ca-certificates gnupg ocl-icd-opencl-dev libhwloc-dev + fi + shell: bash + + - name: Download Go modules + run: go mod download + shell: bash + + # Cache FFI build based on submodule commit + - name: Generate FFI cache key + id: ffi-cache-key + run: | + FFI_COMMIT=$(git -C extern/filecoin-ffi rev-parse HEAD 2>/dev/null || echo "unknown") + echo "key=ffi-${{ runner.os }}-${{ inputs.go-version }}-${FFI_COMMIT}" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache FFI build + id: cache-ffi + uses: actions/cache@v4 + with: + path: | + extern/filecoin-ffi/.install-filcrypto + extern/filecoin-ffi/filcrypto.h + extern/filecoin-ffi/libfilcrypto.a + extern/filecoin-ffi/filcrypto.pc + build/.filecoin-install + build/.blst-install + extern/supraseal/.install-blst + extern/supraseal/deps/blst + key: ${{ steps.ffi-cache-key.outputs.key }} + + - name: Build FFI + if: steps.cache-ffi.outputs.cache-hit != 'true' + run: make deps + shell: bash + + - name: Restore FFI marker files + if: steps.cache-ffi.outputs.cache-hit == 'true' + run: | + mkdir -p build + touch build/.filecoin-install build/.blst-install || true + shell: bash diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml deleted file mode 100644 index b23c1bea5..000000000 --- a/.github/actions/setup-go/action.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: 'Setup Go' -description: 'Setup Go environment' - -inputs: - go-version: - description: 'Go version to use' - required: true - -runs: - using: 'composite' - steps: - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: ${{ inputs.go-version }} - cache: false diff --git a/.github/images/build-env/Dockerfile b/.github/images/build-env/Dockerfile new file mode 100644 index 000000000..cb7096bed --- /dev/null +++ b/.github/images/build-env/Dockerfile @@ -0,0 +1,41 @@ +# Build environment for Curio CI +# Contains: Go, system dependencies, and common tools +# Rebuild when: Go version changes, system deps change, go.mod/go.sum change + +ARG GO_VERSION=1.24.7 + +FROM golang:${GO_VERSION}-bookworm + +# Install system dependencies (same as install-deps action) +RUN apt-get update && apt-get install -y \ + curl \ + ca-certificates \ + gnupg \ + ocl-icd-opencl-dev \ + libhwloc-dev \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Install Docker CLI (for test-itest to communicate with host Docker daemon) +RUN install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \ + chmod a+r /etc/apt/keyrings/docker.asc && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" > /etc/apt/sources.list.d/docker.list && \ + apt-get update && \ + apt-get install -y docker-ce-cli && \ + rm -rf /var/lib/apt/lists/* + +# Set up workspace +WORKDIR /workspace + +# Copy go.mod and go.sum to pre-download modules +COPY go.mod go.sum ./ + +# Pre-download Go modules (will be cached in /go/pkg/mod) +RUN go mod download + +# Install common dev/CI tools (from tools/tools.go) +RUN go install golang.org/x/tools/cmd/goimports@latest && \ + go install github.com/hannahhoward/cbor-gen-for@latest && \ + go install github.com/swaggo/swag/cmd/swag@latest + diff --git a/.github/workflows/build-env-image.yml b/.github/workflows/build-env-image.yml new file mode 100644 index 000000000..50cfaefc5 --- /dev/null +++ b/.github/workflows/build-env-image.yml @@ -0,0 +1,51 @@ +name: Build CI Environment Image + +on: + push: + branches: [main] + paths: + - '.github/images/build-env/**' + - '.github/workflows/build-env-image.yml' + - 'go.mod' + - 'go.sum' + workflow_dispatch: # Allow manual trigger + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/build-env + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v4 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest + type=sha,prefix= + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: .github/images/build-env/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3be08478f..7285142f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,575 +9,448 @@ on: - '**' env: - GO_VERSION: 1.24 + GO_VERSION: 1.24.7 jobs: - ci-lint: + # Fast checks - no FFI needed, run immediately + quick-checks: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - name: Setup Go - uses: ./.github/actions/setup-go + uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} + cache: true + cache-dependency-path: go.sum - - name: Install actionlint - run: go install github.com/rhysd/actionlint/cmd/actionlint@latest - - - name: Run actionlint - run: actionlint -shellcheck= -pyflakes= - - setup-params: - runs-on: [self-hosted, docker] - needs: [ci-lint] - steps: - - name: Fetch parameters - run: lotus fetch-params 8388608 - shell: bash - - build-mainnet: + - name: Run checks in parallel + run: | + # gofmt check + ( + go fmt ./... + git diff --quiet || { echo "gofmt needed"; exit 1; } + ) & + FMT_PID=$! + + # actionlint + ( + go install github.com/rhysd/actionlint/cmd/actionlint@latest + actionlint -shellcheck= -pyflakes= + ) & + LINT_PID=$! + + # mod tidy check + ( + go mod tidy -v + git diff --quiet go.mod go.sum || { echo "go mod tidy needed"; exit 1; } + ) & + TIDY_PID=$! + + wait $FMT_PID || exit 1 + wait $LINT_PID || exit 1 + wait $TIDY_PID || exit 1 + + # Lint - runs immediately, no dependencies + lint: runs-on: ubuntu-latest - needs: [ci-lint] steps: - uses: actions/checkout@v4 + with: + submodules: recursive - - name: Setup Go - uses: ./.github/actions/setup-go + - name: Setup build environment + uses: ./.github/actions/setup-build-env with: go-version: ${{ env.GO_VERSION }} - - name: Install Dependencies - uses: ./.github/actions/install-deps - - - name: Install FFI - env: - GITHUB_TOKEN: ${{ github.token }} + - name: Install and run golangci-lint run: | - make deps - shell: bash - - - name: Build Go - run: make build - shell: bash + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.4.0 + golangci-lint run -v --timeout 15m --concurrency 4 - build-calibnet: + # Build variants - run in parallel with matrix + build: runs-on: ubuntu-latest - needs: [ ci-lint ] + strategy: + fail-fast: false + matrix: + variant: [debug, build, calibnet, 2k] steps: - uses: actions/checkout@v4 + with: + submodules: recursive - - name: Setup Go - uses: ./.github/actions/setup-go + - name: Setup build environment + uses: ./.github/actions/setup-build-env with: go-version: ${{ env.GO_VERSION }} - - name: Install Dependencies - uses: ./.github/actions/install-deps - - - name: Install FFI - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - make deps - shell: bash - - - name: Build Go - run: make build - shell: bash + - name: Build ${{ matrix.variant }} + run: make ${{ matrix.variant }} - build-debug: - runs-on: ubuntu-latest - needs: [ ci-lint ] + # Unit tests - no database needed + test-unit: + runs-on: [self-hosted, docker] steps: - uses: actions/checkout@v4 + with: + submodules: recursive - - name: Setup Go - uses: ./.github/actions/setup-go + - name: Setup build environment + uses: ./.github/actions/setup-build-env with: go-version: ${{ env.GO_VERSION }} - - name: Install Dependencies - uses: ./.github/actions/install-deps + - name: Run unit tests + run: go test -v --tags=debug -timeout 30m $(go list ./... | grep -v curio/itests) - - name: Install FFI - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - make deps - shell: bash - - - name: Build Go - run: make build - shell: bash - - build-2k: - runs-on: ubuntu-latest - needs: [ ci-lint ] + # Integration tests - curio_test.go (heavy, runs separately) + test-itest-curio: + runs-on: [self-hosted, docker] + env: + YB_CONTAINER: yugabyte-curio-${{ github.run_id }} steps: - - uses: actions/checkout@v4 + - name: Check local proof parameters + id: local-params + run: | + PARAMS_DIR="/var/tmp/filecoin-proof-parameters" + if [ -d "$PARAMS_DIR" ] && [ "$(ls -A $PARAMS_DIR 2>/dev/null)" ]; then + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi - - name: Setup Go - uses: ./.github/actions/setup-go + - name: Restore proof parameters from cache + if: steps.local-params.outputs.exists != 'true' + id: cache-params + uses: actions/cache/restore@v4 with: - go-version: ${{ env.GO_VERSION }} - - - name: Install Dependencies - uses: ./.github/actions/install-deps + path: /var/tmp/filecoin-proof-parameters + key: proof-params-2k-v1 + restore-keys: proof-params- - - name: Install FFI - env: - GITHUB_TOKEN: ${{ github.token }} + - name: Fetch proof parameters + if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' run: | - make deps - shell: bash + echo "Fetching proof parameters (cache miss)..." + lotus fetch-params 8388608 - - name: Build Go - run: make build - shell: bash + - name: Save proof parameters to cache + if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + uses: actions/cache/save@v4 + with: + path: /var/tmp/filecoin-proof-parameters + key: proof-params-2k-v1 - build-forest: - runs-on: ubuntu-latest - needs: [ ci-lint ] - steps: - uses: actions/checkout@v4 + with: + submodules: recursive - - name: Setup Go - uses: ./.github/actions/setup-go + - name: Start YugabyteDB + id: start-yb + run: | + docker stop $YB_CONTAINER 2>/dev/null || true + docker rm $YB_CONTAINER 2>/dev/null || true + docker run --rm --name $YB_CONTAINER -d yugabytedb/yugabyte:2024.1.2.0-b77 bin/yugabyted start --daemon=false + for i in {1..60}; do + if docker exec $YB_CONTAINER bin/yugabyted status 2>/dev/null | grep -q Running; then + echo "YugabyteDB is ready" + break + fi + sleep 1 + done + YB_IP=$(docker inspect $YB_CONTAINER --format '{{ .NetworkSettings.Networks.bridge.IPAddress }}') + echo "yb_ip=$YB_IP" >> $GITHUB_OUTPUT + + - name: Setup build environment + uses: ./.github/actions/setup-build-env with: go-version: ${{ env.GO_VERSION }} - - name: Install Dependencies - uses: ./.github/actions/install-deps - - - name: Install FFI + - name: Run curio integration test env: - GITHUB_TOKEN: ${{ github.token }} - run: | - make deps - shell: bash + CURIO_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }} + LOTUS_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }} + run: go test -v --tags=debug -timeout 60m ./itests/curio_test.go - - name: Build Forest - run: make forest-test - shell: bash + - name: Stop YugabyteDB + if: always() + run: docker stop $YB_CONTAINER 2>/dev/null || true - test: + # Integration tests - other parallel tests + test-itest-other: runs-on: [self-hosted, docker] - needs: [setup-params] env: - CONTAINER_NAME: yugabyte-${{ github.run_id }}-${{ matrix.test-suite.name }} - strategy: - fail-fast: false # Continue running even if one test fails - matrix: - test-suite: - - name: test-itest-curio - target: "./itests/curio_test.go" - - name: test-all - target: "`go list ./... | grep -v curio/itests`" - - name: test-itest-harmonyDB - target: "./itests/harmonydb_test.go" - - name: test-itest-alertnow - target: "./itests/alertnow_test.go" - - name: test-itest-pdp-prove - target: "./itests/pdp_prove_test.go" + YB_CONTAINER: yugabyte-other-${{ github.run_id }} steps: - - uses: actions/checkout@v4 + - name: Check local proof parameters + id: local-params + run: | + PARAMS_DIR="/var/tmp/filecoin-proof-parameters" + if [ -d "$PARAMS_DIR" ] && [ "$(ls -A $PARAMS_DIR 2>/dev/null)" ]; then + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi - - name: Setup Go - uses: ./.github/actions/setup-go + - name: Restore proof parameters from cache + if: steps.local-params.outputs.exists != 'true' + id: cache-params + uses: actions/cache/restore@v4 with: - go-version: ${{ env.GO_VERSION }} - - - name: Install Dependencies - uses: ./.github/actions/install-deps + path: /var/tmp/filecoin-proof-parameters + key: proof-params-2k-v1 + restore-keys: proof-params- - - name: Install FFI - env: - GITHUB_TOKEN: ${{ github.token }} + - name: Fetch proof parameters + if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' run: | - make deps - shell: bash + echo "Fetching proof parameters (cache miss)..." + lotus fetch-params 8388608 - - name: Start YugabyteDB container with dynamic ports - id: start-yugabyte - run: | - # Start YugabyteDB container with dynamic port mapping for PostgreSQL and YCQL - docker run --rm --name ${{ env.CONTAINER_NAME }} -d yugabytedb/yugabyte:2024.1.2.0-b77 bin/yugabyted start --daemon=false + - name: Save proof parameters to cache + if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + uses: actions/cache/save@v4 + with: + path: /var/tmp/filecoin-proof-parameters + key: proof-params-2k-v1 - - name: Wait for YugabyteDB to start - run: | - while true; do - status=$(docker exec ${{ env.CONTAINER_NAME }} bin/yugabyted status); - echo $status; - echo $status | grep Running && break; - sleep 1; - done - shell: bash + - uses: actions/checkout@v4 + with: + submodules: recursive - - name: Get YugabyteDB container IP - id: get-yb-ip + - name: Start YugabyteDB + id: start-yb run: | - # Retrieve internal bridge IP of YugabyteDB container - YB_IP=$(docker inspect $CONTAINER_NAME --format '{{ .NetworkSettings.Networks.bridge.IPAddress }}') + docker stop $YB_CONTAINER 2>/dev/null || true + docker rm $YB_CONTAINER 2>/dev/null || true + docker run --rm --name $YB_CONTAINER -d yugabytedb/yugabyte:2024.1.2.0-b77 bin/yugabyted start --daemon=false + for i in {1..60}; do + if docker exec $YB_CONTAINER bin/yugabyted status 2>/dev/null | grep -q Running; then + echo "YugabyteDB is ready" + break + fi + sleep 1 + done + YB_IP=$(docker inspect $YB_CONTAINER --format '{{ .NetworkSettings.Networks.bridge.IPAddress }}') echo "yb_ip=$YB_IP" >> $GITHUB_OUTPUT - - name: Run tests + - name: Setup build environment + uses: ./.github/actions/setup-build-env + with: + go-version: ${{ env.GO_VERSION }} + + - name: Run other integration tests env: - CURIO_HARMONYDB_HOSTS: ${{ steps.get-yb-ip.outputs.yb_ip }} # Use internal IP for DB host - LOTUS_HARMONYDB_HOSTS: ${{ steps.get-yb-ip.outputs.yb_ip }} + CURIO_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }} + LOTUS_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }} run: | - echo "Using YugabyteDB Container IP: ${{env.CURIO_HARMONYDB_HOSTS}}" - export CURIO_HARMONYDB_HOSTS=${{ env.CURIO_HARMONYDB_HOSTS }} - export LOTUS_HARMONYDB_HOSTS=${{ env.CURIO_HARMONYDB_HOSTS }} - go test -v --tags=debug -timeout 30m ${{ matrix.test-suite.target }} + go test -v --tags=debug -timeout 30m -parallel 4 \ + ./itests/harmonydb_test.go \ + ./itests/pdp_prove_test.go \ + ./itests/indexstore_test.go \ + ./itests/move_shared_test.go \ + ./itests/local_test.go + + - name: Stop YugabyteDB + if: always() + run: docker stop $YB_CONTAINER 2>/dev/null || true + + # Integration tests - serial tests (modify global state) + test-itest-serial: + runs-on: [self-hosted, docker] + env: + YB_CONTAINER: yugabyte-serial-${{ github.run_id }} + steps: + - name: Check local proof parameters + id: local-params + run: | + PARAMS_DIR="/var/tmp/filecoin-proof-parameters" + if [ -d "$PARAMS_DIR" ] && [ "$(ls -A $PARAMS_DIR 2>/dev/null)" ]; then + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi - - name: Stop YugabyteDB container - if: always() # Ensure this runs even if the tests fail - run: docker stop ${{ env.CONTAINER_NAME }} + - name: Restore proof parameters from cache + if: steps.local-params.outputs.exists != 'true' + id: cache-params + uses: actions/cache/restore@v4 + with: + path: /var/tmp/filecoin-proof-parameters + key: proof-params-2k-v1 + restore-keys: proof-params- - lint: - runs-on: ubuntu-latest - needs: [ci-lint] - steps: - - uses: actions/checkout@v4 + - name: Fetch proof parameters + if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' + run: | + echo "Fetching proof parameters (cache miss)..." + lotus fetch-params 8388608 - - name: Setup Go - uses: ./.github/actions/setup-go + - name: Save proof parameters to cache + if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + uses: actions/cache/save@v4 with: - go-version: ${{ env.GO_VERSION }} + path: /var/tmp/filecoin-proof-parameters + key: proof-params-2k-v1 - - name: Install Dependencies - uses: ./.github/actions/install-deps + - uses: actions/checkout@v4 + with: + submodules: recursive - - name: Install FFI - env: - GITHUB_TOKEN: ${{ github.token }} + - name: Start YugabyteDB + id: start-yb run: | - make deps - shell: bash + docker stop $YB_CONTAINER 2>/dev/null || true + docker rm $YB_CONTAINER 2>/dev/null || true + docker run --rm --name $YB_CONTAINER -d yugabytedb/yugabyte:2024.1.2.0-b77 bin/yugabyted start --daemon=false + for i in {1..60}; do + if docker exec $YB_CONTAINER bin/yugabyted status 2>/dev/null | grep -q Running; then + echo "YugabyteDB is ready" + break + fi + sleep 1 + done + YB_IP=$(docker inspect $YB_CONTAINER --format '{{ .NetworkSettings.Networks.bridge.IPAddress }}') + echo "yb_ip=$YB_IP" >> $GITHUB_OUTPUT - - name: Install golangci-lint - run: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.4.0 - shell: bash + - name: Setup build environment + uses: ./.github/actions/setup-build-env + with: + go-version: ${{ env.GO_VERSION }} - - name: Lint - run: | - golangci-lint run -v --timeout 15m --concurrency 4 - shell: bash + - name: Run serial integration tests + env: + CURIO_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }} + LOTUS_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }} + run: go test -v --tags=debug,serial -timeout 30m ./itests/serial/... + + - name: Stop YugabyteDB + if: always() + run: docker stop $YB_CONTAINER 2>/dev/null || true - gofmt: + # Gen check - verify generated code is up to date + # Optimized: run independent gen steps in parallel, then build once + gen-check: runs-on: ubuntu-latest - needs: [ci-lint] steps: - uses: actions/checkout@v4 + with: + submodules: recursive - - name: Setup Go - uses: ./.github/actions/setup-go + - name: Setup build environment + uses: ./.github/actions/setup-build-env with: go-version: ${{ env.GO_VERSION }} - - name: Check gofmt + - name: Install Go tools in parallel + run: | + go install golang.org/x/tools/cmd/goimports@latest & + go install github.com/hannahhoward/cbor-gen-for@latest & + go install github.com/swaggo/swag/cmd/swag@latest & + wait + + - name: Run code generation (parallel where possible) + run: | + # These can run in parallel - they don't depend on each other + make api-gen & + API_PID=$! + + make cfgdoc-gen & + CFG_PID=$! + + make marketgen & + MKT_PID=$! + + wait $API_PID || exit 1 + wait $CFG_PID || exit 1 + wait $MKT_PID || exit 1 + + # go-generate depends on api-gen completing + make go-generate + + - name: Generate docs (requires binaries) run: | - go fmt ./... - shell: bash + # docsgen builds docgen-md and docgen-openrpc binaries + make docsgen + # docsgen-cli builds curio + sptool, then generates CLI docs + make docsgen-cli - - name: Git diff check - run: git --no-pager diff - shell: bash + - name: Fix imports and tidy + run: | + go run ./scripts/fiximports + go mod tidy - - name: Git diff quiet - run: git --no-pager diff --quiet - shell: bash + - name: Check for changes + run: git diff --quiet || { git diff; exit 1; } + # Supraseal build (kept separate - long running, different runner) build-supraseal-ubuntu24: runs-on: ubuntu-24.04 - needs: [ci-lint] - - env: - GCC_VERSION: "12" - steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: submodules: recursive - + - name: Free up disk space run: | - # Remove unnecessary packages to free up space for CUDA installation sudo apt-get clean - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - + sudo rm -rf /usr/share/dotnet /opt/ghc "/usr/local/share/boost" "$AGENT_TOOLSDIRECTORY" + - name: Install system dependencies run: | sudo apt-get update sudo apt-get install -y \ - build-essential \ - gcc-12 g++-12 \ - nasm \ - pkg-config \ - autoconf automake libtool \ - libssl-dev \ - libnuma-dev \ - uuid-dev \ - libaio-dev \ - libfuse3-dev \ - libarchive-dev \ - libkeyutils-dev \ - libncurses-dev \ - python3 python3-pip python3-dev \ - curl wget git \ - xxd - - - name: Set up Python virtual environment - run: | - # Python tools will be installed in venv by build.sh - # Just ensure python3-venv is available - python3 -m venv --help > /dev/null || sudo apt-get install -y python3-venv - - - name: Set up GCC 12 as default + build-essential gcc-12 g++-12 nasm pkg-config \ + autoconf automake libtool libssl-dev libnuma-dev \ + uuid-dev libaio-dev libfuse3-dev libarchive-dev \ + libkeyutils-dev libncurses-dev libgmp-dev libconfig++-dev \ + python3 python3-pip python3-dev curl wget git xxd + + - name: Set up GCC 12 run: | sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 - sudo update-alternatives --set gcc /usr/bin/gcc-12 - sudo update-alternatives --set g++ /usr/bin/g++-12 - gcc --version - g++ --version - + - name: Cache CUDA installation id: cache-cuda uses: actions/cache@v4 with: - path: | - /usr/local/cuda - /usr/local/cuda-* - key: cuda-toolkit-ubuntu-24.04-${{ runner.os }}-v1 - - - name: Install CUDA Toolkit from NVIDIA Repository + path: /usr/local/cuda + key: cuda-toolkit-ubuntu-24.04-v1 + + - name: Install CUDA Toolkit if: steps.cache-cuda.outputs.cache-hit != 'true' run: | - # Install CUDA using official NVIDIA repository for Ubuntu 24.04 - # Source: https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=24.04&target_type=deb_local - - # Download and install the CUDA keyring package wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb sudo dpkg -i cuda-keyring_1.1-1_all.deb - rm cuda-keyring_1.1-1_all.deb - - # Update package list and install CUDA toolkit - sudo apt-get update - sudo apt-get -y install cuda-toolkit - - # Verify installation and find CUDA location - if [ -d "/usr/local/cuda" ]; then - echo "CUDA installed at /usr/local/cuda" - ls -la /usr/local/cuda*/bin/nvcc || true - else - echo "ERROR: CUDA installation not found" - exit 1 - fi - + sudo apt-get update && sudo apt-get -y install cuda-toolkit + - name: Set up CUDA environment run: | - # Verify CUDA installation exists - if [ ! -d "/usr/local/cuda" ]; then - echo "ERROR: /usr/local/cuda not found" - exit 1 - fi - - # Export PATH locally to verify nvcc works - export PATH="/usr/local/cuda/bin:$PATH" - export CUDA_HOME=/usr/local/cuda - export LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" - - # Verify nvcc is available - nvcc --version - - # Set environment for subsequent steps echo "/usr/local/cuda/bin" >> $GITHUB_PATH echo "CUDA_HOME=/usr/local/cuda" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV - - - name: Install libconfig++ - run: | - sudo apt-get install -y libconfig++-dev || { - # If not available in repos, build from source - wget https://hyperrealm.github.io/libconfig/dist/libconfig-1.7.3.tar.gz - tar -xzf libconfig-1.7.3.tar.gz - cd libconfig-1.7.3 - ./configure - make -j$(nproc) - sudo make install - sudo ldconfig - cd .. - rm -rf libconfig-1.7.3* - } - - - name: Install GMP library - run: | - sudo apt-get install -y libgmp-dev - - - name: Cache Python venv - id: cache-venv - uses: actions/cache@v4 - with: - path: extern/supraseal/.venv - key: supraseal-venv-ubuntu24-${{ hashFiles('extern/supraseal/build.sh') }} - restore-keys: | - supraseal-venv-ubuntu24- - - - name: Cache SPDK build - id: cache-spdk + + - name: Cache Python venv and SPDK uses: actions/cache@v4 with: - path: extern/supraseal/deps/spdk-v24.05 - key: spdk-v24.05-gcc12-ubuntu24-${{ hashFiles('extern/supraseal/build.sh') }} - restore-keys: | - spdk-v24.05-gcc12-ubuntu24- - + path: | + extern/supraseal/.venv + extern/supraseal/deps/spdk-v24.05 + key: supraseal-deps-ubuntu24-${{ hashFiles('extern/supraseal/build.sh') }} + - name: Build Supraseal working-directory: extern/supraseal run: | - # Ensure we're using GCC 12 and CUDA - export CC=gcc-12 - export CXX=g++-12 - export CUDA=/usr/local/cuda + export CC=gcc-12 CXX=g++-12 CUDA=/usr/local/cuda export PATH=/usr/local/cuda/bin:$PATH - export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH - - # Verify CUDA is accessible - which nvcc - nvcc --version - - # Run the build script (creates and uses Python venv internally) ./build.sh - - - name: Verify binaries - working-directory: extern/supraseal - run: | - echo "=== Built binaries ===" - ls -lh bin/ - - echo "" - echo "=== Verifying binaries exist ===" - test -f bin/seal && echo "✓ seal binary created" || exit 1 - test -f bin/pc2 && echo "✓ pc2 binary created" || exit 1 - test -f bin/tree_r && echo "✓ tree_r binary created" || exit 1 - test -f bin/tree_r_cpu && echo "✓ tree_r_cpu binary created" || exit 1 - test -f bin/tree_d_cpu && echo "✓ tree_d_cpu binary created" || exit 1 - - echo "" - echo "=== Binary sizes ===" - du -h bin/* - - echo "" - echo "✅ All binaries built successfully!" - - - name: Upload build artifacts - uses: actions/upload-artifact@v4 - with: - name: supraseal-binaries-ubuntu24-gcc12-cuda - path: | - extern/supraseal/bin/seal - extern/supraseal/bin/pc2 - extern/supraseal/bin/tree_r - extern/supraseal/bin/tree_r_cpu - extern/supraseal/bin/tree_d_cpu - retention-days: 30 - - - name: Upload library artifact - uses: actions/upload-artifact@v4 - with: - name: supraseal-library-ubuntu24-gcc12-cuda - path: extern/supraseal/obj/libsupraseal.a - retention-days: 30 - - - name: Build summary - run: | - echo "### 🎉 Supraseal Build Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Configuration:**" >> $GITHUB_STEP_SUMMARY - echo "- OS: Ubuntu 24.04" >> $GITHUB_STEP_SUMMARY - echo "- GCC: $(gcc --version | head -1)" >> $GITHUB_STEP_SUMMARY - echo "- CUDA: $(nvcc --version | grep release)" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Built Binaries:**" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - ls -lh extern/supraseal/bin/ >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "✅ All binaries compiled successfully with GCC 12 and CUDA!" >> $GITHUB_STEP_SUMMARY - - gen-check: - runs-on: ubuntu-latest - needs: [ci-lint] - steps: - - uses: actions/checkout@v4 - - name: Setup Go - uses: ./.github/actions/setup-go - with: - go-version: ${{ env.GO_VERSION }} - - - name: Install Dependencies - uses: ./.github/actions/install-deps - - - name: Install goimports - run: go install golang.org/x/tools/cmd/goimports - shell: bash - - - name: Install cbor-gen-for - run: go install github.com/hannahhoward/cbor-gen-for - shell: bash - - - name: Install swag cli - run: go install github.com/swaggo/swag/cmd/swag@v1.16.4 - shell: bash - -# - name: Install gotext -# run: go install golang.org/x/text/cmd/gotext -# shell: bash - - - name: Install FFI - env: - GITHUB_TOKEN: ${{ github.token }} + - name: Verify binaries run: | - make deps - shell: bash - - - name: Generate Code - env: - LANG: en-US - run: make gen - shell: bash - - - name: Git diff check - run: git --no-pager diff - shell: bash - - - name: Git diff quiet - run: git --no-pager diff --quiet - shell: bash - - mod-tidy-check: - runs-on: ubuntu-latest - needs: [ci-lint] - steps: - - uses: actions/checkout@v4 - - - name: Setup Go - uses: ./.github/actions/setup-go - with: - go-version: ${{ env.GO_VERSION }} - - - name: Install Dependencies - uses: ./.github/actions/install-deps - - - name: Run mod tidy check - run: go mod tidy -v - shell: bash + for bin in seal pc2 tree_r tree_r_cpu tree_d_cpu; do + test -f extern/supraseal/bin/$bin || exit 1 + done + echo "✅ All Supraseal binaries built" diff --git a/cmd/curio/ffi.go b/cmd/curio/ffi.go index 976fecf93..22fb2dcf3 100644 --- a/cmd/curio/ffi.go +++ b/cmd/curio/ffi.go @@ -1,13 +1,13 @@ package main import ( + "context" "fmt" "os" "github.com/ipfs/go-cid" "github.com/snadrus/must" "github.com/urfave/cli/v2" - "golang.org/x/net/context" "github.com/filecoin-project/go-jsonrpc" diff --git a/deps/config/dynamic.go b/deps/config/dynamic.go index 85cd438f5..0723f93cb 100644 --- a/deps/config/dynamic.go +++ b/deps/config/dynamic.go @@ -91,17 +91,33 @@ type cfgRoot[T any] struct { layers []string treeCopy T fixupFn func(string, T) error + ctx context.Context + done chan struct{} } +// StopFunc is returned by EnableChangeDetectionWithContext and should be called +// to stop the change monitor goroutine and wait for it to exit. +type StopFunc func() + func EnableChangeDetection[T any](db *harmonydb.DB, obj T, layers []string, fixupFn func(string, T) error) error { + _, err := EnableChangeDetectionWithContext(context.Background(), db, obj, layers, fixupFn) + return err +} + +// EnableChangeDetectionWithContext starts a goroutine that monitors config changes. +// It returns a StopFunc that cancels the context and waits for the goroutine to exit. +// Call the StopFunc before cleaning up database resources. +func EnableChangeDetectionWithContext[T any](ctx context.Context, db *harmonydb.DB, obj T, layers []string, fixupFn func(string, T) error) (StopFunc, error) { var err error - r := &cfgRoot[T]{db: db, treeCopy: obj, layers: layers, fixupFn: fixupFn} + r := &cfgRoot[T]{db: db, treeCopy: obj, layers: layers, fixupFn: fixupFn, ctx: ctx, done: make(chan struct{})} r.treeCopy, err = CopyWithOriginalDynamics(obj) if err != nil { - return err + return nil, err } go r.changeMonitor() - return nil + return func() { + <-r.done // Wait for goroutine to exit + }, nil } // CopyWithOriginalDynamics copies the original dynamics from the original object to the new object. @@ -181,12 +197,26 @@ func isDynamicType(t reflect.Type) bool { } func (r *cfgRoot[T]) changeMonitor() { + defer close(r.done) // Signal that goroutine has exited + lastTimestamp := time.Time{} // lets do a read at startup for { + // Check if context is cancelled + select { + case <-r.ctx.Done(): + return + default: + } + configCount := 0 - err := r.db.QueryRow(context.Background(), `SELECT COUNT(*) FROM harmony_config WHERE timestamp > $1 AND title IN ($2)`, lastTimestamp, strings.Join(r.layers, ",")).Scan(&configCount) + err := r.db.QueryRow(r.ctx, `SELECT COUNT(*) FROM harmony_config WHERE timestamp > $1 AND title IN ($2)`, lastTimestamp, strings.Join(r.layers, ",")).Scan(&configCount) if err != nil { + // Exit if context was cancelled, pool was closed, or table doesn't exist yet (shutdown/startup condition) + errStr := err.Error() + if r.ctx.Err() != nil || strings.Contains(errStr, "closed pool") || strings.Contains(errStr, "does not exist") { + return + } logger.Errorf("error selecting configs: %s", err) continue } @@ -196,8 +226,13 @@ func (r *cfgRoot[T]) changeMonitor() { lastTimestamp = time.Now() // 1. get all configs - configs, err := GetConfigs(context.Background(), r.db, r.layers) + configs, err := GetConfigs(r.ctx, r.db, r.layers) if err != nil { + // Exit if context was cancelled, pool was closed, or table doesn't exist yet (shutdown/startup condition) + errStr := err.Error() + if r.ctx.Err() != nil || strings.Contains(errStr, "closed pool") || strings.Contains(errStr, "does not exist") { + return + } logger.Errorf("error getting configs: %s", err) continue } @@ -208,13 +243,19 @@ func (r *cfgRoot[T]) changeMonitor() { func() { dynamicLocker.Lock() defer dynamicLocker.Unlock() - err = ApplyLayers(context.Background(), r.treeCopy, configs, r.fixupFn) + err = ApplyLayers(r.ctx, r.treeCopy, configs, r.fixupFn) if err != nil { logger.Errorf("dynamic config failed to ApplyLayers: %s", err) return } }() - time.Sleep(30 * time.Second) + + // Sleep with context cancellation support + select { + case <-r.ctx.Done(): + return + case <-time.After(30 * time.Second): + } } } diff --git a/go.mod b/go.mod index f065850a9..523bd8d29 100644 --- a/go.mod +++ b/go.mod @@ -97,6 +97,7 @@ require ( github.com/prometheus/client_golang v1.23.2 github.com/puzpuzpuz/xsync/v2 v2.5.1 github.com/raulk/clock v1.1.0 + github.com/rhysd/actionlint v1.7.9 github.com/samber/lo v1.47.0 github.com/schollz/progressbar/v3 v3.18.0 github.com/sirupsen/logrus v1.9.3 @@ -112,13 +113,12 @@ require ( go.opencensus.io v0.24.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.43.0 + golang.org/x/crypto v0.45.0 golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b - golang.org/x/net v0.46.0 - golang.org/x/sync v0.17.0 - golang.org/x/sys v0.37.0 - golang.org/x/term v0.36.0 - golang.org/x/text v0.30.0 + golang.org/x/sync v0.18.0 + golang.org/x/sys v0.38.0 + golang.org/x/term v0.37.0 + golang.org/x/text v0.31.0 golang.org/x/tools v0.38.0 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da ) @@ -140,6 +140,7 @@ require ( github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.20.0 // indirect + github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -273,7 +274,8 @@ require ( github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.17 // indirect + github.com/mattn/go-shellwords v1.0.12 // indirect github.com/mattn/go-sqlite3 v1.14.32 // indirect github.com/miekg/dns v1.1.68 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect @@ -323,6 +325,7 @@ require ( github.com/quic-go/webtransport-go v0.9.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect @@ -368,8 +371,10 @@ require ( go.uber.org/fx v1.24.0 // indirect go.uber.org/mock v0.6.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.3 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/telemetry v0.0.0-20251009181524-91c411e14f39 // indirect golang.org/x/time v0.14.0 // indirect gonum.org/v1/gonum v0.16.0 // indirect @@ -382,6 +387,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect lukechampine.com/blake3 v1.4.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi diff --git a/go.sum b/go.sum index 0a9ac8d38..895e05ead 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,8 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= +github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= @@ -993,8 +995,10 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= -github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.17 h1:78v8ZlW0bP43XfmAfPsdXcoNCelfMHsDmd/pkENfrjQ= +github.com/mattn/go-runewidth v0.0.17/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= +github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -1241,10 +1245,14 @@ github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= +github.com/rhysd/actionlint v1.7.9 h1:oq4uFwcW6pRTk8BhAS4+RhYoUddUkbvRMcqndja0CT0= +github.com/rhysd/actionlint v1.7.9/go.mod h1:H3q8YpD2es7K4c+mibw3OhTXGQQ7HkZX1u+DXaHLwfE= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -1535,6 +1543,8 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= +go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= @@ -1564,8 +1574,8 @@ golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= -golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1660,8 +1670,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= -golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1686,8 +1696,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1767,8 +1777,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20251009181524-91c411e14f39 h1:jHQt1JBuPc+c/cAlupnkce8or0E04hX2Oqmnqq1XCVA= golang.org/x/telemetry v0.0.0-20251009181524-91c411e14f39/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1780,8 +1790,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= -golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1794,8 +1804,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2013,5 +2023,7 @@ lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/harmony/harmonydb/harmonydb.go b/harmony/harmonydb/harmonydb.go index 3e3d0ff35..3cab019d1 100644 --- a/harmony/harmonydb/harmonydb.go +++ b/harmony/harmonydb/harmonydb.go @@ -86,13 +86,18 @@ func envElse(env, els string) string { } func NewFromConfigWithITestID(t *testing.T, id ITestID) (*DB, error) { - fmt.Printf("CURIO_HARMONYDB_HOSTS: %s\n", os.Getenv("CURIO_HARMONYDB_HOSTS")) + // Look up the database name from the registry, or fall back to default + database := "yugabyte" + if v, ok := itestDatabaseRegistry.Load(string(id)); ok { + database = v.(string) + } + db, err := New( []string{envElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, - "yugabyte", - "yugabyte", - "yugabyte", - "5433", + envElse("CURIO_HARMONYDB_USERNAME", "yugabyte"), + envElse("CURIO_HARMONYDB_PASSWORD", "yugabyte"), + database, + envElse("CURIO_HARMONYDB_PORT", "5433"), false, id, ) @@ -196,6 +201,15 @@ func New(hosts []string, username, password, database, port string, loadBalance return &db, db.upgrade() } +// Close releases the underlying connection pool. It is safe to call multiple times. +func (db *DB) Close() { + if db == nil || db.pgx == nil { + return + } + db.pgx.Close() + db.pgx = nil +} + type tracer struct { } @@ -285,11 +299,25 @@ func (db *DB) ITestDeleteAll() { return } defer db.pgx.Close() - _, err := db.pgx.Exec(context.Background(), "DROP SCHEMA "+db.schema+" CASCADE") - if err != nil { - fmt.Println("warning: unclean itest shutdown: cannot delete schema: " + err.Error()) - return + + // Retry with exponential backoff for YugabyteDB serialization errors + retryWait := 100 * time.Millisecond + maxRetries := 5 + for i := 0; i < maxRetries; i++ { + _, err := db.pgx.Exec(context.Background(), "DROP SCHEMA "+db.schema+" CASCADE") + if err == nil { + return + } + // Check if it's a serialization error (40001) + if !strings.Contains(err.Error(), "40001") { + fmt.Println("warning: unclean itest shutdown: cannot delete schema: " + err.Error()) + return + } + // Serialization error - retry after backoff + time.Sleep(retryWait) + retryWait *= 2 } + fmt.Println("warning: unclean itest shutdown: cannot delete schema after retries") } var schemaREString = "^[A-Za-z0-9_]+$" diff --git a/harmony/harmonydb/itest_registry.go b/harmony/harmonydb/itest_registry.go new file mode 100644 index 000000000..f7879036f --- /dev/null +++ b/harmony/harmonydb/itest_registry.go @@ -0,0 +1,16 @@ +package harmonydb + +import "sync" + +var itestDatabaseRegistry sync.Map + +// RegisterITestDatabase allows test helpers to declare that a given ITestID +// should connect to the provided physical database instead of creating a fresh +// schema inside the default database. This is used by the integration-test +// template cloning logic. +func RegisterITestDatabase(id ITestID, database string) { + if id == "" || database == "" { + return + } + itestDatabaseRegistry.Store(string(id), database) +} diff --git a/harmony/harmonydb/sql/20231110-mining_tasks.sql b/harmony/harmonydb/sql/20231110-mining_tasks.sql index 13c8419dd..43d289c74 100644 --- a/harmony/harmonydb/sql/20231110-mining_tasks.sql +++ b/harmony/harmonydb/sql/20231110-mining_tasks.sql @@ -38,4 +38,4 @@ create table if not exists mining_base_block unique (sp_id, task_id, block_cid) ); -CREATE UNIQUE INDEX mining_base_block_cid_k ON mining_base_block (sp_id, block_cid) WHERE no_win = false; +CREATE UNIQUE INDEX IF NOT EXISTS mining_base_block_cid_k ON mining_base_block (sp_id, block_cid) WHERE no_win = false; diff --git a/harmony/harmonydb/sql/20240404-machine_detail.sql b/harmony/harmonydb/sql/20240404-machine_detail.sql index 128d1ceda..c9a4784a9 100644 --- a/harmony/harmonydb/sql/20240404-machine_detail.sql +++ b/harmony/harmonydb/sql/20240404-machine_detail.sql @@ -8,5 +8,5 @@ CREATE TABLE IF NOT EXISTS harmony_machine_details ( FOREIGN KEY (machine_id) REFERENCES harmony_machines(id) ON DELETE CASCADE ); -CREATE UNIQUE INDEX machine_details_machine_id ON harmony_machine_details(machine_id); +CREATE UNIQUE INDEX IF NOT EXISTS machine_details_machine_id ON harmony_machine_details(machine_id); diff --git a/harmony/harmonydb/sql/20240507-sdr-pipeline-fk-drop.sql b/harmony/harmonydb/sql/20240507-sdr-pipeline-fk-drop.sql index daf7a4429..cbb0c2bd3 100644 --- a/harmony/harmonydb/sql/20240507-sdr-pipeline-fk-drop.sql +++ b/harmony/harmonydb/sql/20240507-sdr-pipeline-fk-drop.sql @@ -1,12 +1,12 @@ -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_commit_msg_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_finalize_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_move_storage_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_porep_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_precommit_msg_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_sdr_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_tree_c_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_tree_d_fkey; -ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT sectors_sdr_pipeline_task_id_tree_r_fkey; +ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT IF EXISTS sectors_sdr_pipeline_task_id_commit_msg_fkey; +ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT IF EXISTS sectors_sdr_pipeline_task_id_finalize_fkey; +ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT IF EXISTS sectors_sdr_pipeline_task_id_move_storage_fkey; +ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT IF EXISTS sectors_sdr_pipeline_task_id_porep_fkey; +ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT IF EXISTS sectors_sdr_pipeline_task_id_precommit_msg_fkey; +ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT IF EXISTS sectors_sdr_pipeline_task_id_sdr_fkey; +ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT IF EXISTS sectors_sdr_pipeline_task_id_tree_c_fkey; +ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT IF EXISTS sectors_sdr_pipeline_task_id_tree_d_fkey; +ALTER TABLE sectors_sdr_pipeline DROP CONSTRAINT IF EXISTS sectors_sdr_pipeline_task_id_tree_r_fkey; -ALTER TABLE parked_pieces DROP CONSTRAINT parked_pieces_cleanup_task_id_fkey; -ALTER TABLE parked_pieces DROP CONSTRAINT parked_pieces_task_id_fkey; +ALTER TABLE parked_pieces DROP CONSTRAINT IF EXISTS parked_pieces_cleanup_task_id_fkey; +ALTER TABLE parked_pieces DROP CONSTRAINT IF EXISTS parked_pieces_task_id_fkey; diff --git a/harmony/harmonydb/sql/20240611-snap-pipeline.sql b/harmony/harmonydb/sql/20240611-snap-pipeline.sql index 813e769d9..c25fd7688 100644 --- a/harmony/harmonydb/sql/20240611-snap-pipeline.sql +++ b/harmony/harmonydb/sql/20240611-snap-pipeline.sql @@ -105,7 +105,8 @@ INSERT INTO sectors_cc_values (reg_seal_proof, cur_unsealed_cid) VALUES (11, 'baga6ea4seaqgl4u6lwmnerwdrm4iz7ag3mpwwaqtapc2fciabpooqmvjypweeha'), (12, 'baga6ea4seaqdsvqopmj2soyhujb72jza76t4wpq5fzifvm3ctz47iyytkewnubq'), (13, 'baga6ea4seaqao7s73y24kcutaosvacpdjgfe5pw76ooefnyqw4ynr3d2y6x2mpq'), - (14, 'baga6ea4seaqomqafu276g53zko4k23xzh4h4uecjwicbmvhsuqi7o4bhthhm4aq'); + (14, 'baga6ea4seaqomqafu276g53zko4k23xzh4h4uecjwicbmvhsuqi7o4bhthhm4aq') +ON CONFLICT DO NOTHING; ALTER TABLE sectors_meta ADD COLUMN IF NOT EXISTS expiration_epoch BIGINT; diff --git a/harmony/harmonydb/sql/20240823-ipni.sql b/harmony/harmonydb/sql/20240823-ipni.sql index a3966b932..031a11a18 100644 --- a/harmony/harmonydb/sql/20240823-ipni.sql +++ b/harmony/harmonydb/sql/20240823-ipni.sql @@ -35,10 +35,10 @@ CREATE TABLE IF NOT EXISTS ipni ( CREATE INDEX IF NOT EXISTS ipni_provider_order_number ON ipni(provider, order_number); -- This index will speed up lookups based on the ad_cid, which is frequently used to identify specific ads -CREATE UNIQUE INDEX ipni_ad_cid ON ipni(ad_cid); +CREATE UNIQUE INDEX IF NOT EXISTS ipni_ad_cid ON ipni(ad_cid); -- This index will speed up lookups based on the ad_cid, which is frequently used to identify specific ads -CREATE UNIQUE INDEX ipni_context_id ON ipni(context_id, ad_cid, is_rm); -- dropped in 20241106-market-fixes.sql +CREATE UNIQUE INDEX IF NOT EXISTS ipni_context_id ON ipni(context_id, ad_cid, is_rm); -- dropped in 20241106-market-fixes.sql -- 20241106-market-fixes.sql: -- CREATE INDEX ipni_context_id ON ipni(context_id, ad_cid, is_rm, is_skip) -- non-unique to allow multiple skips -- CREATE INDEX ipni_entries_skip ON ipni(entries, is_skip, piece_cid); diff --git a/harmony/harmonydb/sql/20240929-chain-sends-eth.sql b/harmony/harmonydb/sql/20240929-chain-sends-eth.sql index fdb270660..4f3d5d5c9 100644 --- a/harmony/harmonydb/sql/20240929-chain-sends-eth.sql +++ b/harmony/harmonydb/sql/20240929-chain-sends-eth.sql @@ -39,7 +39,7 @@ COMMENT ON COLUMN message_sends_eth.send_time IS 'Time when the send task was ex COMMENT ON COLUMN message_sends_eth.send_success IS 'Whether this transaction was broadcasted to the network already, NULL if not yet attempted, TRUE if successful, FALSE if failed'; COMMENT ON COLUMN message_sends_eth.send_error IS 'Error message if send_success is FALSE'; -CREATE UNIQUE INDEX message_sends_eth_success_index +CREATE UNIQUE INDEX IF NOT EXISTS message_sends_eth_success_index ON message_sends_eth (from_address, nonce) WHERE send_success IS NOT FALSE; diff --git a/harmony/harmonydb/sql/20241104-piece-info.sql b/harmony/harmonydb/sql/20241104-piece-info.sql index 49e36694f..a83a31a76 100644 --- a/harmony/harmonydb/sql/20241104-piece-info.sql +++ b/harmony/harmonydb/sql/20241104-piece-info.sql @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS piece_summary ( ); -- Insert the initial row -INSERT INTO piece_summary (id) VALUES (TRUE); +INSERT INTO piece_summary (id) VALUES (TRUE) ON CONFLICT DO NOTHING; -- Function to update piece_summary when a new entry is added to market_piece_metadata CREATE OR REPLACE FUNCTION update_piece_summary() diff --git a/harmony/harmonydb/sql/20241106-market-fixes.sql b/harmony/harmonydb/sql/20241106-market-fixes.sql index c48723e6a..249d0a345 100644 --- a/harmony/harmonydb/sql/20241106-market-fixes.sql +++ b/harmony/harmonydb/sql/20241106-market-fixes.sql @@ -1,13 +1,13 @@ -ALTER TABLE ipni_peerid ADD UNIQUE (sp_id); +CREATE UNIQUE INDEX IF NOT EXISTS ipni_peerid_sp_id_unique ON ipni_peerid (sp_id); -CREATE UNIQUE INDEX sectors_pipeline_events_task_history_id_uindex +CREATE UNIQUE INDEX IF NOT EXISTS sectors_pipeline_events_task_history_id_uindex ON sectors_pipeline_events (task_history_id, sp_id, sector_number); -CREATE UNIQUE INDEX market_piece_deal_piece_cid_id_uindex +CREATE UNIQUE INDEX IF NOT EXISTS market_piece_deal_piece_cid_id_uindex ON market_piece_deal (piece_cid, id); -alter table market_mk12_deals - add proposal_cid text not null; +ALTER TABLE market_mk12_deals + ADD COLUMN IF NOT EXISTS proposal_cid text not null; CREATE INDEX IF NOT EXISTS market_mk12_deals_proposal_cid_index ON market_mk12_deals (proposal_cid); diff --git a/harmony/harmonydb/sql/20250115-proofshare.sql b/harmony/harmonydb/sql/20250115-proofshare.sql index 51358d3a9..f0761139e 100644 --- a/harmony/harmonydb/sql/20250115-proofshare.sql +++ b/harmony/harmonydb/sql/20250115-proofshare.sql @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS proofshare_meta ( COMMENT ON COLUMN proofshare_meta.enabled IS 'Setting to TRUE indicates acceptance of provider TOS in lib/proofsvc/tos/provider.md and privacy.md'; -INSERT INTO proofshare_meta (singleton, enabled, wallet) VALUES (TRUE, FALSE, NULL); +INSERT INTO proofshare_meta (singleton, enabled, wallet) VALUES (TRUE, FALSE, NULL) ON CONFLICT DO NOTHING; CREATE TABLE IF NOT EXISTS proofshare_provider_payments ( provider_id BIGINT NOT NULL, -- wallet id @@ -106,7 +106,7 @@ CREATE TABLE IF NOT EXISTS proofshare_client_settings ( COMMENT ON COLUMN proofshare_client_settings.enabled IS 'Setting to TRUE indicates acceptance of client TOS in lib/proofsvc/tos/client.md and privacy.md'; -INSERT INTO proofshare_client_settings (enabled, sp_id, wallet, minimum_pending_seconds, do_porep, do_snap) VALUES (FALSE, 0, NULL, 0, FALSE, FALSE); +INSERT INTO proofshare_client_settings (enabled, sp_id, wallet, minimum_pending_seconds, do_porep, do_snap) VALUES (FALSE, 0, NULL, 0, FALSE, FALSE) ON CONFLICT DO NOTHING; CREATE TABLE IF NOT EXISTS proofshare_client_requests ( task_id BIGINT NOT NULL, diff --git a/harmony/harmonydb/sql/20250505-market-mk20.sql b/harmony/harmonydb/sql/20250505-market-mk20.sql index 9489cfe33..d07b4a3ec 100644 --- a/harmony/harmonydb/sql/20250505-market-mk20.sql +++ b/harmony/harmonydb/sql/20250505-market-mk20.sql @@ -144,7 +144,16 @@ BEGIN END $$; -- The order_number column must be completely sequential -ALTER SEQUENCE ipni_order_number_seq CACHE 1; +DO $$ +DECLARE + seq_name TEXT; +BEGIN + -- Find the sequence for ipni.order_number column (handles cloned schemas with different sequence names) + SELECT pg_get_serial_sequence('ipni', 'order_number') INTO seq_name; + IF seq_name IS NOT NULL THEN + EXECUTE format('ALTER SEQUENCE %s CACHE 1', seq_name); + END IF; +END $$; -- This function is used to insert piece metadata and piece deal (piece indexing) -- This makes it easy to keep the logic of how table is updated and fast (in DB). @@ -248,29 +257,21 @@ $$ LANGUAGE plpgsql; -- Update raw_size for existing deals (One time backfill migration) DO $$ BEGIN - UPDATE market_mk12_deals d - SET raw_size = mpd.raw_size - FROM market_piece_deal mpd - WHERE d.uuid = mpd.id; - - UPDATE market_direct_deals d - SET raw_size = mpd.raw_size - FROM market_piece_deal mpd - WHERE d.uuid = mpd.id; - - UPDATE market_mk12_deals d - SET raw_size = p.raw_size - FROM market_mk12_deal_pipeline p - WHERE d.uuid = p.uuid - AND d.raw_size IS NULL - AND p.raw_size IS NOT NULL; - - UPDATE market_direct_deals d - SET raw_size = p.raw_size - FROM market_mk12_deal_pipeline p - WHERE d.uuid = p.uuid - AND d.raw_size IS NULL - AND p.raw_size IS NOT NULL; + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'market_mk12_deals' AND column_name = 'raw_size') THEN + EXECUTE 'UPDATE market_mk12_deals d SET raw_size = mpd.raw_size FROM market_piece_deal mpd WHERE d.uuid = mpd.id'; + END IF; + + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'market_direct_deals' AND column_name = 'raw_size') THEN + EXECUTE 'UPDATE market_direct_deals d SET raw_size = mpd.raw_size FROM market_piece_deal mpd WHERE d.uuid = mpd.id'; + END IF; + + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'market_mk12_deals' AND column_name = 'raw_size') THEN + EXECUTE 'UPDATE market_mk12_deals d SET raw_size = p.raw_size FROM market_mk12_deal_pipeline p WHERE d.uuid = p.uuid AND d.raw_size IS NULL AND p.raw_size IS NOT NULL'; + END IF; + + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'market_direct_deals' AND column_name = 'raw_size') THEN + EXECUTE 'UPDATE market_direct_deals d SET raw_size = p.raw_size FROM market_mk12_deal_pipeline p WHERE d.uuid = p.uuid AND d.raw_size IS NULL AND p.raw_size IS NOT NULL'; + END IF; END $$; -- This is main MK20 Deal table. Rows are added per deal and some diff --git a/harmony/harmonydb/sql/20250727-balancemgr.sql b/harmony/harmonydb/sql/20250727-balancemgr.sql index b44adf651..b5d49217f 100644 --- a/harmony/harmonydb/sql/20250727-balancemgr.sql +++ b/harmony/harmonydb/sql/20250727-balancemgr.sql @@ -28,8 +28,14 @@ CREATE TABLE IF NOT EXISTS balance_manager_addresses ( CREATE INDEX IF NOT EXISTS balance_manager_addresses_last_msg_cid_idx ON balance_manager_addresses (last_msg_cid); -ALTER TABLE balance_manager_addresses ADD CONSTRAINT subject_not_equal_second CHECK (subject_address != second_address); -ALTER TABLE balance_manager_addresses ADD CONSTRAINT balance_manager_addresses_subject_address_second_address_unique UNIQUE (subject_address, second_address, action_type); +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'subject_not_equal_second') THEN + ALTER TABLE balance_manager_addresses ADD CONSTRAINT subject_not_equal_second CHECK (subject_address != second_address); + END IF; +END $$; + +CREATE UNIQUE INDEX IF NOT EXISTS balance_manager_addresses_subject_address_second_address_unique ON balance_manager_addresses (subject_address, second_address, action_type); CREATE OR REPLACE FUNCTION update_balance_manager_from_message_waits() RETURNS trigger AS $$ diff --git a/harmony/harmonydb/sql/20250801-proofshare-pipeline.sql b/harmony/harmonydb/sql/20250801-proofshare-pipeline.sql index e5b18b2d4..f90ae4473 100644 --- a/harmony/harmonydb/sql/20250801-proofshare-pipeline.sql +++ b/harmony/harmonydb/sql/20250801-proofshare-pipeline.sql @@ -9,10 +9,20 @@ CREATE TABLE IF NOT EXISTS proofshare_client_sender ALTER TABLE proofshare_client_requests ADD COLUMN IF NOT EXISTS request_type TEXT NOT NULL DEFAULT 'porep'; -ALTER TABLE proofshare_client_requests DROP CONSTRAINT proofshare_client_requests_pkey; -ALTER TABLE proofshare_client_requests ADD PRIMARY KEY (sp_id, sector_num, request_type); +ALTER TABLE proofshare_client_requests DROP CONSTRAINT IF EXISTS proofshare_client_requests_pkey; -ALTER TABLE proofshare_client_requests DROP COLUMN task_id; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.table_constraints + WHERE table_name = 'proofshare_client_requests' + AND constraint_type = 'PRIMARY KEY' + ) THEN + ALTER TABLE proofshare_client_requests ADD PRIMARY KEY (sp_id, sector_num, request_type); + END IF; +END $$; + +ALTER TABLE proofshare_client_requests DROP COLUMN IF EXISTS task_id; ALTER TABLE proofshare_client_requests ADD COLUMN IF NOT EXISTS task_id_upload BIGINT; ALTER TABLE proofshare_client_requests ADD COLUMN IF NOT EXISTS task_id_poll BIGINT; diff --git a/harmony/harmonydb/sql/20250803-wallet-exporter.sql b/harmony/harmonydb/sql/20250803-wallet-exporter.sql index 5121bbac8..283908d0b 100644 --- a/harmony/harmonydb/sql/20250803-wallet-exporter.sql +++ b/harmony/harmonydb/sql/20250803-wallet-exporter.sql @@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS wallet_exporter_processing ( processed_until TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() ); -INSERT INTO wallet_exporter_processing (singleton) VALUES (TRUE); +INSERT INTO wallet_exporter_processing (singleton) VALUES (TRUE) ON CONFLICT DO NOTHING; -- presence of a message in this table means that we've already accounted the basic send CREATE TABLE IF NOT EXISTS wallet_exporter_watched_msgs ( diff --git a/harmony/harmonydb/sql/20250926-harmony_config_timestamp.sql b/harmony/harmonydb/sql/20250926-harmony_config_timestamp.sql index 05e025097..2fc9324db 100644 --- a/harmony/harmonydb/sql/20250926-harmony_config_timestamp.sql +++ b/harmony/harmonydb/sql/20250926-harmony_config_timestamp.sql @@ -1 +1 @@ -ALTER TABLE harmony_config ADD COLUMN timestamp TIMESTAMP NOT NULL DEFAULT NOW(); \ No newline at end of file +ALTER TABLE harmony_config ADD COLUMN IF NOT EXISTS timestamp TIMESTAMP NOT NULL DEFAULT NOW(); \ No newline at end of file diff --git a/harmony/harmonydb/testutil/setup.go b/harmony/harmonydb/testutil/setup.go new file mode 100644 index 000000000..c098e2dd5 --- /dev/null +++ b/harmony/harmonydb/testutil/setup.go @@ -0,0 +1,367 @@ +package testutil + +import ( + "context" + "fmt" + "net/url" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/yugabyte/pgx/v5" + + "github.com/filecoin-project/curio/harmony/harmonydb" +) + +const ( + templateSchemaID harmonydb.ITestID = "template" + testDBName string = "curio_itest" +) + +var ( + templateOnce sync.Once + templateErr error + baseConnCfg connConfig + cloneMutex sync.Mutex // Serializes schema cloning to avoid YugabyteDB conflicts +) + +type connConfig struct { + host string + port string + username string + password string + baseDB string +} + +// SetupTestDB prepares a reusable template schema once, then rapidly clones it +// for every test invocation using CREATE TABLE ... (LIKE ... INCLUDING ALL). +// YugabyteDB doesn't support custom database templates, so we use schema-based +// isolation within a single shared test database. +// It returns an ITestID that can be passed to harmonydb.NewFromConfigWithITestID. +func SetupTestDB(t *testing.T) harmonydb.ITestID { + t.Helper() + + templateOnce.Do(func() { + baseConnCfg = loadConnConfig() + templateErr = prepareTemplateSchema() + }) + if templateErr != nil { + t.Fatalf("preparing template schema: %v", templateErr) + } + + id := harmonydb.ITestNewID() + if err := cloneTemplateSchema(id); err != nil { + t.Fatalf("cloning template schema: %v", err) + } + + harmonydb.RegisterITestDatabase(id, testDBName) + return id +} + +func loadConnConfig() connConfig { + return connConfig{ + host: firstNonEmpty(splitFirst(os.Getenv("CURIO_HARMONYDB_HOSTS")), os.Getenv("CURIO_DB_HOST"), "127.0.0.1"), + port: firstNonEmpty(os.Getenv("CURIO_HARMONYDB_PORT"), os.Getenv("CURIO_DB_PORT"), "5433"), + username: firstNonEmpty(os.Getenv("CURIO_HARMONYDB_USERNAME"), os.Getenv("CURIO_DB_USER"), "yugabyte"), + password: firstNonEmpty(os.Getenv("CURIO_HARMONYDB_PASSWORD"), os.Getenv("CURIO_DB_PASSWORD"), "yugabyte"), + baseDB: firstNonEmpty(os.Getenv("CURIO_HARMONYDB_NAME"), os.Getenv("CURIO_DB_NAME"), "yugabyte"), + } +} + +// prepareTemplateSchema creates the shared test database (if needed) and +// applies all migrations to a template schema that will be cloned for each test. +func prepareTemplateSchema() error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // Create the shared test database if it doesn't exist + adminConn, err := pgx.Connect(ctx, baseConnCfg.connString(baseConnCfg.baseDB)) + if err != nil { + return fmt.Errorf("connecting to admin database: %w", err) + } + + var exists bool + err = adminConn.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1)", testDBName).Scan(&exists) + if err != nil { + _ = adminConn.Close(ctx) + return fmt.Errorf("checking if test database exists: %w", err) + } + + if !exists { + _, err := adminConn.Exec(ctx, "CREATE DATABASE "+quoteIdentifier(testDBName)) + // Ignore "already exists" errors (race condition with parallel processes) + if err != nil && !strings.Contains(err.Error(), "already exists") { + _ = adminConn.Close(ctx) + return fmt.Errorf("creating test database: %w", err) + } + } + _ = adminConn.Close(ctx) + + // Connect to the test database and drop old template schema if it exists + testConn, err := pgx.Connect(ctx, baseConnCfg.connString(testDBName)) + if err != nil { + return fmt.Errorf("connecting to test database: %w", err) + } + templateSchema := fmt.Sprintf("itest_%s", templateSchemaID) + _, _ = testConn.Exec(ctx, "DROP SCHEMA IF EXISTS "+quoteIdentifier(templateSchema)+" CASCADE") + _ = testConn.Close(ctx) + + // Use harmonydb.New to create the template schema and apply all migrations + db, err := harmonydb.New([]string{baseConnCfg.host}, baseConnCfg.username, baseConnCfg.password, testDBName, baseConnCfg.port, false, templateSchemaID) + if err != nil { + return fmt.Errorf("initializing template schema: %w", err) + } + db.Close() + + return nil +} + +// cloneTemplateSchema creates a new schema for the test by copying all table +// structures, data, and functions from the template schema. This includes seed +// data that was inserted during migrations (e.g., harmony_config entries). +// Uses a mutex to serialize cloning and avoid YugabyteDB transaction conflicts. +func cloneTemplateSchema(id harmonydb.ITestID) error { + cloneMutex.Lock() + defer cloneMutex.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + conn, err := pgx.Connect(ctx, baseConnCfg.connString(testDBName)) + if err != nil { + return fmt.Errorf("connecting to test database: %w", err) + } + defer func() { _ = conn.Close(ctx) }() + + templateSchema := fmt.Sprintf("itest_%s", templateSchemaID) + newSchema := fmt.Sprintf("itest_%s", id) + + // Drop schema if it exists from a previous failed attempt + _, _ = conn.Exec(ctx, "DROP SCHEMA IF EXISTS "+quoteIdentifier(newSchema)+" CASCADE") + + // Create the new schema + if _, err := conn.Exec(ctx, "CREATE SCHEMA "+quoteIdentifier(newSchema)); err != nil { + return fmt.Errorf("creating schema: %w", err) + } + + // Get all tables from template schema + rows, err := conn.Query(ctx, ` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = $1 AND table_type = 'BASE TABLE' + `, templateSchema) + if err != nil { + return fmt.Errorf("querying template tables: %w", err) + } + + var tables []string + for rows.Next() { + var tableName string + if err := rows.Scan(&tableName); err != nil { + rows.Close() + return fmt.Errorf("scanning table name: %w", err) + } + tables = append(tables, tableName) + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating template tables: %w", err) + } + + // Clone each table structure + for _, table := range tables { + createSQL := fmt.Sprintf( + "CREATE TABLE %s.%s (LIKE %s.%s INCLUDING ALL)", + quoteIdentifier(newSchema), quoteIdentifier(table), + quoteIdentifier(templateSchema), quoteIdentifier(table), + ) + if _, err := conn.Exec(ctx, createSQL); err != nil { + return fmt.Errorf("cloning table %s: %w", table, err) + } + } + + // Copy data from all tables (includes migration tracking in 'base' and seed data from migrations) + for _, table := range tables { + _, err = conn.Exec(ctx, fmt.Sprintf( + "INSERT INTO %s.%s SELECT * FROM %s.%s", + quoteIdentifier(newSchema), quoteIdentifier(table), + quoteIdentifier(templateSchema), quoteIdentifier(table), + )) + if err != nil { + return fmt.Errorf("copying data for table %s: %w", table, err) + } + } + + // Clone functions from template schema to new schema + if err := cloneFunctions(ctx, conn, templateSchema, newSchema); err != nil { + return fmt.Errorf("cloning functions: %w", err) + } + + // Clone triggers from template schema to new schema + if err := cloneTriggers(ctx, conn, templateSchema, newSchema); err != nil { + return fmt.Errorf("cloning triggers: %w", err) + } + + return nil +} + +// cloneFunctions copies all functions from the template schema to the new schema. +// It retrieves function definitions using pg_get_functiondef and recreates them +// in the new schema by replacing the schema name in the function definition. +func cloneFunctions(ctx context.Context, conn *pgx.Conn, templateSchema, newSchema string) error { + // Query all functions in the template schema + rows, err := conn.Query(ctx, ` + SELECT p.oid, p.proname + FROM pg_proc p + JOIN pg_namespace n ON p.pronamespace = n.oid + WHERE n.nspname = $1 + `, templateSchema) + if err != nil { + return fmt.Errorf("querying functions: %w", err) + } + + type funcInfo struct { + oid uint32 + name string + } + var functions []funcInfo + for rows.Next() { + var f funcInfo + if err := rows.Scan(&f.oid, &f.name); err != nil { + rows.Close() + return fmt.Errorf("scanning function: %w", err) + } + functions = append(functions, f) + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating functions: %w", err) + } + + // Recreate each function in the new schema + for _, f := range functions { + var funcDef string + err := conn.QueryRow(ctx, "SELECT pg_get_functiondef($1)", f.oid).Scan(&funcDef) + if err != nil { + return fmt.Errorf("getting definition for function %s: %w", f.name, err) + } + + // Replace schema name in the function definition + // The function definition starts with "CREATE OR REPLACE FUNCTION schema.funcname" + // Try both quoted and unquoted schema names since pg_get_functiondef output varies + funcDef = strings.Replace(funcDef, + quoteIdentifier(templateSchema)+".", + quoteIdentifier(newSchema)+".", + 1) + funcDef = strings.Replace(funcDef, + templateSchema+".", + newSchema+".", + 1) + + if _, err := conn.Exec(ctx, funcDef); err != nil { + return fmt.Errorf("creating function %s: %w", f.name, err) + } + } + + return nil +} + +// cloneTriggers copies all triggers from the template schema to the new schema. +// It retrieves trigger definitions and recreates them in the new schema. +func cloneTriggers(ctx context.Context, conn *pgx.Conn, templateSchema, newSchema string) error { + // Query all triggers in the template schema + rows, err := conn.Query(ctx, ` + SELECT + t.tgname AS trigger_name, + c.relname AS table_name, + pg_get_triggerdef(t.oid) AS trigger_def + FROM pg_trigger t + JOIN pg_class c ON t.tgrelid = c.oid + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE n.nspname = $1 + AND NOT t.tgisinternal + `, templateSchema) + if err != nil { + return fmt.Errorf("querying triggers: %w", err) + } + + type triggerInfo struct { + name string + tableName string + def string + } + var triggers []triggerInfo + for rows.Next() { + var t triggerInfo + if err := rows.Scan(&t.name, &t.tableName, &t.def); err != nil { + rows.Close() + return fmt.Errorf("scanning trigger: %w", err) + } + triggers = append(triggers, t) + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating triggers: %w", err) + } + + // Recreate each trigger in the new schema + for _, t := range triggers { + // Replace schema references in the trigger definition + // Try both quoted and unquoted schema names + triggerDef := t.def + triggerDef = strings.ReplaceAll(triggerDef, + quoteIdentifier(templateSchema)+".", + quoteIdentifier(newSchema)+".") + triggerDef = strings.ReplaceAll(triggerDef, + templateSchema+".", + newSchema+".") + + if _, err := conn.Exec(ctx, triggerDef); err != nil { + return fmt.Errorf("creating trigger %s on %s: %w", t.name, t.tableName, err) + } + } + + return nil +} + +func (c connConfig) connString(database string) string { + u := url.URL{ + Scheme: "postgresql", + Host: fmt.Sprintf("%s:%s", c.host, c.port), + Path: "/" + database, + RawQuery: "sslmode=disable", + } + if c.password == "" { + u.User = url.User(c.username) + } else { + u.User = url.UserPassword(c.username, c.password) + } + return u.String() +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func splitFirst(hosts string) string { + if hosts == "" { + return "" + } + for _, part := range strings.Split(hosts, ",") { + part = strings.TrimSpace(part) + if part != "" { + return part + } + } + return "" +} + +func quoteIdentifier(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} diff --git a/itests/curio_test.go b/itests/curio_test.go index 10f8f47ec..3831baa14 100644 --- a/itests/curio_test.go +++ b/itests/curio_test.go @@ -31,6 +31,7 @@ import ( "github.com/filecoin-project/curio/deps" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonydb/testutil" "github.com/filecoin-project/curio/lib/ffiselect" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/lib/testutils" @@ -47,6 +48,7 @@ import ( ) func TestCurioHappyPath(t *testing.T) { + t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -73,7 +75,7 @@ func TestCurioHappyPath(t *testing.T) { fapi := fmt.Sprintf("%s:%s", string(token), full.ListenAddr) - sharedITestID := harmonydb.ITestNewID() + sharedITestID := testutil.SetupTestDB(t) t.Logf("sharedITestID: %s", sharedITestID) db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID) diff --git a/itests/harmonydb_test.go b/itests/harmonydb_test.go index b62b17883..445fa21f6 100644 --- a/itests/harmonydb_test.go +++ b/itests/harmonydb_test.go @@ -11,13 +11,15 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonydb/testutil" ) func TestCrud(t *testing.T) { + t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - sharedITestID := harmonydb.ITestNewID() + sharedITestID := testutil.SetupTestDB(t) cdb, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID) require.NoError(t, err) @@ -35,21 +37,24 @@ func TestCrud(t *testing.T) { Animal string `db:"content"` Unpopulated int } - err = cdb.Select(ctx, &ints, "SELECT content, some_int FROM itest_scratch") + err = cdb.Select(ctx, &ints, "SELECT content, some_int FROM itest_scratch ORDER BY some_int DESC") require.NoError(t, err) require.Len(t, ints, 2, "unexpected count of returns. Want 2, Got ", len(ints)) - require.True(t, ints[0].Count == 11 || ints[1].Count == 5, "expected [11,5] got ", ints) - require.True(t, ints[0].Animal == "cows" || ints[1].Animal == "cats", "expected, [cows, cats] ", ints) + require.Equal(t, 11, ints[0].Count, "expected first row count to be 11") + require.Equal(t, 5, ints[1].Count, "expected second row count to be 5") + require.Equal(t, "cows", ints[0].Animal, "expected first row animal to be cows") + require.Equal(t, "cats", ints[1].Animal, "expected second row animal to be cats") fmt.Println("test completed") } func TestTransaction(t *testing.T) { + t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - testID := harmonydb.ITestNewID() + testID := testutil.SetupTestDB(t) cdb, err := harmonydb.NewFromConfigWithITestID(t, testID) require.NoError(t, err) _, err = cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)") @@ -96,10 +101,11 @@ func TestTransaction(t *testing.T) { } func TestPartialWalk(t *testing.T) { + t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - testID := harmonydb.ITestNewID() + testID := testutil.SetupTestDB(t) cdb, err := harmonydb.NewFromConfigWithITestID(t, testID) require.NoError(t, err) _, err = cdb.Exec(ctx, ` diff --git a/market/indexstore/indexstore_test.go b/itests/indexstore_test.go similarity index 83% rename from market/indexstore/indexstore_test.go rename to itests/indexstore_test.go index eafcd6594..84b5b3302 100644 --- a/market/indexstore/indexstore_test.go +++ b/itests/indexstore_test.go @@ -1,4 +1,4 @@ -package indexstore +package itests import ( "context" @@ -18,22 +18,17 @@ import ( "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/lib/savecache" "github.com/filecoin-project/curio/lib/testutils" + "github.com/filecoin-project/curio/market/indexstore" ) -func envElse(env, els string) string { - if v := os.Getenv(env); v != "" { - return v - } - return els -} - func TestNewIndexStore(t *testing.T) { + t.Parallel() // Set up the indexStore for testing ctx := context.Background() cfg := config.DefaultCurioConfig() - idxStore := NewIndexStore([]string{envElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, 9042, cfg) + idxStore := indexstore.NewIndexStore([]string{testutils.EnvElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, 9042, cfg) err := idxStore.Start(ctx, true) require.NoError(t, err) @@ -89,7 +84,7 @@ func TestNewIndexStore(t *testing.T) { dealCfg := cfg.Market.StorageMarketConfig chanSize := dealCfg.Indexing.InsertConcurrency * dealCfg.Indexing.InsertBatchSize - recs := make(chan Record, chanSize) + recs := make(chan indexstore.Record, chanSize) opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} blockReader, err := carv2.NewBlockReader(f, opts...) require.NoError(t, err) @@ -109,7 +104,7 @@ func TestNewIndexStore(t *testing.T) { if i == 0 { m = blockMetadata.Hash() } - recs <- Record{ + recs <- indexstore.Record{ Cid: blockMetadata.Cid, Offset: blockMetadata.SourceOffset, Size: blockMetadata.Size, @@ -141,10 +136,8 @@ func TestNewIndexStore(t *testing.T) { err = idxStore.RemoveIndexes(ctx, pcids[0].PieceCid) require.NoError(t, err) - err = idxStore.session.Query("SELECT * FROM piece_by_aggregate").Exec() - require.NoError(t, err) - - aggrRec := []Record{ + // Test aggregate index + aggrRec := []indexstore.Record{ { Cid: pcid1, Offset: 0, @@ -174,9 +167,9 @@ func TestNewIndexStore(t *testing.T) { require.NoError(t, err) // Test PDP layer - leafs := make([]NodeDigest, len(layer)) + leafs := make([]indexstore.NodeDigest, len(layer)) for i, s := range layer { - leafs[i] = NodeDigest{ + leafs[i] = indexstore.NodeDigest{ Layer: layerIdx, Hash: s.Hash, Index: int64(i), @@ -213,16 +206,4 @@ func TestNewIndexStore(t *testing.T) { err = idxStore.DeletePDPLayer(ctx, pcid2) require.NoError(t, err) - - // Drop the tables - err = idxStore.session.Query("DROP TABLE PayloadToPieces").Exec() - require.NoError(t, err) - err = idxStore.session.Query("DROP TABLE PieceBlockOffsetSize").Exec() - require.NoError(t, err) - err = idxStore.session.Query("DROP TABLE aggregate_by_piece").Exec() - require.NoError(t, err) - err = idxStore.session.Query("DROP TABLE piece_by_aggregate").Exec() - require.NoError(t, err) - err = idxStore.session.Query("DROP TABLE pdp_cache_layer").Exec() - require.NoError(t, err) } diff --git a/lib/paths/local_test.go b/itests/local_test.go similarity index 82% rename from lib/paths/local_test.go rename to itests/local_test.go index 24282c4df..0ba5c0e05 100644 --- a/lib/paths/local_test.go +++ b/itests/local_test.go @@ -1,4 +1,4 @@ -package paths +package itests import ( "context" @@ -11,6 +11,8 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonydb/testutil" + "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/storage/sealer/fsutil" @@ -50,7 +52,7 @@ func (t *TestingLocalStorage) init(subpath string) error { return err } - metaFile := filepath.Join(path, MetaFile) + metaFile := filepath.Join(path, paths.MetaFile) meta := &storiface.LocalStorageMeta{ ID: storiface.ID(uuid.New().String()), @@ -71,9 +73,10 @@ func (t *TestingLocalStorage) init(subpath string) error { return nil } -var _ LocalStorage = &TestingLocalStorage{} +var _ paths.LocalStorage = &TestingLocalStorage{} func TestLocalStorage(t *testing.T) { + t.Parallel() ctx := context.TODO() root := t.TempDir() @@ -82,14 +85,14 @@ func TestLocalStorage(t *testing.T) { root: root, } - sharedITestID := harmonydb.ITestNewID() + sharedITestID := testutil.SetupTestDB(t) db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID) require.NoError(t, err) - index := NewDBIndex(nil, db) + index := paths.NewDBIndex(nil, db) - st, err := NewLocal(ctx, tstor, index, "") + st, err := paths.NewLocal(ctx, tstor, index, "") require.NoError(t, err) p1 := "1" diff --git a/itests/move_shared_test.go b/itests/move_shared_test.go new file mode 100644 index 000000000..1e3b202fc --- /dev/null +++ b/itests/move_shared_test.go @@ -0,0 +1,138 @@ +package itests + +import ( + "context" + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" + "github.com/gorilla/mux" + logging "github.com/ipfs/go-log/v2" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonydb/testutil" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" +) + +const metaFile = "sectorstore.json" + +func createTestStorage(t *testing.T, p string, seal bool, att ...*paths.Local) storiface.ID { + if err := os.MkdirAll(p, 0755); err != nil { + if !os.IsExist(err) { + require.NoError(t, err) + } + } + + cfg := &storiface.LocalStorageMeta{ + ID: storiface.ID(uuid.New().String()), + Weight: 10, + CanSeal: seal, + CanStore: !seal, + } + + b, err := json.MarshalIndent(cfg, "", " ") + require.NoError(t, err) + + require.NoError(t, os.WriteFile(filepath.Join(p, metaFile), b, 0644)) + + for _, s := range att { + require.NoError(t, s.OpenPath(context.Background(), p)) + } + + return cfg.ID +} + +func TestMoveShared(t *testing.T) { + t.Parallel() + logging.SetAllLoggers(logging.LevelDebug) + + sharedITestID := testutil.SetupTestDB(t) + + db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID) + require.NoError(t, err) + + index := paths.NewDBIndex(nil, db) + + ctx := context.Background() + + dir := t.TempDir() + + openRepo := func(dir string) paths.LocalStorage { + bls := &paths.BasicLocalStorage{PathToJSON: filepath.Join(t.TempDir(), "storage.json")} + return bls + } + + // setup two repos with two storage paths: + // repo 1 with both paths + // repo 2 with one path (shared) + + lr1 := openRepo(filepath.Join(dir, "l1")) + lr2 := openRepo(filepath.Join(dir, "l2")) + + mux1 := mux.NewRouter() + mux2 := mux.NewRouter() + hs1 := httptest.NewServer(mux1) + hs2 := httptest.NewServer(mux2) + + ls1, err := paths.NewLocal(ctx, lr1, index, hs1.URL+"/remote") + require.NoError(t, err) + ls2, err := paths.NewLocal(ctx, lr2, index, hs2.URL+"/remote") + require.NoError(t, err) + + dirStor := filepath.Join(dir, "stor") + dirSeal := filepath.Join(dir, "seal") + + id1 := createTestStorage(t, dirStor, false, ls1, ls2) + id2 := createTestStorage(t, dirSeal, true, ls1) + + rs1, err := paths.NewRemote(ls1, index, nil, 20, &paths.DefaultPartialFileHandler{}) + require.NoError(t, err) + rs2, err := paths.NewRemote(ls2, index, nil, 20, &paths.DefaultPartialFileHandler{}) + require.NoError(t, err) + _ = rs2 + mux1.PathPrefix("/").Handler(&paths.FetchHandler{Local: ls1, PfHandler: &paths.DefaultPartialFileHandler{}}) + mux2.PathPrefix("/").Handler(&paths.FetchHandler{Local: ls2, PfHandler: &paths.DefaultPartialFileHandler{}}) + + // add a sealed replica file to the sealing (non-shared) path + + s1ref := storiface.SectorRef{ + ID: abi.SectorID{ + Miner: 12, + Number: 1, + }, + ProofType: abi.RegisteredSealProof_StackedDrg2KiBV1, + } + + sp, sid, err := rs1.AcquireSector(ctx, s1ref, storiface.FTNone, storiface.FTSealed, storiface.PathSealing, storiface.AcquireMove) + require.NoError(t, err) + require.Equal(t, id2, storiface.ID(sid.Sealed)) + + data := make([]byte, 2032) + data[1] = 54 + require.NoError(t, os.WriteFile(sp.Sealed, data, 0666)) + fmt.Println("write to ", sp.Sealed) + + require.NoError(t, index.StorageDeclareSector(ctx, storiface.ID(sid.Sealed), s1ref.ID, storiface.FTSealed, true)) + + // move to the shared path from the second node (remote move / delete) + + require.NoError(t, rs2.MoveStorage(ctx, s1ref, storiface.FTSealed)) + + // check that the file still exists + sp, sid, err = rs2.AcquireSector(ctx, s1ref, storiface.FTSealed, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) + require.NoError(t, err) + require.Equal(t, id1, storiface.ID(sid.Sealed)) + fmt.Println("read from ", sp.Sealed) + + read, err := os.ReadFile(sp.Sealed) + require.NoError(t, err) + require.EqualValues(t, data, read) +} diff --git a/itests/pdp_prove_test.go b/itests/pdp_prove_test.go index cde14bbca..62644b1de 100644 --- a/itests/pdp_prove_test.go +++ b/itests/pdp_prove_test.go @@ -26,6 +26,7 @@ import ( // TestPDPProving verifies the functionality of generating and validating PDP proofs with a random file created in a temporary directory. func TestPDPProving(t *testing.T) { + t.Parallel() ctx := context.Background() cfg := config.DefaultCurioConfig() idxStore := indexstore.NewIndexStore([]string{testutils.EnvElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, 9042, cfg) diff --git a/itests/alertnow_test.go b/itests/serial/alertnow_test.go similarity index 80% rename from itests/alertnow_test.go rename to itests/serial/alertnow_test.go index 0eef39730..d0a22ff19 100644 --- a/itests/alertnow_test.go +++ b/itests/serial/alertnow_test.go @@ -1,4 +1,6 @@ -package itests +//go:build serial + +package serial import ( "testing" @@ -10,17 +12,19 @@ import ( "github.com/filecoin-project/curio/alertmanager/plugin" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonydb/testutil" ) +// TestAlertNow tests the alerting system. +// NOTE: Cannot run in parallel - modifies global variables: +// plugin.TestPlugins and alertmanager.AlertFuncs func TestAlertNow(t *testing.T) { - // TestAlertNow tests alerting system - tp := &testPlugin{} plugin.TestPlugins = []plugin.Plugin{ tp, } // Create dependencies - sharedITestID := harmonydb.ITestNewID() + sharedITestID := testutil.SetupTestDB(t) db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID) require.NoError(t, err) diff --git a/itests/dyncfg_test.go b/itests/serial/dyncfg_test.go similarity index 67% rename from itests/dyncfg_test.go rename to itests/serial/dyncfg_test.go index 8b70249f1..449be5dad 100644 --- a/itests/dyncfg_test.go +++ b/itests/serial/dyncfg_test.go @@ -1,4 +1,6 @@ -package itests +//go:build serial + +package serial import ( "context" @@ -10,13 +12,16 @@ import ( "github.com/filecoin-project/curio/deps" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonydb/testutil" ) +// TestDynamicConfig tests the dynamic configuration change detection. +// NOTE: Cannot run in parallel - EnableChangeDetection starts a background +// goroutine that persists after the test and can interfere with other tests. func TestDynamicConfig(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - sharedITestID := harmonydb.ITestNewID() + sharedITestID := testutil.SetupTestDB(t) cdb, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID) require.NoError(t, err) @@ -41,7 +46,15 @@ func TestDynamicConfig(t *testing.T) { require.NoError(t, setTestConfig(ctx, cdb, databaseContents)) // "Start the server". This will immediately poll for a config update. - require.NoError(t, config.EnableChangeDetection(cdb, databaseContents, []string{"testcfg"}, config.FixTOML)) + // Get the stop function to properly shut down the goroutine before test cleanup + stopFn, err := config.EnableChangeDetectionWithContext(ctx, cdb, databaseContents, []string{"testcfg"}, config.FixTOML) + require.NoError(t, err) + + // Ensure we stop the change monitor BEFORE database cleanup happens + defer func() { + cancel() // Signal context cancellation + stopFn() // Wait for goroutine to exit + }() // Positive Test: the runtime config should have the new value require.Eventually(t, func() bool { @@ -58,6 +71,7 @@ func setTestConfig(ctx context.Context, cdb *harmonydb.DB, cfg *config.CurioConf if err != nil { return err } - _, err = cdb.Exec(ctx, `INSERT INTO harmony_config (title, config) VALUES ($1, $2)`, "testcfg", string(tomlData)) + _, err = cdb.Exec(ctx, `INSERT INTO harmony_config (title, config) VALUES ($1, $2) + ON CONFLICT (title) DO UPDATE SET config = EXCLUDED.config`, "testcfg", string(tomlData)) return err } diff --git a/itests/serial/sql_idempotent_test.go b/itests/serial/sql_idempotent_test.go new file mode 100644 index 000000000..34b0d2cac --- /dev/null +++ b/itests/serial/sql_idempotent_test.go @@ -0,0 +1,57 @@ +//go:build serial + +package serial + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "github.com/yugabyte/pgx/v5/pgxpool" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonydb/testutil" +) + +// TestSQLIdempotent tests that the SQL DDL files are idempotent. +// The upgrader will fail unless everything has "IF NOT EXISTS" or "IF EXISTS" statements. +// Or equivalent safety checks. +// NOTE: Cannot run in parallel - modifies global harmonydb.ITestUpgradeFunc +func TestSQLIdempotent(t *testing.T) { + defer func() { + harmonydb.ITestUpgradeFunc = nil + }() + + // Use SetupTestDB to get a cloned schema quickly (all structures already exist) + testID := testutil.SetupTestDB(t) + cdb, err := harmonydb.NewFromConfigWithITestID(t, testID) + require.NoError(t, err) + + // Clear migration tracking so migrations will re-run + ctx := context.Background() + _, err = cdb.Exec(ctx, `DELETE FROM base`) + require.NoError(t, err) + + // Set up idempotency check - each migration SQL will be run twice + harmonydb.ITestUpgradeFunc = func(pool *pgxpool.Pool, name string, sql string) { + _, err := pool.Exec(context.Background(), sql) + if err != nil { + require.NoError(t, fmt.Errorf("SQL DDL file failed idempotent check: %s, %w", name, err)) + } + } + + // Create second connection - migrations re-run on existing structures (tests idempotency) + // Keep both connections open - cleanup handles closing + _, err = harmonydb.NewFromConfigWithITestID(t, testID) + require.NoError(t, err) + + _, err = cdb.Exec(ctx, ` + INSERT INTO + itest_scratch (content, some_int) + VALUES + ('andy was here', 5), + ('lotus is awesome', 6) + `) + require.NoError(t, err) +} diff --git a/itests/sql_idempotent_test.go b/itests/sql_idempotent_test.go deleted file mode 100644 index 2d11aa450..000000000 --- a/itests/sql_idempotent_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package itests - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/require" - "github.com/yugabyte/pgx/v5/pgxpool" - - "github.com/filecoin-project/curio/harmony/harmonydb" -) - -// TestSQLIdempotent tests that the SQL DDL files are idempotent. -// The upgrader will fail unless everything has "IF NOT EXISTS" or "IF EXISTS" statements. -// Or equivalent safety checks. -func TestSQLIdempotent(t *testing.T) { - harmonydb.ITestUpgradeFunc = func(db *pgxpool.Pool, name string, sql string) { - _, err := db.Exec(context.Background(), sql) - require.NoError(t, fmt.Errorf("SQL DDL file failed idempotent check: %s, %w", name, err)) - } - - testID := harmonydb.ITestNewID() - cdb, err := harmonydb.NewFromConfigWithITestID(t, testID) - require.NoError(t, err) - - ctx := context.Background() - _, err = cdb.Exec(ctx, ` - INSERT INTO - itest_scratch (content, some_int) - VALUES - ('andy was here', 5), - ('lotus is awesome', 6) - `) - require.NoError(t, err) -} diff --git a/lib/paths/remote_test.go b/lib/paths/remote_test.go index 682ded8da..a18069773 100644 --- a/lib/paths/remote_test.go +++ b/lib/paths/remote_test.go @@ -3,17 +3,14 @@ package paths_test import ( "context" - "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "os" - "path/filepath" "testing" "github.com/golang/mock/gomock" - "github.com/google/uuid" "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" @@ -21,127 +18,12 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/partialfile" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/paths/mocks" "github.com/filecoin-project/curio/lib/storiface" ) -const metaFile = "sectorstore.json" - -func createTestStorage(t *testing.T, p string, seal bool, att ...*paths.Local) storiface.ID { - if err := os.MkdirAll(p, 0755); err != nil { - if !os.IsExist(err) { - require.NoError(t, err) - } - } - - cfg := &storiface.LocalStorageMeta{ - ID: storiface.ID(uuid.New().String()), - Weight: 10, - CanSeal: seal, - CanStore: !seal, - } - - b, err := json.MarshalIndent(cfg, "", " ") - require.NoError(t, err) - - require.NoError(t, os.WriteFile(filepath.Join(p, metaFile), b, 0644)) - - for _, s := range att { - require.NoError(t, s.OpenPath(context.Background(), p)) - } - - return cfg.ID -} - -func TestMoveShared(t *testing.T) { - logging.SetAllLoggers(logging.LevelDebug) - - sharedITestID := harmonydb.ITestNewID() - - db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID) - require.NoError(t, err) - - index := paths.NewDBIndex(nil, db) - - ctx := context.Background() - - dir := t.TempDir() - - openRepo := func(dir string) paths.LocalStorage { - bls := &paths.BasicLocalStorage{PathToJSON: filepath.Join(t.TempDir(), "storage.json")} - return bls - } - - // setup two repos with two storage paths: - // repo 1 with both paths - // repo 2 with one path (shared) - - lr1 := openRepo(filepath.Join(dir, "l1")) - lr2 := openRepo(filepath.Join(dir, "l2")) - - mux1 := mux.NewRouter() - mux2 := mux.NewRouter() - hs1 := httptest.NewServer(mux1) - hs2 := httptest.NewServer(mux2) - - ls1, err := paths.NewLocal(ctx, lr1, index, hs1.URL+"/remote") - require.NoError(t, err) - ls2, err := paths.NewLocal(ctx, lr2, index, hs2.URL+"/remote") - require.NoError(t, err) - - dirStor := filepath.Join(dir, "stor") - dirSeal := filepath.Join(dir, "seal") - - id1 := createTestStorage(t, dirStor, false, ls1, ls2) - id2 := createTestStorage(t, dirSeal, true, ls1) - - rs1, err := paths.NewRemote(ls1, index, nil, 20, &paths.DefaultPartialFileHandler{}) - require.NoError(t, err) - rs2, err := paths.NewRemote(ls2, index, nil, 20, &paths.DefaultPartialFileHandler{}) - require.NoError(t, err) - _ = rs2 - mux1.PathPrefix("/").Handler(&paths.FetchHandler{Local: ls1, PfHandler: &paths.DefaultPartialFileHandler{}}) - mux2.PathPrefix("/").Handler(&paths.FetchHandler{Local: ls2, PfHandler: &paths.DefaultPartialFileHandler{}}) - - // add a sealed replica file to the sealing (non-shared) path - - s1ref := storiface.SectorRef{ - ID: abi.SectorID{ - Miner: 12, - Number: 1, - }, - ProofType: abi.RegisteredSealProof_StackedDrg2KiBV1, - } - - sp, sid, err := rs1.AcquireSector(ctx, s1ref, storiface.FTNone, storiface.FTSealed, storiface.PathSealing, storiface.AcquireMove) - require.NoError(t, err) - require.Equal(t, id2, storiface.ID(sid.Sealed)) - - data := make([]byte, 2032) - data[1] = 54 - require.NoError(t, os.WriteFile(sp.Sealed, data, 0666)) - fmt.Println("write to ", sp.Sealed) - - require.NoError(t, index.StorageDeclareSector(ctx, storiface.ID(sid.Sealed), s1ref.ID, storiface.FTSealed, true)) - - // move to the shared path from the second node (remote move / delete) - - require.NoError(t, rs2.MoveStorage(ctx, s1ref, storiface.FTSealed)) - - // check that the file still exists - sp, sid, err = rs2.AcquireSector(ctx, s1ref, storiface.FTSealed, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) - require.NoError(t, err) - require.Equal(t, id1, storiface.ID(sid.Sealed)) - fmt.Println("read from ", sp.Sealed) - - read, err := os.ReadFile(sp.Sealed) - require.NoError(t, err) - require.EqualValues(t, data, read) -} - func TestReader(t *testing.T) { //stm: @STORAGE_INFO_001 logging.SetAllLoggers(logging.LevelDebug) diff --git a/tools/tools.go b/tools/tools.go new file mode 100644 index 000000000..80a64c8b5 --- /dev/null +++ b/tools/tools.go @@ -0,0 +1,14 @@ +//go:build tools + +// Package tools tracks dev/CI tool dependencies. +// This file is not compiled into the binary but ensures tools are tracked in go.mod. +// Install all tools with: go install ./tools/... +// Or individually: go install golang.org/x/tools/cmd/goimports +package tools + +import ( + _ "github.com/hannahhoward/cbor-gen-for" + _ "github.com/rhysd/actionlint/cmd/actionlint" + _ "github.com/swaggo/swag/cmd/swag" + _ "golang.org/x/tools/cmd/goimports" +)