Merge pull request #524 from MilesCranmer/skip-mooncake

Refactor test suite into projects

Author: MilesCranmer

.github/workflows/CI.yml

@@ -17,48 +17,34 @@ concurrency:
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 
 jobs:
-  test:
-    name: Julia ${{ matrix.julia-version }}-${{ matrix.os }}-${{ matrix.test }}-${{ github.event_name }}
+  unit:
+    name: u/${{ matrix.group }}-${{ matrix.julia-version }}-${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 240
+    timeout-minutes: 60
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
     strategy:
       fail-fast: false
       matrix:
-        test:
-          - "part1"
-          - "part2"
-          - "part3"
-        julia-version:
-          - "1.10"
-          - "1"
-        os:
-          - ubuntu-latest
-        include:
-          - os: windows-latest
-            julia-version: "1"
-            test: "part1"
-          - os: windows-latest
-            julia-version: "1"
-            test: "part2"
-          - os: windows-latest
-            julia-version: "1"
-            test: "part3"
-          - os: macOS-latest
-            julia-version: "1"
-            test: "part1"
-          - os: macOS-latest
-            julia-version: "1"
-            test: "part2"
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+        julia-version: ["1.10", "1"]
+        group:
+          - basic
+          - expressions
+          - templates
+          - operators-core
+          - constraints
+          - evolution-core
+          - evaluation
+          - dataset
+          - parallel
+          - precompile
+          - misc
+        exclude:
+          # Only run Julia 1.10 on ubuntu
           - os: macOS-latest
-            julia-version: "1"
-            test: "part3"
-          - os: ubuntu-latest
-            julia-version: "1"
-            test: "enzyme"
-          - os: ubuntu-latest
-            julia-version: "1"
-            test: "jet"
+            julia-version: "1.10"
+          - os: windows-latest
+            julia-version: "1.10"
 
     steps:
       - uses: actions/checkout@v5
@@ -69,35 +55,93 @@ jobs:
       - name: "Cache dependencies"
         uses: julia-actions/cache@v2
         with:
-          cache-name: julia-cache;workflow=${{ github.workflow }};job=${{ github.job }};os=${{ matrix.os }};julia=${{ matrix.julia-version }};project=${{ hashFiles('**/Project.toml') }}
+          cache-name: julia-cache;workflow=${{ github.workflow }};job=${{ github.job }};os=${{ matrix.os }};julia=${{ matrix.julia-version }};group=unit-${{ matrix.group }};project=${{ hashFiles('test/Project.toml') }}
       - name: "Build package"
         uses: julia-actions/julia-buildpkg@v1
-      - name: "Install Enzyme (if needed)"
-        if: matrix.test == 'enzyme'
-        run: julia --color=yes --project=test -e 'import Pkg; Pkg.add("Enzyme")'
-      - name: "Install JET (if needed)"
-        if: matrix.test == 'jet'
-        run: julia --color=yes --project=test -e 'import Pkg; Pkg.add("JET")'
-      - name: "Run tests"
+      - name: "Run unit tests (${{ matrix.group }})"
+        env:
+          TEST_GROUP: unit/${{ matrix.group }}
+        run: |
+          julia --color=yes -e 'import Pkg; Pkg.add("Coverage")'
+          julia --color=yes --project=. --threads=auto --check-bounds=yes --depwarn=yes --code-coverage=user -e 'import Coverage; import Pkg; Pkg.test(coverage=true)'
+          julia --color=yes coverage.jl
+        shell: bash
+      - name: "Sanitize artifact group"
+        id: sanitize_unit_group
+        shell: bash
+        run: |
+          group='${{ matrix.group }}'
+          safe_group="${group//\//-}"
+          echo "group=$safe_group" >> "$GITHUB_OUTPUT"
+      - name: "Upload coverage artifacts"
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-unit-${{ steps.sanitize_unit_group.outputs.group }}-${{ matrix.julia-version }}-${{ matrix.os }}
+          path: lcov.info
+          retention-days: 1
+
+  integration:
+    name: i/${{ matrix.group }}-${{ matrix.julia-version }}-ubuntu-latest
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      fail-fast: false
+      matrix:
+        julia-version: ["1.10", "1"]
+        group:
+          - ad/forwarddiff
+          - ad/zygote
+          - ad/enzyme
+          - ad/mooncake
+          - aqua
+          - jet
+          - ext/mlj/core
+          - ext/mlj/mti
+          - ext/mlj/templates
+          - ext/symbolicutils
+          - ext/json3_recorder
+          - ext/dynamicquantities_units
+          - ext/loopvectorization
+        exclude:
+          - julia-version: "1"
+            group: ad/mooncake
+    steps:
+      - uses: actions/checkout@v5
+      - name: "Set up Julia"
+        uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.julia-version }}
+      - name: "Sanitize group"
+        id: sanitize_integration_group
+        shell: bash
+        run: |
+          group='${{ matrix.group }}'
+          safe_group="${group//\//-}"
+          echo "group=$safe_group" >> "$GITHUB_OUTPUT"
+      - name: "Cache dependencies"
+        uses: julia-actions/cache@v2
+        with:
+          cache-name: julia-cache;workflow=${{ github.workflow }};job=${{ github.job }};group=integration-${{ steps.sanitize_integration_group.outputs.group }};julia=${{ matrix.julia-version }};project=${{ hashFiles(format('test/integration/{0}/Project.toml', matrix.group)) }}
+      - name: "Run integration test"
         env:
-          SYMBOLIC_REGRESSION_TEST_SUITE: ${{ matrix.test }}
+          TEST_GROUP: integration/${{ matrix.group }}
         run: |
           julia --color=yes -e 'import Pkg; Pkg.add("Coverage")'
-          julia --color=yes --threads=auto --check-bounds=yes --depwarn=yes --code-coverage=user -e 'import Coverage; import Pkg; Pkg.activate("."); Pkg.test(coverage=true)'
+          julia --color=yes --project=. --threads=auto --check-bounds=yes --code-coverage=user -e 'import Coverage; import Pkg; Pkg.test(coverage=true)'
           julia --color=yes coverage.jl
         shell: bash
       - name: "Upload coverage artifacts"
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-${{ matrix.julia-version }}-${{ matrix.os }}-${{ matrix.test }}
+          name: coverage-integration-${{ steps.sanitize_integration_group.outputs.group }}-${{ matrix.julia-version }}
           path: lcov.info
           retention-days: 1
 
   upload-coverage:
     name: Upload Coverage to Codecov
-    needs: test
+    needs: [unit, integration]
     runs-on: ubuntu-latest
-    # Only run on pushes to master or pull requests
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - uses: actions/checkout@v5

.gitignore

@@ -25,6 +25,8 @@ docs/node_modules/
 docs/package-lock.json
 
 LocalPreferences.toml
+!test/LocalPreferences.toml
+!test/**/LocalPreferences.toml
 private
 
 # Generated examples:

docs/make.jl

@@ -247,7 +247,9 @@ function generate_favicons()
 
     for (filename, size) in favicon_configs
         output_path = joinpath(public_dir, filename)
-        run(`magick $(logo_path) -resize $(size) -background none -gravity center -extent $(size) $(output_path)`)
+        # Use 'convert' for ImageMagick 6.x (Ubuntu default), 'magick' for ImageMagick 7.x
+        magick_cmd = Sys.which("magick") !== nothing ? "magick" : "convert"
+        run(`$(magick_cmd) $(logo_path) -resize $(size) -background none -gravity center -extent $(size) $(output_path)`)
         @info "Generated: $filename"
     end
 
@@ -311,7 +313,7 @@ DocMeta.setdocmeta!(
 makedocs(;
     sitename="SymbolicRegression.jl",
     authors="Miles Cranmer",
-    current_version=current_version,
+    version=current_version,
     doctest=true,
     clean=get(ENV, "DOCUMENTER_PRODUCTION", "false") == "true",
     warnonly=[:docs_block, :cross_references, :missing_docs],

docs/utils.jl

@@ -3,9 +3,9 @@ using Literate: Literate
 # Function to process literate blocks in test files
 function process_literate_blocks(base_path="test")
     test_dir = joinpath(@__DIR__, "..", base_path)
-    for file in readdir(test_dir)
-        if endswith(file, ".jl")
-            process_file(joinpath(test_dir, file))
+    for (dirpath, _, files) in walkdir(test_dir)
+        for file in files
+            endswith(file, ".jl") && process_file(joinpath(dirpath, file))
         end
     end
 end

examples/custom_types.jl

@@ -12,7 +12,7 @@ Let's actually try this. Let's evolve an _expression over strings_.
 First, we mock up a dataset. Say that we wish to find the expression
 
 ```math
-y = \text{zip}(
+y = \text{interleave}(
     \text{concat}(x_1, \text{concat}(\text{``abc''}, x_2)),
     \text{concat}(
         \text{concat}(\text{tail}(x_3), \text{reverse}(x_4)),
@@ -39,7 +39,7 @@ tail(s::String) = length(s) == 0 ? "" : join(collect(s)[max(1, div(length(s), 2)
 concat(a::String, b::String) = a * b
 
 """Interleaves characters from two strings."""
-function zip(a::String, b::String)
+function interleave(a::String, b::String)
     total_length = length(a) + length(b)
     result = Vector{Char}(undef, total_length)
     i_a = firstindex(a)
@@ -71,7 +71,7 @@ function single_instance(rng=Random.default_rng())
     x_4 = join(rand(rng, 'a':'z', rand(rng, 1:10)))
 
     ## True formula:
-    y = zip(x_1 * "abc" * x_2, tail(x_3) * reverse(x_4) * "xyz")
+    y = interleave(x_1 * "abc" * x_2, tail(x_3) * reverse(x_4) * "xyz")
     return (; X=(; x_1, x_2, x_3, x_4), y)
 end
 
@@ -222,7 +222,7 @@ because we are dealing with non-numeric types.
 We also need to manually define the `loss_type`, since it's not inferrable from
 `loss_type`.
 =#
-binary_operators = (concat, zip)
+binary_operators = (concat, interleave)
 unary_operators = (head, tail, reverse)
 hparams = (;
     batching=true,

src/SymbolicRegression.jl

@@ -231,6 +231,8 @@ using DispatchDoctor: @stable
     include("TemplateExpression.jl")
     include("TemplateExpressionMacro.jl")
     include("ParametricExpression.jl")
+
+    __dispatch_doctor_unsable_test() = Val(rand(1:10))
 end
 
 using .CoreModule:

test/LocalPreferences.toml

@@ -1,36 +1,22 @@
 [deps]
-Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
-Bumper = "8ce10254-0962-460f-a3d8-1f77fea1446e"
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
-DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
-DispatchDoctor = "8d63f2c5-f18a-4cf2-ba9d-b3f60fc568c8"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 DynamicExpressions = "a40a106e-89c9-4ca8-8020-a735e8728b6b"
-DynamicQuantities = "06fc5a27-2a28-4c7c-a15d-362465fb6821"
-ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
-LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
-LoopVectorization = "bdcacae8-1622-11e9-2a5c-532679323890"
-MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
-MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea"
-MLJTestInterface = "72560011-54dd-4dc2-94f3-c5de45b75ecd"
-Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6"
 Optim = "429524aa-4258-5aef-a3af-852621145aeb"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 StyledStrings = "f489334b-da3d-4c2e-b8f0-e476e12c162b"
 Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
-SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b"
-TensorBoardLogger = "899adc3e-224a-11e9-021f-63837185c80f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
 TestItems = "1c621080-faea-4a02-84b6-bbd5e436b8fe"
-Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [preferences.SymbolicRegression]
+dispatch_doctor_codegen_level = "min"
 dispatch_doctor_mode = "error"

test/Project.toml

@@ -0,0 +1,18 @@
+# Common imports and TEST_GROUP selection
+using Test
+using TestItems
+using TestItemRunner
+using Preferences: set_preferences!
+
+ENV["SYMBOLIC_REGRESSION_IS_TESTING"] = "true"
+
+# Allow TEST_GROUP via ENV or the first CLI arg, default to "unit/basic"
+const TEST_GROUP = let g = get(ENV, "TEST_GROUP", nothing)
+    g === nothing && length(ARGS) > 0 ? ARGS[1] : (g === nothing ? "unit/basic" : g)
+end
+
+ENV["SYMBOLIC_REGRESSION_TEST"] = "true"
+
+if TEST_GROUP == "integration/jet"
+    set_preferences!("SymbolicRegression", "dispatch_doctor_mode" => "disable"; force=true)
+end