Merge pull request #92 from SymbolicML/multi-expressions

Algebra for `AbstractExpression` objects; and `StructuredExpression`

Author: MilesCranmer

.github/workflows/CI.yml

@@ -83,13 +83,16 @@ jobs:
       - uses: julia-actions/cache@v1
       - uses: julia-actions/julia-buildpkg@v1
       - name: Run tests
+        id: run-tests
+        continue-on-error: ${{ matrix.test_name == 'enzyme' }}
         run: |
             julia --color=yes -e 'import Pkg; Pkg.add("Coverage")'
             SR_TEST=${{ matrix.test_name }} julia --color=yes --threads=auto --check-bounds=yes --depwarn=yes --code-coverage=user -e 'import Coverage; import Pkg; Pkg.activate("."); Pkg.test(coverage=true)'
             julia --color=yes coverage.jl
         shell: bash
       - name: Coveralls
         uses: coverallsapp/github-action@v2
+        if: steps.run-tests.outcome == 'success'
         with:
           parallel: true
           path-to-lcov: lcov.info

docs/Project.toml

@@ -1,3 +1,5 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 DynamicExpressions = "a40a106e-89c9-4ca8-8020-a735e8728b6b"
+Interfaces = "85a1e053-f937-4924-92a5-1367d23b7b87"
+Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"

docs/make.jl

@@ -1,6 +1,18 @@
 using Documenter
 using DynamicExpressions
 using Random: AbstractRNG
+using Literate: markdown
+
+####################################
+# Literate #########################
+####################################
+
+include("utils.jl")
+process_literate_blocks()
+
+####################################
+# index.md #########################
+####################################
 
 readme = joinpath(@__DIR__, "..", "README.md")
 
@@ -28,9 +40,6 @@ index_content = let r = read(readme, String)
     bottom_part = """
     ## Contents
 
-    ```@contents
-    Pages = ["utils.md", "api.md", "eval.md"]
-    ```
     """
 
     join((top_part, r, bottom_part), "\n")
@@ -41,13 +50,26 @@ open(index_md, "w") do f
     write(f, index_content)
 end
 
+####################################
+
 makedocs(;
     sitename="DynamicExpressions.jl",
     authors="Miles Cranmer",
-    doctest=false,
     clean=true,
-    format=Documenter.HTML(),
-    warnonly=true,
+    format=Documenter.HTML(;
+        canonical="https://symbolicml.org/DynamicExpressions.jl/stable"
+    ),
+    pages=[
+        "Home" => "index.md",
+        "Examples" => [
+            "examples/base_operations.md", # Defined by `test/test_base_2.jl`
+            "examples/expression.md", # Defined by `test/test_expression.jl`
+            "examples/structured_expression.md", # Defined by `test/test_structured_expression.jl`
+        ],
+        "Eval" => "eval.md",
+        "Utils" => "utils.md",
+        "API" => "api.md",
+    ],
 )
 
 # Forward links from old docs:

docs/src/api.md

@@ -166,6 +166,33 @@ ParametricExpression
 ParametricNode
 ```
 
+Another example is the `StructuredExpression` type, for defining rigid
+predefined operations in an expression tree:
+
+```@docs
+StructuredExpression
+```
+
+You may use operators directly on `AbstractExpression` objects to create a new object
+containing the combined expression tree, so long as those objects have identical operators
+in their metadata.
+
+You can extract and set contents and metadata with a few utility functions, including:
+
+```@docs
+get_contents
+with_contents
+get_metadata
+with_metadata
+get_tree
+```
+
+To declare a new operator for expressions, you may use:
+
+```@docs
+@declare_expression_operator
+```
+
 ## Interfaces
 
 The interfaces for `AbstractExpression` and `AbstractExpressionNode` are

docs/utils.jl

@@ -0,0 +1,70 @@
+
+# Function to process literate blocks in test files
+function process_literate_blocks()
+    test_dir = joinpath(@__DIR__, "..", "test")
+    for file in readdir(test_dir)
+        if endswith(file, ".jl")
+            process_file(joinpath(test_dir, file))
+        end
+    end
+end
+
+function process_file(filepath)
+    content = read(filepath, String)
+    blocks = match_literate_blocks(content)
+    for (output_file, block_content) in blocks
+        process_literate_block(output_file, block_content, filepath)
+    end
+end
+
+function match_literate_blocks(content)
+    pattern = r"^(\s*)#literate_begin\s+file=\"(.*?)\"\n(.*?)#literate_end"sm
+    matches = collect(eachmatch(pattern, content))
+    return Dict(
+        m.captures[2] => process_block_content(m.captures[1], m.captures[3]) for
+        m in matches
+    )
+end
+
+function process_block_content(indent, block_content)
+    if isempty(block_content)
+        return ""
+    end
+    indent_length = length(indent)
+    lines = split(block_content, '\n')
+    stripped_lines = [
+        if length(line) > indent_length
+            line[(indent_length + 1):end]
+        else
+            ""
+        end for line in lines
+    ]
+    return strip(join(stripped_lines, '\n'))
+end
+
+function process_literate_block(output_file, content, source_file)
+    # Create a temporary .jl file
+    temp_file = tempname() * ".jl"
+    write(temp_file, content)
+
+    # Process the temporary file with Literate.markdown
+    output_dir = joinpath(@__DIR__, "src", "examples")
+    base_name = first(splitext(basename(output_file))) # Remove any existing extension
+
+    markdown(temp_file, output_dir; name=base_name, documenter=true)
+
+    # Generate the relative path for EditURL
+    edit_path = relpath(source_file, output_dir)
+
+    # Read the generated markdown file
+    md_file = joinpath(output_dir, base_name * ".md")
+    md_content = read(md_file, String)
+
+    # Replace the existing EditURL with the correct one
+    new_content = replace(md_content, r"EditURL = .*" => "EditURL = \"$edit_path\"")
+
+    # Write the updated content back to the file
+    write(md_file, new_content)
+
+    @info "Processed literate block to $md_file with EditURL set to $edit_path"
+end

src/DynamicExpressions.jl

@@ -17,9 +17,11 @@ using DispatchDoctor: @stable, @unstable
     include("Simplify.jl")
     include("OperatorEnumConstruction.jl")
     include("Expression.jl")
+    include("ExpressionAlgebra.jl")
     include("Random.jl")
     include("Parse.jl")
     include("ParametricExpression.jl")
+    include("StructuredExpression.jl")
 end
 
 import PackageExtensionCompat: @require_extensions
@@ -76,12 +78,20 @@ import .NodeModule:
 @reexport import .ExtensionInterfaceModule: node_to_symbolic, symbolic_to_node
 @reexport import .RandomModule: NodeSampler
 @reexport import .ExpressionModule:
-    AbstractExpression, Expression, with_contents, with_metadata, get_contents, get_metadata
+    AbstractExpression,
+    Expression,
+    with_contents,
+    with_metadata,
+    get_contents,
+    get_metadata,
+    get_tree
 import .ExpressionModule:
-    get_tree, get_operators, get_variable_names, Metadata, default_node_type, node_type
+    get_operators, get_variable_names, Metadata, default_node_type, node_type
+@reexport import .ExpressionAlgebraModule: @declare_expression_operator
 @reexport import .ParseModule: @parse_expression, parse_expression
 import .ParseModule: parse_leaf
 @reexport import .ParametricExpressionModule: ParametricExpression, ParametricNode
+@reexport import .StructuredExpressionModule: StructuredExpression
 
 @stable default_mode = "disable" begin
     include("Interfaces.jl")

src/Expression.jl

@@ -36,11 +36,13 @@ Base.propertynames(x::Metadata) = propertynames(_data(x))
 @unstable @inline Base.getproperty(x::Metadata, f::Symbol) = getproperty(_data(x), f)
 Base.show(io::IO, x::Metadata) = print(io, "Metadata(", _data(x), ")")
 @inline _copy(x) = copy(x)
+@inline _copy(x::NamedTuple) = copy_named_tuple(x)
 @inline _copy(x::Nothing) = nothing
+@inline function copy_named_tuple(nt::NamedTuple)
+    return NamedTuple{keys(nt)}(map(_copy, values(nt)))
+end
 @inline function Base.copy(metadata::Metadata)
-    nt = _data(metadata)
-    copied_nt = NamedTuple{keys(nt)}(map(_copy, values(nt)))
-    return Metadata(copied_nt)
+    return Metadata(_copy(_data(metadata)))
 end
 @inline Base.:(==)(x::Metadata, y::Metadata) = _data(x) == _data(y)
 @inline Base.hash(x::Metadata, h::UInt) = hash(_data(x), h)
@@ -101,26 +103,11 @@ default_node_type(::Type{<:AbstractExpression{T}}) where {T} = Node{T}
 ########################################################
 # Abstract interface ###################################
 ########################################################
-"""
-    get_operators(ex::AbstractExpression, operators::Union{Nothing,Any})
-
-which will return the operators to be passed to internal functions
-such as `eval_tree_array` or `string_tree`, either from the expression itself,
-or `cur_operators` if it is not `nothing`. If left as default,
-it requires `cur_operators` to not be `nothing`.
-`cur_operators` would typically be an `OperatorEnum`.
-"""
 function get_operators(
     ex::AbstractExpression, operators::Union{AbstractOperatorEnum,Nothing}=nothing
 )
     return error("`get_operators` function must be implemented for $(typeof(ex)) types.")
 end
-
-"""
-    get_variable_names(ex::AbstractExpression, variable_names::Union{Nothing,AbstractVector{<:AbstractString}})
-
-The same as `operators`, but for variable names.
-"""
 function get_variable_names(
     ex::AbstractExpression,
     variable_names::Union{Nothing,AbstractVector{<:AbstractString}}=nothing,
@@ -130,12 +117,6 @@ function get_variable_names(
     )
 end
 
-"""
-    get_tree(ex::AbstractExpression)
-
-A method that extracts the expression tree from `AbstractExpression`
-and should return an `AbstractExpressionNode`.
-"""
 function get_tree(ex::AbstractExpression)
     return error("`get_tree` function must be implemented for $(typeof(ex)) types.")
 end
@@ -167,6 +148,50 @@ function get_metadata(ex::AbstractExpression)
 end
 ########################################################
 
+"""
+    get_operators(ex::AbstractExpression, operators::Union{Nothing,Any})
+
+which will return the operators to be passed to internal functions
+such as `eval_tree_array` or `string_tree`, either from the expression itself,
+or `cur_operators` if it is not `nothing`. If left as default,
+it requires `cur_operators` to not be `nothing`.
+`cur_operators` would typically be an `OperatorEnum`.
+"""
+get_operators
+
+"""
+    get_variable_names(ex::AbstractExpression, variable_names::Union{Nothing,AbstractVector{<:AbstractString}})
+
+The same as `operators`, but for variable names.
+"""
+get_variable_names
+
+"""
+    get_tree(ex::AbstractExpression)
+
+A method that extracts the expression tree from `AbstractExpression`
+and should return an `AbstractExpressionNode`.
+"""
+get_tree
+
+"""
+    get_contents(ex::AbstractExpression)
+
+Get the contents of the expression, which might be a plain
+`AbstractExpressionNode`, or some combination of them, or other data.
+This should include everything other than that returned by [`get_metadata`](@ref).
+"""
+get_contents
+
+"""
+    get_metadata(ex::AbstractExpression)
+
+Get the metadata of the expression, which might be a plain
+`NamedTuple`, or some combination of them, or other data.
+This should include everything other than that returned by [`get_contents`](@ref).
+"""
+get_metadata
+
 """
     with_contents(ex::AbstractExpression, tree::AbstractExpressionNode)
     with_contents(ex::AbstractExpression, tree::AbstractExpression)
@@ -317,7 +342,7 @@ for io in ((), (:(io::IO),))
     end
 end
 
-function Base.show(io::IO, ::MIME"text/plain", ex::AbstractExpression)
+function Base.show(io::IO, ex::AbstractExpression)
     return print(io, string_tree(ex))
 end