Add test-JuliaLowering_stdlibs target for precompiling stdlibs with JuliaLowering (#60194)

This adds a new test target: `make test-JuliaLowering_stdlibs`, which
precompiles a supported set of standard libraries using JuliaLowering.

To accomplish this, it also adds a basic incremental sysimage build
target (`julia-sysimg-JL-release` / `julia-sysimg-JL-debug`) that
modifies the base sysimage to include JuliaLowering.

Currently 38 / 52 pkgimage-based stdlibs can be compiled with
JuliaLowering:

- Pre-compiling: ArgTools, Future, LibCURL_jll, Logging, OpenSSL_jll,
Tar, dSFMT_jll, Base64, GMP_jll, LibGit2_jll, MPFR_jll, PCRE2_jll,
UUIDs, libLLVM_jll, CRC32c, LLD_jll, LibSSH2_jll, MozillaCACerts_jll,
Printf, Unicode, Mmap, nghttp2_jll, Distributed, LLVMLibUnwind_jll,
LibUV_jll, NetworkOptions, Serialization, Zlib_jll, p7zip_jll, LibCURL,
LibUnwind_jll, OpenLibm_jll, SuiteSparse_jll, Zstd_jll, Dates,
DelimitedFiles, SharedArrays, Downloads
- Failing: Statistics, StyledStrings, SparseArrays, SuiteSparse,
Profile, JuliaSyntaxHighlighting, Markdown, LibGit2, InteractiveUtils,
Test, REPL, TOML, Pkg, LazyArtifacts

Author: topolarity

Makefile

@@ -123,6 +123,10 @@ julia-cli-release julia-cli-debug: julia-cli-% : julia-deps
 julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir)
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-$*
 
+.PHONY: julia-sysimg-JL-release julia-sysimg-JL-debug
+julia-sysimg-JL-release julia-sysimg-JL-debug : julia-sysimg-JL-% : julia-sysimg-% julia-stdlib | $(build_private_libdir)
+	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-JL-$*
+
 # Useful for cross-bootstrapping
 .PHONY: julia-sysbase-release julia-sysbase-debug
 julia-sysbase-release julia-sysbase-debug : julia-sysbase-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir)

base/Base.jl

@@ -319,6 +319,9 @@ a_method_to_overwrite_in_test() = inferencebarrier(1)
 Core.println("JuliaSyntax/src/JuliaSyntax.jl")
 include(@__MODULE__, string(DATAROOT, "julia/JuliaSyntax/src/JuliaSyntax.jl"))
 
+# May be replaced in incremental sysimage build after-the-fact
+const JuliaLowering = nothing
+
 end_base_include = time_ns()
 
 const _sysimage_modules = PkgId[]
@@ -397,6 +400,11 @@ function __init__()
     if get_bool_env("JULIA_USE_FLISP_PARSER", false) === false
         JuliaSyntax.enable_in_core!()
     end
+    if JuliaLowering !== nothing && get_bool_env("JULIA_USE_FLISP_LOWERING", true) === false
+        # This is not available by default, but JuliaLowering can be added to
+        # Base after-the-fact via an incremental sysimage build.
+        JuliaLowering.activate!()
+    end
 
     CoreLogging.global_logger(CoreLogging.ConsoleLogger())
     nothing

sysimage.mk

@@ -11,6 +11,8 @@ sysimg-ji: $(build_private_libdir)/sysbase.ji
 sysimg-bc: $(build_private_libdir)/sys-bc.a
 sysimg-release: $(build_private_libdir)/sys.$(SHLIB_EXT)
 sysimg-debug: $(build_private_libdir)/sys-debug.$(SHLIB_EXT)
+sysimg-JL-release: $(build_private_libdir)/sys-JL.$(SHLIB_EXT)
+sysimg-JL-debug: $(build_private_libdir)/sys-JL-debug.$(SHLIB_EXT)
 sysbase-release: $(build_private_libdir)/sysbase.$(SHLIB_EXT)
 sysbase-debug: $(build_private_libdir)/sysbase-debug.$(SHLIB_EXT)
 
@@ -147,7 +149,31 @@ $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(buil
 .SECONDARY: $$(build_private_libdir)/sys$1-o.a $(build_private_libdir)/sys$1-bc.a # request Make to keep these files around
 .SECONDARY: $$(build_private_libdir)/sysbase$1-o.a $(build_private_libdir)/sysbase$1-bc.a # request Make to keep these files around
 endef
+
+JULIALOWERING_SRCS := $(shell find $(build_datarootdir)/julia/JuliaLowering/src -name \*.jl)
+
+define JL_sysimg_builder
+$$(build_private_libdir)/sys-JL$1-o.a $$(build_private_libdir)/sys-JL$1-bc.a : $$(build_private_libdir)/sys-JL$1-%.a : $$(build_private_libdir)/sys$1.$$(SHLIB_EXT) $$(JULIALOWERING_SRCS)
+	@$$(call PRINT_JULIA, cd $$(JULIAHOME)/JuliaLowering/src/ && \
+	if ! JULIA_BINDIR=$$(call cygpath_w,$(build_bindir)) \
+		 WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \
+		 JULIA_LOAD_PATH='@stdlib' \
+		 JULIA_PROJECT= \
+		 JULIA_DEPOT_PATH=':' \
+		 JULIA_NUM_THREADS=1 \
+			$$(call spawn, $3) $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \
+			$(bootstrap_julia_flags) \
+			--startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) -e "Core.include(Base, raw\"$$(call cygpath_w,$$(BUILDROOT)/JuliaLowering/src/JuliaLowering.jl)\")"; then \
+		echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \
+		false; \
+	fi )
+	@mv $$@.tmp $$@
+.SECONDARY: $$(build_private_libdir)/sys-JL$1-o.a $(build_private_libdir)/sys-JL$1-bc.a # request Make to keep these files around
+endef
+
 $(eval $(call base_builder,,-O1,$(JULIA_EXECUTABLE_release)))
 $(eval $(call base_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))
 $(eval $(call sysimg_builder,,-O3,$(JULIA_EXECUTABLE_release)))
 $(eval $(call sysimg_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))
+$(eval $(call JL_sysimg_builder,,-O3,$(JULIA_EXECUTABLE_release)))
+$(eval $(call JL_sysimg_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))

test/JuliaLowering_stdlibs.jl

@@ -0,0 +1,79 @@
+import Libdl
+
+### 38 / 52 (non-sysimage) stdlibs precompile successfully with JuliaLowering ###
+const INCOMPATIBLE_STDLIBS = String[
+    "Statistics", "JuliaSyntaxHighlighting", "Markdown", "LibGit2",
+    "InteractiveUtils", "Test", "REPL", "Pkg", "LazyArtifacts", "SparseArrays",
+    "TOML", "StyledStrings", "Profile", "SuiteSparse",
+]
+
+const JULIA_EXECUTABLE = Base.unsafe_string(Base.JLOptions().julia_bin)
+const JULIA_CPU_TARGET = get(ENV, "JULIA_CPU_TARGET", Base.unsafe_string(Base.JLOptions().cpu_target))
+const debug = get(ENV, "JULIA_BUILD_MODE", "release") == "debug" ? "-debug" : ""
+const JL_sysimage = joinpath(dirname(unsafe_string(Base.JLOptions().image_file)), "sys-JL$(debug).$(Libdl.dlext)")
+
+function compile_JL_sysimage(output_filepath)
+    sysimage = unsafe_string(Base.JLOptions().image_file)
+    output_sysimage = abspath(output_filepath)
+    output_object = "$(splitext(output_sysimage)[1])-o.a"
+
+    package_root = joinpath(Sys.STDLIB, "..", "..", "JuliaLowering")
+    cmd = `$(JULIA_EXECUTABLE) -C "$(JULIA_CPU_TARGET)" --output-o $(output_object)
+           --startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $(sysimage)
+           -e "Core.include(Base, $(repr(joinpath(package_root, "src", "JuliaLowering.jl"))))"`
+    cmd = addenv(
+        Base.Cmd(cmd; dir = joinpath(package_root, "src")),
+        "JULIA_BINDIR" => unsafe_string(Base.JLOptions().julia_bindir),
+        "JULIA_LOAD_PATH" => "@stdlib",
+        "JULIA_PROJECT" => nothing,
+        "JULIA_DEPOT_PATH" => ":",
+        "JULIA_NUM_THREADS" => "1",
+    )
+    println("Compiling incremental sysimage with JuliaLowering...")
+    success(run(cmd))
+
+    cmd = Base.Linking.link_image_cmd(output_object, output_sysimage)
+    success(run(cmd))
+
+    return nothing
+end
+
+# ensure JL-inclusive sysimage is built / available
+if "BUILDROOT" in keys(ENV)
+    # Running via Makefile, use sysimage.mk with its built-in caching / file tracking
+    run(`$(ENV["MAKE"]) -C $(ENV["BUILDROOT"]) -f sysimage.mk sysimg-JL-$(ENV["JULIA_BUILD_MODE"])`)
+else
+    # Standalone test run (CI), compile every time
+    compile_JL_sysimage(JL_sysimage)
+end
+stdlibs_to_test = filter(name -> !in(name, INCOMPATIBLE_STDLIBS), readdir(Sys.STDLIB))
+
+configs = [
+    ``=>Base.CacheFlags(check_bounds=0, debug_level=2, opt_level=3),
+    ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3),
+]
+setupproject_command = "using Pkg; Pkg.add($(stdlibs_to_test))"
+compilecache_command = "using Base: CacheFlags; Base.Precompilation.precompilepkgs($(stdlibs_to_test); configs=$(configs))"
+
+# pre-compile stdlibs (into temporary depot)
+mktempdir() do tmp_depot
+    # first setup the project / environment
+    env_dir = joinpath(tmp_depot, "environments", "v$(VERSION.major).$(VERSION.minor)")
+    cmd = addenv(
+        `$(JULIA_EXECUTABLE) --startup-file=no --project=$env_dir -e $setupproject_command`,
+        ; inherit = true
+    )
+    success(run(cmd))
+
+    # now actually perform the precompilation
+    cmd = addenv(
+        `$(JULIA_EXECUTABLE) --sysimage $(JL_sysimage) --startup-file=no -e $compilecache_command`,
+        "JULIA_LOAD_PATH" => "@stdlib$(Base.Linking.pathsep)$(env_dir)",
+        "JULIA_CPU_TARGET" => "sysimage",
+        "JULIA_USE_FLISP_LOWERING" => "0",
+        "JULIA_USE_FALLBACK_REPL" => "0",
+        "JULIA_DEPOT_PATH" => tmp_depot,
+        ; inherit = true
+    )
+    success(run(cmd))
+end

test/Makefile

@@ -11,6 +11,11 @@ export JULIA_LOAD_PATH := @$(PATHSEP)@stdlib
 unexport JULIA_PROJECT :=
 unexport JULIA_BINDIR :=
 
+# used by JuliaLowering_stdlibs.jl test (invokes `make` for JL sysimage)
+export MAKE
+export BUILDROOT
+export JULIA_BUILD_MODE
+
 TESTGROUPS = unicode strings compiler Compiler JuliaSyntax JuliaLowering
 TESTS = all default stdlib $(TESTGROUPS) \
 		$(patsubst $(STDLIBDIR)/%/,%,$(dir $(wildcard $(STDLIBDIR)/*/.))) \
@@ -33,7 +38,7 @@ default:
 
 .PHONY: $(TESTS)
 $(TESTS):
-	@cd $(SRCDIR) && \
+	+@cd $(SRCDIR) && \
 	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@)
 
 .PHONY: install-revise-deps

test/choosetests.jl

@@ -31,7 +31,7 @@ const TESTNAMES = [
         "smallarrayshrink", "opaque_closure", "filesystem", "download",
         "scopedvalues", "compileall", "rebinding",
         "faulty_constructor_method_should_not_cause_stack_overflows",
-        "JuliaSyntax", "JuliaLowering",
+        "JuliaSyntax", "JuliaLowering", "JuliaLowering_stdlibs",
 ]
 
 const INTERNET_REQUIRED_LIST = [