[JuliaLowering] Reconstruct macro name before, not during, macro expansion (#60170)

On top of #60162. This
simplification lets macro expansion handle the macro name as an
identifier rather than the nonterminal `K"macro_name"` node. That and
the old `MacroName` node (as well as `CmdMacroName`, `StrMacroName`) are
mainly the result of restrictions on raw parser output: we can't parse
`@Base.x` to a non-contiguous `@x`, even though that's what we want in
the AST in almost every case.

Author: mlechu

JuliaLowering/src/ast.jl

@@ -162,8 +162,7 @@ end
 function newleaf(ctx, srcref, k::Kind, @nospecialize(value))
     leaf = makeleaf(ctx, srcref, k)
     if k == K"Identifier" || k == K"core" || k == K"top" || k == K"Symbol" ||
-            k == K"globalref" || k == K"Placeholder" ||
-            k == K"StrMacroName" || k == K"CmdMacroName"
+            k == K"globalref" || k == K"Placeholder"
         setattr!(leaf._graph, leaf._id, :name_val, value)
     elseif k == K"BindingId"
         setattr!(leaf._graph, leaf._id, :var_id, value)

JuliaLowering/src/compat.jl

@@ -267,14 +267,10 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
             src = child_exprs[2]
         end
         deleteat!(child_exprs, 2)
-        if a1 isa Symbol
-            child_exprs[1] = a1_esc(Expr(:macro_name, a1))
-        elseif a1 isa Expr && a1.head === :(.)
-            a12,a12_esc = unwrap_esc(a1.args[2])
-            if a12 isa QuoteNode
-                child_exprs[1] = a1_esc(Expr(:(.), a1.args[1],
-                                             Expr(:macro_name, a12_esc(a12.value))))
-            end
+        if a1 isa Symbol && a1 === Symbol("@__dot__")
+            child_exprs[1] = Symbol("@.")
+        elseif a1 isa Expr && nargs === 2 && a1.args[2] === Symbol("@__dot__")
+            child_exprs[1] = Expr(a1.head, a1.args[1], Symbol("@."))
         elseif a1 isa GlobalRef && a1.mod === Core
             # Syntax-introduced macrocalls are listed here for reference.  We
             # probably don't need to convert these.
@@ -412,16 +408,15 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
         # (do (call f args...) (-> (tuple lam_args...) (block ...)))
         # SyntaxTree:
         # (call f args... (do (tuple lam_args...) (block ...)))
-        callargs = collect_expr_parameters(e.args[1], 2)
         if e.args[1].head === :macrocall
             st_k = K"macrocall"
+            callargs = collect_expr_parameters(e.args[1], 3)
             if callargs[2] isa LineNumberNode
                 src = callargs[2]
             end
             deleteat!(callargs, 2)
-            c1,c1_esc = unwrap_esc(callargs[1])
-            callargs[1] = c1_esc(Expr(:macro_name, c1))
         else
+            callargs = collect_expr_parameters(e.args[1], 2)
             st_k = K"call"
         end
         child_exprs = Any[callargs..., Expr(:do_lambda, e.args[2].args...)]
@@ -538,20 +533,7 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
 
     #---------------------------------------------------------------------------
     # Possibly-temporary heads introduced by us converting the parent expr
-    if e.head === :macro_name
-        @assert nargs === 1
-        # Trim `@` for a correct SyntaxTree, although we need to add it back
-        # later for finding the macro
-        if e.args[1] === :(.)
-            mac_name = string(e.args[1][2])
-            mac_name = mac_name == "@__dot__" ? "." : mac_name[2:end]
-            child_exprs[1] = Expr(:(.), e.args[1][1], Symbol(mac_name))
-        else
-            mac_name = string(e.args[1])
-            mac_name = mac_name == "@__dot__" ? "." : mac_name[2:end]
-            child_exprs[1] = Symbol(mac_name)
-        end
-    elseif e.head === :catch_var_placeholder
+    if e.head === :catch_var_placeholder
         st_k = K"Placeholder"
         st_attrs[:name_val] = ""
         child_exprs = nothing

JuliaLowering/src/macro_expansion.jl

@@ -138,27 +138,6 @@ function Base.showerror(io::IO, exc::MacroExpansionError)
     end
 end
 
-function fixup_macro_name(ctx::MacroExpansionContext, ex::SyntaxTree)
-    k = kind(ex)
-    if k == K"StrMacroName" || k == K"CmdMacroName"
-        layerid = get(ex, :scope_layer, current_layer_id(ctx))
-        newname = JuliaSyntax.lower_identifier_name(ex.name_val, k)
-        makeleaf(ctx, ex, ex, [:kind=>K"Identifier", :scope_layer=>layerid,
-                               :name_val=>newname])
-    elseif k == K"macro_name"
-        @chk numchildren(ex) === 1
-        if kind(ex[1]) === K"."
-            @ast ctx ex [K"." ex[1][1] [K"macro_name" ex[1][2]]]
-        else
-            layerid = get(ex, :scope_layer, current_layer_id(ctx))
-            newname = JuliaSyntax.lower_identifier_name(ex[1].name_val, K"macro_name")
-            makeleaf(ctx, ex[1], ex[1], [:kind=>kind(ex[1]), :name_val=>newname])
-        end
-    else
-        mapchildren(e->fixup_macro_name(ctx,e), ctx, ex)
-    end
-end
-
 function _eval_dot(world::UInt, mod, ex::SyntaxTree)
     if kind(ex) === K"."
         mod = _eval_dot(world, mod, ex[1])
@@ -175,7 +154,7 @@ end
 # isn't clear the language is meant to support this).
 function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex0::SyntaxTree)
     mod = current_layer(ctx).mod
-    ex = fixup_macro_name(ctx, expand_forms_1(ctx, ex0))
+    ex = expand_forms_1(ctx, ex0)
     try
         if kind(ex) === K"Value"
             !(ex.value isa GlobalRef) ? ex.value :
@@ -433,10 +412,6 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree)
             scope_layer = get(ex, :scope_layer, current_layer_id(ctx))
             makeleaf(ctx, ex, ex, [:kind=>k, :scope_layer=>scope_layer])
         end
-    elseif k == K"StrMacroName" || k == K"CmdMacroName" || k == K"macro_name"
-        # These can appear outside of a macrocall, e.g. in `import`
-        e2 = fixup_macro_name(ctx, ex)
-        expand_forms_1(ctx, e2)
     elseif k == K"var" || k == K"char" || k == K"parens"
         # Strip "container" nodes
         @chk numchildren(ex) == 1
@@ -512,7 +487,7 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree)
         @ast ctx ex [K"." expand_forms_1(ctx, ex[1]) e2]
     elseif k == K"cmdstring"
         @chk numchildren(ex) == 1
-        e2 = @ast ctx ex [K"macrocall" [K"macro_name" "cmd"::K"core"] ex[1]]
+        e2 = @ast ctx ex [K"macrocall" "@cmd"::K"core" ex[1]]
         expand_macro(ctx, e2)
     elseif (k == K"call" || k == K"dotcall")
         # Do some initial desugaring of call and dotcall here to simplify

JuliaLowering/src/syntax_graph.jl

@@ -1,7 +1,8 @@
 # TODO: This whole file should probably be moved to JuliaSyntax.
 import .JuliaSyntax: ParseStream, RedTreeCursor, reverse_toplevel_siblings,
     has_toplevel_siblings, _unsafe_wrap_substring, parse_julia_literal, is_trivia,
-    is_prefix_op_call, @isexpr, SyntaxHead, COLON_QUOTE, is_syntactic_operator
+    is_prefix_op_call, @isexpr, SyntaxHead, COLON_QUOTE, is_syntactic_operator,
+    lower_identifier_name
 
 const NodeId = Int
 
@@ -417,7 +418,7 @@ attrsummary(name, value::Number) = "$name=$value"
 
 function _value_string(ex)
     k = kind(ex)
-    str = k in KSet"Identifier StrMacroName CmdMacroName" || is_operator(k) ? ex.name_val :
+    str = k == K"Identifier" || is_operator(k) ? ex.name_val :
           k == K"Placeholder" ? ex.name_val           :
           k == K"SSAValue"    ? "%"                   :
           k == K"BindingId"   ? "#"                   :
@@ -584,17 +585,11 @@ function _find_SyntaxTree_macro(ex, line)
         # We're in the line range. Either
         if firstline == line && kind(c) == K"macrocall" && begin
                     name = c[1]
-                    if kind(name) == K"macro_name"
-                        name = name[1]
-                    end
                     if kind(name) == K"."
                         name = name[2]
-                        if kind(name) == K"macro_name"
-                            name = name[1]
-                        end
                     end
                     @assert kind(name) == K"Identifier"
-                    name.name_val == "SyntaxTree"
+                    name.name_val == "@SyntaxTree"
                 end
             # We find the node we're looking for. NB: Currently assuming a max
             # of one @SyntaxTree invocation per line. Though we could relax
@@ -891,6 +886,24 @@ function _green_to_ast(parent::Kind, ex::SyntaxTree; eq_to_kw=false)
     elseif k === K"=" && eq_to_kw
         setattr!(makenode(graph, ex, ex, _map_green_to_ast(k, children(ex))),
                  :kind, K"kw")
+    elseif k === K"CmdMacroName" || k === K"StrMacroName"
+        name = lower_identifier_name(ex.name_val, k)
+        setattr!(makeleaf(graph, ex, K"Identifier"),
+                 :name_val, name)
+    elseif k === K"macro_name"
+        # M.@x parses to (. M (macro_name x))
+        # @M.x parses to (macro_name (. M x))
+        # We want (. M @x) (both identifiers) in either case
+        @assert numchildren(ex) === 2 && kind(ex[1]) === K"@"
+        id = ex[2]
+        mname_raw = (kind(id) === K"." ? id[2] : id).name_val
+        mac_id = setattr!(makeleaf(graph, ex, K"Identifier"), :name_val,
+                          lower_identifier_name(mname_raw, K"macro_name"))
+        if kind(id) === K"."
+            makenode(graph, ex, ex, NodeId[id[1]._id, mac_id._id])
+        else
+            mac_id
+        end
     elseif is_leaf(ex)
         return ex
     else

JuliaLowering/test/compat.jl

@@ -494,7 +494,7 @@ const JL = JuliaLowering
         # `@mac x` with macro name escaped
         @test JuliaLowering.expr_to_syntaxtree(Expr(:macrocall, esc(Symbol("@mac")), nothing, :x)) ≈
             @ast_ [K"macrocall"
-                [K"escape" [K"macro_name" "mac"::K"Identifier"]]
+                [K"escape" "@mac"::K"Identifier"]
                 "x"::K"Identifier"
             ]
 
@@ -505,7 +505,7 @@ const JL = JuliaLowering
             [K"escape"
                 [K"."
                     "A"::K"Identifier"
-                    [K"macro_name" "mac"::K"Identifier"]
+                    "@mac"::K"Identifier"
                 ]
             ]
             "x"::K"Identifier"
@@ -577,7 +577,7 @@ const JL = JuliaLowering
             Expr(:macrocall, Expr(:var"hygienic-scope", Symbol("@mac"), :other, :args), nothing, :x)) ≈
             @ast_ [K"macrocall"
                 [K"hygienic_scope"
-                    [K"macro_name" "mac"::K"Identifier"]
+                    "@mac"::K"Identifier"
                     "other"::K"Identifier" # (<- normally a Module)
                     "args"::K"Identifier" # (<- normally a LineNumberNode)
                 ]
@@ -587,7 +587,7 @@ const JL = JuliaLowering
         # One example of double escaping
         @test JuliaLowering.expr_to_syntaxtree(Expr(:macrocall, esc(esc(Symbol("@mac"))), nothing, :x)) ≈
             @ast_ [K"macrocall"
-                [K"escape" [K"escape" [K"macro_name" "mac"::K"Identifier"]]]
+                [K"escape" [K"escape" "@mac"::K"Identifier"]]
                 "x"::K"Identifier"
             ]
 
@@ -600,7 +600,7 @@ const JL = JuliaLowering
             @ast_ [K"macrocall"
                 [K"hygienic_scope"
                     [K"escape"
-                        [K"macro_name" "mac"::K"Identifier"]
+                        "@mac"::K"Identifier"
                     ]
                     "other"::K"Identifier" # (<- normally a Module)
                     "args"::K"Identifier" # (<- normally a LineNumberNode)

JuliaLowering/test/macros.jl

@@ -347,6 +347,14 @@ end
         GC.@preserve x unsafe_load(p)
     end""") === 101 # Expr(:gc_preserve)
 
+    # JuliaLowering.jl/issues/121
+    @test JuliaLowering.include_string(test_mod, """
+    GC.@preserve @static if true @__MODULE__ else end
+    """) isa Module
+    @test JuliaLowering.include_string(test_mod, """
+    GC.@preserve @static if true v"1.14" else end
+    """) isa VersionNumber
+
     # only invokelatest produces :isglobal now, so MWE here
     Base.eval(test_mod, :(macro isglobal(x); esc(Expr(:isglobal, x)); end))
     @test JuliaLowering.include_string(test_mod, """

JuliaLowering/test/macros_ir.jl

@@ -147,7 +147,7 @@ _never_exist = @m_not_exist 42
 #---------------------
 MacroExpansionError while expanding @m_not_exist in module Main.TestMod:
 _never_exist = @m_not_exist 42
-#               └─────────┘ ── Macro not found
+#              └──────────┘ ── Macro not found
 Caused by:
 UndefVarError: `@m_not_exist` not defined in `Main.TestMod`
 Suggestion: check for spelling errors or missing imports.

JuliaLowering/test/utils.jl

@@ -167,7 +167,8 @@ end
 function format_ir_for_test(mod, case)
     ex = parsestmt(SyntaxTree, case.input)
     try
-        if kind(ex) == K"macrocall" && kind(ex[1]) == K"macro_name" && ex[1][1].name_val == "ast_"
+        if (kind(ex) == K"macrocall" && kind(ex[1]) == K"Identifier" &&
+            ex[1].name_val == "@ast_")
             # Total hack, until @ast_ can be implemented in terms of new-style
             # macros.
             ex = Base.eval(mod, Expr(ex))

test/JuliaLowering_stdlibs.jl

@@ -1,10 +1,19 @@
 import Libdl
 
-### 38 / 52 (non-sysimage) stdlibs precompile successfully with JuliaLowering ###
+# known precompilation failures under JL
 const INCOMPATIBLE_STDLIBS = String[
-    "Statistics", "JuliaSyntaxHighlighting", "Markdown", "LibGit2",
-    "InteractiveUtils", "Test", "REPL", "Pkg", "LazyArtifacts", "SparseArrays",
-    "TOML", "StyledStrings", "Profile", "SuiteSparse",
+    "InteractiveUtils"
+    "LazyArtifacts"
+    "LibGit2"
+    "Pkg"
+    "REPL"
+    "REPLExt"
+    "SparseArrays"
+    "SparseArraysExt"
+    "Statistics"
+    "SuiteSparse"
+    "TOML"
+    "Test"
 ]
 
 const JULIA_EXECUTABLE = Base.unsafe_string(Base.JLOptions().julia_bin)