diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 332a8251ee3..6f5765976d0 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -312,7 +312,12 @@ defmodule Code do @available_compiler_options @boolean_compiler_options ++ @list_compiler_options ++ - [:on_undefined_variable, :infer_signatures, :no_warn_undefined] + [ + :on_undefined_variable, + :infer_signatures, + :no_warn_undefined, + :module_definition + ] @doc """ Lists all required files. @@ -1702,9 +1707,6 @@ defmodule Code do Available options are: - * `:docs` - when `true`, retains documentation in the compiled module. - Defaults to `true`. - * `:debug_info` - when `true`, retains debug information in the compiled module. This option can also be overridden per module using the `@compile` directive. Defaults to `true`. @@ -1720,6 +1722,9 @@ defmodule Code do via the `:test_elixirc_options` project configuration, as there is typically no need to store debug chunks for test files. + * `:docs` - when `true`, retains documentation in the compiled module. + Defaults to `true`. + * `:ignore_already_consolidated` (since v1.10.0) - when `true`, does not warn when a protocol has already been consolidated and a new implementation is added. Defaults to `false`. @@ -1738,17 +1743,33 @@ defmodule Code do via the `:test_elixirc_options` project configuration, as there is typically no need to infer signatures for test files. - * `:relative_paths` - when `true`, uses relative paths in quoted nodes, - warnings, and errors generated by the compiler. Note disabling this option - won't affect runtime warnings and errors. Defaults to `true`. + * `:module_definition` (since v1.20.0) - stores if the module definition should + be `:compiled` (the default) or `:interpreted`. Note this does not affect the + `.beam` file written to disk, only how the contents inside `defmodule` are + executed. Using the `:interpreted` mode may offer better compilation times for + large projects, especially on machines with high core count, however, it comes + with some downsides: + + * Errors during compilation may have less precise stacktraces + + * Anonymous functions within `defmodule` can have only up to 20 arguments. + If this is an issue, you can use maps or tuples to group the data. + Note the functions themselves inside `defmodule`, such as the ones defined + inside `def` and friends, can still have up to 255 arguments * `:no_warn_undefined` (since v1.10.0) - list of modules and `{Mod, fun, arity}` tuples that will not emit warnings that the module or function does not exist at compilation time. Pass atom `:all` to skip warning for all undefined functions. This can be useful when doing dynamic compilation. Defaults to `[]`. - * `:tracers` (since v1.10.0) - a list of tracers (modules) to be used during - compilation. See the module docs for more information. Defaults to `[]`. + * `:on_undefined_variable` (since v1.15.0) - either `:raise` or `:warn`. + When `:raise` (the default), undefined variables will trigger a compilation + error. You may be set it to `:warn` if you want undefined variables to + emit a warning and expand as to a local call to the zero-arity function + of the same name (for example, `node` would be expanded as `node()`). + This `:warn` behavior only exists for compatibility reasons when working + with old dependencies, its usage is discouraged and it will be removed + in future releases. * `:parser_options` (since v1.10.0) - a keyword list of options to be given to the parser when compiling files. It accepts the same options as @@ -1759,14 +1780,12 @@ defmodule Code do and `compile_file/2` but not `string_to_quoted/2` and friends, as the latter is used for other purposes beyond compilation. - * `:on_undefined_variable` (since v1.15.0) - either `:raise` or `:warn`. - When `:raise` (the default), undefined variables will trigger a compilation - error. You may be set it to `:warn` if you want undefined variables to - emit a warning and expand as to a local call to the zero-arity function - of the same name (for example, `node` would be expanded as `node()`). - This `:warn` behavior only exists for compatibility reasons when working - with old dependencies, its usage is discouraged and it will be removed - in future releases. + * `:relative_paths` - when `true`, uses relative paths in quoted nodes, + warnings, and errors generated by the compiler. Note disabling this option + won't affect runtime warnings and errors. Defaults to `true`. + + * `:tracers` (since v1.10.0) - a list of tracers (modules) to be used during + compilation. See the module docs for more information. Defaults to `[]`. It always returns `:ok`. Raises an error for invalid options. @@ -1806,6 +1825,15 @@ defmodule Code do :ok end + def put_compiler_option(:module_definition, value) do + if value not in [:interpreted, :compiled] do + raise "compiler option :module_definition should be either :interpreted or :compiled, got: #{inspect(value)}" + end + + :elixir_config.put(:module_definition, value) + :ok + end + def put_compiler_option(:infer_signatures, value) do value = cond do diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl index 1702dcb3265..9fd8bc1269b 100644 --- a/lib/elixir/src/elixir.erl +++ b/lib/elixir/src/elixir.erl @@ -10,9 +10,8 @@ -export([start/2, stop/1, config_change/3]). -export([ string_to_tokens/5, tokens_to_quoted/3, string_to_quoted/5, 'string_to_quoted!'/5, - env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3, - eval_quoted/4, eval_local_handler/2, eval_external_handler/3, - emit_warnings/3 + env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3, eval_quoted/4, + erl_eval/3, eval_local_handler/2, eval_external_handler/3, emit_warnings/3 ]). -include("elixir.hrl"). -define(system, 'Elixir.System'). @@ -92,16 +91,17 @@ start(_Type, _Args) -> {no_halt, false}, %% Compiler options + {debug_info, true}, {docs, true}, {ignore_already_consolidated, false}, {ignore_module_conflict, false}, - {initial_dbg_callback, InitialDbgCallback}, {infer_signatures, [elixir]}, + {initial_dbg_callback, InitialDbgCallback}, + {module_definition, compiled}, + {no_warn_undefined, []}, {on_undefined_variable, raise}, {parser_options, [{columns, true}]}, - {debug_info, true}, {relative_paths, true}, - {no_warn_undefined, []}, {tracers, []} | URIConfig ], @@ -324,33 +324,7 @@ eval_forms(Tree, Binding, OrigE, Opts) -> end; _ -> - Exprs = - case Erl of - {block, _, BlockExprs} -> BlockExprs; - _ -> [Erl] - end, - - %% We use remote names so eval works across Elixir versions. - LocalHandler = {value, fun ?MODULE:eval_local_handler/2}, - ExternalHandler = {value, fun ?MODULE:eval_external_handler/3}, - - {value, Value, NewBinding} = - try - %% ?elixir_eval_env is used by the external handler. - %% - %% The reason why we use the process dictionary to pass the environment - %% is because we want to avoid passing closures to erl_eval, as that - %% would effectively tie the eval code to the Elixir version and it is - %% best if it depends solely on Erlang/OTP. - %% - %% The downside is that functions that escape the eval context will no - %% longer have the original environment they came from. - erlang:put(?elixir_eval_env, NewE), - erl_eval:exprs(Exprs, ErlBinding, LocalHandler, ExternalHandler) - after - erlang:erase(?elixir_eval_env) - end, - + {value, Value, NewBinding} = erl_eval(Erl, ErlBinding, NewE), PruneBefore = if Prune -> length(Binding); true -> -1 end, {DumpedBinding, DumpedVars} = @@ -359,6 +333,28 @@ eval_forms(Tree, Binding, OrigE, Opts) -> {Value, DumpedBinding, NewE#{versioned_vars := DumpedVars}} end. +%% Evaluate Erlang code with careful handling of local and external functions +erl_eval(Expr, Binding, Env) -> + %% We use remote names so eval works across Elixir versions + LocalHandler = {value, fun ?MODULE:eval_local_handler/2}, + ExternalHandler = {value, fun ?MODULE:eval_external_handler/3}, + + try + %% ?elixir_eval_env is used by the external handler. + %% + %% The reason why we use the process dictionary to pass the environment + %% is because we want to avoid passing closures to erl_eval, as that + %% would effectively tie the eval code to the Elixir version and it is + %% best if it depends solely on Erlang/OTP. + %% + %% The downside is that functions that escape the eval context will no + %% longer have the original environment they came from. + erlang:put(?elixir_eval_env, Env), + erl_eval:expr(Expr, Binding, LocalHandler, ExternalHandler) + after + erlang:erase(?elixir_eval_env) + end. + eval_local_handler(FunName, Args) -> {current_stacktrace, Stack} = erlang:process_info(self(), current_stacktrace), Opts = [{module, nil}, {function, FunName}, {arity, length(Args)}, {reason, 'undefined local'}], @@ -405,10 +401,8 @@ eval_external_handler(Ann, FunOrModFun, Args) -> %% Add file+line information at the bottom Bottom = case erlang:get(?elixir_eval_env) of - #{file := File} -> - [{elixir_eval, '__FILE__', 1, - [{file, elixir_utils:characters_to_list(File)}, {line, erl_anno:line(Ann)}]}]; - + #{'__struct__' := 'Elixir.Macro.Env'} = E -> + 'Elixir.Macro.Env':stacktrace(E#{line := erl_anno:line(Ann)}); _ -> [] end, diff --git a/lib/elixir/src/elixir_compiler.erl b/lib/elixir/src/elixir_compiler.erl index 803231ba054..6629f0688f7 100644 --- a/lib/elixir/src/elixir_compiler.erl +++ b/lib/elixir/src/elixir_compiler.erl @@ -4,7 +4,7 @@ %% Elixir compiler front-end to the Erlang backend. -module(elixir_compiler). --export([string/3, quoted/3, bootstrap/0, file/2, compile/4]). +-export([string/3, quoted/3, bootstrap/0, file/2, compile/4, interpret/3]). -include("elixir.hrl"). string(Contents, File, Callback) -> @@ -20,7 +20,7 @@ quoted(Forms, File, Callback) -> elixir_lexical:run( Env, - fun (LexicalEnv) -> maybe_fast_compile(Forms, LexicalEnv) end, + fun (LexicalEnv) -> optimize_defmodule(Forms, LexicalEnv) end, fun (#{lexical_tracker := Pid}) -> Callback(File, Pid) end ), @@ -33,17 +33,37 @@ file(File, Callback) -> {ok, Bin} = file:read_file(File), string(elixir_utils:characters_to_list(Bin), File, Callback). -%% Evaluates the given code through the Erlang compiler. -%% It may end-up evaluating the code if it is deemed a -%% more efficient strategy depending on the code snippet. -maybe_fast_compile(Forms, E) -> - case (?key(E, module) == nil) andalso allows_fast_compilation(Forms) andalso +%% In case the forms only holds defmodules, we optimize +%% it by expanding them directly. +optimize_defmodule(Forms, E) -> + case (?key(E, module) == nil) andalso only_defmodule(Forms) andalso (not elixir_config:is_bootstrap()) of - true -> fast_compile(Forms, E); + true -> expand_defmodule(Forms, E); false -> compile(Forms, [], [], E) end, ok. +%% A version of compilation that uses eval (interpreted) +interpret(Quoted, ArgsList, #{line := Line} = E) -> + {Expanded, SE, EE} = elixir_expand:expand(Quoted, elixir_env:env_to_ex(E), E), + elixir_env:check_unused_vars(SE, EE), + + {Vars, TS} = elixir_erl_var:from_env(E), + {ErlExprs, _} = elixir_erl_pass:translate(Expanded, erl_anno:new(Line), TS), + + ListBinding = lists:zipwith(fun({_, Var}, Arg) -> {Var, Arg} end, Vars, ArgsList), + Binding = maps:from_list(ListBinding), + + {value, Result, _} = + try + elixir:erl_eval(ErlExprs, Binding, E) + catch + Kind:Reason:Stacktrace -> + erlang:raise(Kind, Reason, Stacktrace ++ 'Elixir.Macro.Env':stacktrace(E)) + end, + + {Result, SE, EE}. + compile(Quoted, ArgsList, CompilerOpts, #{line := Line} = E) -> Block = no_tail_optimize([{line, Line}], Quoted), {Expanded, SE, EE} = elixir_expand:expand(Block, elixir_env:env_to_ex(E), E), @@ -107,16 +127,16 @@ retrieve_compiler_module() -> return_compiler_module(Module, Purgeable) -> elixir_code_server:cast({return_compiler_module, Module, Purgeable}). -allows_fast_compilation({'__block__', _, Exprs}) -> - lists:all(fun allows_fast_compilation/1, Exprs); -allows_fast_compilation({defmodule, _, [_, [{do, _}]]}) -> +only_defmodule({'__block__', _, Exprs}) -> + lists:all(fun only_defmodule/1, Exprs); + only_defmodule({defmodule, _, [_, [{do, _}]]}) -> true; -allows_fast_compilation(_) -> +only_defmodule(_) -> false. -fast_compile({'__block__', _, Exprs}, E) -> - lists:foldl(fun(Expr, _) -> fast_compile(Expr, E) end, nil, Exprs); -fast_compile({defmodule, Meta, [Mod, [{do, Block}]]}, NoLineE) -> +expand_defmodule({'__block__', _, Exprs}, E) -> + lists:foldl(fun(Expr, _) -> expand_defmodule(Expr, E) end, nil, Exprs); +expand_defmodule({defmodule, Meta, [Mod, [{do, Block}]]}, NoLineE) -> E = NoLineE#{line := ?line(Meta)}, Expanded = case Mod of diff --git a/lib/elixir/src/elixir_module.erl b/lib/elixir/src/elixir_module.erl index 8d978aa4594..88166254eae 100644 --- a/lib/elixir/src/elixir_module.erl +++ b/lib/elixir/src/elixir_module.erl @@ -472,8 +472,12 @@ build(Module, Line, File, E) -> eval_form(Line, Module, DataBag, Block, Vars, Prune, E) -> %% Given Elixir modules can get very long to compile due to metaprogramming, - %% we disable expansions that take linear time to code size. - {Value, ExS, EE} = elixir_compiler:compile(Block, Vars, [no_bool_opt, no_ssa_opt], E), + %% we disable expansions that have linear time to code size. + {Value, ExS, EE} = + case elixir_config:get(module_definition) of + interpreted -> elixir_compiler:interpret(Block, Vars, E); + compiled -> elixir_compiler:compile(Block, Vars, [no_bool_opt, no_ssa_opt], E) + end, elixir_overridable:store_not_overridden(Module), EV = (elixir_env:reset_vars(EE))#{line := Line}, EC = eval_callbacks(Line, DataBag, before_compile, [EV], EV), diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs index 4542008208f..0d1144fef3b 100644 --- a/lib/elixir/test/elixir/code_test.exs +++ b/lib/elixir/test/elixir/code_test.exs @@ -751,6 +751,36 @@ defmodule Code.SyncTest do defp refute_cached(_path), do: :ok end + test "evaluates module definitions" do + Code.put_compiler_option(:module_definition, :interpreted) + + defmodule CodeTest.EvalModule do + {:current_stacktrace, stacktrace} = Process.info(self(), :current_stacktrace) + assert Enum.find(stacktrace, &(elem(&1, 0) == :erl_eval)) + end + after + Code.put_compiler_option(:module_definition, :compiled) + end + + test "evaluates module definitions with stacktraces" do + Code.put_compiler_option(:module_definition, :interpreted) + + try do + defmodule CodeTest.EvalModuleRaise do + Enum.map(1..10, fn x -> x <> "example" end) + end + rescue + e -> + assert e.__struct__ == ArgumentError + assert Enum.find(__STACKTRACE__, &(elem(&1, 0) == Code.SyncTest.CodeTest.EvalModuleRaise)) + assert Enum.find(__STACKTRACE__, &(elem(&1, 0) == :erl_eval)) + else + _ -> flunk("defmodule should have failed") + end + after + Code.put_compiler_option(:module_definition, :compiled) + end + test "prepend_path" do path = Path.join(__DIR__, "fixtures") true = Code.prepend_path(path) diff --git a/lib/iex/lib/iex/evaluator.ex b/lib/iex/lib/iex/evaluator.ex index 8c0b8370d3a..6830916c5b6 100644 --- a/lib/iex/lib/iex/evaluator.ex +++ b/lib/iex/lib/iex/evaluator.ex @@ -448,7 +448,7 @@ defmodule IEx.Evaluator do defp prune_stacktrace(stack) do stack |> Enum.reverse() - |> Enum.drop_while(&(elem(&1, 0) != :elixir_eval)) + |> Enum.drop_while(&(elem(&1, 0) != :elixir_compiler)) |> Enum.reverse() |> case do [] -> stack diff --git a/lib/iex/test/iex/interaction_test.exs b/lib/iex/test/iex/interaction_test.exs index 26635c4f005..f351da9345e 100644 --- a/lib/iex/test/iex/interaction_test.exs +++ b/lib/iex/test/iex/interaction_test.exs @@ -174,10 +174,9 @@ defmodule IEx.InteractionTest do test "exception" do exception = Regex.escape("** (ArithmeticError) bad argument in arithmetic expression") - assert capture_iex("1 + :atom\n:this_is_still_working") =~ - ~r/^#{exception}.+\n:this_is_still_working$/s - - refute capture_iex("1 + :atom\n:this_is_still_working") =~ ~r/erl_eval/s + result = capture_iex("1 + :atom\n:this_is_still_working") + assert result =~ ~r/^#{exception}.+\n:this_is_still_working$/s + refute result =~ ~r/erl_eval/s end test "exception while invoking conflicting helpers" do