Infer types from guards and do post-inference on stdlib (#15032)

We now infer types from guards although we don't
yet infer conditional types (such as on `or`).

Author: josevalim

.github/workflows/ci.yml

@@ -102,12 +102,15 @@ jobs:
           path: cover/*
 
       - name: Check reproducible builds
+        if: ${{ matrix.deterministic }}
+        run: taskset 1 make check_reproducible
+
+      - name: Check git is not required
         if: ${{ matrix.deterministic }}
         run: |
           rm -rf .git
-          # Recompile System without .git
-          cd lib/elixir && ../../bin/elixirc -o ebin lib/system.ex && cd -
-          taskset 1 make check_reproducible
+          cd lib/elixir
+          elixirc --ignore-module-conflict -o ebin "lib/**/*.ex"
 
   test_windows:
     name: Windows Server 2022, OTP ${{ matrix.otp_version }}

Makefile

@@ -107,8 +107,10 @@ $(KERNEL): lib/elixir/src/* lib/elixir/lib/*.ex lib/elixir/lib/*/*.ex lib/elixir
 	fi
 	@ echo "==> elixir (compile)";
 	$(Q) cd lib/elixir && ../../$(ELIXIRC_MIN_SIG) "lib/**/*.ex" -o ebin;
+	$(Q) $(GENERATE_APP) $(VERSION)
+	$(Q) bin/elixir lib/elixir/scripts/infer.exs;
 
-$(APP): lib/elixir/src/elixir.app.src lib/elixir/ebin VERSION $(GENERATE_APP)
+$(APP): lib/elixir/src/elixir.app.src $(GENERATE_APP)
 	$(Q) $(GENERATE_APP) $(VERSION)
 
 unicode: $(UNICODE)

lib/elixir/lib/access.ex

@@ -873,11 +873,6 @@ defmodule Access do
       ...> end)
       {[], [%{name: "john", salary: 10}, %{name: "francine", salary: 30}]}
 
-  An error is raised if the predicate is not a function or is of the incorrect arity:
-
-      iex> get_in([], [Access.filter(5)])
-      ** (FunctionClauseError) no function clause matching in Access.filter/1
-
   An error is raised if the accessed structure is not a list:
 
       iex> get_in(%{}, [Access.filter(fn a -> a == 10 end)])
@@ -1154,11 +1149,6 @@ defmodule Access do
       ...> end)
       {nil, [%{name: "john", salary: 10}, %{name: "francine", salary: 30}]}
 
-  An error is raised if the predicate is not a function or is of the incorrect arity:
-
-      iex> get_in([], [Access.find(5)])
-      ** (FunctionClauseError) no function clause matching in Access.find/1
-
   An error is raised if the accessed structure is not a list:
 
       iex> get_in(%{}, [Access.find(fn a -> a == 10 end)])

lib/elixir/lib/code.ex

@@ -1741,10 +1741,10 @@ defmodule Code do
       module. Type checking will be executed regardless of the value of this option.
       Defaults to `true`, which is equivalent to setting it to `[:elixir]` only.
 
-      When setting this option, we recommend running `mix clean` so the current module
-      may be compiled from scratch. `mix test` automatically disables this option via
-      the `:test_elixirc_options` project configuration, as there is typically no need
-      to infer signatures for test files.
+      When setting this option, we recommend running `mix clean` so the modules can be
+      recompiled with the new behaviour. `mix test` automatically disables this option
+      via the `:test_elixirc_options` project configuration, as there is typically no
+      need to infer signatures for test files.
 
     * `:relative_paths` - when `true`, uses relative paths in quoted nodes,
       warnings, and errors generated by the compiler. Note disabling this option

lib/elixir/lib/exception.ex

@@ -1930,8 +1930,8 @@ defmodule FunctionClauseError do
 
   For example:
 
-      iex> URI.parse(:wrong_argument)
-      ** (FunctionClauseError) no function clause matching in URI.parse/1
+      iex> List.duplicate(:ok, -3)
+      ** (FunctionClauseError) no function clause matching in List.duplicate/2
 
   The following fields of this exception are public and can be accessed freely:
 

lib/elixir/lib/kernel.ex

@@ -4764,8 +4764,8 @@ defmodule Kernel do
   defp in_range(left, first, last, step) do
     quoted =
       quote do
-        :erlang.is_integer(unquote(left)) and :erlang.is_integer(unquote(first)) and
-          :erlang.is_integer(unquote(last)) and
+        unquote(generated_is_integer(left)) and unquote(generated_is_integer(first)) and
+          unquote(generated_is_integer(last)) and
           ((:erlang.>(unquote(step), 0) and
               unquote(increasing_compare(left, first, last))) or
              (:erlang.<(unquote(step), 0) and
@@ -4782,8 +4782,8 @@ defmodule Kernel do
   defp in_range_literal(left, first, last, step) when step > 0 do
     quoted =
       quote do
-        :erlang.andalso(
-          :erlang.is_integer(unquote(left)),
+        Kernel.and(
+          unquote(generated_is_integer(left)),
           unquote(increasing_compare(left, first, last))
         )
       end
@@ -4794,8 +4794,8 @@ defmodule Kernel do
   defp in_range_literal(left, first, last, step) when step < 0 do
     quoted =
       quote do
-        :erlang.andalso(
-          :erlang.is_integer(unquote(left)),
+        Kernel.and(
+          unquote(generated_is_integer(left)),
           unquote(decreasing_compare(left, first, last))
         )
       end
@@ -4809,7 +4809,7 @@ defmodule Kernel do
 
   defp in_range_step(quoted, left, first, step) do
     quote do
-      :erlang.andalso(
+      Kernel.and(
         unquote(quoted),
         :erlang."=:="(:erlang.rem(unquote(left) - unquote(first), unquote(step)), 0)
       )
@@ -4818,7 +4818,7 @@ defmodule Kernel do
 
   defp in_list(left, head, tail, expand, right, in_body?) do
     [head | tail] = :lists.map(&comp(left, &1, expand, right, in_body?), [head | tail])
-    :lists.foldl(&quote(do: :erlang.orelse(unquote(&2), unquote(&1))), head, tail)
+    :lists.foldl(&quote(do: Kernel.or(unquote(&2), unquote(&1))), head, tail)
   end
 
   defp comp(left, {:|, _, [head, tail]}, expand, right, in_body?) do
@@ -4828,15 +4828,15 @@ defmodule Kernel do
 
       [tail_head | tail] ->
         quote do
-          :erlang.orelse(
+          Kernel.or(
             :erlang."=:="(unquote(left), unquote(head)),
             unquote(in_list(left, tail_head, tail, expand, right, in_body?))
           )
         end
 
       tail when in_body? ->
         quote do
-          :erlang.orelse(
+          Kernel.or(
             :erlang."=:="(unquote(left), unquote(head)),
             :lists.member(unquote(left), unquote(tail))
           )
@@ -4851,9 +4851,13 @@ defmodule Kernel do
     quote(do: :erlang."=:="(unquote(left), unquote(right)))
   end
 
+  defp generated_is_integer(arg) do
+    quote generated: true, do: :erlang.is_integer(unquote(arg))
+  end
+
   defp increasing_compare(var, first, last) do
     quote do
-      :erlang.andalso(
+      Kernel.and(
         :erlang.>=(unquote(var), unquote(first)),
         :erlang."=<"(unquote(var), unquote(last))
       )
@@ -4862,7 +4866,7 @@ defmodule Kernel do
 
   defp decreasing_compare(var, first, last) do
     quote do
-      :erlang.andalso(
+      Kernel.and(
         :erlang."=<"(unquote(var), unquote(first)),
         :erlang.>=(unquote(var), unquote(last))
       )

lib/elixir/lib/list.ex

@@ -187,9 +187,10 @@ defmodule List do
   """
   @spec duplicate(any, 0) :: []
   @spec duplicate(elem, pos_integer) :: [elem, ...] when elem: var
-  def duplicate(elem, n) do
-    :lists.duplicate(n, elem)
-  end
+  def duplicate(elem, n) when is_integer(n) and n >= 0, do: duplicate(n, elem, [])
+
+  defp duplicate(0, _elem, acc), do: acc
+  defp duplicate(n, elem, acc), do: duplicate(n - 1, elem, [elem | acc])
 
   @doc """
   Flattens the given `list` of nested lists.

lib/elixir/lib/module/parallel_checker.ex

@@ -220,14 +220,6 @@ defmodule Module.ParallelChecker do
     end
   end
 
-  @doc """
-  Test cache.
-  """
-  def test_cache do
-    {:ok, cache} = start_link()
-    cache
-  end
-
   @doc """
   Returns the export kind and deprecation reason for the given MFA from
   the cache. If the module does not exist return `:badmodule`,

lib/elixir/lib/module/types.ex

@@ -24,20 +24,17 @@ defmodule Module.Types do
   #
   #   * :infer - Same as :dynamic but skips remote calls.
   #
-  #   * :traversal - Focused mostly on traversing AST, skips most type system
-  #     operations. Used by macros and when skipping inference.
-  #
   # The mode may also control exhaustiveness checks in the future (to be decided).
   # We may also want for applications with subtyping in dynamic mode to always
   # intersect with dynamic, but this mode may be too lax (to be decided based on
   # feedback).
-  @modes [:static, :dynamic, :infer, :traversal]
+  @modes [:static, :dynamic, :infer]
 
   # These functions are not inferred because they are added/managed by the compiler
   @no_infer [behaviour_info: 1]
 
   @doc false
-  def infer(module, file, attrs, defs, private, used_private, env, {_, cache}) do
+  def infer(module, file, attrs, defs, used_private, env, {_, cache}) do
     # We don't care about inferring signatures for protocols,
     # those will be replaced anyway. There is also nothing to
     # infer if there is no cache system, we only do traversals.
@@ -75,11 +72,12 @@ defmodule Module.Types do
 
     stack = stack(:infer, file, module, {:__info__, 1}, env, cache, handler)
 
-    {types, %{local_sigs: reachable_sigs} = context} =
-      for {fun_arity, kind, meta, _clauses} = def <- defs,
-          kind in [:def, :defmacro],
-          reduce: {[], context()} do
-        {types, context} ->
+    # In case there are loops, the other we traverse matters,
+    # so we sort the definitions for determinism
+    {types, private, %{local_sigs: reachable_sigs} = context} =
+      for {fun_arity, kind, meta, _clauses} = def <- Enum.sort(defs),
+          reduce: {[], [], context()} do
+        {types, private, context} when kind in [:def, :defmacro] ->
           # Optimized version of finder, since we already have the definition
           finder = fn _ ->
             default_domain(infer_mode(kind, infer_signatures?), def, fun_arity, impl)
@@ -88,10 +86,13 @@ defmodule Module.Types do
           {_kind, inferred, context} = local_handler(meta, fun_arity, stack, context, finder)
 
           if infer_signatures? and kind == :def and fun_arity not in @no_infer do
-            {[{fun_arity, inferred} | types], context}
+            {[{fun_arity, inferred} | types], private, context}
           else
-            {types, context}
+            {types, private, context}
           end
+
+        {types, private, context} ->
+          {types, [def | private], context}
       end
 
     # Now traverse all used privates to find any other private that have been used by them.
@@ -105,8 +106,8 @@ defmodule Module.Types do
 
     {unreachable, _context} =
       Enum.reduce(private, {[], context}, fn
-        {fun_arity, kind, _meta, _defaults} = info, {unreachable, context} ->
-          warn_unused_def(info, used_sigs, env)
+        {fun_arity, kind, meta, _clauses}, {unreachable, context} ->
+          warn_unused_def(fun_arity, kind, meta, used_sigs, env)
 
           # Find anything undefined within unused functions
           {_kind, _inferred, context} = local_handler([], fun_arity, stack, context, finder)
@@ -125,7 +126,7 @@ defmodule Module.Types do
   end
 
   defp infer_mode(kind, infer_signatures?) do
-    if infer_signatures? and kind in [:def, :defp], do: :infer, else: :traversal
+    if infer_signatures? and kind in [:def, :defp], do: :infer, else: :traverse
   end
 
   defp protocol?(attrs) do
@@ -154,7 +155,7 @@ defmodule Module.Types do
         | List.duplicate(Descr.dynamic(), arity - 1)
       ]
 
-      {fun_arity, kind, meta, clauses} = def
+      {_fun_arity, kind, meta, clauses} = def
 
       clauses =
         for {meta, args, guards, body} <- clauses do
@@ -173,29 +174,30 @@ defmodule Module.Types do
     :elixir_errors.module_error(Helpers.with_span(meta, fun), env, __MODULE__, tuple)
   end
 
-  defp warn_unused_def({_fun_arity, _kind, false, _}, _used, _env) do
-    :ok
-  end
+  defp warn_unused_def(fun_arity, kind, meta, used, env) do
+    default = Keyword.get(meta, :defaults, 0)
 
-  defp warn_unused_def({fun_arity, kind, meta, 0}, used, env) do
-    case is_map_key(used, fun_arity) do
-      true -> :ok
-      false -> :elixir_errors.file_warn(meta, env, __MODULE__, {:unused_def, fun_arity, kind})
-    end
+    cond do
+      Keyword.get(meta, :context) != nil ->
+        :ok
 
-    :ok
-  end
+      default == 0 ->
+        case is_map_key(used, fun_arity) do
+          true -> :ok
+          false -> :elixir_errors.file_warn(meta, env, __MODULE__, {:unused_def, fun_arity, kind})
+        end
 
-  defp warn_unused_def({tuple, kind, meta, default}, used, env) when default > 0 do
-    {name, arity} = tuple
-    min = arity - default
-    max = arity
+      default > 0 ->
+        {name, arity} = fun_arity
+        min = arity - default
+        max = arity
 
-    case min_reachable_default(max, min, :none, name, used) do
-      :none -> :elixir_errors.file_warn(meta, env, __MODULE__, {:unused_def, tuple, kind})
-      ^min -> :ok
-      ^max -> :elixir_errors.file_warn(meta, env, __MODULE__, {:unused_args, tuple})
-      diff -> :elixir_errors.file_warn(meta, env, __MODULE__, {:unused_args, tuple, diff})
+        case min_reachable_default(max, min, :none, name, used) do
+          :none -> :elixir_errors.file_warn(meta, env, __MODULE__, {:unused_def, fun_arity, kind})
+          ^min -> :ok
+          ^max -> :elixir_errors.file_warn(meta, env, __MODULE__, {:unused_args, fun_arity})
+          diff -> :elixir_errors.file_warn(meta, env, __MODULE__, {:unused_args, fun_arity, diff})
+        end
     end
 
     :ok
@@ -291,7 +293,7 @@ defmodule Module.Types do
             context = put_in(context.local_sigs, Map.put(local_sigs, fun_arity, kind))
 
             {inferred, mapping, context} =
-              local_handler(fun_arity, kind, meta, clauses, expected, mode, stack, context)
+              local_handler(mode, fun_arity, kind, meta, clauses, expected, stack, context)
 
             context =
               update_in(context.local_sigs, &Map.put(&1, fun_arity, {kind, inferred, mapping}))
@@ -304,7 +306,17 @@ defmodule Module.Types do
     end
   end
 
-  defp local_handler(fun_arity, kind, meta, clauses, expected, mode, stack, context) do
+  defp local_handler(:traverse, {_, arity}, _kind, _meta, clauses, _expected, stack, context) do
+    context =
+      Enum.reduce(clauses, context, fn {_meta, _args, _guards, body}, context ->
+        Module.Types.Traverse.of_expr(body, stack, context)
+      end)
+
+    inferred = {:infer, nil, [{List.duplicate(Descr.term(), arity), Descr.dynamic()}]}
+    {inferred, [{0, 0}], context}
+  end
+
+  defp local_handler(mode, fun_arity, kind, meta, clauses, expected, stack, context) do
     {fun, _arity} = fun_arity
     stack = stack |> fresh_stack(mode, fun_arity) |> with_file_meta(meta)
 
@@ -320,12 +332,7 @@ defmodule Module.Types do
             {return_type, context} =
               Expr.of_expr(body, Descr.term(), body, stack, context)
 
-            args_types =
-              if stack.mode == :traversal do
-                expected
-              else
-                Pattern.of_domain(trees, context)
-              end
+            args_types = Pattern.of_domain(trees, context)
 
             {type_index, inferred} =
               add_inferred(inferred, args_types, return_type, total - 1, [])
@@ -442,7 +449,9 @@ defmodule Module.Types do
       warnings: [],
       # All vars and their types
       vars: %{},
-      # Variables and arguments from patterns
+      # Variables that are specific to the current environment/conditional
+      conditional_vars: nil,
+      # Track metadata specific to matches and guards
       pattern_info: nil,
       # If type checking has found an error/failure
       failed: false,