Avoid hardcoding names of tests which are to be generated

erszcz · erszcz · commit 06e50662c8bc · 2024-06-05T16:00:02.000+02:00
diff --git a/test/gradualizer_dynamic_suite.erl b/test/gradualizer_dynamic_suite.erl
@@ -7,19 +7,25 @@
 
 reload(Config) ->
     Module = ?config(dynamic_suite_module, Config),
-    Path = ?config(dynamic_suite_test_path, Config),
     ?assert(Module /= undefined),
-    ?assert(Path /= undefined),
-    Forms = get_forms(Module),
-    FilesForms = map_erl_files(fun (File) ->
-                                       make_test_form(Forms, File, Config)
-                               end, Path),
-    {TestFiles, TestForms} = lists:unzip(FilesForms),
-    TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ],
-    ct:pal("All tests found under ~s:\n~p\n", [Path, TestNames]),
-    NewForms = Forms ++ TestForms ++ [{eof, 0}],
-    {ok, _} = merl:compile_and_load(NewForms),
-    {ok, TestNames}.
+    case erlang:function_exported(Module, generated_tests, 0) of
+        true ->
+            {ok, Module:generated_tests()};
+        false ->
+            Path = ?config(dynamic_suite_test_path, Config),
+            ?assert(Path /= undefined),
+            Forms = get_forms(Module),
+            FilesForms = map_erl_files(fun (File) ->
+                                               make_test_form(Forms, File, Config)
+                                       end, Path),
+            {TestFiles, TestForms} = lists:unzip(FilesForms),
+            TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ],
+            ct:pal("All tests found under ~s:\n~p\n", [Path, TestNames]),
+            GeneratedTestsForm = make_generated_tests_form(TestNames),
+            NewForms = Forms ++ TestForms ++ [GeneratedTestsForm, {eof, 0}],
+            {ok, _} = merl:compile_and_load(NewForms),
+            {ok, TestNames}
+    end.
 
 map_erl_files(Fun, Dir) ->
     Files = filelib:wildcard(filename:join(Dir, "*.erl")),
@@ -39,6 +45,10 @@ make_test_form(Forms, File, Config) ->
               ],
     erl_syntax:revert(merl:subst(TestTemplate, TestEnv)).
 
+make_generated_tests_form(TestNames) ->
+    Template = merl:quote("generated_tests() -> _@Body."),
+    erl_syntax:revert(merl:subst(Template, [{'Body', merl:term(TestNames)}])).
+
 get_forms(Module) ->
     ModPath = code:which(Module),
     {ok, {Module, [Abst]}} = beam_lib:chunks(ModPath, [abstract_code]),
diff --git a/test/known_problems_should_fail_SUITE.erl b/test/known_problems_should_fail_SUITE.erl
@@ -16,20 +16,21 @@
 suite() ->
     [{timetrap, {minutes, 10}}].
 
-init_per_suite(Config0) ->
-    AppBase = code:lib_dir(gradualizer),
+all() ->
+    [{group, all_tests}].
+
+groups() ->
     Config = [
               {dynamic_suite_module, ?MODULE},
-              {dynamic_suite_test_path, filename:join(AppBase, "test/known_problems/should_fail")},
+              {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/known_problems/should_fail")},
               {dynamic_test_template, known_problems_should_fail_template}
-             ] ++ Config0,
+             ],
+    {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config),
+    [{all_tests, [parallel], GeneratedTests}].
+
+init_per_suite(Config) ->
     {ok, _} = application:ensure_all_started(gradualizer),
-    ok = load_prerequisites(AppBase),
-    {ok, TestNames} = gradualizer_dynamic_suite:reload(Config),
-    case all_tests() of
-        TestNames -> ok;
-        _ -> ct:fail("Please update all_tests/0 to list all tests")
-    end,
+    ok = load_prerequisites(code:lib_dir(gradualizer)),
     Config.
 
 load_prerequisites(AppBase) ->
@@ -53,20 +54,6 @@ init_per_testcase(_TestCase, Config) ->
 end_per_testcase(_TestCase, _Config) ->
     ok.
 
-all() ->
-    [{group, all_tests}].
-
-groups() ->
-    [{all_tests, [parallel], all_tests()}].
-
-all_tests() ->
-    [arith_op,binary_comprehension,case_pattern_should_fail,
-     exhaustive_argumentwise,exhaustive_expr,exhaustive_map_variants,
-     exhaustive_remote_map_variants,guard_should_fail,infer_any_pattern,
-     intersection_with_any_should_fail,intersection_with_unreachable,
-     lambda_wrong_args,map_refinement_fancy,poly_lists_map_should_fail,
-     poly_should_fail,recursive_types_should_fail,refine_ty_vars,sample].
-
 known_problems_should_fail_template(_@File) ->
     Result = safe_type_check_file(_@File, [return_errors]),
     case Result of
diff --git a/test/known_problems_should_pass_SUITE.erl b/test/known_problems_should_pass_SUITE.erl
@@ -16,20 +16,21 @@
 suite() ->
     [{timetrap, {minutes, 10}}].
 
-init_per_suite(Config0) ->
-    AppBase = code:lib_dir(gradualizer),
+all() ->
+    [{group, all_tests}].
+
+groups() ->
     Config = [
               {dynamic_suite_module, ?MODULE},
-              {dynamic_suite_test_path, filename:join(AppBase, "test/known_problems/should_pass")},
+              {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/known_problems/should_pass")},
               {dynamic_test_template, known_problems_should_pass_template}
-             ] ++ Config0,
+             ],
+    {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config),
+    [{all_tests, [parallel], GeneratedTests}].
+
+init_per_suite(Config) ->
     {ok, _} = application:ensure_all_started(gradualizer),
-    ok = load_prerequisites(AppBase),
-    {ok, TestNames} = gradualizer_dynamic_suite:reload(Config),
-    case all_tests() of
-        TestNames -> ok;
-        _ -> ct:fail("Please update all_tests/0 to list all tests")
-    end,
+    ok = load_prerequisites(code:lib_dir(gradualizer)),
     Config.
 
 load_prerequisites(_AppBase) ->
@@ -51,23 +52,6 @@ init_per_testcase(_TestCase, Config) ->
 end_per_testcase(_TestCase, _Config) ->
     ok.
 
-all() ->
-    [{group, all_tests}].
-
-groups() ->
-    [{all_tests, [parallel], all_tests()}].
-
-all_tests() ->
-    [arith_op_arg_types,binary_exhaustiveness_checking_should_pass,
-     call_intersection_function_with_union_arg_should_pass,
-     different_normalization_levels,elixir_list_first,error_in_guard,
-     fun_subtyping,generator_var_shadow,inner_union_subtype_of_root_union,
-     intersection_should_pass,intersection_with_any,list_concat_op_should_pass,
-     list_tail,map_pattern_duplicate_key,maybe_expr,poly_should_pass,
-     poly_type_vars,recursive_types,refine_bound_var_on_mismatch,
-     refine_bound_var_with_guard_should_pass,refine_comparison_should_pass,
-     refine_list_tail,union_fun].
-
 known_problems_should_pass_template(_@File) ->
     {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []),
     ExpectedErrors = typechecker:number_of_exported_functions(Forms),
diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl
@@ -16,20 +16,21 @@
 suite() ->
     [{timetrap, {minutes, 10}}].
 
-init_per_suite(Config0) ->
-    AppBase = code:lib_dir(gradualizer),
+all() ->
+    [{group, all_tests}].
+
+groups() ->
     Config = [
               {dynamic_suite_module, ?MODULE},
-              {dynamic_suite_test_path, filename:join(AppBase, "test/should_fail")},
+              {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/should_fail")},
               {dynamic_test_template, should_fail_template}
-             ] ++ Config0,
+             ],
+    {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config),
+    [{all_tests, [parallel], GeneratedTests}].
+
+init_per_suite(Config) ->
     {ok, _} = application:ensure_all_started(gradualizer),
-    ok = load_prerequisites(AppBase),
-    {ok, TestNames} = gradualizer_dynamic_suite:reload(Config),
-    case all_tests() of
-        TestNames -> ok;
-        _ -> ct:fail("Please update all_tests/0 to list all tests")
-    end,
+    ok = load_prerequisites(code:lib_dir(gradualizer)),
     Config.
 
 load_prerequisites(AppBase) ->
@@ -56,48 +57,6 @@ init_per_testcase(_TestCase, Config) ->
 end_per_testcase(_TestCase, _Config) ->
     ok.
 
-all() ->
-    [{group, all_tests}].
-
-groups() ->
-    [{all_tests, [parallel], all_tests()}].
-
-all_tests() ->
-    [annotated_types_fail,arg,arith_op_fail,arity_mismatch,
-     bc_fail,bin_expression,bin_type_error,branch,branch2,call,
-     call_intersection_function_with_union_arg_fail,case_pattern,
-     case_pattern2,catch_expr_fail,cons,covariant_map_keys_fail,
-     cyclic_type_vars,depth,exhaustive,exhaustive_float,
-     exhaustive_list_variants,exhaustive_refinable_map_variants,
-     exhaustive_remote_user_type,exhaustive_string_variants,
-     exhaustive_type,exhaustive_user_type,
-     exhaustiveness_check_toggling,generator,guard_fail,
-     imported_undef,infer_enabled,intersection_check,
-     intersection_fail,intersection_infer,
-     intersection_with_any_fail,iodata_fail,lambda_not_fun,
-     lc_generator_not_none_fail,lc_not_list,list_infer_fail,
-     list_op,list_op_should_fail,list_union_fail,
-     lists_map_nonempty_fail,literal_char,literal_patterns,
-     logic_op,map_entry,map_fail,map_failing_expr,
-     map_failing_subtyping,map_field_invalid_update,map_literal,
-     map_pattern_fail,map_refinement_fail,map_type_error,match,
-     messaging_fail,module_info_fail,named_fun_fail,
-     named_fun_infer_fail,nil,no_idempotent_xor,
-     non_neg_plus_pos_is_pos_fail,
-     nonempty_list_match_in_head_nonexhaustive,
-     nonempty_string_fail,opaque_fail,operator_pattern_fail,
-     pattern,pattern_record_fail,poly_fail,poly_lists_map_fail,
-     poly_union_lower_bound_fail,pp_intersection,record,
-     record_exhaustive,record_field,record_index,
-     record_info_fail,record_refinement_fail,record_update,
-     record_wildcard_fail,recursive_type_fail,
-     recursive_types_failing,rel_op,return_fun_fail,
-     rigid_type_variables_fail,send_fail,shortcut_ops_fail,
-     spec_and_fun_clause_intersection_fail,string_literal,
-     tuple_union_arg_fail,tuple_union_fail,tuple_union_pattern,
-     tuple_union_refinement,type_refinement_fail,unary_op,
-     unary_plus_fail,union_with_any,unreachable_after_refinement].
-
 should_fail_template(_@File) ->
     Errors = gradualizer:type_check_file(_@File, [return_errors]),
     Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors],
diff --git a/test/should_pass_SUITE.erl b/test/should_pass_SUITE.erl
@@ -16,20 +16,21 @@
 suite() ->
     [{timetrap, {minutes, 10}}].
 
-init_per_suite(Config0) ->
-    AppBase = code:lib_dir(gradualizer),
-    Config = [
-              {dynamic_suite_module, ?MODULE},
-              {dynamic_suite_test_path, filename:join(AppBase, "test/should_pass")},
-              {dynamic_test_template, should_pass_template}
-             ] ++ Config0,
+all() ->
+    [{group, all_tests}].
+
+groups() ->
+    Opts = [
+            {dynamic_suite_module, ?MODULE},
+            {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/should_pass")},
+            {dynamic_test_template, should_pass_template}
+           ],
+    {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Opts),
+    [{all_tests, [parallel], GeneratedTests}].
+
+init_per_suite(Config) ->
     {ok, _} = application:ensure_all_started(gradualizer),
-    ok = load_prerequisites(AppBase),
-    {ok, TestNames} = gradualizer_dynamic_suite:reload(Config),
-    case all_tests() of
-        TestNames -> ok;
-        _ -> ct:fail("Please update all_tests/0 to list all tests")
-    end,
+    ok = load_prerequisites(code:lib_dir(gradualizer)),
     Config.
 
 load_prerequisites(AppBase) ->
@@ -57,51 +58,5 @@ init_per_testcase(_TestCase, Config) ->
 end_per_testcase(_TestCase, _Config) ->
     ok.
 
-all() ->
-    [{group, all_tests}].
-
-groups() ->
-    [{all_tests, [parallel], all_tests()}].
-
-all_tests() ->
-    [alias_in_pattern,andalso_any,ann_types,annotated_types,any,
-     any_doesnt_have_type_none_pass,any_pattern,bc_pass,
-     binary_exhaustiveness_checking,binary_in_union,binary_literal_pattern,
-     bitstring,block_scope,bool,bounded_funs,
-     call_intersection_function_with_union_arg_pass,'case',
-     case_of_record_with_user_defined,catch_expr_pass,covariant_map_keys_pass,
-     cyclic_otp_specs,erlang_error_args_none_pass,exhaustiveness_union_types,
-     factorial,float,flow,fun_capture,fun_spec,guard,guard_sequences_pass,if_expr,
-     imported,int,intersection_pass,intersection_with_any_pass,iodata,issue131,lc,
-     lc_generator_not_none,lc_var_binds_in_filters,list,list_concat_op_pass,
-     list_exhaustiveness_checking_regressions,
-     list_exhaustiveness_checking_regressions2,
-     list_exhaustiveness_checking_unreachable_clause_regression,list_infer_pass,
-     list_op_pass,listsspecs,map,map_as_argument_update,map_creation,
-     map_field_valid_update,map_infer_pass,map_passing_expr,map_passing_subtyping,
-     map_pattern,map_refinement,map_update,map_update_with_record_field,
-     messaging_pass,minimised_gradualizer_fmt,minus,module_info_higher_arity,
-     module_info_pass,named_fun_infer_pass,named_fun_pass,negate_none,
-     nested_pattern_match,non_neg_plus_pos_is_pos_pass,nonempty_cons,
-     nonempty_list_match_in_head_exhaustive,nonempty_string,
-     nonexhaustive_record_pattern,opaque,operator_pattern_pass,operator_subtypes,
-     other_module,pattern_bind_reuse,pattern_record,pattern_with_ty_vars,
-     poly_lists_map_constraints_pass,poly_lists_map_pass,poly_map_pattern,
-     poly_pass,poly_pass_infer,poly_pass_no_solve_constraints,
-     poly_union_lower_bound_pass,preludes,qlc_test,record_info,record_refinement,
-     record_union_pass,record_union_with_any_should_pass,record_var,
-     record_wildcard_pass,record_with_user_defined,records,
-     recursive_call_with_remote_union_return_type_pass,recursive_types_passing,
-     refine_comparison,refine_mismatch_using_guard_bifs,remote_types,
-     remote_types_pass,return_fun,rigid_type_variables,rigid_type_variables_pass,
-     scope,send_pass,sets_set,shortcut_ops_pass,
-     spec_and_fun_clause_intersection_pass,stuff_as_top,'try',try_expr,tuple,
-     tuple_union_pass,tuple_union_pat,tuple_union_pattern_pass,type_decl,
-     type_pattern,type_refinement_pass,type_variable,type_vars_term,
-     typed_record_field_access,unary_negate_union_with_user_type_pass,unary_plus,
-     underscore,user_type_in_pattern_body,user_types,var,var_fun,varbind_in_block,
-     varbind_in_case,varbind_in_function_head,varbind_in_lc,variable_binding,
-     variable_binding_leaks].
-
 should_pass_template(_@File) ->
     ?assertEqual(ok, gradualizer:type_check_file(_@File, [{form_check_timeout_ms, 2000}])).
