Add parameterized tests

lib/ex_unit/lib/ex_unit.ex

@@ -101,9 +101,10 @@ defmodule ExUnit do
       * `:time` - the duration in microseconds of the test's runtime
       * `:tags` - the test tags
       * `:logs` - the captured logs
+      * `:parameters` - the test parameters
 
     """
-    defstruct [:name, :case, :module, :state, time: 0, tags: %{}, logs: ""]
+    defstruct [:name, :case, :module, :state, time: 0, tags: %{}, logs: "", parameters: %{}]
 
     # TODO: Remove the `:case` field on v2.0
     @type t :: %__MODULE__{

lib/ex_unit/lib/ex_unit/case.ex

@@ -189,6 +189,8 @@ defmodule ExUnit.Case do
 
     * `:capture_log` - see the "Log Capture" section below
 
+    * `:parameterize` - see the "Parameterize tests" section below
+
     * `:skip` - skips the test with the given reason
 
     * `:timeout` - customizes the test timeout in milliseconds (defaults to 60000).
@@ -227,6 +229,36 @@ defmodule ExUnit.Case do
   Keep in mind that all tests are included by default, so unless they are
   excluded first, the `include` option has no effect.
 
+  ## Parameterized tests
+
+  Sometimes you want to run the same tests but with different parameters.
+  In ExUnit, it is possible to do by returning a `:parameterize` key in
+  your `setup_all` context. The value must be a list of maps which will be
+  the parameters merged into the test context.
+
+  For example, Elixir has a module called `Registry`, which can have type
+  `:unique` or `:duplicate`, and can control its concurrency factor using
+  the `:partitions` option. If you have a number of tests that *behave the
+  same* across all of those values, I can parameterize those tests with:
+
+      setup_all do
+        parameters =
+          for kind <- [:unique, :duplicate],
+              partitions <- [1, 8],
+              do: %{kind: kind, partitions: partitions}
+
+        [parameterize: parameters]
+      end
+
+  Use parameterized tests with care:
+
+  * Abuse of parameterized tests may make your test suite considerably slower
+
+  * If you use parameterized tests and then find yourself adding conditionals
+    in your tests to deal with different parameters, then parameterized tests
+    may be the wrong solution to your problem. Consider creating separated
+    tests and sharing logic between them using regular functions
+
   ## Log Capture
 
   ExUnit can optionally suppress printing of log messages that are generated

lib/ex_unit/lib/ex_unit/cli_formatter.ex

@@ -213,7 +213,7 @@ defmodule ExUnit.CLIFormatter do
   end
 
   defp trace_test_started(test) do
-    String.replace("  * #{test.name}", "\n", " ")
+    String.replace("  * #{trace_test_name(test)}", "\n", " ")
   end
 
   defp trace_test_result(test) do
@@ -229,13 +229,21 @@ defmodule ExUnit.CLIFormatter do
   end
 
   defp trace_aborted(%ExUnit.Test{} = test) do
-    "* #{test.name} [#{trace_test_file_line(test)}]"
+    "* #{trace_test_name(test)} [#{trace_test_file_line(test)}]"
   end
 
   defp trace_aborted(%ExUnit.TestModule{name: name, file: file}) do
     "* #{inspect(name)} [#{Path.relative_to_cwd(file)}]"
   end
 
+  defp trace_test_name(%{name: name, parameters: parameters}) do
+    if parameters == %{} do
+      Atom.to_string(name)
+    else
+      "#{name} (parameters: #{inspect(parameters)})"
+    end
+  end
+
   defp normalize_us(us) do
     div(us, 1000)
   end

lib/ex_unit/lib/ex_unit/formatter.ex

@@ -94,6 +94,7 @@ defmodule ExUnit.Formatter do
           | :error_info
           | :test_module_info
           | :test_info
+          | :parameters_info
           | :location_info
           | :stacktrace_info
           | :blame_diff
@@ -119,7 +120,7 @@ defmodule ExUnit.Formatter do
     * `:diff_insert` and `:diff_insert_whitespace` - Should format a diff insertion,
       with or without whitespace respectively.
 
-    * `:extra_info` - Should format extra information, such as the `"code: "` label
+    * `:extra_info` - Should format optional extra labels, such as the `"code: "` label
       that precedes code to show.
 
     * `:error_info` - Should format error information.
@@ -129,6 +130,8 @@ defmodule ExUnit.Formatter do
 
     * `:test_info` - Should format test information.
 
+    * `:parameters_info` - Should format test parameters.
+
     * `:location_info` - Should format test location information.
 
     * `:stacktrace_info` - Should format stacktrace information.
@@ -266,9 +269,10 @@ defmodule ExUnit.Formatter do
         ) :: String.t()
         when failure: {atom, term, Exception.stacktrace()}
   def format_test_failure(test, failures, counter, width, formatter) do
-    %ExUnit.Test{name: name, module: module, tags: tags} = test
+    %ExUnit.Test{name: name, module: module, tags: tags, parameters: parameters} = test
 
     test_info(with_counter(counter, "#{name} (#{inspect(module)})"), formatter) <>
+      test_parameters(parameters, formatter) <>
       test_location(with_location(tags), formatter) <>
       Enum.map_join(Enum.with_index(failures), "", fn {{kind, reason, stack}, index} ->
         {text, stack} = format_kind_reason(test, kind, reason, stack, width, formatter)
@@ -711,6 +715,15 @@ defmodule ExUnit.Formatter do
   defp test_info(msg, nil), do: msg <> "\n"
   defp test_info(msg, formatter), do: test_info(formatter.(:test_info, msg), nil)
 
+  defp test_parameters(params, _formatter) when params == %{}, do: ""
+  defp test_parameters(params, nil) when is_binary(params), do: "     " <> params <> "\n"
+
+  defp test_parameters(params, nil) when is_map(params),
+    do: test_parameters("Parameters: #{inspect(params)}", nil)
+
+  defp test_parameters(params, formatter),
+    do: test_parameters(formatter.(:parameters_info, params), nil)
+
   defp test_location(msg, nil), do: "     " <> msg <> "\n"
   defp test_location(msg, formatter), do: test_location(formatter.(:location_info, msg), nil)
 

lib/ex_unit/lib/ex_unit/runner.ex

@@ -327,7 +327,7 @@ defmodule ExUnit.Runner do
 
       result =
         try do
-          {:ok, module.__ex_unit__(:setup_all, test_module.tags)}
+          validate_setup_all(module.__ex_unit__(:setup_all, test_module.tags))
         catch
           kind, error ->
             failed = failed(kind, error, prune_stacktrace(__STACKTRACE__))
@@ -355,7 +355,26 @@ defmodule ExUnit.Runner do
     end
   end
 
+  defp validate_setup_all(%{parameterize: parameterize} = context) do
+    if is_list(parameterize) and Enum.all?(parameterize, &is_map/1) do
+      {:ok, context}
+    else
+      raise ":parameterize key must return a list of maps to be merged into the context"
+    end
+  end
+
+  defp validate_setup_all(context), do: {:ok, context}
+
   defp run_tests(config, tests, context) do
+    {parametrize, context} = Map.pop(context, :parameterize)
+
+    tests =
+      if parametrize do
+        for parameters <- parametrize, test <- tests, do: %{test | parameters: parameters}
+      else
+        tests
+      end
+
     Enum.reduce_while(tests, [], fn test, acc ->
       Process.put(@current_key, test)
 
@@ -401,14 +420,15 @@ defmodule ExUnit.Runner do
     spawn_monitor(fn ->
       ExUnit.OnExitHandler.register(self())
       generate_test_seed(seed, test, rand_algorithm)
-      capture_log = Map.get(test.tags, :capture_log, capture_log)
+      context = Map.merge(context, Map.merge(test.tags, test.parameters))
+      capture_log = Map.get(context, :capture_log, capture_log)
 
       {time, test} =
         :timer.tc(
           maybe_capture_log(capture_log, test, fn ->
-            tags = maybe_create_tmp_dir(test.tags, test)
+            context = maybe_create_tmp_dir(context, test)
 
-            case exec_test_setup(test, Map.merge(context, tags)) do
+            case exec_test_setup(test, context) do
               {:ok, context} -> exec_test(test, context)
               {:error, test} -> test
             end

lib/ex_unit/test/ex_unit/formatter_test.exs

@@ -230,6 +230,31 @@ defmodule ExUnit.FormatterTest do
            """
   end
 
+  test "formats test errors with parameters" do
+    failure = [{:error, catch_error(raise "oops"), []}]
+
+    assert format_test_failure(%{test() | parameters: %{foo: :bar}}, failure, 1, 80, &formatter/2) =~
+             """
+               1) world (Hello)
+                  Parameters: %{foo: :bar}
+                  test/ex_unit/formatter_test.exs:1
+                  ** (RuntimeError) oops
+             """
+
+    formatter = fn
+      :parameters_info, map -> Map.put(map, :more, :keys)
+      key, val -> formatter(key, val)
+    end
+
+    assert format_test_failure(%{test() | parameters: %{foo: :bar}}, failure, 1, 80, formatter) =~
+             """
+               1) world (Hello)
+                  Parameters: #{inspect(%{foo: :bar, more: :keys})}
+                  test/ex_unit/formatter_test.exs:1
+                  ** (RuntimeError) oops
+             """
+  end
+
   test "formats stacktraces" do
     stacktrace = [{Oops, :wrong, 1, [file: "formatter_test.exs", line: 1]}]
     failure = [{:error, catch_error(raise "oops"), stacktrace}]