Add parameterized tests

lib/ex_unit/lib/ex_unit.ex

@@ -101,9 +101,10 @@ defmodule ExUnit do
       * `:time` - the duration in microseconds of the test's runtime
       * `:tags` - the test tags
       * `:logs` - the captured logs
+      * `:parameters` - the test parameters
 
     """
-    defstruct [:name, :case, :module, :state, time: 0, tags: %{}, logs: ""]
+    defstruct [:name, :case, :module, :state, time: 0, tags: %{}, logs: "", parameters: %{}]
 
     # TODO: Remove the `:case` field on v2.0
     @type t :: %__MODULE__{
@@ -131,8 +132,10 @@ defmodule ExUnit do
 
       * `:tests` - all tests in this module
 
+      * `:parameters` - the test module parameters
+
     """
-    defstruct [:file, :name, :state, tags: %{}, tests: []]
+    defstruct [:file, :name, :state, tags: %{}, tests: [], parameters: %{}]
 
     @type t :: %__MODULE__{
             file: binary(),
@@ -404,10 +407,12 @@ defmodule ExUnit do
     for module <- additional_modules do
       module_attributes = module.__info__(:attributes)
 
-      if true in Keyword.get(module_attributes, :ex_unit_async, []) do
-        ExUnit.Server.add_async_module(module)
-      else
-        ExUnit.Server.add_sync_module(module)
+      case Keyword.get(module_attributes, :ex_unit_module) do
+        [config] ->
+          ExUnit.Server.add_module(module, config)
+
+        _ ->
+          raise(ArgumentError, "#{inspect(module)} is not a ExUnit.Case module")
       end
     end
 

lib/ex_unit/lib/ex_unit/case.ex

@@ -38,6 +38,11 @@ defmodule ExUnit.Case do
     * `:register` - when `false`, does not register this module within
       ExUnit server. This means the module won't run when ExUnit suite runs.
 
+    * `:parameterize` - a list of maps to parameterize tests. If both
+      `:async` and `:parameterize` are given, the different parameters
+      run concurrently. See the "Parameterized tests" section below for
+      more information.
+
   > #### `use ExUnit.Case` {: .info}
   >
   > When you `use ExUnit.Case`, it will import the functionality
@@ -196,6 +201,41 @@ defmodule ExUnit.Case do
 
     * `:tmp_dir` - (since v1.11.0) see the "Tmp Dir" section below
 
+  ## Parameterized tests
+
+  Sometimes you want to run the same tests but with different parameters.
+  In ExUnit, it is possible to do so by passing a `:parameterize` key to
+  `ExUnit.Case`. The value must be a list of maps which will be the
+  parameters merged into the test context.
+
+  For example, Elixir has a module called `Registry`, which can have type
+  `:unique` or `:duplicate`, and can control its concurrency factor using
+  the `:partitions` option. If you have a number of tests that *behave the
+  same* across all of those values, you can parameterize those tests with:
+
+      use ExUnit.Case,
+        async: true,
+        parameterize:
+          for(kind <- [:unique, :duplicate],
+              partitions <- [1, 8],
+              do: %{kind: kind, partitions: partitions})
+
+  Then, in your tests, you can access the parameters as part of the context:
+
+      test "starts a registry", %{kind: kind, partitions: partitions} do
+        ...
+      end
+
+  Use parameterized tests with care:
+
+    * Although parameterized tests run concurrently when `async: true` is also given,
+      abuse of parameterized tests may make your test suite slower
+
+    * If you use parameterized tests and then find yourself adding conditionals
+      in your tests to deal with different parameters, then parameterized tests
+      may be the wrong solution to your problem. Consider creating separated
+      tests and sharing logic between them using regular functions
+
   ## Filters
 
   Tags can also be used to identify specific tests, which can then
@@ -301,6 +341,18 @@ defmodule ExUnit.Case do
               ~s(got: #{inspect(opts)})
     end
 
+    {register?, opts} = Keyword.pop(opts, :register, true)
+    {async?, opts} = Keyword.pop(opts, :async, false)
+    {parameterize, opts} = Keyword.pop(opts, :parameterize, nil)
+
+    unless parameterize == nil or (is_list(parameterize) and Enum.all?(parameterize, &is_map/1)) do
+      raise ArgumentError, ":parameterize must be a list of maps, got: #{inspect(parameterize)}"
+    end
+
+    if opts != [] do
+      IO.warn("unknown options given to ExUnit.Case: #{inspect(opts)}")
+    end
+
     registered? = Module.has_attribute?(module, :ex_unit_tests)
 
     unless registered? do
@@ -322,23 +374,18 @@ defmodule ExUnit.Case do
 
       Enum.each(accumulate_attributes, &Module.register_attribute(module, &1, accumulate: true))
 
-      persisted_attributes = [:ex_unit_async]
+      persisted_attributes = [:ex_unit_module]
 
       Enum.each(persisted_attributes, &Module.register_attribute(module, &1, persist: true))
 
-      if Keyword.get(opts, :register, true) do
+      if register? do
         Module.put_attribute(module, :after_compile, ExUnit.Case)
       end
 
       Module.put_attribute(module, :before_compile, ExUnit.Case)
     end
 
-    async? = opts[:async]
-
-    if is_boolean(async?) or not registered? do
-      Module.put_attribute(module, :ex_unit_async, async? || false)
-    end
-
+    Module.put_attribute(module, :ex_unit_module, {async?, parameterize})
     registered?
   end
 
@@ -521,21 +568,22 @@ defmodule ExUnit.Case do
   end
 
   @doc false
-  defmacro __before_compile__(env) do
+  defmacro __before_compile__(%{module: module} = env) do
     tests =
-      env.module
+      module
       |> Module.get_attribute(:ex_unit_tests)
       |> Enum.reverse()
       |> Macro.escape()
 
-    moduletag = Module.get_attribute(env.module, :moduletag)
+    moduletag = Module.get_attribute(module, :moduletag)
+    {async?, _parameterize} = Module.get_attribute(module, :ex_unit_module)
 
     tags =
       moduletag
       |> normalize_tags()
       |> validate_tags()
       |> Map.new()
-      |> Map.merge(%{module: env.module, case: env.module})
+      |> Map.merge(%{module: module, case: env.module, async: async?})
 
     quote do
       def __ex_unit__ do
@@ -552,17 +600,16 @@ defmodule ExUnit.Case do
   @doc false
   def __after_compile__(%{module: module}, _) do
     cond do
-      Process.whereis(ExUnit.Server) == nil ->
-        unless Code.can_await_module_compilation?() do
-          raise "cannot use ExUnit.Case without starting the ExUnit application, " <>
-                  "please call ExUnit.start() or explicitly start the :ex_unit app"
-        end
+      Process.whereis(ExUnit.Server) ->
+        config = Module.get_attribute(module, :ex_unit_module)
+        ExUnit.Server.add_module(module, config)
 
-      Module.get_attribute(module, :ex_unit_async) ->
-        ExUnit.Server.add_async_module(module)
+      Code.can_await_module_compilation?() ->
+        :ok
 
       true ->
-        ExUnit.Server.add_sync_module(module)
+        raise "cannot use ExUnit.Case without starting the ExUnit application, " <>
+                "please call ExUnit.start() or explicitly start the :ex_unit app"
     end
   end
 
@@ -600,7 +647,6 @@ defmodule ExUnit.Case do
 
     moduletag = Module.get_attribute(mod, :moduletag)
     tag = Module.delete_attribute(mod, :tag)
-    async = Module.get_attribute(mod, :ex_unit_async)
 
     {name, describe, describe_line, describetag} =
       case Module.get_attribute(mod, :ex_unit_describe) do
@@ -625,7 +671,6 @@ defmodule ExUnit.Case do
         line: line,
         file: file,
         registered: registered,
-        async: async,
         describe: describe,
         describe_line: describe_line,
         test_type: test_type

lib/ex_unit/lib/ex_unit/cli_formatter.ex

@@ -137,9 +137,14 @@ defmodule ExUnit.CLIFormatter do
     {:noreply, update_test_timings(config, test)}
   end
 
-  def handle_cast({:module_started, %ExUnit.TestModule{name: name, file: file}}, config) do
+  def handle_cast({:module_started, %ExUnit.TestModule{} = module}, config) do
     if config.trace do
+      %{name: name, file: file, parameters: parameters} = module
       IO.puts("\n#{inspect(name)} [#{Path.relative_to_cwd(file)}]")
+
+      if parameters != %{} do
+        IO.puts("Parameters: #{inspect(parameters)}")
+      end
     end
 
     {:noreply, config}

lib/ex_unit/lib/ex_unit/formatter.ex

@@ -94,6 +94,7 @@ defmodule ExUnit.Formatter do
           | :error_info
           | :test_module_info
           | :test_info
+          | :parameters_info
           | :location_info
           | :stacktrace_info
           | :blame_diff
@@ -119,7 +120,7 @@ defmodule ExUnit.Formatter do
     * `:diff_insert` and `:diff_insert_whitespace` - Should format a diff insertion,
       with or without whitespace respectively.
 
-    * `:extra_info` - Should format extra information, such as the `"code: "` label
+    * `:extra_info` - Should format optional extra labels, such as the `"code: "` label
       that precedes code to show.
 
     * `:error_info` - Should format error information.
@@ -129,6 +130,8 @@ defmodule ExUnit.Formatter do
 
     * `:test_info` - Should format test information.
 
+    * `:parameters_info` - Should format test parameters.
+
     * `:location_info` - Should format test location information.
 
     * `:stacktrace_info` - Should format stacktrace information.
@@ -266,9 +269,10 @@ defmodule ExUnit.Formatter do
         ) :: String.t()
         when failure: {atom, term, Exception.stacktrace()}
   def format_test_failure(test, failures, counter, width, formatter) do
-    %ExUnit.Test{name: name, module: module, tags: tags} = test
+    %ExUnit.Test{name: name, module: module, tags: tags, parameters: parameters} = test
 
     test_info(with_counter(counter, "#{name} (#{inspect(module)})"), formatter) <>
+      test_parameters(parameters, formatter) <>
       test_location(with_location(tags), formatter) <>
       Enum.map_join(Enum.with_index(failures), "", fn {{kind, reason, stack}, index} ->
         {text, stack} = format_kind_reason(test, kind, reason, stack, width, formatter)
@@ -305,9 +309,10 @@ defmodule ExUnit.Formatter do
         ) :: String.t()
         when failure: {atom, term, Exception.stacktrace()}
   def format_test_all_failure(test_module, failures, counter, width, formatter) do
-    name = test_module.name
+    %{name: name, parameters: parameters} = test_module
 
     test_module_info(with_counter(counter, "#{inspect(name)}: "), formatter) <>
+      test_parameters(parameters, formatter) <>
       Enum.map_join(Enum.with_index(failures), "", fn {{kind, reason, stack}, index} ->
         {text, stack} = format_kind_reason(test_module, kind, reason, stack, width, formatter)
         failure_header(failures, index) <> text <> format_stacktrace(stack, name, nil, formatter)
@@ -711,6 +716,15 @@ defmodule ExUnit.Formatter do
   defp test_info(msg, nil), do: msg <> "\n"
   defp test_info(msg, formatter), do: test_info(formatter.(:test_info, msg), nil)
 
+  defp test_parameters(params, _formatter) when params == %{}, do: ""
+  defp test_parameters(params, nil) when is_binary(params), do: "     " <> params <> "\n"
+
+  defp test_parameters(params, nil) when is_map(params),
+    do: test_parameters("Parameters: #{inspect(params)}", nil)
+
+  defp test_parameters(params, formatter),
+    do: test_parameters(formatter.(:parameters_info, params), nil)
+
   defp test_location(msg, nil), do: "     " <> msg <> "\n"
   defp test_location(msg, formatter), do: test_location(formatter.(:location_info, msg), nil)