Merge pull request #1030 from ericmj/report-setup-all-error

Report setup_all errors and mark tests as invalid

Author: José Valim

lib/ex_unit/lib/ex_unit.ex

@@ -1,12 +1,20 @@
 defmodule ExUnit do
-  defrecord Test, [:name, :case, :failure] do
+  defrecord Test, [:name, :case, :failure, :invalid] do
     @moduledoc """
     A record that keeps information about the test.
     It is received by formatters and also accessible
     in the metadata under the key `:test`.
     """
   end
 
+  defrecord TestCase, [:name, :failure] do
+    @moduledoc """
+    A record that keeps information about the test case.
+    It is received by formatters and also accessible
+    in the metadata under the key `:case`.
+    """
+  end
+
   @moduledoc """
   Basic unit test structure for Elixir.
 

lib/ex_unit/lib/ex_unit/cli_formatter.ex

@@ -9,7 +9,7 @@ defmodule ExUnit.CLIFormatter do
   use GenServer.Behaviour
 
   import Exception, only: [format_entry: 2]
-  defrecord Config, counter: 0, failures: []
+  defrecord Config, counter: 0, test_failures: [], case_failures: []
 
   ## Behaviour
 
@@ -26,8 +26,8 @@ defmodule ExUnit.CLIFormatter do
     :ok
   end
 
-  def case_finished(_id, _test_case) do
-    :ok
+  def case_finished(id, test_case) do
+    :gen_server.cast(id, { :case_finished, test_case })
   end
 
   def test_started(_id, _test) do
@@ -45,14 +45,20 @@ defmodule ExUnit.CLIFormatter do
   end
 
   def handle_call({ :suite_finished, ms }, _from, config) do
-    print_suite(config.counter, config.failures, ms)
-    { :stop, :normal, length(config.failures), config }
+    print_suite(config.counter, config.test_failures, config.case_failures, ms)
+    { :stop, :normal, length(config.test_failures), config }
   end
 
   def handle_call(_, _, _) do
     super
   end
 
+  def handle_cast({ :test_finished, test = ExUnit.Test[failure: nil, invalid: true] }, config) do
+    IO.write invalid("?")
+    { :noreply, config.update_counter(&1 + 1).
+        update_test_failures([test|&1]) }
+  end
+
   def handle_cast({ :test_finished, ExUnit.Test[failure: nil] }, config) do
     IO.write success(".")
     { :noreply, config.update_counter(&1 + 1) }
@@ -61,30 +67,61 @@ defmodule ExUnit.CLIFormatter do
   def handle_cast({ :test_finished, test }, config) do
     IO.write failure("F")
     { :noreply, config.update_counter(&1 + 1).
-        update_failures([test|&1]) }
+        update_test_failures([test|&1]) }
+  end
+
+  def handle_cast({ :case_finished, test_case }, config) do
+    if test_case.failure do
+      { :noreply, config.update_case_failures([test_case|&1]) }
+    else
+      { :noreply, config }
+    end
   end
 
   def handle_cast(_, _) do
     super
   end
 
-  defp print_suite(counter, [], ms) do
+  defp print_suite(counter, [], [], ms) do
     IO.write "\n\n"
     IO.puts "Finished in #{format_ms ms} seconds"
     IO.puts success("#{counter} tests, 0 failures")
   end
 
-  defp print_suite(counter, failures, ms) do
+  defp print_suite(counter, test_failures, case_failures, ms) do
     IO.write "\n\nFailures:\n\n"
-    Enum.reduce Enum.reverse(failures), 1, print_failure(&1, &2, File.cwd!)
+    num_fails = Enum.reduce Enum.reverse(test_failures), 1, print_test_failure(&1, &2, File.cwd!)
+    Enum.reduce Enum.reverse(case_failures), num_fails, print_case_failure(&1, &2, File.cwd!)
+    num_invalids = Enum.count test_failures, fn test -> test.invalid end
+
     IO.puts "Finished in #{format_ms ms} seconds"
-    IO.puts failure("#{counter} tests, #{length(failures)} failures")
+
+    num_fails = num_fails - 1
+    message = "#{counter} tests, #{num_fails} failures"
+    if num_invalids > 0, do: message = message <>  ", #{num_invalids} invalid"
+    cond do
+      num_fails > 0    -> IO.puts failure(message)
+      num_invalids > 0 -> IO.puts invalid(message)
+      true             -> IO.puts success(message)
+    end
+  end
+
+  defp print_test_failure(ExUnit.Test[failure: nil], acc, _cwd) do
+    acc
+  end
+
+  defp print_test_failure(ExUnit.Test[case: test_case, name: test, failure: { kind, reason, stacktrace }], acc, cwd) do
+    IO.puts "  #{acc}) #{test} (#{inspect test_case.name})"
+    print_kind_reason(kind, reason)
+    print_stacktrace(stacktrace, test_case.name, test, cwd)
+    IO.write "\n"
+    acc + 1
   end
 
-  defp print_failure(ExUnit.Test[case: test_case, name: test, failure: { kind, reason, stacktrace }], acc, cwd) do
-    IO.puts "  #{acc}) #{test} (#{inspect test_case})"
+  defp print_case_failure(ExUnit.TestCase[name: case_name, failure: { kind, reason, stacktrace }], acc, cwd) do
+    IO.puts "  #{acc}) #{inspect case_name}"
     print_kind_reason(kind, reason)
-    print_stacktrace(stacktrace, test_case, test, cwd)
+    print_stacktrace(stacktrace, case_name, nil, cwd)
     IO.write "\n"
     acc + 1
   end
@@ -164,6 +201,10 @@ defmodule ExUnit.CLIFormatter do
     IO.ANSI.escape("%{green}" <>  msg)
   end
 
+  defp invalid(msg) do
+    IO.ANSI.escape("%{yellow}" <>  msg)
+  end
+
   defp failure(msg) do
     IO.ANSI.escape("%{red}" <>  msg)
   end
@@ -179,4 +220,4 @@ defmodule ExUnit.CLIFormatter do
   defp stacktrace_info(msg) do
     "       " <> msg
   end
-end
+end

lib/ex_unit/lib/ex_unit/runner.ex

@@ -71,31 +71,71 @@ defmodule ExUnit.Runner do
     end
   end
 
-  defp run_tests(config, pid, test_case) do
+  defp run_tests(config, pid, case_name) do
+    test_case = ExUnit.TestCase[name: case_name]
     config.formatter.case_started(config.formatter_id, test_case)
 
-    try do
-      tests = tests_for(test_case)
-      context = test_case.__exunit__(:setup_all, [case: test_case])
-      Enum.each tests, run_test(config, pid, test_case, &1, context)
-      test_case.__exunit__(:teardown_all, context)
-    after
-      pid <- { self, :case_finished, test_case }
+    self_pid = self
+    { case_pid, case_ref } = Process.spawn_monitor fn ->
+      { test_case, context } = try do
+        context = case_name.__exunit__(:setup_all, [case: test_case])
+        { test_case, context }
+      rescue
+        error ->
+          { test_case.failure({ :error, error, filtered_stacktrace }), nil }
+      catch
+        kind, error ->
+          { test_case.failure({ kind, error, filtered_stacktrace }), nil }
+      end
+
+      tests = tests_for(case_name)
+      if test_case.failure do
+        Enum.each tests, fn test_name ->
+          test = ExUnit.Test[name: test_name, case: test_case, invalid: true]
+          pid <- { self, :test_finished, test }
+        end
+        self_pid <- { self, :case_finished, test_case }
+
+      else
+        Enum.each tests, run_test(config, pid, test_case, &1, context)
+
+        test_case = try do
+          case_name.__exunit__(:teardown_all, context)
+          test_case
+        rescue
+          error ->
+            test_case.failure { :error, error, filtered_stacktrace }
+        catch
+          kind, error ->
+            test_case.failure { kind, error, filtered_stacktrace }
+        end
+
+        self_pid <- { self, :case_finished, test_case }
+      end
+    end
+
+    receive do
+      { ^case_pid, :case_finished, test_case } ->
+        pid <- { case_pid, :case_finished, test_case }
+      { :DOWN, ^case_ref, :process, ^case_pid, { error, stacktrace } } ->
+        test_case = test_case.failure { :EXIT, error, filter_stacktrace(stacktrace) }
+        pid <- { case_pid, :case_finished, test_case }
     end
   end
 
   defp run_test(config, pid, test_case, test_name, context) do
     test = ExUnit.Test[name: test_name, case: test_case]
+    ExUnit.TestCase[name: case_name] = test_case
     config.formatter.test_started(config.formatter_id, test)
 
     # Run test in a new process so that we can trap exits for a single test
     self_pid = self
     { test_pid, test_ref } = Process.spawn_monitor fn ->
       test = try do
-        context = test_case.__exunit__(:setup, Keyword.put(context, :test, test))
+        context = case_name.__exunit__(:setup, Keyword.put(context, :test, test))
 
         test = try do
-          apply test_case, test_name, [context]
+          apply case_name, test_name, [context]
           test
         rescue
           error1 ->
@@ -105,7 +145,7 @@ defmodule ExUnit.Runner do
             test.failure { kind1, error1, filtered_stacktrace }
         end
 
-        test_case.__exunit__(:teardown, Keyword.put(context, :test, test))
+        case_name.__exunit__(:teardown, Keyword.put(context, :test, test))
         test
       rescue
         error2 ->