Start running validate_solutions script for Travis CI (#3215)

* Removed print error_msgs at the end of test:

This was done only to reduce the message clutter produced by 60
failing tests. As that is fixed, we can produce the traceback in
short form and allow pytest to print the captured error message
output at the end of test.

* Start validate_solutions script for Travis CI

I am separating out the solution testing and doctest as validating
the solutions for the current number of solutions present is
taking 2 minutes to run.

Author: dhruvmanila

Diff for: .travis.yml

@@ -12,11 +12,14 @@ jobs:
         - pytest --doctest-modules --ignore=project_euler/ --durations=10 --cov-report=term-missing:skip-covered --cov=. .
     - name: Project Euler
       install:
-        - pip install pytest-cov pytest-subtests
-      before_script:
-        - pytest --tb=no --no-summary --capture=no project_euler/validate_solutions.py || true  # fail fast on wrong solution
+        - pip install pytest-cov
       script:
         - pytest --doctest-modules --durations=10 --cov-report=term-missing:skip-covered --cov=project_euler/ project_euler/
+    - name: Project Euler Solution
+      install:
+        - pip install pytest-subtests
+      script:
+        - pytest --tb=short project_euler/validate_solutions.py
 after_success:
   - scripts/build_directory_md.py 2>&1 | tee DIRECTORY.md
 notifications:

Diff for: project_euler/validate_solutions.py

@@ -15,8 +15,6 @@
 with open(PROJECT_EULER_ANSWERS_PATH) as file_handle:
     PROBLEM_ANSWERS = json.load(file_handle)
 
-error_msgs = []
-
 
 def generate_solution_modules(
     dir_path: pathlib.Path,
@@ -48,20 +46,9 @@ def test_project_euler(subtests, problem_number: int, expected: str):
                     answer = str(solution_module.solution())
                     assert answer == expected, f"Expected {expected} but got {answer}"
                 except (AssertionError, AttributeError, TypeError) as err:
-                    error_msgs.append(
-                        f"problem_{problem_number}/{solution_module.__name__}: {err}"
+                    print(
+                        f"problem_{problem_number:02}/{solution_module.__name__}: {err}"
                     )
-                    raise  # We still want pytest to know that this test failed
+                    raise
     else:
         pytest.skip(f"Solution {problem_number} does not exist yet.")
-
-
-# Run this function at the end of all the tests
-# https://stackoverflow.com/a/52873379
-@pytest.fixture(scope="session", autouse=True)
-def custom_print_message(request):
-    def print_error_messages():
-        if error_msgs:
-            print("\n" + "\n".join(error_msgs))
-
-    request.addfinalizer(print_error_messages)