WIP/CI: Clean up to test calls

.travis.yml

@@ -104,15 +104,4 @@ script:
   - echo "script start"
   - source activate pandas-dev
   - ci/build_docs.sh
-  - ci/run_tests.sh
-
-after_script:
-  - echo "after_script start"
-  - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
-  - if [ -e test-data-single.xml ]; then
-        ci/print_skipped.py test-data-single.xml;
-    fi
-  - if [ -e test-data-multiple.xml ]; then
-        ci/print_skipped.py test-data-multiple.xml;
-    fi
-  - echo "after_script done"
+  - python ci/run_tests.py

ci/azure/posix.yml

@@ -43,7 +43,7 @@ jobs:
           ENV_FILE: ci/deps/azure-37-numpydev.yaml
           CONDA_PY: "37"
           PATTERN: "not slow and not network and not db"
-          TEST_ARGS: "-W error"
+          WARNINGS_ARE_ERRORS: "true"
           PANDAS_TESTING_MODE: "deprecate"
           EXTRA_APT: "xsel"
 
@@ -64,33 +64,21 @@ jobs:
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
-        ci/run_tests.sh
+        python ci/run_tests.py
       displayName: 'Test'
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
     - task: PublishTestResults@2
       inputs:
-        testResultsFiles: 'test-data-*.xml'
+        testResultsFiles: 'test-data.xml'
         testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }}
     - powershell: |
-        $junitXml = "test-data-single.xml"
+        $junitXml = "test-data.xml"
         $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
         if ($matches[1] -eq 0)
         {
-          Write-Host "No test failures in test-data-single"
-        }
-        else
-        {
-          # note that this will produce $LASTEXITCODE=1
-          Write-Error "$($matches[1]) tests failed"
-        }
-
-        $junitXmlMulti = "test-data-multiple.xml"
-        $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"'
-        if ($matches[1] -eq 0)
-        {
-          Write-Host "No test failures in test-data-multi"
+          Write-Host "No test failures in test-data"
         }
         else
         {

ci/azure/windows.yml

@@ -11,10 +11,12 @@ jobs:
       py36_np14:
         ENV_FILE: ci/deps/azure-windows-36.yaml
         CONDA_PY: "36"
+        PATTERN: "not slow and not network and not db"
 
       py27_np121:
         ENV_FILE: ci/deps/azure-windows-27.yaml
         CONDA_PY: "27"
+        PATTERN: "not slow and not network and not db"
 
   steps:
     - task: CondaEnvironment@1
@@ -38,7 +40,7 @@ jobs:
       displayName: 'Build'
     - script: |
         call activate pandas-dev
-        pytest -m "not slow and not network and not db" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
+        python ci\\run_tests.py
       displayName: 'Test'
     - task: PublishTestResults@2
       inputs:

ci/run_tests.py

@@ -0,0 +1,143 @@
+#!/bin/env python
+import os
+import subprocess
+import sys
+import random
+import tempfile
+import time
+import warnings
+import xml.etree.ElementTree
+try:
+    from urllib.request import urlretrieve
+except ImportError:  # py2
+    from urllib import urlretrieve
+
+
+def set_environ(pattern, locale):
+    """
+    Set environment variables needed for running the tests.
+    """
+    # Workaround for pytest-xdist flaky collection order
+    # https://github.com/pytest-dev/pytest/issues/920
+    # https://github.com/pytest-dev/pytest/issues/1075
+    os.environ['PYTHONHASHSEED'] = str(random.randint(1, 4294967295))
+
+    if locale:
+        os.environ['LC_ALL'] = os.environ['LANG'] = locale
+        import pandas
+        pandas_locale = pandas.get_option('display.encoding')
+        if pandas_locale != locale:
+            # TODO raise exception instead of warning when
+            # https://github.com/pandas-dev/pandas/issues/23923 is fixed
+            warnings.warn(('pandas could not detect the locale. '
+                           'System locale: {}, '
+                           'pandas detected: {}').format(locale,
+                                                         pandas_locale))
+
+    if 'not network' in pattern:
+        os.environ['http_proxy'] = os.environ['https_proxy'] = 'http://1.2.3.4'
+
+
+def skipped_tests(fname):
+    """
+    Yield the list of skipped tests, including a header to be printed.
+    """
+    root = xml.etree.ElementTree.parse(fname).getroot()
+    for item in root.findall('testcase'):
+        for skipped in item.findall('skipped'):
+            yield (item.attrib['classname'],
+                   item.attrib['name'],
+                   skipped.attrib['message'])
+
+
+def pytest_command(pattern, junit_xml, coverage_file):
+    """
+    Build and return the pytest command to run.
+    """
+    cmd = ['pytest', '--junitxml={}'.format(junit_xml)]
+
+    if pattern:
+        cmd += ['-m', pattern]
+
+    if coverage_file:
+        cmd += ['--cov=pandas', '--cov-report=xml:{}'.format(coverage_file)]
+
+    test_jobs = os.environ.get('TESTS_JOBS', 'auto')
+    if test_jobs != '0':
+        cmd += ['-n', test_jobs, '--dist', 'loadfile']
+
+    if os.environ.get('WARNINGS_ARE_ERRORS'):
+        cmd += ['-W', 'error']
+
+    return cmd + ['pandas']
+
+
+def upload_coverage(coverage_file):
+    """
+    Download codecov.io script and run it to upload coverage for coverage_file.
+    """
+    script_fname = os.path.join(os.path.dirname(coverage_file),
+                                'codecov_script.sh')
+    urlretrieve('https://codecov.io/bash', script_fname)
+    upload_coverage_cmd = ['bash',
+                           script_fname,
+                           '-Z',
+                           '-c',
+                           '-f',
+                           coverage_file]
+    sys.stderr.write('{}\n'.format(' '.join(upload_coverage_cmd)))
+    subprocess.check_call(upload_coverage_cmd.split())
+    os.remove(script_fname)
+    os.remove(coverage_file)
+
+
+def run_tests(pattern, locale=None, coverage_file=False):
+    """
+    Run tests with the specified environment.
+
+    Parameters
+    ----------
+    pattern : str
+        Tests to execute based on pytest markers (e.g. "slow and not network").
+    locale : str, optional
+        Locale to use instead of the system defaule (e.g. "it_IT.UTF8").
+    coverage_file : str, optional
+        If provided, the file path where to save the coverage.
+    """
+    if os.environ.get('DOC'):
+        sys.stdout.write('We are not running pytest as this is a doc-build\n')
+        return
+    junit_xml = 'test-data.xml'
+    set_environ(pattern, locale)
+    pytest_cmd = pytest_command(pattern, junit_xml, coverage_file)
+    sys.stderr.write('{}\n'.format(' '.join(pytest_cmd)))
+    start = time.time()
+    subprocess.check_call(pytest_cmd)
+    tests_run_in_seconds = int(time.time() - start)
+
+    prev_class = None
+    for i, (class_, name, msg) in enumerate(skipped_tests(junit_xml)):
+        if prev_class is not None and class_ != prev_class:
+            sys.stdout.write('{}\n'.format('-' * 100))
+        sys.stdout.write('#{} {}.{}: {}\n'.format(i + 1, class_, name, msg))
+        prev_class = class_
+    sys.stdout.write('{}\n'.format('=' * 100))
+    import pandas
+    pandas.show_versions()
+    sys.stdout.write('{}\n'.format('=' * 100))
+    sys.stdout.write('Tests run in {} seconds\n'.format(tests_run_in_seconds))
+
+    if coverage_file:
+        upload_coverage(coverage_file)
+
+
+if __name__ == '__main__':
+    pattern = os.environ.get('PATTERN', '')
+    locale = os.environ.get('LOCALE_OVERRIDE')
+    coverage_file = None
+    if os.environ.get('COVERAGE', '') != '':
+        if sys.platform == 'win32':
+            raise RuntimeError('Coverage can not be uploaded from Windows')
+        coverage_file = os.path.join(tempfile.gettempdir(),
+                                     'pandas-coverage.xml')
+    run_tests(pattern, locale, coverage_file)

pandas/tests/frame/test_rank.py

@@ -310,7 +310,7 @@ def test_rank_pct_true(self, method, exp):
         expected = DataFrame(exp)
         tm.assert_frame_equal(result, expected)
 
-    @pytest.mark.single
+    @pytest.mark.serial
     def test_pct_max_many_rows(self):
         # GH 18271
         df = DataFrame({'A': np.arange(2**24 + 1),

pandas/tests/io/json/test_pandas.py

@@ -846,7 +846,7 @@ def test_misc_example(self):
         assert_frame_equal(result, expected)
 
     @network
-    @pytest.mark.single
+    @pytest.mark.serial
     def test_round_trip_exception_(self):
         # GH 3867
         csv = 'https://raw.github.com/hayd/lahman2012/master/csvs/Teams.csv'
@@ -857,7 +857,7 @@ def test_round_trip_exception_(self):
             index=df.index, columns=df.columns), df)
 
     @network
-    @pytest.mark.single
+    @pytest.mark.serial
     def test_url(self):
         url = 'https://api.github.com/repos/pandas-dev/pandas/issues?per_page=5'  # noqa
         result = read_json(url, convert_dates=True)

pandas/tests/io/test_clipboard.py

@@ -115,7 +115,7 @@ def test_mock_clipboard(mock_clipboard):
     assert result == "abc"
 
 
-@pytest.mark.single
+@pytest.mark.serial
 @pytest.mark.clipboard
 @pytest.mark.skipif(not _DEPS_INSTALLED,
                     reason="clipboard primitives not installed")

pandas/tests/io/test_feather.py

@@ -15,7 +15,7 @@
 pyarrow_version = LooseVersion(pyarrow.__version__)
 
 
-@pytest.mark.single
+@pytest.mark.serial
 class TestFeather(object):
 
     def check_error_on_write(self, df, exc):

pandas/tests/io/test_gbq.py

@@ -109,7 +109,7 @@ def test_read_gbq_without_dialect_warns_future_change(monkeypatch):
         pd.read_gbq("SELECT 1")
 
 
-@pytest.mark.single
+@pytest.mark.serial
 class TestToGBQIntegrationWithServiceAccountKeyPath(object):
 
     @classmethod