Skip to content

Commit 0b8d2ca

Browse files
jrebackjreback
committed
TST: use xdist for multiple cpu testing
closes #15369
1 parent f87db63 commit 0b8d2ca

File tree

10 files changed

+223
-181
lines changed

10 files changed

+223
-181
lines changed

‎.travis.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,8 @@ before_script:
320320
script:
321321
- echo "script start"
322322
- ci/run_build_docs.sh
323-
- ci/script.sh
323+
- ci/script_single.sh
324+
- ci/script_multi.sh
324325
- ci/lint.sh
325326
- echo "script done"
326327

‎ci/script_multi.sh

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
echo "[script multi]"
4+
5+
source activate pandas
6+
7+
# don't run the tests for the doc build
8+
if [ x"$DOC_BUILD" != x"" ]; then
9+
exit 0
10+
fi
11+
12+
if [ -n "$LOCALE_OVERRIDE" ]; then
13+
export LC_ALL="$LOCALE_OVERRIDE";
14+
echo "Setting LC_ALL to $LOCALE_OVERRIDE"
15+
16+
pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))'
17+
python -c "$pycmd"
18+
fi
19+
20+
if [ "$BUILD_TEST" ]; then
21+
echo "We are not running pytest as this is simply a build test."
22+
elif [ "$COVERAGE" ]; then
23+
echo pytest -s -n 4 -m "not single" --cov=pandas --cov-append --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
24+
pytest -s -n 4 -m "not single" --cov=pandas --cov-append --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
25+
else
26+
echo pytest -n 4 -m "not single" $TEST_ARGS pandas
27+
pytest -n 4 -m "not single" $TEST_ARGS pandas # TODO: doctest
28+
fi
29+
30+
RET="$?"
31+
32+
exit "$RET"

‎ci/script.sh renamed to ‎ci/script_single.sh

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
echo "inside $0"
3+
echo "[script_single]"
44

55
source activate pandas
66

@@ -20,11 +20,11 @@ fi
2020
if [ "$BUILD_TEST" ]; then
2121
echo "We are not running pytest as this is simply a build test."
2222
elif [ "$COVERAGE" ]; then
23-
echo pytest -s --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
24-
pytest -s --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
23+
echo pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
24+
pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
2525
else
26-
echo pytest $TEST_ARGS pandas
27-
pytest $TEST_ARGS pandas # TODO: doctest
26+
echo pytest -m "single" $TEST_ARGS pandas
27+
pytest -m "single" $TEST_ARGS pandas # TODO: doctest
2828
fi
2929

3030
RET="$?"

‎pandas/tests/indexes/datetimes/test_ops.py

+123-121
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1+
import pytest
12
import warnings
23
import numpy as np
34
from datetime import timedelta
45

6+
from itertools import product
57
import pandas as pd
68
import pandas.tslib as tslib
79
import pandas.util.testing as tm
@@ -958,134 +960,134 @@ def test_second(self):
958960
tm.assert_index_equal(r1, r2)
959961

960962

961-
class TestDatetimeIndex(tm.TestCase):
962-
963-
# GH 10699
964-
def test_datetime64_with_DateOffset(self):
965-
for klass, assert_func in zip([Series, DatetimeIndex],
966-
[self.assert_series_equal,
967-
tm.assert_index_equal]):
968-
s = klass(date_range('2000-01-01', '2000-01-31'), name='a')
969-
result = s + pd.DateOffset(years=1)
970-
result2 = pd.DateOffset(years=1) + s
971-
exp = klass(date_range('2001-01-01', '2001-01-31'), name='a')
963+
# GH 10699
964+
@pytest.mark.parametrize('klass,assert_func', zip([Series, DatetimeIndex],
965+
[tm.assert_series_equal,
966+
tm.assert_index_equal]))
967+
def test_datetime64_with_DateOffset(klass, assert_func):
968+
s = klass(date_range('2000-01-01', '2000-01-31'), name='a')
969+
result = s + pd.DateOffset(years=1)
970+
result2 = pd.DateOffset(years=1) + s
971+
exp = klass(date_range('2001-01-01', '2001-01-31'), name='a')
972+
assert_func(result, exp)
973+
assert_func(result2, exp)
974+
975+
result = s - pd.DateOffset(years=1)
976+
exp = klass(date_range('1999-01-01', '1999-01-31'), name='a')
977+
assert_func(result, exp)
978+
979+
s = klass([Timestamp('2000-01-15 00:15:00', tz='US/Central'),
980+
pd.Timestamp('2000-02-15', tz='US/Central')], name='a')
981+
result = s + pd.offsets.Day()
982+
result2 = pd.offsets.Day() + s
983+
exp = klass([Timestamp('2000-01-16 00:15:00', tz='US/Central'),
984+
Timestamp('2000-02-16', tz='US/Central')], name='a')
985+
assert_func(result, exp)
986+
assert_func(result2, exp)
987+
988+
s = klass([Timestamp('2000-01-15 00:15:00', tz='US/Central'),
989+
pd.Timestamp('2000-02-15', tz='US/Central')], name='a')
990+
result = s + pd.offsets.MonthEnd()
991+
result2 = pd.offsets.MonthEnd() + s
992+
exp = klass([Timestamp('2000-01-31 00:15:00', tz='US/Central'),
993+
Timestamp('2000-02-29', tz='US/Central')], name='a')
994+
assert_func(result, exp)
995+
assert_func(result2, exp)
996+
997+
# array of offsets - valid for Series only
998+
if klass is Series:
999+
with tm.assert_produces_warning(PerformanceWarning):
1000+
s = klass([Timestamp('2000-1-1'), Timestamp('2000-2-1')])
1001+
result = s + Series([pd.offsets.DateOffset(years=1),
1002+
pd.offsets.MonthEnd()])
1003+
exp = klass([Timestamp('2001-1-1'), Timestamp('2000-2-29')
1004+
])
9721005
assert_func(result, exp)
973-
assert_func(result2, exp)
9741006

975-
result = s - pd.DateOffset(years=1)
976-
exp = klass(date_range('1999-01-01', '1999-01-31'), name='a')
1007+
# same offset
1008+
result = s + Series([pd.offsets.DateOffset(years=1),
1009+
pd.offsets.DateOffset(years=1)])
1010+
exp = klass([Timestamp('2001-1-1'), Timestamp('2001-2-1')])
9771011
assert_func(result, exp)
9781012

979-
s = klass([Timestamp('2000-01-15 00:15:00', tz='US/Central'),
980-
pd.Timestamp('2000-02-15', tz='US/Central')], name='a')
981-
result = s + pd.offsets.Day()
982-
result2 = pd.offsets.Day() + s
983-
exp = klass([Timestamp('2000-01-16 00:15:00', tz='US/Central'),
984-
Timestamp('2000-02-16', tz='US/Central')], name='a')
985-
assert_func(result, exp)
986-
assert_func(result2, exp)
987-
988-
s = klass([Timestamp('2000-01-15 00:15:00', tz='US/Central'),
989-
pd.Timestamp('2000-02-15', tz='US/Central')], name='a')
990-
result = s + pd.offsets.MonthEnd()
991-
result2 = pd.offsets.MonthEnd() + s
992-
exp = klass([Timestamp('2000-01-31 00:15:00', tz='US/Central'),
993-
Timestamp('2000-02-29', tz='US/Central')], name='a')
994-
assert_func(result, exp)
995-
assert_func(result2, exp)
996-
997-
# array of offsets - valid for Series only
998-
if klass is Series:
999-
with tm.assert_produces_warning(PerformanceWarning):
1000-
s = klass([Timestamp('2000-1-1'), Timestamp('2000-2-1')])
1001-
result = s + Series([pd.offsets.DateOffset(years=1),
1002-
pd.offsets.MonthEnd()])
1003-
exp = klass([Timestamp('2001-1-1'), Timestamp('2000-2-29')
1004-
])
1005-
assert_func(result, exp)
1006-
1007-
# same offset
1008-
result = s + Series([pd.offsets.DateOffset(years=1),
1009-
pd.offsets.DateOffset(years=1)])
1010-
exp = klass([Timestamp('2001-1-1'), Timestamp('2001-2-1')])
1011-
assert_func(result, exp)
1012-
1013-
s = klass([Timestamp('2000-01-05 00:15:00'),
1013+
s = klass([Timestamp('2000-01-05 00:15:00'),
1014+
Timestamp('2000-01-31 00:23:00'),
1015+
Timestamp('2000-01-01'),
1016+
Timestamp('2000-03-31'),
1017+
Timestamp('2000-02-29'),
1018+
Timestamp('2000-12-31'),
1019+
Timestamp('2000-05-15'),
1020+
Timestamp('2001-06-15')])
1021+
1022+
# DateOffset relativedelta fastpath
1023+
relative_kwargs = [('years', 2), ('months', 5), ('days', 3),
1024+
('hours', 5), ('minutes', 10), ('seconds', 2),
1025+
('microseconds', 5)]
1026+
for i, kwd in enumerate(relative_kwargs):
1027+
op = pd.DateOffset(**dict([kwd]))
1028+
assert_func(klass([x + op for x in s]), s + op)
1029+
assert_func(klass([x - op for x in s]), s - op)
1030+
op = pd.DateOffset(**dict(relative_kwargs[:i + 1]))
1031+
assert_func(klass([x + op for x in s]), s + op)
1032+
assert_func(klass([x - op for x in s]), s - op)
1033+
1034+
# assert these are equal on a piecewise basis
1035+
offsets = ['YearBegin', ('YearBegin', {'month': 5}),
1036+
'YearEnd', ('YearEnd', {'month': 5}),
1037+
'MonthBegin', 'MonthEnd',
1038+
'SemiMonthEnd', 'SemiMonthBegin',
1039+
'Week', ('Week', {'weekday': 3}),
1040+
'BusinessDay', 'BDay', 'QuarterEnd', 'QuarterBegin',
1041+
'CustomBusinessDay', 'CDay', 'CBMonthEnd',
1042+
'CBMonthBegin', 'BMonthBegin', 'BMonthEnd',
1043+
'BusinessHour', 'BYearBegin', 'BYearEnd',
1044+
'BQuarterBegin', ('LastWeekOfMonth', {'weekday': 2}),
1045+
('FY5253Quarter', {'qtr_with_extra_week': 1,
1046+
'startingMonth': 1,
1047+
'weekday': 2,
1048+
'variation': 'nearest'}),
1049+
('FY5253', {'weekday': 0,
1050+
'startingMonth': 2,
1051+
'variation':
1052+
'nearest'}),
1053+
('WeekOfMonth', {'weekday': 2,
1054+
'week': 2}),
1055+
'Easter', ('DateOffset', {'day': 4}),
1056+
('DateOffset', {'month': 5})]
1057+
1058+
with warnings.catch_warnings(record=True):
1059+
for normalize in (True, False):
1060+
for do in offsets:
1061+
if isinstance(do, tuple):
1062+
do, kwargs = do
1063+
else:
1064+
do = do
1065+
kwargs = {}
1066+
1067+
for n in [0, 5]:
1068+
if (do in ['WeekOfMonth', 'LastWeekOfMonth',
1069+
'FY5253Quarter', 'FY5253'] and n == 0):
1070+
continue
1071+
op = getattr(pd.offsets, do)(n,
1072+
normalize=normalize,
1073+
**kwargs)
1074+
assert_func(klass([x + op for x in s]), s + op)
1075+
assert_func(klass([x - op for x in s]), s - op)
1076+
assert_func(klass([op + x for x in s]), op + s)
1077+
1078+
1079+
@pytest.mark.parametrize('years,months', product([-1, 0, 1], [-2, 0, 2]))
1080+
def test_shift_months(years, months):
1081+
s = DatetimeIndex([Timestamp('2000-01-05 00:15:00'),
10141082
Timestamp('2000-01-31 00:23:00'),
10151083
Timestamp('2000-01-01'),
1016-
Timestamp('2000-03-31'),
10171084
Timestamp('2000-02-29'),
1018-
Timestamp('2000-12-31'),
1019-
Timestamp('2000-05-15'),
1020-
Timestamp('2001-06-15')])
1021-
1022-
# DateOffset relativedelta fastpath
1023-
relative_kwargs = [('years', 2), ('months', 5), ('days', 3),
1024-
('hours', 5), ('minutes', 10), ('seconds', 2),
1025-
('microseconds', 5)]
1026-
for i, kwd in enumerate(relative_kwargs):
1027-
op = pd.DateOffset(**dict([kwd]))
1028-
assert_func(klass([x + op for x in s]), s + op)
1029-
assert_func(klass([x - op for x in s]), s - op)
1030-
op = pd.DateOffset(**dict(relative_kwargs[:i + 1]))
1031-
assert_func(klass([x + op for x in s]), s + op)
1032-
assert_func(klass([x - op for x in s]), s - op)
1033-
1034-
# assert these are equal on a piecewise basis
1035-
offsets = ['YearBegin', ('YearBegin', {'month': 5}), 'YearEnd',
1036-
('YearEnd', {'month': 5}), 'MonthBegin', 'MonthEnd',
1037-
'SemiMonthEnd', 'SemiMonthBegin',
1038-
'Week', ('Week', {
1039-
'weekday': 3
1040-
}), 'BusinessDay', 'BDay', 'QuarterEnd', 'QuarterBegin',
1041-
'CustomBusinessDay', 'CDay', 'CBMonthEnd',
1042-
'CBMonthBegin', 'BMonthBegin', 'BMonthEnd',
1043-
'BusinessHour', 'BYearBegin', 'BYearEnd',
1044-
'BQuarterBegin', ('LastWeekOfMonth', {
1045-
'weekday': 2
1046-
}), ('FY5253Quarter', {'qtr_with_extra_week': 1,
1047-
'startingMonth': 1,
1048-
'weekday': 2,
1049-
'variation': 'nearest'}),
1050-
('FY5253', {'weekday': 0,
1051-
'startingMonth': 2,
1052-
'variation':
1053-
'nearest'}), ('WeekOfMonth', {'weekday': 2,
1054-
'week': 2}),
1055-
'Easter', ('DateOffset', {'day': 4}),
1056-
('DateOffset', {'month': 5})]
1057-
1058-
with warnings.catch_warnings(record=True):
1059-
for normalize in (True, False):
1060-
for do in offsets:
1061-
if isinstance(do, tuple):
1062-
do, kwargs = do
1063-
else:
1064-
do = do
1065-
kwargs = {}
1066-
1067-
for n in [0, 5]:
1068-
if (do in ['WeekOfMonth', 'LastWeekOfMonth',
1069-
'FY5253Quarter', 'FY5253'] and n == 0):
1070-
continue
1071-
op = getattr(pd.offsets, do)(n,
1072-
normalize=normalize,
1073-
**kwargs)
1074-
assert_func(klass([x + op for x in s]), s + op)
1075-
assert_func(klass([x - op for x in s]), s - op)
1076-
assert_func(klass([op + x for x in s]), op + s)
1077-
1078-
def test_shift_months(self):
1079-
s = DatetimeIndex([Timestamp('2000-01-05 00:15:00'), Timestamp(
1080-
'2000-01-31 00:23:00'), Timestamp('2000-01-01'), Timestamp(
1081-
'2000-02-29'), Timestamp('2000-12-31')])
1082-
for years in [-1, 0, 1]:
1083-
for months in [-2, 0, 2]:
1084-
actual = DatetimeIndex(tslib.shift_months(s.asi8, years * 12 +
1085-
months))
1086-
expected = DatetimeIndex([x + offsets.DateOffset(
1087-
years=years, months=months) for x in s])
1088-
tm.assert_index_equal(actual, expected)
1085+
Timestamp('2000-12-31')])
1086+
actual = DatetimeIndex(tslib.shift_months(s.asi8, years * 12 +
1087+
months))
1088+
expected = DatetimeIndex([x + offsets.DateOffset(
1089+
years=years, months=months) for x in s])
1090+
tm.assert_index_equal(actual, expected)
10891091

10901092

10911093
class TestBusinessDatetimeIndex(tm.TestCase):

‎pandas/tests/io/test_clipboard.py

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
_DEPS_INSTALLED = 0
2121

2222

23+
@pytest.mark.single
2324
@pytest.mark.skipif(not _DEPS_INSTALLED,
2425
reason="clipboard primitives not installed")
2526
class TestClipboard(tm.TestCase):

‎pandas/tests/io/test_pytables.py

+1-6
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,6 @@
3636
from pandas import concat, Timestamp
3737
from pandas import compat
3838
from pandas.compat import range, lrange, u
39-
40-
try:
41-
import tables
42-
except ImportError:
43-
pytest.skip('no pytables')
44-
4539
from distutils.version import LooseVersion
4640

4741
_default_compressor = ('blosc' if LooseVersion(tables.__version__) >= '2.2'
@@ -165,6 +159,7 @@ def tearDown(self):
165159
pass
166160

167161

162+
@pytest.mark.single
168163
class TestHDFStore(Base, tm.TestCase):
169164

170165
def test_factory_fun(self):

0 commit comments

Comments
 (0)