Skip to content

Commit 0612230

Browse files
jrebackjreback
committed
TST/PERF: have moments slow tests run faster (GH8450)
1 parent daf8761 commit 0612230

File tree

2 files changed

+105
-68
lines changed

2 files changed

+105
-68
lines changed

‎pandas/computation/tests/test_eval.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -134,9 +134,11 @@ def tearDown(self):
134134

135135
@slow
136136
def test_complex_cmp_ops(self):
137-
for lhs, cmp1, rhs, binop, cmp2 in product(self.lhses, self.cmp_ops,
137+
cmp_ops = ('not in', 'in', '!=', '==', '<=', '>=', '<', '>')
138+
cmp2_ops = ('>', '<')
139+
for lhs, cmp1, rhs, binop, cmp2 in product(self.lhses, cmp_ops,
138140
self.rhses, self.bin_ops,
139-
self.cmp2_ops):
141+
cmp2_ops):
140142
self.check_complex_cmp_op(lhs, cmp1, rhs, binop, cmp2)
141143

142144
def test_simple_cmp_ops(self):

‎pandas/stats/tests/test_moments.py

+101-66
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import numpy as np
1010
from distutils.version import LooseVersion
1111

12-
from pandas import Series, DataFrame, Panel, bdate_range, isnull, notnull
12+
from pandas import Series, DataFrame, Panel, bdate_range, isnull, notnull, concat
1313
from pandas.util.testing import (
1414
assert_almost_equal, assert_series_equal, assert_frame_equal, assert_panel_equal, assert_index_equal
1515
)
@@ -20,15 +20,14 @@
2020

2121
N, K = 100, 10
2222

23-
24-
class TestMoments(tm.TestCase):
23+
class Base(tm.TestCase):
2524

2625
_multiprocess_can_split_ = True
2726

2827
_nan_locs = np.arange(20, 40)
2928
_inf_locs = np.array([])
3029

31-
def setUp(self):
30+
def _create_data(self):
3231
arr = randn(N)
3332
arr[self._nan_locs] = np.NaN
3433

@@ -40,6 +39,10 @@ def setUp(self):
4039
self.frame = DataFrame(randn(N, K), index=self.rng,
4140
columns=np.arange(K))
4241

42+
class TestMoments(Base):
43+
44+
def setUp(self):
45+
self._create_data()
4346
warnings.simplefilter("ignore", category=FutureWarning)
4447

4548
def test_centered_axis_validation(self):
@@ -71,7 +74,7 @@ def test_cmov_mean(self):
7174
# GH 8238
7275
tm._skip_if_no_scipy()
7376

74-
vals = np.array([6.95, 15.21, 4.72, 9.12, 13.81, 13.49,
77+
vals = np.array([6.95, 15.21, 4.72, 9.12, 13.81, 13.49,
7578
16.68, 9.48, 10.63, 14.48])
7679
xp = np.array([np.nan, np.nan, 9.962, 11.27 , 11.564, 12.516,
7780
12.818, 12.952, np.nan, np.nan])
@@ -87,7 +90,7 @@ def test_cmov_window(self):
8790
# GH 8238
8891
tm._skip_if_no_scipy()
8992

90-
vals = np.array([6.95, 15.21, 4.72, 9.12, 13.81,
93+
vals = np.array([6.95, 15.21, 4.72, 9.12, 13.81,
9194
13.49, 16.68, 9.48, 10.63, 14.48])
9295
xp = np.array([np.nan, np.nan, 9.962, 11.27 , 11.564, 12.516,
9396
12.818, 12.952, np.nan, np.nan])
@@ -173,21 +176,21 @@ def test_cmov_window_regular(self):
173176
vals = np.array([6.95, 15.21, 4.72, 9.12, 13.81,
174177
13.49, 16.68, 9.48, 10.63, 14.48])
175178
xps = {
176-
'hamming': [np.nan, np.nan, 8.71384, 9.56348, 12.38009,
177-
14.03687, 13.8567, 11.81473, np.nan, np.nan],
178-
'triang': [np.nan, np.nan, 9.28667, 10.34667, 12.00556,
179-
13.33889, 13.38, 12.33667, np.nan, np.nan],
180-
'barthann': [np.nan, np.nan, 8.4425, 9.1925, 12.5575,
181-
14.3675, 14.0825, 11.5675, np.nan, np.nan],
182-
'bohman': [np.nan, np.nan, 7.61599, 9.1764, 12.83559,
183-
14.17267, 14.65923, 11.10401, np.nan, np.nan],
184-
'blackmanharris': [np.nan, np.nan, 6.97691, 9.16438, 13.05052,
179+
'hamming': [np.nan, np.nan, 8.71384, 9.56348, 12.38009,
180+
14.03687, 13.8567, 11.81473, np.nan, np.nan],
181+
'triang': [np.nan, np.nan, 9.28667, 10.34667, 12.00556,
182+
13.33889, 13.38, 12.33667, np.nan, np.nan],
183+
'barthann': [np.nan, np.nan, 8.4425, 9.1925, 12.5575,
184+
14.3675, 14.0825, 11.5675, np.nan, np.nan],
185+
'bohman': [np.nan, np.nan, 7.61599, 9.1764, 12.83559,
186+
14.17267, 14.65923, 11.10401, np.nan, np.nan],
187+
'blackmanharris': [np.nan, np.nan, 6.97691, 9.16438, 13.05052,
185188
14.02156, 15.10512, 10.74574, np.nan, np.nan],
186-
'nuttall': [np.nan, np.nan, 7.04618, 9.16786, 13.02671,
187-
14.03559, 15.05657, 10.78514, np.nan, np.nan],
188-
'blackman': [np.nan, np.nan, 7.73345, 9.17869, 12.79607,
189-
14.20036, 14.57726, 11.16988, np.nan, np.nan],
190-
'bartlett': [np.nan, np.nan, 8.4425, 9.1925, 12.5575,
189+
'nuttall': [np.nan, np.nan, 7.04618, 9.16786, 13.02671,
190+
14.03559, 15.05657, 10.78514, np.nan, np.nan],
191+
'blackman': [np.nan, np.nan, 7.73345, 9.17869, 12.79607,
192+
14.20036, 14.57726, 11.16988, np.nan, np.nan],
193+
'bartlett': [np.nan, np.nan, 8.4425, 9.1925, 12.5575,
191194
14.3675, 14.0825, 11.5675, np.nan, np.nan]}
192195

193196
for wt in win_types:
@@ -219,25 +222,25 @@ def test_cmov_window_regular_missing_data(self):
219222
win_types = ['triang', 'blackman', 'hamming', 'bartlett', 'bohman',
220223
'blackmanharris', 'nuttall', 'barthann']
221224

222-
vals = np.array([6.95, 15.21, 4.72, 9.12, 13.81,
225+
vals = np.array([6.95, 15.21, 4.72, 9.12, 13.81,
223226
13.49, 16.68, np.nan, 10.63, 14.48])
224227
xps = {
225-
'bartlett': [np.nan, np.nan, 9.70333, 10.5225, 8.4425,
226-
9.1925, 12.5575, 14.3675, 15.61667, 13.655],
227-
'blackman': [np.nan, np.nan, 9.04582, 11.41536, 7.73345,
228-
9.17869, 12.79607, 14.20036, 15.8706, 13.655],
229-
'barthann': [np.nan, np.nan, 9.70333, 10.5225, 8.4425,
230-
9.1925, 12.5575, 14.3675, 15.61667, 13.655],
231-
'bohman': [np.nan, np.nan, 8.9444, 11.56327, 7.61599,
232-
9.1764, 12.83559, 14.17267, 15.90976, 13.655],
233-
'hamming': [np.nan, np.nan, 9.59321, 10.29694, 8.71384,
234-
9.56348, 12.38009, 14.20565, 15.24694, 13.69758],
235-
'nuttall': [np.nan, np.nan, 8.47693, 12.2821, 7.04618,
236-
9.16786, 13.02671, 14.03673, 16.08759, 13.65553],
237-
'triang': [np.nan, np.nan, 9.33167, 9.76125, 9.28667,
238-
10.34667, 12.00556, 13.82125, 14.49429, 13.765],
228+
'bartlett': [np.nan, np.nan, 9.70333, 10.5225, 8.4425,
229+
9.1925, 12.5575, 14.3675, 15.61667, 13.655],
230+
'blackman': [np.nan, np.nan, 9.04582, 11.41536, 7.73345,
231+
9.17869, 12.79607, 14.20036, 15.8706, 13.655],
232+
'barthann': [np.nan, np.nan, 9.70333, 10.5225, 8.4425,
233+
9.1925, 12.5575, 14.3675, 15.61667, 13.655],
234+
'bohman': [np.nan, np.nan, 8.9444, 11.56327, 7.61599,
235+
9.1764, 12.83559, 14.17267, 15.90976, 13.655],
236+
'hamming': [np.nan, np.nan, 9.59321, 10.29694, 8.71384,
237+
9.56348, 12.38009, 14.20565, 15.24694, 13.69758],
238+
'nuttall': [np.nan, np.nan, 8.47693, 12.2821, 7.04618,
239+
9.16786, 13.02671, 14.03673, 16.08759, 13.65553],
240+
'triang': [np.nan, np.nan, 9.33167, 9.76125, 9.28667,
241+
10.34667, 12.00556, 13.82125, 14.49429, 13.765],
239242
'blackmanharris': [np.nan, np.nan, 8.42526, 12.36824, 6.97691,
240-
9.16438, 13.05052, 14.02175, 16.1098,
243+
9.16438, 13.05052, 14.02175, 16.1098,
241244
13.65509]
242245
}
243246

@@ -258,14 +261,14 @@ def test_cmov_window_special(self):
258261
13.49, 16.68, 9.48, 10.63, 14.48])
259262

260263
xps = {
261-
'gaussian': [np.nan, np.nan, 8.97297, 9.76077, 12.24763,
262-
13.89053, 13.65671, 12.01002, np.nan, np.nan],
263-
'general_gaussian': [np.nan, np.nan, 9.85011, 10.71589,
264-
11.73161, 13.08516, 12.95111, 12.74577,
265-
np.nan, np.nan],
266-
'slepian': [np.nan, np.nan, 9.81073, 10.89359, 11.70284,
267-
12.88331, 12.96079, 12.77008, np.nan, np.nan],
268-
'kaiser': [np.nan, np.nan, 9.86851, 11.02969, 11.65161,
264+
'gaussian': [np.nan, np.nan, 8.97297, 9.76077, 12.24763,
265+
13.89053, 13.65671, 12.01002, np.nan, np.nan],
266+
'general_gaussian': [np.nan, np.nan, 9.85011, 10.71589,
267+
11.73161, 13.08516, 12.95111, 12.74577,
268+
np.nan, np.nan],
269+
'slepian': [np.nan, np.nan, 9.81073, 10.89359, 11.70284,
270+
12.88331, 12.96079, 12.77008, np.nan, np.nan],
271+
'kaiser': [np.nan, np.nan, 9.86851, 11.02969, 11.65161,
269272
12.75129, 12.90702, 12.83757, np.nan, np.nan]
270273
}
271274

@@ -635,7 +638,7 @@ def test_ewma(self):
635638
self.assertTrue(np.abs(result - 1) < 1e-2)
636639

637640
s = Series([1.0, 2.0, 4.0, 8.0])
638-
641+
639642
expected = Series([1.0, 1.6, 2.736842, 4.923077])
640643
for f in [lambda s: mom.ewma(s, com=2.0, adjust=True),
641644
lambda s: mom.ewma(s, com=2.0, adjust=True, ignore_na=False),
@@ -783,7 +786,10 @@ def _check_ew_structures(self, func):
783786
frame_result = func(self.frame, com=10)
784787
self.assertEqual(type(frame_result), DataFrame)
785788

786-
def _test_series(self):
789+
# create the data only once as we are not setting it
790+
def _create_consistency_data():
791+
792+
def create_series():
787793
return [Series(),
788794
Series([np.nan]),
789795
Series([np.nan, np.nan]),
@@ -804,18 +810,38 @@ def _test_series(self):
804810
Series(range(20, 0, -2)),
805811
]
806812

807-
def _test_dataframes(self):
813+
def create_dataframes():
808814
return [DataFrame(),
809815
DataFrame(columns=['a']),
810816
DataFrame(columns=['a', 'a']),
811817
DataFrame(columns=['a', 'b']),
812818
DataFrame(np.arange(10).reshape((5, 2))),
813819
DataFrame(np.arange(25).reshape((5, 5))),
814820
DataFrame(np.arange(25).reshape((5, 5)), columns=['a', 'b', 99, 'd', 'd']),
815-
] + [DataFrame(s) for s in self._test_series()]
821+
] + [DataFrame(s) for s in create_series()]
822+
823+
def is_constant(x):
824+
values = x.values.ravel()
825+
return len(set(values[notnull(values)])) == 1
826+
827+
def no_nans(x):
828+
return x.notnull().all().all()
829+
830+
# data is a tuple(object, is_contant, no_nans)
831+
data = create_series() + create_dataframes()
832+
833+
return [ (x, is_constant(x), no_nans(x)) for x in data ]
834+
_consistency_data = _create_consistency_data()
816835

817-
def _test_data(self):
818-
return self._test_series() + self._test_dataframes()
836+
class TestMomentsConsistency(Base):
837+
838+
def _create_data(self):
839+
super(TestMomentsConsistency, self)._create_data()
840+
self.data = _consistency_data
841+
842+
def setUp(self):
843+
self._create_data()
844+
warnings.simplefilter("ignore", category=FutureWarning)
819845

820846
def _test_moments_consistency(self,
821847
min_periods,
@@ -825,11 +851,11 @@ def _test_moments_consistency(self,
825851
var_debiasing_factors=None):
826852

827853
def _non_null_values(x):
828-
return set([v for v in x.values.reshape(x.values.size) if notnull(v)])
854+
values = x.values.ravel()
855+
return set(values[notnull(values)].tolist())
829856

830-
for x in self._test_data():
857+
for (x, is_constant, no_nans) in self.data:
831858
assert_equal = assert_series_equal if isinstance(x, Series) else assert_frame_equal
832-
is_constant = (len(_non_null_values(x)) == 1)
833859
count_x = count(x)
834860
mean_x = mean(x)
835861

@@ -861,7 +887,7 @@ def _non_null_values(x):
861887

862888
for (std, var, cov) in [(std_biased, var_biased, cov_biased),
863889
(std_unbiased, var_unbiased, cov_unbiased)]:
864-
890+
865891
# check that var(x), std(x), and cov(x) are all >= 0
866892
var_x = var(x)
867893
std_x = std(x)
@@ -873,7 +899,7 @@ def _non_null_values(x):
873899

874900
# check that var(x) == cov(x, x)
875901
assert_equal(var_x, cov_x_x)
876-
902+
877903
# check that var(x) == std(x)^2
878904
assert_equal(var_x, std_x * std_x)
879905

@@ -892,7 +918,7 @@ def _non_null_values(x):
892918
assert_equal(var_x, expected)
893919

894920
if isinstance(x, Series):
895-
for y in self._test_data():
921+
for (y, is_constant, no_nans) in self.data:
896922
if not x.isnull().equals(y.isnull()):
897923
# can only easily test two Series with similar structure
898924
continue
@@ -907,7 +933,7 @@ def _non_null_values(x):
907933
cov_x_y = cov(x, y)
908934
cov_y_x = cov(y, x)
909935
assert_equal(cov_x_y, cov_y_x)
910-
936+
911937
# check that cov(x, y) == (var(x+y) - var(x) - var(y)) / 2
912938
var_x_plus_y = var(x + y)
913939
var_y = var(y)
@@ -928,9 +954,15 @@ def test_ewm_consistency(self):
928954

929955
def _weights(s, com, adjust, ignore_na):
930956
if isinstance(s, DataFrame):
931-
w = DataFrame(index=s.index, columns=s.columns)
932-
for i, _ in enumerate(s.columns):
933-
w.iloc[:, i] = _weights(s.iloc[:, i], com=com, adjust=adjust, ignore_na=ignore_na)
957+
if not len(s.columns):
958+
return DataFrame(index=s.index, columns=s.columns)
959+
w = concat([ _weights(s.iloc[:, i],
960+
com=com,
961+
adjust=adjust,
962+
ignore_na=ignore_na) for i, _ in enumerate(s.columns) ],
963+
axis=1)
964+
w.index=s.index
965+
w.columns=s.columns
934966
return w
935967

936968
w = Series(np.nan, index=s.index)
@@ -1053,11 +1085,12 @@ def test_expanding_consistency(self):
10531085

10541086
# test consistency between expanding_xyz() and either (a) expanding_apply of Series.xyz(),
10551087
# or (b) expanding_apply of np.nanxyz()
1056-
for x in self._test_data():
1088+
for (x, is_constant, no_nans) in self.data:
10571089
assert_equal = assert_series_equal if isinstance(x, Series) else assert_frame_equal
10581090
functions = base_functions
1091+
10591092
# GH 8269
1060-
if x.notnull().all().all():
1093+
if no_nans:
10611094
functions = base_functions + no_nan_functions
10621095
for (expanding_f, f, require_min_periods) in functions:
10631096
if require_min_periods and (min_periods is not None) and (min_periods < require_min_periods):
@@ -1085,6 +1118,7 @@ def test_expanding_consistency(self):
10851118

10861119
@slow
10871120
def test_rolling_consistency(self):
1121+
10881122
base_functions = [
10891123
(mom.rolling_count, lambda v: Series(v).count(), None),
10901124
(mom.rolling_max, lambda v: Series(v).max(), None),
@@ -1150,11 +1184,12 @@ def test_rolling_consistency(self):
11501184

11511185
# test consistency between rolling_xyz() and either (a) rolling_apply of Series.xyz(),
11521186
# or (b) rolling_apply of np.nanxyz()
1153-
for x in self._test_data():
1187+
for (x, is_constant, no_nans) in self.data:
1188+
11541189
assert_equal = assert_series_equal if isinstance(x, Series) else assert_frame_equal
11551190
functions = base_functions
11561191
# GH 8269
1157-
if x.notnull().all().all():
1192+
if no_nans:
11581193
functions = base_functions + no_nan_functions
11591194
for (rolling_f, f, require_min_periods) in functions:
11601195
if require_min_periods and (min_periods is not None) and (min_periods < require_min_periods):
@@ -1183,7 +1218,7 @@ def test_rolling_consistency(self):
11831218
expected.iloc[:, i, j] = rolling_f(x.iloc[:, i], x.iloc[:, j],
11841219
window=window, min_periods=min_periods, center=center)
11851220
assert_panel_equal(rolling_f_result, expected)
1186-
1221+
11871222
# binary moments
11881223
def test_rolling_cov(self):
11891224
A = self.series
@@ -1608,7 +1643,7 @@ def test_expanding_corr_pairwise_diff_length(self):
16081643
assert_frame_equal(result2, expected)
16091644
assert_frame_equal(result3, expected)
16101645
assert_frame_equal(result4, expected)
1611-
1646+
16121647
def test_pairwise_stats_column_names_order(self):
16131648
# GH 7738
16141649
df1s = [DataFrame([[2,4],[1,2],[5,2],[8,1]], columns=[0,1]),

0 commit comments

Comments
 (0)