Skip to content

Commit 3fcb774

Browse files
author
Brendan Boerner
committed
2nd draft of test case for pandas-dev#8014 
1 parent a6d0ec3 commit 3fcb774

File tree

1 file changed

+204
-61
lines changed

1 file changed

+204
-61
lines changed

‎pandas/io/tests/test_pytables2.py

+204-61
Original file line numberDiff line numberDiff line change
@@ -145,27 +145,21 @@ def setUp(self):
145145
def tearDown(self):
146146
pass
147147

148-
def test_select_iterator_8014(self):
148+
def xtest_select_iterator_8014_0(self):
149149

150150
# single table
151151
with ensure_clean_store(self.path) as store:
152152

153-
frames = []
154-
df = tm.makeTimeDataFrame(200000, 'S')
153+
df = tm.makeTimeDataFrame(10064, 'S')
155154
_maybe_remove(store, 'df')
156-
store.append('df', df)
157-
frames.append(df)
158-
df = tm.makeTimeDataFrame(58689, 'S')
159-
store.append('df', df)
160-
frames.append(df)
161-
df = tm.makeTimeDataFrame(41375, 'S')
162-
frames.append(df)
163-
store.append('df', df)
164-
expected = concat(frames)
155+
i = 0; ln = 5927
156+
store.append('df', df[i:i+ln])
157+
i = i+ln; ln = 4137
158+
store.append('df', df[i:i+ln])
159+
expected = df
165160

166161
beg_dt = expected.index[0]
167162
end_dt = expected.index[-1]
168-
#expected = store.select('df')
169163

170164
# select w/o iteration and no where clause works
171165
result = store.select('df')
@@ -194,19 +188,96 @@ def test_select_iterator_8014(self):
194188
# fails
195189
where = "index >= '%s' & index <= '%s'" % (beg_dt, end_dt)
196190
result = store.select('df',where=where)
197-
tm.assert_frame_equal(expected, result)
191+
#tm.assert_frame_equal(expected, result)
198192

199193
#
194+
# with iterator
200195
#
196+
197+
# select w/iterator and where clause, single term, begin
198+
# of range, fails
199+
where = "index >= '%s'" % beg_dt
200+
results = []
201+
for s in store.select('df',where=where,iterator=True, chunksize=1e2):
202+
results.append(s)
203+
result = concat(results)
204+
tm.assert_frame_equal(expected, result)
205+
206+
# select w/iterator and where clause, single term, end of
207+
# range, fails
208+
where = "index <= '%s'" % end_dt
209+
results = []
210+
for s in store.select('df',where=where,iterator=True, chunksize=1e2):
211+
results.append(s)
212+
result = concat(results)
213+
#tm.assert_frame_equal(expected, result)
214+
215+
# select w/iterator and where clause, inclusive range, fails
216+
where = "index >= '%s' & index <= '%s'" % (beg_dt, end_dt)
217+
results = []
218+
for s in store.select('df',where=where,iterator=True, chunksize=1e2):
219+
results.append(s)
220+
result = concat(results)
221+
#tm.assert_frame_equal(expected, result)
222+
223+
def test_select_iterator_8014_1(self):
224+
225+
# single table
226+
with ensure_clean_store(self.path) as store:
227+
228+
df = tm.makeTimeDataFrame(100064, 'S')
229+
_maybe_remove(store, 'df')
230+
i = 0; ln = 58689
231+
store.append('df', df[i:i+ln])
232+
i = i+ln; ln = 41375
233+
store.append('df', df[i:i+ln])
234+
expected = df
235+
236+
beg_dt = expected.index[0]
237+
end_dt = expected.index[-1]
238+
239+
#
240+
# w/o iterator
201241
#
202242

243+
# select w/o iteration and no where clause works
244+
result = store.select('df')
245+
tm.assert_frame_equal(expected, result)
246+
247+
# select w/o iterator and where clause, single term, begin
248+
# of range, works
249+
where = "index >= '%s'" % beg_dt
250+
result = store.select('df',where=where)
251+
tm.assert_frame_equal(expected, result)
203252

253+
# select w/o iterator and where clause, single term, end
254+
# of range, works
255+
where = "index <= '%s'" % end_dt
256+
result = store.select('df',where=where)
257+
tm.assert_frame_equal(expected, result)
258+
259+
# select w/o iterator and where clause, inclusive range,
260+
# works
261+
where = "index >= '%s' & index <= '%s'" % (beg_dt, end_dt)
262+
result = store.select('df',where=where)
263+
tm.assert_frame_equal(expected, result)
264+
265+
#
266+
# with iterator
267+
#
268+
269+
# select w/iterator and no where clause works
270+
results = []
271+
for s in store.select('df',iterator=True):
272+
results.append(s)
273+
result = concat(results)
274+
tm.assert_frame_equal(expected, result)
204275

205276
# select w/iterator and where clause, single term, begin
206277
# of range, fails
207278
where = "index >= '%s'" % beg_dt
208279
results = []
209-
for s in store.select('df',where=where,iterator=True):
280+
for s in store.select('df',where=where,iterator=True, chunksize=1e4):
210281
results.append(s)
211282
result = concat(results)
212283
tm.assert_frame_equal(expected, result)
@@ -215,99 +286,171 @@ def test_select_iterator_8014(self):
215286
# range, fails
216287
where = "index <= '%s'" % end_dt
217288
results = []
218-
for s in store.select('df',where=where,iterator=True):
289+
for s in store.select('df',where=where,iterator=True, chunksize=1e4):
219290
results.append(s)
220291
result = concat(results)
221292
tm.assert_frame_equal(expected, result)
222293

223294
# select w/iterator and where clause, inclusive range, fails
224295
where = "index >= '%s' & index <= '%s'" % (beg_dt, end_dt)
225296
results = []
226-
for s in store.select('df',where=where,iterator=True):
297+
for s in store.select('df',where=where,iterator=True, chunksize=1e4):
227298
results.append(s)
228299
result = concat(results)
229300
tm.assert_frame_equal(expected, result)
230301

231-
def test_select_iterator(self):
302+
def xtest_select_iterator_8014_2(self):
232303

233304
# single table
234305
with ensure_clean_store(self.path) as store:
235306

236-
df = tm.makeTimeDataFrame(500)
307+
df = tm.makeTimeDataFrame(200064, 'S')
237308
_maybe_remove(store, 'df')
238-
store.append('df', df)
309+
i = 0; ln = 100000
310+
store.append('df', df[i:i+ln])
311+
i = i+ln; ln = 58689
312+
store.append('df', df[i:i+ln])
313+
i = i+ln; ln = 41375
314+
store.append('df', df[i:i+ln])
315+
expected = df
239316

240-
expected = store.select('df')
317+
beg_dt = expected.index[0]
318+
end_dt = expected.index[-1]
319+
320+
# select w/o iteration and no where clause works
321+
result = store.select('df')
322+
tm.assert_frame_equal(expected, result)
241323

324+
# select w/iterator and no where clause works
242325
results = []
243326
for s in store.select('df',iterator=True):
244327
results.append(s)
245328
result = concat(results)
246329
tm.assert_frame_equal(expected, result)
330+
331+
# select w/o iterator and where clause, single term, begin
332+
# of range, works
333+
where = "index >= '%s'" % beg_dt
334+
result = store.select('df',where=where)
335+
tm.assert_frame_equal(expected, result)
336+
337+
# select w/o iterator and where clause, single term, end
338+
# of range, works
339+
where = "index <= '%s'" % end_dt
340+
result = store.select('df',where=where)
341+
tm.assert_frame_equal(expected, result)
342+
343+
# select w/o iterator and where clause, inclusive range,
344+
# fails
345+
where = "index >= '%s' & index <= '%s'" % (beg_dt, end_dt)
346+
result = store.select('df',where=where)
347+
tm.assert_frame_equal(expected, result)
348+
349+
#
350+
# with iterator
351+
#
352+
353+
# select w/iterator and where clause, single term, begin
354+
# of range, works
355+
where = "index >= '%s'" % beg_dt
247356
results = []
248-
for s in store.select('df',chunksize=100):
357+
for s in store.select('df',where=where,iterator=True, chunksize=1e4):
249358
results.append(s)
250-
self.assertEqual(len(results), 5)
251359
result = concat(results)
252360
tm.assert_frame_equal(expected, result)
253361

362+
# select w/iterator and where clause, single term, end of
363+
# range, fails
364+
where = "index <= '%s'" % end_dt
254365
results = []
255-
for s in store.select('df',chunksize=150):
366+
for s in store.select('df',where=where,iterator=True, chunksize=1e4):
256367
results.append(s)
257368
result = concat(results)
258-
tm.assert_frame_equal(result, expected)
369+
#tm.assert_frame_equal(expected, result)
370+
371+
# select w/iterator and where clause, inclusive range, fails
372+
where = "index >= '%s' & index <= '%s'" % (beg_dt, end_dt)
373+
results = []
374+
for s in store.select('df',where=where,iterator=True, chunksize=1e4):
375+
results.append(s)
376+
result = concat(results)
377+
#tm.assert_frame_equal(expected, result)
259378

260-
with ensure_clean_path(self.path) as path:
379+
def xtest_select_iterator_8014_3(self):
380+
381+
# single table
382+
with ensure_clean_store(self.path) as store:
261383

262-
df = tm.makeTimeDataFrame(500)
263-
df.to_hdf(path,'df_non_table')
264-
self.assertRaises(TypeError, read_hdf, path,'df_non_table',chunksize=100)
265-
self.assertRaises(TypeError, read_hdf, path,'df_non_table',iterator=True)
384+
df = tm.makeTimeDataFrame(300064, 'S')
385+
_maybe_remove(store, 'df')
386+
i = 0; ln = 200000
387+
store.append('df', df[i:i+ln])
388+
i = i+ln; ln = 58689
389+
store.append('df', df[i:i+ln])
390+
i = i+ln; ln = 41375
391+
store.append('df', df[i:i+ln])
392+
expected = df
266393

267-
with ensure_clean_path(self.path) as path:
394+
beg_dt = expected.index[0]
395+
end_dt = expected.index[-1]
268396

269-
df = tm.makeTimeDataFrame(500)
270-
df.to_hdf(path,'df',format='table')
397+
# select w/o iteration and no where clause works
398+
result = store.select('df')
399+
tm.assert_frame_equal(expected, result)
271400

401+
# select w/iterator and no where clause works
272402
results = []
273-
for x in read_hdf(path,'df',chunksize=100):
274-
results.append(x)
275-
276-
self.assertEqual(len(results), 5)
403+
for s in store.select('df',iterator=True):
404+
results.append(s)
277405
result = concat(results)
278-
tm.assert_frame_equal(result, df)
279-
tm.assert_frame_equal(result, read_hdf(path,'df'))
406+
tm.assert_frame_equal(expected, result)
280407

281-
# multiple
408+
# select w/o iterator and where clause, single term, begin
409+
# of range, works
410+
where = "index >= '%s'" % beg_dt
411+
result = store.select('df',where=where)
412+
tm.assert_frame_equal(expected, result)
282413

283-
with ensure_clean_store(self.path) as store:
414+
# select w/o iterator and where clause, single term, end
415+
# of range, works
416+
where = "index <= '%s'" % end_dt
417+
result = store.select('df',where=where)
418+
tm.assert_frame_equal(expected, result)
419+
420+
# select w/o iterator and where clause, inclusive range,
421+
# works
422+
where = "index >= '%s' & index <= '%s'" % (beg_dt, end_dt)
423+
result = store.select('df',where=where)
424+
tm.assert_frame_equal(expected, result)
284425

285-
df1 = tm.makeTimeDataFrame(500)
286-
store.append('df1',df1,data_columns=True)
287-
df2 = tm.makeTimeDataFrame(500).rename(columns=lambda x: "%s_2" % x)
288-
df2['foo'] = 'bar'
289-
store.append('df2',df2)
426+
#
427+
# with iterator
428+
#
290429

291-
df = concat([df1, df2], axis=1)
430+
# select w/iterator and where clause, single term, begin
431+
# of range, fails
432+
where = "index >= '%s'" % beg_dt
433+
results = []
434+
for s in store.select('df',where=where,iterator=True):
435+
results.append(s)
436+
result = concat(results)
437+
#tm.assert_frame_equal(expected, result)
292438

293-
# full selection
294-
expected = store.select_as_multiple(
295-
['df1', 'df2'], selector='df1')
439+
# select w/iterator and where clause, single term, end of
440+
# range, fails
441+
where = "index <= '%s'" % end_dt
296442
results = []
297-
for s in store.select_as_multiple(
298-
['df1', 'df2'], selector='df1', chunksize=150):
443+
for s in store.select('df',where=where,iterator=True):
299444
results.append(s)
300445
result = concat(results)
301-
tm.assert_frame_equal(expected, result)
446+
#tm.assert_frame_equal(expected, result)
302447

303-
# where selection
304-
#expected = store.select_as_multiple(
305-
# ['df1', 'df2'], where= Term('A>0'), selector='df1')
306-
#results = []
307-
#for s in store.select_as_multiple(
308-
# ['df1', 'df2'], where= Term('A>0'), selector='df1', chunksize=25):
309-
# results.append(s)
310-
#result = concat(results)
448+
# select w/iterator and where clause, inclusive range, fails
449+
where = "index >= '%s' & index <= '%s'" % (beg_dt, end_dt)
450+
results = []
451+
for s in store.select('df',where=where,iterator=True):
452+
results.append(s)
453+
result = concat(results)
311454
#tm.assert_frame_equal(expected, result)
312455

313456

0 commit comments

Comments
 (0)