|
25 | 25 | class TestBuf(TestBase): |
26 | 26 |
|
27 | 27 | def test_basics(self): |
28 | | - fc = FileCreator(self.k_window_test_size, "buffer_test") |
29 | | - |
30 | | - # invalid paths fail upon construction |
31 | | - c = man_optimal.make_cursor(fc.path) |
32 | | - self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor |
33 | | - self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large |
34 | | - |
35 | | - buf = SlidingWindowMapBuffer() # can create uninitailized buffers |
36 | | - assert buf.cursor() is None |
37 | | - |
38 | | - # can call end access any time |
39 | | - buf.end_access() |
40 | | - buf.end_access() |
41 | | - assert len(buf) == 0 |
42 | | - |
43 | | - # begin access can revive it, if the offset is suitable |
44 | | - offset = 100 |
45 | | - assert buf.begin_access(c, fc.size) == False |
46 | | - assert buf.begin_access(c, offset) == True |
47 | | - assert len(buf) == fc.size - offset |
48 | | - assert buf.cursor().is_valid() |
49 | | - |
50 | | - # empty begin access keeps it valid on the same path, but alters the offset |
51 | | - assert buf.begin_access() == True |
52 | | - assert len(buf) == fc.size |
53 | | - assert buf.cursor().is_valid() |
54 | | - |
55 | | - # simple access |
56 | | - with open(fc.path, 'rb') as fp: |
57 | | - data = fp.read() |
58 | | - assert data[offset] == buf[0] |
59 | | - assert data[offset:offset * 2] == buf[0:offset] |
60 | | - |
61 | | - # negative indices, partial slices |
62 | | - assert buf[-1] == buf[len(buf) - 1] |
63 | | - assert buf[-10:] == buf[len(buf) - 10:len(buf)] |
64 | | - |
65 | | - # end access makes its cursor invalid |
66 | | - buf.end_access() |
67 | | - assert not buf.cursor().is_valid() |
68 | | - assert buf.cursor().is_associated() # but it remains associated |
69 | | - |
70 | | - # an empty begin access fixes it up again |
71 | | - assert buf.begin_access() == True and buf.cursor().is_valid() |
72 | | - del(buf) # ends access automatically |
73 | | - del(c) |
74 | | - |
75 | | - assert man_optimal.num_file_handles() == 1 |
76 | | - |
77 | | - # PERFORMANCE |
78 | | - # blast away with random access and a full mapping - we don't want to |
79 | | - # exaggerate the manager's overhead, but measure the buffer overhead |
80 | | - # We do it once with an optimal setting, and with a worse manager which |
81 | | - # will produce small mappings only ! |
82 | | - max_num_accesses = 100 |
83 | | - fd = os.open(fc.path, os.O_RDONLY) |
84 | | - for item in (fc.path, fd): |
85 | | - for manager, man_id in ((man_optimal, 'optimal'), |
86 | | - (man_worst_case, 'worst case'), |
87 | | - (static_man, 'static optimal')): |
88 | | - buf = SlidingWindowMapBuffer(manager.make_cursor(item)) |
89 | | - assert manager.num_file_handles() == 1 |
90 | | - for access_mode in range(2): # single, multi |
91 | | - num_accesses_left = max_num_accesses |
92 | | - num_bytes = 0 |
93 | | - fsize = fc.size |
94 | | - |
95 | | - st = time() |
96 | | - buf.begin_access() |
97 | | - while num_accesses_left: |
98 | | - num_accesses_left -= 1 |
99 | | - if access_mode: # multi |
100 | | - ofs_start = randint(0, fsize) |
101 | | - ofs_end = randint(ofs_start, fsize) |
102 | | - d = buf[ofs_start:ofs_end] |
103 | | - assert len(d) == ofs_end - ofs_start |
104 | | - assert d == data[ofs_start:ofs_end] |
105 | | - num_bytes += len(d) |
106 | | - del d |
107 | | - else: |
108 | | - pos = randint(0, fsize) |
109 | | - assert buf[pos] == data[pos] |
110 | | - num_bytes += 1 |
111 | | - # END handle mode |
112 | | - # END handle num accesses |
113 | | - |
114 | | - buf.end_access() |
115 | | - assert manager.num_file_handles() |
116 | | - assert manager.collect() |
117 | | - assert manager.num_file_handles() == 0 |
118 | | - elapsed = max(time() - st, 0.001) # prevent zero division errors on windows |
119 | | - mb = float(1000 * 1000) |
120 | | - mode_str = (access_mode and "slice") or "single byte" |
121 | | - print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" |
122 | | - % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), |
123 | | - file=sys.stderr) |
124 | | - # END handle access mode |
125 | | - del buf |
126 | | - # END for each manager |
127 | | - # END for each input |
128 | | - os.close(fd) |
| 28 | + with FileCreator(self.k_window_test_size, "buffer_test") as fc: |
| 29 | + |
| 30 | + # invalid paths fail upon construction |
| 31 | + c = man_optimal.make_cursor(fc.path) |
| 32 | + self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor |
| 33 | + self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large |
| 34 | + |
| 35 | + buf = SlidingWindowMapBuffer() # can create uninitailized buffers |
| 36 | + assert buf.cursor() is None |
| 37 | + |
| 38 | + # can call end access any time |
| 39 | + buf.end_access() |
| 40 | + buf.end_access() |
| 41 | + assert len(buf) == 0 |
| 42 | + |
| 43 | + # begin access can revive it, if the offset is suitable |
| 44 | + offset = 100 |
| 45 | + assert buf.begin_access(c, fc.size) == False |
| 46 | + assert buf.begin_access(c, offset) == True |
| 47 | + assert len(buf) == fc.size - offset |
| 48 | + assert buf.cursor().is_valid() |
| 49 | + |
| 50 | + # empty begin access keeps it valid on the same path, but alters the offset |
| 51 | + assert buf.begin_access() == True |
| 52 | + assert len(buf) == fc.size |
| 53 | + assert buf.cursor().is_valid() |
| 54 | + |
| 55 | + # simple access |
| 56 | + with open(fc.path, 'rb') as fp: |
| 57 | + data = fp.read() |
| 58 | + assert data[offset] == buf[0] |
| 59 | + assert data[offset:offset * 2] == buf[0:offset] |
| 60 | + |
| 61 | + # negative indices, partial slices |
| 62 | + assert buf[-1] == buf[len(buf) - 1] |
| 63 | + assert buf[-10:] == buf[len(buf) - 10:len(buf)] |
| 64 | + |
| 65 | + # end access makes its cursor invalid |
| 66 | + buf.end_access() |
| 67 | + assert not buf.cursor().is_valid() |
| 68 | + assert buf.cursor().is_associated() # but it remains associated |
| 69 | + |
| 70 | + # an empty begin access fixes it up again |
| 71 | + assert buf.begin_access() == True and buf.cursor().is_valid() |
| 72 | + del(buf) # ends access automatically |
| 73 | + del(c) |
| 74 | + |
| 75 | + assert man_optimal.num_file_handles() == 1 |
| 76 | + |
| 77 | + # PERFORMANCE |
| 78 | + # blast away with random access and a full mapping - we don't want to |
| 79 | + # exaggerate the manager's overhead, but measure the buffer overhead |
| 80 | + # We do it once with an optimal setting, and with a worse manager which |
| 81 | + # will produce small mappings only ! |
| 82 | + max_num_accesses = 100 |
| 83 | + fd = os.open(fc.path, os.O_RDONLY) |
| 84 | + for item in (fc.path, fd): |
| 85 | + for manager, man_id in ((man_optimal, 'optimal'), |
| 86 | + (man_worst_case, 'worst case'), |
| 87 | + (static_man, 'static optimal')): |
| 88 | + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) |
| 89 | + assert manager.num_file_handles() == 1 |
| 90 | + for access_mode in range(2): # single, multi |
| 91 | + num_accesses_left = max_num_accesses |
| 92 | + num_bytes = 0 |
| 93 | + fsize = fc.size |
| 94 | + |
| 95 | + st = time() |
| 96 | + buf.begin_access() |
| 97 | + while num_accesses_left: |
| 98 | + num_accesses_left -= 1 |
| 99 | + if access_mode: # multi |
| 100 | + ofs_start = randint(0, fsize) |
| 101 | + ofs_end = randint(ofs_start, fsize) |
| 102 | + d = buf[ofs_start:ofs_end] |
| 103 | + assert len(d) == ofs_end - ofs_start |
| 104 | + assert d == data[ofs_start:ofs_end] |
| 105 | + num_bytes += len(d) |
| 106 | + del d |
| 107 | + else: |
| 108 | + pos = randint(0, fsize) |
| 109 | + assert buf[pos] == data[pos] |
| 110 | + num_bytes += 1 |
| 111 | + # END handle mode |
| 112 | + # END handle num accesses |
| 113 | + |
| 114 | + buf.end_access() |
| 115 | + assert manager.num_file_handles() |
| 116 | + assert manager.collect() |
| 117 | + assert manager.num_file_handles() == 0 |
| 118 | + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows |
| 119 | + mb = float(1000 * 1000) |
| 120 | + mode_str = (access_mode and "slice") or "single byte" |
| 121 | + print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" |
| 122 | + % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), |
| 123 | + file=sys.stderr) |
| 124 | + # END handle access mode |
| 125 | + del buf |
| 126 | + # END for each manager |
| 127 | + # END for each input |
| 128 | + os.close(fd) |
0 commit comments