Skip to content

Commit b2eafcc

Browse files
committed
fix(MapWindow): unicode foes in read_into_memory() used by gitpython TCs
Drop Windows only codepath bypassing memory-mapping due to some leaks in the past. Now Appveyor proves everything run ok. Additionally, this codepath had unicode problems on PY3. So deleting it, fixes 2 TCs in gitpython: + TestRepo.test_file_handle_leaks() + TestObjDbPerformance.test_random_access() See gitpython-developers/GitPython#525
1 parent ac5df70 commit b2eafcc

File tree

2 files changed

+13
-28
lines changed

2 files changed

+13
-28
lines changed

Diff for: smmap/mman.py

+10-6
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@ class WindowCursor(object):
3131
__slots__ = (
3232
'_manager', # the manger keeping all file regions
3333
'_rlist', # a regions list with regions for our file
34-
'_region', # our current region or None
35-
'_ofs', # relative offset from the actually mapped area to our start area
36-
'_size' # maximum size we should provide
34+
'_region', # our current class:`MapRegion` or None
35+
'_ofs', # relative offset from the actually mapped area to our start area
36+
'_size' # maximum size we should provide
3737
)
3838

3939
def __init__(self, manager=None, regions=None):
@@ -308,10 +308,14 @@ def _collect_lru_region(self, size):
308308
if 0, we try to free any available region
309309
:return: Amount of freed regions
310310
311-
**Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation.
312-
If the system runs out of memory, it will tell.
311+
.. Note::
312+
We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation.
313+
If the system runs out of memory, it will tell.
313314
314-
**todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force"""
315+
.. TODO::
316+
implement a case where all unusued regions are discarded efficiently.
317+
Currently its only brute force
318+
"""
315319
num_found = 0
316320
while (size == 0) or (self._memory_size + size > self._max_memory_size):
317321
lru_region = None

Diff for: smmap/util.py

+3-22
Original file line numberDiff line numberDiff line change
@@ -116,16 +116,13 @@ class MapRegion(object):
116116
'_size', # cached size of our memory map
117117
'__weakref__'
118118
]
119-
_need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6
119+
_need_compat_layer = sys.version_info[:2] < (2, 6)
120120

121121
if _need_compat_layer:
122122
__slots__.append('_mfb') # mapped memory buffer to provide offset
123123
# END handle additional slot
124124

125125
#{ Configuration
126-
# Used for testing only. If True, all data will be loaded into memory at once.
127-
# This makes sure no file handles will remain open.
128-
_test_read_into_memory = False
129126
#} END configuration
130127

131128
def __init__(self, path_or_fd, ofs, size, flags=0):
@@ -160,10 +157,7 @@ def __init__(self, path_or_fd, ofs, size, flags=0):
160157
# bark that the size is too large ... many extra file accesses because
161158
# if this ... argh !
162159
actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size)
163-
if self._test_read_into_memory:
164-
self._mf = self._read_into_memory(fd, ofs, actual_size)
165-
else:
166-
self._mf = mmap(fd, actual_size, **kwargs)
160+
self._mf = mmap(fd, actual_size, **kwargs)
167161
# END handle memory mode
168162

169163
self._size = len(self._mf)
@@ -179,19 +173,6 @@ def __init__(self, path_or_fd, ofs, size, flags=0):
179173
# We assume the first one to use us keeps us around
180174
self.increment_client_count()
181175

182-
def _read_into_memory(self, fd, offset, size):
183-
""":return: string data as read from the given file descriptor, offset and size """
184-
os.lseek(fd, offset, os.SEEK_SET)
185-
mf = ''
186-
bytes_todo = size
187-
while bytes_todo:
188-
chunk = 1024 * 1024
189-
d = os.read(fd, chunk)
190-
bytes_todo -= len(d)
191-
mf += d
192-
# END loop copy items
193-
return mf
194-
195176
def __repr__(self):
196177
return "MapRegion<%i, %i>" % (self._b, self.size())
197178

@@ -267,7 +248,7 @@ class MapRegionList(list):
267248
"""List of MapRegion instances associating a path with a list of regions."""
268249
__slots__ = (
269250
'_path_or_fd', # path or file descriptor which is mapped by all our regions
270-
'_file_size' # total size of the file we map
251+
'_file_size' # total size of the file we map
271252
)
272253

273254
def __new__(cls, path):

0 commit comments

Comments
 (0)