COMPAT: free parser memory at close() for non-refcnt gc

relying on __dealloc__ to clean up malloc() ed memory can lead
to a perceived "leak" on PyPy since the garbage collector will not
necessarily collect the object as soon as its refcnt reaches 0.
Instead, pre-emptively release memory when close() is called
The code still maintains backward compatibility for the case where
close() is never called

Author: mattip <[email protected]>

Closes pandas-dev#15665 from mattip/pypy-compat and squashes the following commits:

eaf50fe [mattip] COMPAT: free parser memory at close() for non-refcnt gc

Author: mattip

pandas/_libs/src/parser/tokenizer.c

@@ -162,6 +162,7 @@ int parser_cleanup(parser_t *self) {
         if (self->cb_cleanup(self->source) < 0) {
             status = -1;
         }
+        self->cb_cleanup = NULL;
     }
 
     return status;
@@ -239,6 +240,9 @@ int parser_init(parser_t *self) {
 void parser_free(parser_t *self) {
     // opposite of parser_init
     parser_cleanup(self);
+}
+
+void parser_del(parser_t *self) {
     free(self);
 }
 

pandas/_libs/src/parser/tokenizer.h

@@ -243,6 +243,8 @@ int parser_set_skipfirstnrows(parser_t *self, int64_t nrows);
 
 void parser_free(parser_t *self);
 
+void parser_del(parser_t *self);
+
 void parser_set_default_options(parser_t *self);
 
 void debug_print_parser(parser_t *self);

pandas/io/parsers.pyx

@@ -214,6 +214,7 @@ cdef extern from "parser/tokenizer.h":
 
     int parser_init(parser_t *self) nogil
     void parser_free(parser_t *self) nogil
+    void parser_del(parser_t *self) nogil
     int parser_add_skiprow(parser_t *self, int64_t row)
 
     int parser_set_skipfirstnrows(parser_t *self, int64_t nrows)
@@ -573,8 +574,13 @@ cdef class TextReader:
 
     def __dealloc__(self):
         parser_free(self.parser)
-        kh_destroy_str(self.true_set)
-        kh_destroy_str(self.false_set)
+        if self.true_set:
+            kh_destroy_str(self.true_set)
+            self.true_set = NULL
+        if self.false_set:
+            kh_destroy_str(self.false_set)
+            self.false_set = NULL
+        parser_del(self.parser)
 
     def close(self):
         # we need to properly close an open derived
@@ -584,6 +590,14 @@ cdef class TextReader:
                 self.handle.close()
             except:
                 pass
+        # also preemptively free all allocated memory
+        parser_free(self.parser)
+        if self.true_set:
+            kh_destroy_str(self.true_set)
+            self.true_set = NULL
+        if self.false_set:
+            kh_destroy_str(self.false_set)
+            self.false_set = NULL
 
     def set_error_bad_lines(self, int status):
         self.parser.error_bad_lines = status