Skip to content

COMPAT: free parser memory at close() for non-refcnt gc #15665

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

COMPAT: free parser memory at close() for non-refcnt gc #15665

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pandas/_libs/src/parser/tokenizer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ int parser_cleanup(parser_t *self) {
if (self->cb_cleanup(self->source) < 0) {
status = -1;
}
self->cb_cleanup = NULL;
}

return status;
Expand Down Expand Up @@ -239,6 +240,9 @@ int parser_init(parser_t *self) {
void parser_free(parser_t *self) {
// opposite of parser_init
parser_cleanup(self);
}

void parser_del(parser_t *self) {
free(self);
}

Expand Down
2 changes: 2 additions & 0 deletions pandas/_libs/src/parser/tokenizer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,8 @@ int parser_set_skipfirstnrows(parser_t *self, int64_t nrows);

void parser_free(parser_t *self);

void parser_del(parser_t *self);

void parser_set_default_options(parser_t *self);

void debug_print_parser(parser_t *self);
Expand Down
18 changes: 16 additions & 2 deletions pandas/io/parsers.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ cdef extern from "parser/tokenizer.h":

int parser_init(parser_t *self) nogil
void parser_free(parser_t *self) nogil
void parser_del(parser_t *self) nogil
int parser_add_skiprow(parser_t *self, int64_t row)

int parser_set_skipfirstnrows(parser_t *self, int64_t nrows)
Expand Down Expand Up @@ -573,8 +574,13 @@ cdef class TextReader:

def __dealloc__(self):
parser_free(self.parser)
kh_destroy_str(self.true_set)
kh_destroy_str(self.false_set)
if self.true_set:
kh_destroy_str(self.true_set)
self.true_set = NULL
if self.false_set:
kh_destroy_str(self.false_set)
self.false_set = NULL
parser_del(self.parser)

def close(self):
# we need to properly close an open derived
Expand All @@ -584,6 +590,14 @@ cdef class TextReader:
self.handle.close()
except:
pass
# also preemptively free all allocated memory
parser_free(self.parser)
if self.true_set:
kh_destroy_str(self.true_set)
self.true_set = NULL
if self.false_set:
kh_destroy_str(self.false_set)
self.false_set = NULL

def set_error_bad_lines(self, int status):
self.parser.error_bad_lines = status
Expand Down