Skip the remainder of stream if there are no word_deletions

roberthdevries · roberthdevries · commit 1b4cc7402691 · 2020-03-11T18:15:33.000+01:00
diff --git a/pandas/_libs/src/parser/tokenizer.c b/pandas/_libs/src/parser/tokenizer.c
@@ -1193,7 +1193,7 @@ int parser_consume_rows(parser_t *self, size_t nrows) {
         char_count = (self->word_starts[word_deletions - 1] +
                       strlen(self->words[word_deletions - 1]) + 1);
     } else {
-        char_count = 1;
+        char_count = self->stream_len;
     }
 
     TRACE(("parser_consume_rows: Deleting %d words, %d chars\n", word_deletions,

Original file line number	Diff line number	Diff line change
`@@ -1193,7 +1193,7 @@ int parser_consume_rows(parser_t *self, size_t nrows) {`
`1193`	`1193`	`char_count = (self->word_starts[word_deletions - 1] +`
`1194`	`1194`	`strlen(self->words[word_deletions - 1]) + 1);`
`1195`	`1195`	`} else {`
`1196`		`- char_count = 1;`
	`1196`	`+ char_count = self->stream_len;`
`1197`	`1197`	`}`
`1198`	`1198`
`1199`	`1199`	`TRACE(("parser_consume_rows: Deleting %d words, %d chars\n", word_deletions,`