BUG: Fixed incorrect stream size check (pandas-dev#14125)

Author: jeffcarey

pandas/io/tests/parser/c_parser_only.py

@@ -395,3 +395,15 @@ def test_float_precision_round_trip_with_text(self):
                            float_precision='round_trip',
                            header=None)
         tm.assert_frame_equal(df, DataFrame({0: ['a']}))
+
+    def test_large_difference_in_columns(self):
+        # gh-14125
+        count = 10000
+        large_row = ('X,' * count)[:-1] + '\n'
+        normal_row = 'XXXXXX XXXXXX,111111111111111\n'
+        test_input = (large_row + normal_row * 6)[:-1]
+        result = self.read_csv(StringIO(test_input), header=None, usecols=[0])
+        rows = test_input.split('\n')
+        expected = DataFrame([row.split(',')[0] for row in rows])
+
+        tm.assert_frame_equal(result, expected)

pandas/src/parser/tokenizer.c

@@ -592,11 +592,11 @@ static int parser_buffer_bytes(parser_t *self, size_t nbytes) {
     TRACE(                                                                    \
         ("PUSH_CHAR: Pushing %c, slen= %d, stream_cap=%zu, stream_len=%zu\n", \
          c, slen, self->stream_cap, self->stream_len))                        \
-    if (slen >= maxstreamsize) {                                              \
-        TRACE(("PUSH_CHAR: ERROR!!! slen(%d) >= maxstreamsize(%d)\n", slen,   \
-               maxstreamsize))                                                \
+    if (slen >= self->stream_cap) {                                           \
+        TRACE(("PUSH_CHAR: ERROR!!! slen(%d) >= stream_cap(%d)\n", slen,      \
+               self->stream_cap))                                             \
         int bufsize = 100;                                                    \
-        self->error_msg = (char *)malloc(bufsize);                            \
+        self->error_msg = (char *) malloc(bufsize);                           \
         snprintf(self->error_msg, bufsize,                                    \
                  "Buffer overflow caught - possible malformed input file.\n");\
         return PARSER_OUT_OF_MEMORY;                                          \
@@ -711,7 +711,6 @@ int skip_this_line(parser_t *self, int64_t rownum) {
 int tokenize_bytes(parser_t *self, size_t line_limit, int start_lines) {
     int i, slen;
     int should_skip;
-    long maxstreamsize;
     char c;
     char *stream;
     char *buf = self->data + self->datapos;
@@ -723,7 +722,6 @@ int tokenize_bytes(parser_t *self, size_t line_limit, int start_lines) {
 
     stream = self->stream + self->stream_len;
     slen = self->stream_len;
-    maxstreamsize = self->stream_cap;
 
     TRACE(("%s\n", buf));