Make read_json with lines=True more memory-efficient

Instead of reading the whole file to memory and then manipulating
it, read and parse it 10k lines at a time.
This only covers some kinds of input to read_json.
This also is much slower than the previous implementation.

Author: louispotok

pandas/io/json/json.py

@@ -1,4 +1,6 @@
 # pylint: disable-msg=E1101,W0613,W0603
+from itertools import islice
+from pandas import concat
 import os
 import numpy as np
 
@@ -334,8 +336,25 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
         if exists:
             fh, handles = _get_handle(filepath_or_buffer, 'r',
                                       encoding=encoding)
-            json = fh.read()
-            fh.close()
+            if lines:
+                return_val = None
+                while True:
+                    lines = list(islice(fh, 10000))
+                    if lines:
+                        lines_json = '[' + ','.join(lines) + ']'
+                        obj = _get_obj(typ, lines_json, orient, dtype, convert_axes, convert_dates, keep_default_dates, numpy, precise_float, date_unit)
+                        if not return_val:
+                            obj = return_val
+                        else:
+                            return_val = concat([return_val, obj])
+
+                    else:
+                        break
+                fh.close()
+                return return_val
+            else:
+                json = fh.read()
+                fh.close()
         else:
             json = filepath_or_buffer
     elif hasattr(filepath_or_buffer, 'read'):
@@ -349,6 +368,11 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
         lines = list(StringIO(json.strip()))
         json = '[' + ','.join(lines) + ']'
 
+    return _get_obj(typ, json, orient, dtype, convert_axes, convert_dates, keep_default_dates, numpy, precise_float, date_unit)
+
+def _get_obj(typ, json, orient, dtype, convert_axes, convert_dates,
+                          keep_default_dates, numpy, precise_float,
+             date_unit):
     obj = None
     if typ == 'frame':
         obj = FrameParser(json, orient, dtype, convert_axes, convert_dates,