Merge branch 'Komnomnomnom-ujson-precise-float'

Author: jreback

doc/source/io.rst

@@ -1060,6 +1060,8 @@ is ``None``. To explicity force ``Series`` parsing, pass ``typ=series``
 - ``keep_default_dates`` : boolean, default True. If parsing dates, then parse the default datelike columns
 - ``numpy`` : direct decoding to numpy arrays. default is False;
   Note that the JSON ordering **MUST** be the same for each term if ``numpy=True``
+- ``precise_float`` : boolean, default ``False``. Set to enable usage of higher precision (strtod) function
+  when decoding string to double values. Default (``False``) is to use fast but less precise builtin functionality
 
 The parser will raise one of ``ValueError/TypeError/AssertionError`` if the JSON is
 not parsable.

doc/source/release.rst

@@ -35,6 +35,9 @@ pandas 0.12
     list of ``DataFrame`` s courtesy of @cpcloud. (:issue:`3477`,
     :issue:`3605`, :issue:`3606`)
   - Support for reading Amazon S3 files. (:issue:`3504`)
+  - Added module for reading and writing JSON strings/files: pandas.io.json
+    includes ``to_json`` DataFrame/Series method, and a ``read_json`` top-level reader
+    various issues (:issue:`1226`, :issue:`3804`, :issue:`3876`, :issue:`3867`, :issue:`1305`)
   - Added module for reading and writing Stata files: pandas.io.stata (:issue:`1512`)
     includes ``to_stata`` DataFrame method, and a ``read_stata`` top-level reader
   - Added support for writing in ``to_csv`` and reading in ``read_csv``,

doc/source/v0.12.0.txt

@@ -206,6 +206,7 @@ I/O Enhancements
   - Added module for reading and writing json format files: ``pandas.io.json``
     accessable via ``read_json`` top-level function for reading,
     and ``to_json`` DataFrame method for writing, :ref:`See the docs<io.json>`
+    various issues (:issue:`1226`, :issue:`3804`, :issue:`3876`, :issue:`3867`, :issue:`1305`)
 
   - ``MultiIndex`` column support for reading and writing csv format files
 

pandas/io/json.py

@@ -16,9 +16,9 @@
 ### interface to/from ###
 
 def to_json(path_or_buf, obj, orient=None, date_format='epoch', double_precision=10, force_ascii=True):
-        
+
     if isinstance(obj, Series):
-        s = SeriesWriter(obj, orient=orient, date_format=date_format, double_precision=double_precision, 
+        s = SeriesWriter(obj, orient=orient, date_format=date_format, double_precision=double_precision,
                          ensure_ascii=force_ascii).write()
     elif isinstance(obj, DataFrame):
         s = FrameWriter(obj, orient=orient, date_format=date_format, double_precision=double_precision,
@@ -41,7 +41,7 @@ def __init__(self, obj, orient, date_format, double_precision, ensure_ascii):
 
         if orient is None:
             orient = self._default_orient
-            
+
         self.orient = orient
         self.date_format = date_format
         self.double_precision = double_precision
@@ -64,7 +64,7 @@ def _format_to_date(self, data):
         if self._needs_to_date(data):
             return data.apply(lambda x: x.isoformat())
         return data
-    
+
     def copy_if_needed(self):
         """ copy myself if necessary """
         if not self.is_copy:
@@ -119,7 +119,8 @@ def _format_dates(self):
                     self.obj[c] = self._format_to_date(self.obj[c])
 
 def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
-              convert_axes=True, convert_dates=True, keep_default_dates=True, numpy=False):
+              convert_axes=True, convert_dates=True, keep_default_dates=True,
+              numpy=False, precise_float=False):
     """
     Convert JSON string to pandas object
 
@@ -154,8 +155,10 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
         default is True
     keep_default_dates : boolean, default True. If parsing dates,
         then parse the default datelike columns
-    numpy: direct decoding to numpy arrays. default is False.Note that the JSON ordering MUST be the same
+    numpy : direct decoding to numpy arrays. default is False.Note that the JSON ordering MUST be the same
         for each term if numpy=True.
+    precise_float : boolean, default False. Set to enable usage of higher precision (strtod) function
+        when decoding string to double values. Default (False) is to use fast but less precise builtin functionality
 
     Returns
     -------
@@ -186,28 +189,31 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
     return obj
 
 class Parser(object):
-    
-    def __init__(self, json, orient, dtype=True, convert_axes=True, convert_dates=True, keep_default_dates=False, numpy=False):
+
+    def __init__(self, json, orient, dtype=True, convert_axes=True,
+                 convert_dates=True, keep_default_dates=False, numpy=False,
+                 precise_float=False):
         self.json = json
 
         if orient is None:
             orient = self._default_orient
-            
+
         self.orient = orient
         self.dtype = dtype
 
         if orient == "split":
             numpy = False
 
         self.numpy = numpy
+        self.precise_float = precise_float
         self.convert_axes  = convert_axes
         self.convert_dates = convert_dates
         self.keep_default_dates = keep_default_dates
         self.obj = None
 
     def parse(self):
 
-        # try numpy 
+        # try numpy
         numpy = self.numpy
         if numpy:
             self._parse_numpy()
@@ -269,7 +275,7 @@ def _try_convert_data(self, name, data, use_dtypes=True, convert_dates=True):
                 pass
 
         if data.dtype == 'float':
-            
+
             # coerce floats to 64
             try:
                 data = data.astype('float64')
@@ -291,7 +297,7 @@ def _try_convert_data(self, name, data, use_dtypes=True, convert_dates=True):
 
         # coerce ints to 64
         if data.dtype == 'int':
-            
+
             # coerce floats to 64
             try:
                 data = data.astype('int64')
@@ -322,7 +328,7 @@ def _try_convert_to_date(self, data):
         if issubclass(new_data.dtype.type,np.number):
             if not ((new_data == iNaT) | (new_data > 31536000000000000L)).all():
                 return data, False
-                
+
         try:
             new_data = to_datetime(new_data)
         except:
@@ -342,29 +348,35 @@ class SeriesParser(Parser):
     _default_orient = 'index'
 
     def _parse_no_numpy(self):
-    
+
         json = self.json
         orient = self.orient
         if orient == "split":
             decoded = dict((str(k), v)
-                           for k, v in loads(json).iteritems())
+                           for k, v in loads(
+                               json,
+                               precise_float=self.precise_float).iteritems())
             self.obj = Series(dtype=None, **decoded)
         else:
-            self.obj = Series(loads(json), dtype=None)
+            self.obj = Series(
+                loads(json, precise_float=self.precise_float), dtype=None)
 
     def _parse_numpy(self):
 
         json = self.json
         orient = self.orient
         if orient == "split":
-            decoded = loads(json, dtype=None, numpy=True)
+            decoded = loads(json, dtype=None, numpy=True,
+                            precise_float=self.precise_float)
             decoded = dict((str(k), v) for k, v in decoded.iteritems())
             self.obj = Series(**decoded)
         elif orient == "columns" or orient == "index":
             self.obj = Series(*loads(json, dtype=None, numpy=True,
-                                     labelled=True))
+                                     labelled=True,
+                                     precise_float=self.precise_float))
         else:
-            self.obj = Series(loads(json, dtype=None, numpy=True))
+            self.obj = Series(loads(json, dtype=None, numpy=True,
+                                    precise_float=self.precise_float))
 
     def _try_convert_types(self):
         if self.obj is None: return
@@ -381,34 +393,43 @@ def _parse_numpy(self):
         orient = self.orient
 
         if orient == "columns":
-            args = loads(json, dtype=None, numpy=True, labelled=True)
+            args = loads(json, dtype=None, numpy=True, labelled=True,
+                         precise_float=self.precise_float)
             if args:
                 args = (args[0].T, args[2], args[1])
             self.obj = DataFrame(*args)
         elif orient == "split":
-            decoded = loads(json, dtype=None, numpy=True)
+            decoded = loads(json, dtype=None, numpy=True,
+                            precise_float=self.precise_float)
             decoded = dict((str(k), v) for k, v in decoded.iteritems())
             self.obj = DataFrame(**decoded)
         elif orient == "values":
-            self.obj = DataFrame(loads(json, dtype=None, numpy=True))
+            self.obj = DataFrame(loads(json, dtype=None, numpy=True,
+                                       precise_float=self.precise_float))
         else:
-            self.obj = DataFrame(*loads(json, dtype=None, numpy=True, labelled=True))
+            self.obj = DataFrame(*loads(json, dtype=None, numpy=True, labelled=True,
+                                        precise_float=self.precise_float))
 
     def _parse_no_numpy(self):
 
         json = self.json
         orient = self.orient
 
         if orient == "columns":
-            self.obj = DataFrame(loads(json), dtype=None)
+            self.obj = DataFrame(
+                loads(json, precise_float=self.precise_float), dtype=None)
         elif orient == "split":
             decoded = dict((str(k), v)
-                           for k, v in loads(json).iteritems())
+                           for k, v in loads(
+                               json,
+                               precise_float=self.precise_float).iteritems())
             self.obj = DataFrame(dtype=None, **decoded)
         elif orient == "index":
-            self.obj = DataFrame(loads(json), dtype=None).T
+            self.obj = DataFrame(
+                loads(json, precise_float=self.precise_float), dtype=None).T
         else:
-            self.obj = DataFrame(loads(json), dtype=None)
+            self.obj = DataFrame(
+                loads(json, precise_float=self.precise_float), dtype=None)
 
     def _try_convert_types(self):
         if self.obj is None: return

pandas/io/tests/test_json/test_pandas.py

@@ -289,6 +289,16 @@ def test_series_to_json_except(self):
         s = Series([1, 2, 3])
         self.assertRaises(ValueError, s.to_json, orient="garbage")
 
+    def test_series_from_json_precise_float(self):
+        s = Series([4.56, 4.56, 4.56])
+        result = read_json(s.to_json(), typ='series', precise_float=True)
+        assert_series_equal(result, s)
+
+    def test_frame_from_json_precise_float(self):
+        df = DataFrame([[4.56, 4.56, 4.56], [4.56, 4.56, 4.56]])
+        result = read_json(df.to_json(), precise_float=True)
+        assert_frame_equal(result, df)
+
     def test_typ(self):
 
         s = Series(range(6), index=['a','b','c','d','e','f'], dtype='int64')