CLN: reorg pandas/io/json to sub-dirs

Author: jreback

pandas/io/json/__init__.py

@@ -0,0 +1,4 @@
+from .json import to_json, read_json, loads, dumps  # noqa
+from .normalize import json_normalize  # noqa
+
+del json, normalize  # noqa

pandas/io/json.py renamed to pandas/io/json/json.py

@@ -1,8 +1,6 @@
 # pylint: disable-msg=E1101,W0613,W0603
 
 import os
-import copy
-from collections import defaultdict
 import numpy as np
 
 import pandas.json as _json
@@ -13,6 +11,7 @@
 from pandas.io.common import get_filepath_or_buffer, _get_handle
 from pandas.core.common import AbstractMethodError
 from pandas.formats.printing import pprint_thing
+from .normalize import _convert_to_line_delimits
 
 loads = _json.loads
 dumps = _json.dumps
@@ -641,246 +640,3 @@ def is_ok(col):
             lambda col, c: self._try_convert_to_date(c),
             lambda col, c: ((self.keep_default_dates and is_ok(col)) or
                             col in convert_dates))
-
-# ---------------------------------------------------------------------
-# JSON normalization routines
-
-
-def _convert_to_line_delimits(s):
-    """Helper function that converts json lists to line delimited json."""
-
-    # Determine we have a JSON list to turn to lines otherwise just return the
-    # json object, only lists can
-    if not s[0] == '[' and s[-1] == ']':
-        return s
-    s = s[1:-1]
-
-    from pandas.lib import convert_json_to_lines
-    return convert_json_to_lines(s)
-
-
-def nested_to_record(ds, prefix="", level=0):
-    """a simplified json_normalize
-
-    converts a nested dict into a flat dict ("record"), unlike json_normalize,
-    it does not attempt to extract a subset of the data.
-
-    Parameters
-    ----------
-    ds : dict or list of dicts
-    prefix: the prefix, optional, default: ""
-    level: the number of levels in the jason string, optional, default: 0
-
-    Returns
-    -------
-    d - dict or list of dicts, matching `ds`
-
-    Examples
-    --------
-
-    IN[52]: nested_to_record(dict(flat1=1,dict1=dict(c=1,d=2),
-                                  nested=dict(e=dict(c=1,d=2),d=2)))
-    Out[52]:
-    {'dict1.c': 1,
-     'dict1.d': 2,
-     'flat1': 1,
-     'nested.d': 2,
-     'nested.e.c': 1,
-     'nested.e.d': 2}
-    """
-    singleton = False
-    if isinstance(ds, dict):
-        ds = [ds]
-        singleton = True
-
-    new_ds = []
-    for d in ds:
-
-        new_d = copy.deepcopy(d)
-        for k, v in d.items():
-            # each key gets renamed with prefix
-            if not isinstance(k, compat.string_types):
-                k = str(k)
-            if level == 0:
-                newkey = k
-            else:
-                newkey = prefix + '.' + k
-
-            # only dicts gets recurse-flattend
-            # only at level>1 do we rename the rest of the keys
-            if not isinstance(v, dict):
-                if level != 0:  # so we skip copying for top level, common case
-                    v = new_d.pop(k)
-                    new_d[newkey] = v
-                continue
-            else:
-                v = new_d.pop(k)
-                new_d.update(nested_to_record(v, newkey, level + 1))
-        new_ds.append(new_d)
-
-    if singleton:
-        return new_ds[0]
-    return new_ds
-
-
-def json_normalize(data, record_path=None, meta=None,
-                   meta_prefix=None,
-                   record_prefix=None,
-                   errors='raise'):
-
-    """
-    "Normalize" semi-structured JSON data into a flat table
-
-    Parameters
-    ----------
-    data : dict or list of dicts
-        Unserialized JSON objects
-    record_path : string or list of strings, default None
-        Path in each object to list of records. If not passed, data will be
-        assumed to be an array of records
-    meta : list of paths (string or list of strings), default None
-        Fields to use as metadata for each record in resulting table
-    record_prefix : string, default None
-        If True, prefix records with dotted (?) path, e.g. foo.bar.field if
-        path to records is ['foo', 'bar']
-    meta_prefix : string, default None
-    errors : {'raise', 'ignore'}, default 'raise'
-
-        * ignore : will ignore KeyError if keys listed in meta are not
-        always present
-        * raise : will raise KeyError if keys listed in meta are not
-        always present
-
-        .. versionadded:: 0.20.0
-
-    Returns
-    -------
-    frame : DataFrame
-
-    Examples
-    --------
-
-    >>> data = [{'state': 'Florida',
-    ...          'shortname': 'FL',
-    ...          'info': {
-    ...               'governor': 'Rick Scott'
-    ...          },
-    ...          'counties': [{'name': 'Dade', 'population': 12345},
-    ...                      {'name': 'Broward', 'population': 40000},
-    ...                      {'name': 'Palm Beach', 'population': 60000}]},
-    ...         {'state': 'Ohio',
-    ...          'shortname': 'OH',
-    ...          'info': {
-    ...               'governor': 'John Kasich'
-    ...          },
-    ...          'counties': [{'name': 'Summit', 'population': 1234},
-    ...                       {'name': 'Cuyahoga', 'population': 1337}]}]
-    >>> from pandas.io.json import json_normalize
-    >>> result = json_normalize(data, 'counties', ['state', 'shortname',
-    ...                                           ['info', 'governor']])
-    >>> result
-             name  population info.governor    state shortname
-    0        Dade       12345    Rick Scott  Florida        FL
-    1     Broward       40000    Rick Scott  Florida        FL
-    2  Palm Beach       60000    Rick Scott  Florida        FL
-    3      Summit        1234   John Kasich     Ohio        OH
-    4    Cuyahoga        1337   John Kasich     Ohio        OH
-
-    """
-    def _pull_field(js, spec):
-        result = js
-        if isinstance(spec, list):
-            for field in spec:
-                result = result[field]
-        else:
-            result = result[spec]
-
-        return result
-
-    # A bit of a hackjob
-    if isinstance(data, dict):
-        data = [data]
-
-    if record_path is None:
-        if any([isinstance(x, dict) for x in compat.itervalues(data[0])]):
-            # naive normalization, this is idempotent for flat records
-            # and potentially will inflate the data considerably for
-            # deeply nested structures:
-            #  {VeryLong: { b: 1,c:2}} -> {VeryLong.b:1 ,VeryLong.c:@}
-            #
-            # TODO: handle record value which are lists, at least error
-            #       reasonably
-            data = nested_to_record(data)
-        return DataFrame(data)
-    elif not isinstance(record_path, list):
-        record_path = [record_path]
-
-    if meta is None:
-        meta = []
-    elif not isinstance(meta, list):
-        meta = [meta]
-
-    for i, x in enumerate(meta):
-        if not isinstance(x, list):
-            meta[i] = [x]
-
-    # Disastrously inefficient for now
-    records = []
-    lengths = []
-
-    meta_vals = defaultdict(list)
-    meta_keys = ['.'.join(val) for val in meta]
-
-    def _recursive_extract(data, path, seen_meta, level=0):
-        if len(path) > 1:
-            for obj in data:
-                for val, key in zip(meta, meta_keys):
-                    if level + 1 == len(val):
-                        seen_meta[key] = _pull_field(obj, val[-1])
-
-                _recursive_extract(obj[path[0]], path[1:],
-                                   seen_meta, level=level + 1)
-        else:
-            for obj in data:
-                recs = _pull_field(obj, path[0])
-
-                # For repeating the metadata later
-                lengths.append(len(recs))
-
-                for val, key in zip(meta, meta_keys):
-                    if level + 1 > len(val):
-                        meta_val = seen_meta[key]
-                    else:
-                        try:
-                            meta_val = _pull_field(obj, val[level:])
-                        except KeyError as e:
-                            if errors == 'ignore':
-                                meta_val = np.nan
-                            else:
-                                raise \
-                                    KeyError("Try running with "
-                                             "errors='ignore' as key "
-                                             "%s is not always present", e)
-                    meta_vals[key].append(meta_val)
-
-                records.extend(recs)
-
-    _recursive_extract(data, record_path, {}, level=0)
-
-    result = DataFrame(records)
-
-    if record_prefix is not None:
-        result.rename(columns=lambda x: record_prefix + x, inplace=True)
-
-    # Data types, a problem
-    for k, v in compat.iteritems(meta_vals):
-        if meta_prefix is not None:
-            k = meta_prefix + k
-
-        if k in result:
-            raise ValueError('Conflicting metadata name %s, '
-                             'need distinguishing prefix ' % k)
-
-        result[k] = np.array(v).repeat(lengths)
-
-    return result