Merge branch 'gh15077' of https://github.com/mralgos/pandas into gh15077

Author: Giacomo Ferroni

appveyor.yml

@@ -65,7 +65,8 @@ install:
 
   # install our build environment
   - cmd: conda config --set show_channel_urls true --set always_yes true --set changeps1 false
-  - cmd: conda update -q conda
+  #- cmd: conda update -q conda
+  - cmd: conda install conda=4.2.15
   - cmd: conda config --set ssl_verify false
 
   # add the pandas channel *before* defaults to have defaults take priority

doc/source/io.rst

@@ -187,6 +187,16 @@ skipinitialspace : boolean, default ``False``
 skiprows : list-like or integer, default ``None``
   Line numbers to skip (0-indexed) or number of lines to skip (int) at the start
   of the file.
+
+  If callable, the callable function will be evaluated against the row
+  indices, returning True if the row should be skipped and False otherwise:
+
+  .. ipython:: python
+
+     data = 'col1,col2,col3\na,b,1\na,b,2\nc,d,3'
+     pd.read_csv(StringIO(data))
+     pd.read_csv(StringIO(data), skiprows=lambda x: x % 2 != 0)
+
 skipfooter : int, default ``0``
   Number of lines at bottom of file to skip (unsupported with engine='c').
 skip_footer : int, default ``0``

doc/source/whatsnew/v0.20.0.txt

@@ -91,6 +91,25 @@ support for bz2 compression in the python 2 c-engine improved (:issue:`14874`).
    df = pd.read_table(url, compression='bz2')  # explicitly specify compression
    df.head(2)
 
+.. _whatsnew_0200.enhancements.uint64_support:
+
+Pandas has significantly improved support for operations involving unsigned,
+or purely non-negative, integers. Previously, handling these integers would
+result in improper rounding or data-type casting, leading to incorrect results.
+Notably, a new numerical index, ``UInt64Index``, has been created (:issue:`14937`)
+
+.. ipython:: python
+
+   idx = pd.UInt64Index([1, 2, 3])
+   df = pd.DataFrame({'A': ['a', 'b', 'c']}, index=idx)
+   df.index
+
+- Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`, :issue:`14982`)
+- Bug in ``Series.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14721`)
+- Bug in ``DataFrame`` construction in which unsigned 64-bit integer elements were being converted to objects (:issue:`14881`)
+- Bug in ``pd.read_csv()`` in which unsigned 64-bit integer elements were being improperly converted to the wrong data types (:issue:`14983`)
+- Bug in ``pd.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14915`)
+
 .. _whatsnew_0200.enhancements.other:
 
 Other enhancements
@@ -110,6 +129,7 @@ Other enhancements
 - ``pd.qcut`` has gained the ``duplicates='raise'|'drop'`` option to control whether to raise on duplicated edges (:issue:`7751`)
 - ``Series`` provides a ``to_excel`` method to output Excel files (:issue:`8825`)
 - The ``usecols`` argument in ``pd.read_csv`` now accepts a callable function as a value  (:issue:`14154`)
+- The ``skiprows`` argument in ``pd.read_csv`` now accepts a callable function as a value  (:issue:`10882`)
 - ``pd.DataFrame.plot`` now prints a title above each subplot if ``suplots=True`` and ``title`` is a list of strings (:issue:`14753`)
 - ``pd.Series.interpolate`` now supports timedelta as an index type with ``method='time'`` (:issue:`6424`)
 - ``pandas.io.json.json_normalize()`` gained the option ``errors='ignore'|'raise'``; the default is ``errors='raise'`` which is backward compatible. (:issue:`14583`)
@@ -244,10 +264,11 @@ Other API Changes
 - ``CParserError`` has been renamed to ``ParserError`` in ``pd.read_csv`` and will be removed in the future (:issue:`12665`)
 - ``SparseArray.cumsum()`` and ``SparseSeries.cumsum()`` will now always return ``SparseArray`` and ``SparseSeries`` respectively (:issue:`12855`)
 - ``DataFrame.applymap()`` with an empty ``DataFrame`` will return a copy of the empty ``DataFrame`` instead of a ``Series`` (:issue:`8222`)
-
+- ``.loc`` has compat with ``.ix`` for accepting iterators, and NamedTuples (:issue:`15120`)
 - ``pd.read_csv()`` will now issue a ``ParserWarning`` whenever there are conflicting values provided by the ``dialect`` parameter and the user (:issue:`14898`)
 - ``pd.read_csv()`` will now raise a ``ValueError`` for the C engine if the quote character is larger than than one byte (:issue:`11592`)
 - ``inplace`` arguments now require a boolean value, else a ``ValueError`` is thrown (:issue:`14189`)
+- ``pandas.api.types.is_datetime64_ns_dtype`` will now report ``True`` on a tz-aware dtype, similar to ``pandas.api.types.is_datetime64_any_dtype``
 
 .. _whatsnew_0200.deprecations:
 
@@ -296,8 +317,6 @@ Bug Fixes
 
 - Bug in ``Index`` power operations with reversed operands (:issue:`14973`)
 - Bug in ``TimedeltaIndex`` addition where overflow was being allowed without error (:issue:`14816`)
-- Bug in ``DataFrame`` construction in which unsigned 64-bit integer elements were being converted to objects (:issue:`14881`)
-- Bug in ``pd.read_csv()`` in which unsigned 64-bit integer elements were being improperly converted to the wrong data types (:issue:`14983`)
 - Bug in ``astype()`` where ``inf`` values were incorrectly converted to integers. Now raises error now with ``astype()`` for Series and DataFrames (:issue:`14265`)
 - Bug in ``DataFrame(..).apply(to_numeric)`` when values are of type decimal.Decimal. (:issue:`14827`)
 - Bug in ``describe()`` when passing a numpy array which does not contain the median to the ``percentiles`` keyword argument (:issue:`14908`)
@@ -318,13 +337,11 @@ Bug Fixes
 - Bug in ``Series`` construction with a datetimetz (:issue:`14928`)
 
 - Bug in compat for passing long integers to ``Timestamp.replace`` (:issue:`15030`)
+- Bug in ``.loc`` that would not return the correct dtype for scalar access for a DataFrame (:issue:`11617`)
 
 
 
 
-- Bug in ``Series.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14721`)
-- Bug in ``pd.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14915`)
-
 
 
 
@@ -348,18 +365,21 @@ Bug Fixes
 
 
 - Require at least 0.23 version of cython to avoid problems with character encodings (:issue:`14699`)
-- Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`, :issue:`14982`)
 - Bug in ``pd.pivot_table()`` where no error was raised when values argument was not in the columns (:issue:`14938`)
+- Bug in ``.to_json()`` where ``lines=True`` and contents (keys or values) contain escaped characters (:issue:`15096`)
 
+- Bug in ``DataFrame.groupby().describe()`` when grouping on ``Index`` containing tuples (:issue:`14848`)
+- Bug in creating a ``MultiIndex`` with tuples and not passing a list of names; this will now raise ``ValueError`` (:issue:`15110`)
 
+- Bug in catching an overflow in ``Timestamp`` + ``Timedelta/Offset`` operations (:issue:`15126`)
 
 
+- Bug in ``pd.merge_asof()`` where ``left_index``/``right_index`` together caused a failure when ``tolerance`` was specified (:issue:`15135`)
 
 
 
 
 
-
-
+- Bug in ``Series`` constructor when both ``copy=True`` and ``dtype`` arguments are provided (:issue:`15125`)
 - Bug in ``pd.read_csv()`` for the C engine where ``usecols`` were being indexed incorrectly with ``parse_dates`` (:issue:`14792`)
 - Incorrect dtyped ``Series`` was returned by comparison methods (e.g., ``lt``, ``gt``, ...) against a constant for an empty ``DataFrame`` (:issue:`15077`)

pandas/api/tests/test_api.py

@@ -53,7 +53,7 @@ class TestPDApi(Base, tm.TestCase):
     classes = ['Categorical', 'CategoricalIndex', 'DataFrame', 'DateOffset',
                'DatetimeIndex', 'ExcelFile', 'ExcelWriter', 'Float64Index',
                'Grouper', 'HDFStore', 'Index', 'Int64Index', 'MultiIndex',
-               'Period', 'PeriodIndex', 'RangeIndex',
+               'Period', 'PeriodIndex', 'RangeIndex', 'UInt64Index',
                'Series', 'SparseArray', 'SparseDataFrame',
                'SparseSeries', 'TimeGrouper', 'Timedelta',
                'TimedeltaIndex', 'Timestamp']

pandas/computation/tests/test_compat.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 
 # flake8: noqa
 

pandas/computation/tests/test_eval.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 
 # flake8: noqa
 

pandas/core/api.py

@@ -10,7 +10,8 @@
 from pandas.core.groupby import Grouper
 from pandas.formats.format import set_eng_float_format
 from pandas.core.index import (Index, CategoricalIndex, Int64Index,
-                               RangeIndex, Float64Index, MultiIndex)
+                               UInt64Index, RangeIndex, Float64Index,
+                               MultiIndex)
 
 from pandas.core.series import Series, TimeSeries
 from pandas.core.frame import DataFrame

pandas/core/indexing.py

@@ -9,6 +9,7 @@
                                  is_categorical_dtype,
                                  is_list_like,
                                  is_sequence,
+                                 is_iterator,
                                  is_scalar,
                                  is_sparse,
                                  _is_unorderable_exception,
@@ -859,15 +860,20 @@ def _convert_for_reindex(self, key, axis=0):
             return labels[key]
         else:
             if isinstance(key, Index):
-                # want Index objects to pass through untouched
-                keyarr = key
+                keyarr = labels._convert_index_indexer(key)
             else:
                 # asarray can be unsafe, NumPy strings are weird
                 keyarr = _asarray_tuplesafe(key)
 
-            if is_integer_dtype(keyarr) and not labels.is_integer():
-                keyarr = _ensure_platform_int(keyarr)
-                return labels.take(keyarr)
+            if is_integer_dtype(keyarr):
+                # Cast the indexer to uint64 if possible so
+                # that the values returned from indexing are
+                # also uint64.
+                keyarr = labels._convert_arr_indexer(keyarr)
+
+                if not labels.is_integer():
+                    keyarr = _ensure_platform_int(keyarr)
+                    return labels.take(keyarr)
 
             return keyarr
 
@@ -1043,11 +1049,10 @@ def _getitem_iterable(self, key, axis=0):
             return self.obj.take(inds, axis=axis, convert=False)
         else:
             if isinstance(key, Index):
-                # want Index objects to pass through untouched
-                keyarr = key
+                keyarr = labels._convert_index_indexer(key)
             else:
-                # asarray can be unsafe, NumPy strings are weird
                 keyarr = _asarray_tuplesafe(key)
+                keyarr = labels._convert_arr_indexer(keyarr)
 
             if is_categorical_dtype(labels):
                 keyarr = labels._shallow_copy(keyarr)
@@ -1300,17 +1305,24 @@ class _LocationIndexer(_NDFrameIndexer):
     _exception = Exception
 
     def __getitem__(self, key):
-        if isinstance(key, tuple):
-            key = tuple(com._apply_if_callable(x, self.obj) for x in key)
-        else:
-            # scalar callable may return tuple
-            key = com._apply_if_callable(key, self.obj)
-
         if type(key) is tuple:
+            key = tuple(com._apply_if_callable(x, self.obj) for x in key)
+            try:
+                if self._is_scalar_access(key):
+                    return self._getitem_scalar(key)
+            except (KeyError, IndexError):
+                pass
             return self._getitem_tuple(key)
         else:
+            key = com._apply_if_callable(key, self.obj)
             return self._getitem_axis(key, axis=0)
 
+    def _is_scalar_access(self, key):
+        raise NotImplementedError()
+
+    def _getitem_scalar(self, key):
+        raise NotImplementedError()
+
     def _getitem_axis(self, key, axis=0):
         raise NotImplementedError()
 
@@ -1389,7 +1401,8 @@ def _has_valid_type(self, key, axis):
                 return True
 
             # TODO: don't check the entire key unless necessary
-            if len(key) and np.all(ax.get_indexer_for(key) < 0):
+            if (not is_iterator(key) and len(key) and
+                    np.all(ax.get_indexer_for(key) < 0)):
 
                 raise KeyError("None of [%s] are in the [%s]" %
                                (key, self.obj._get_axis_name(axis)))
@@ -1420,6 +1433,36 @@ def error():
 
         return True
 
+    def _is_scalar_access(self, key):
+        # this is a shortcut accessor to both .loc and .iloc
+        # that provide the equivalent access of .at and .iat
+        # a) avoid getting things via sections and (to minimize dtype changes)
+        # b) provide a performant path
+        if not hasattr(key, '__len__'):
+            return False
+
+        if len(key) != self.ndim:
+            return False
+
+        for i, k in enumerate(key):
+            if not is_scalar(k):
+                return False
+
+            ax = self.obj.axes[i]
+            if isinstance(ax, MultiIndex):
+                return False
+
+            if not ax.is_unique:
+                return False
+
+        return True
+
+    def _getitem_scalar(self, key):
+        # a fast-path to scalar access
+        # if not, raise
+        values = self.obj.get_value(*key)
+        return values
+
     def _get_partial_string_timestamp_match_key(self, key, labels):
         """Translate any partial string timestamp matches in key, returning the
         new key (GH 10331)"""
@@ -1536,6 +1579,33 @@ def _has_valid_type(self, key, axis):
     def _has_valid_setitem_indexer(self, indexer):
         self._has_valid_positional_setitem_indexer(indexer)
 
+    def _is_scalar_access(self, key):
+        # this is a shortcut accessor to both .loc and .iloc
+        # that provide the equivalent access of .at and .iat
+        # a) avoid getting things via sections and (to minimize dtype changes)
+        # b) provide a performant path
+        if not hasattr(key, '__len__'):
+            return False
+
+        if len(key) != self.ndim:
+            return False
+
+        for i, k in enumerate(key):
+            if not is_integer(k):
+                return False
+
+            ax = self.obj.axes[i]
+            if not ax.is_unique:
+                return False
+
+        return True
+
+    def _getitem_scalar(self, key):
+        # a fast-path to scalar access
+        # if not, raise
+        values = self.obj.get_value(*key, takeable=True)
+        return values
+
     def _is_valid_integer(self, key, axis):
         # return a boolean if we have a valid integer indexer
 

pandas/core/series.py

@@ -237,7 +237,8 @@ def __init__(self, data=None, index=None, dtype=None, name=None,
             # create/copy the manager
             if isinstance(data, SingleBlockManager):
                 if dtype is not None:
-                    data = data.astype(dtype=dtype, raise_on_error=False)
+                    data = data.astype(dtype=dtype, raise_on_error=False,
+                                       copy=copy)
                 elif copy:
                     data = data.copy()
             else:

pandas/indexes/api.py

@@ -4,7 +4,7 @@
 from pandas.indexes.category import CategoricalIndex  # noqa
 from pandas.indexes.multi import MultiIndex  # noqa
 from pandas.indexes.numeric import (NumericIndex, Float64Index,  # noqa
-                                    Int64Index)
+                                    Int64Index, UInt64Index)
 from pandas.indexes.range import RangeIndex  # noqa
 
 import pandas.core.common as com
@@ -13,7 +13,7 @@
 # TODO: there are many places that rely on these private methods existing in
 # pandas.core.index
 __all__ = ['Index', 'MultiIndex', 'NumericIndex', 'Float64Index', 'Int64Index',
-           'CategoricalIndex', 'RangeIndex',
+           'CategoricalIndex', 'RangeIndex', 'UInt64Index',
            'InvalidIndexError',
            '_new_Index',
            '_ensure_index', '_get_na_value', '_get_combined_index',