Merge remote-tracking branch 'upstream/master'

Author: m-charlton

asv_bench/benchmarks/period.py

@@ -49,3 +49,28 @@ def time_value_counts_pindex(self):
         self.i.value_counts()
 
 
+class period_standard_indexing(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.index = PeriodIndex(start='1985', periods=1000, freq='D')
+        self.series = Series(range(1000), index=self.index)
+        self.period = self.index[500]
+
+    def time_get_loc(self):
+        self.index.get_loc(self.period)
+
+    def time_shape(self):
+        self.index.shape
+
+    def time_shallow_copy(self):
+        self.index._shallow_copy()
+
+    def time_series_loc(self):
+        self.series.loc[self.period]
+
+    def time_align(self):
+        pd.DataFrame({'a': self.series, 'b': self.series[:500]})
+
+    def time_intersection(self):
+        self.index[:750].intersection(self.index[250:])

doc/source/timeseries.rst

@@ -358,8 +358,8 @@ See :ref:`here <timeseries.oob>` for ways to represent data outside these bound.
 
 .. _timeseries.datetimeindex:
 
-DatetimeIndex
--------------
+Indexing
+--------
 
 One of the main uses for ``DatetimeIndex`` is as an index for pandas objects.
 The ``DatetimeIndex`` class contains many timeseries related optimizations:
@@ -399,8 +399,8 @@ intelligent functionality like selection, slicing, etc.
 
 .. _timeseries.partialindexing:
 
-DatetimeIndex Partial String Indexing
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Partial String Indexing
+~~~~~~~~~~~~~~~~~~~~~~~
 
 You can pass in dates and strings that parse to dates as indexing parameters:
 
@@ -457,22 +457,6 @@ We are stopping on the included end-point as it is part of the index
 
    dft['2013-1-15':'2013-1-15 12:30:00']
 
-.. warning::
-
-   The following selection will raise a ``KeyError``; otherwise this selection methodology
-   would be inconsistent with other selection methods in pandas (as this is not a *slice*, nor does it
-   resolve to one)
-
-   .. code-block:: python
-
-      dft['2013-1-15 12:30:00']
-
-   To select a single row, use ``.loc``
-
-   .. ipython:: python
-
-      dft.loc['2013-1-15 12:30:00']
-
 .. versionadded:: 0.18.0
 
 DatetimeIndex Partial String Indexing also works on DataFrames with a ``MultiIndex``. For example:
@@ -491,12 +475,86 @@ DatetimeIndex Partial String Indexing also works on DataFrames with a ``MultiInd
    dft2 = dft2.swaplevel(0, 1).sort_index()
    dft2.loc[idx[:, '2013-01-05'], :]
 
-Datetime Indexing
-~~~~~~~~~~~~~~~~~
+.. _timeseries.slice_vs_exact_match:
+
+Slice vs. exact match
+~~~~~~~~~~~~~~~~~~~~~
+
+.. versionchanged:: 0.20.0
+
+The same string used as an indexing parameter can be treated either as a slice or as an exact match depending on the resolution of an index. If the string is less accurate than the index, it will be treated as a slice, otherwise as an exact match.
+
+For example, let us consider ``Series`` object which index has minute resolution.
+
+.. ipython:: python
+
+    series_minute = pd.Series([1, 2, 3],
+                              pd.DatetimeIndex(['2011-12-31 23:59:00',
+                                                '2012-01-01 00:00:00',
+                                                '2012-01-01 00:02:00']))
+    series_minute.index.resolution
+
+A Timestamp string less accurate than a minute gives a ``Series`` object.
+
+.. ipython:: python
+
+    series_minute['2011-12-31 23']
+
+A Timestamp string with minute resolution (or more accurate), gives a scalar instead, i.e. it is not casted to a slice.
+
+.. ipython:: python
+
+    series_minute['2011-12-31 23:59']
+    series_minute['2011-12-31 23:59:00']
+
+If index resolution is second, then, the minute-accurate timestamp gives a ``Series``.
 
-Indexing a ``DateTimeIndex`` with a partial string depends on the "accuracy" of the period, in other words how specific the interval is in relation to the frequency of the index. In contrast, indexing with datetime objects is exact, because the objects have exact meaning. These also follow the semantics of *including both endpoints*.
+.. ipython:: python
+
+    series_second = pd.Series([1, 2, 3],
+                              pd.DatetimeIndex(['2011-12-31 23:59:59',
+                                                '2012-01-01 00:00:00',
+                                                '2012-01-01 00:00:01']))
+    series_second.index.resolution
+    series_second['2011-12-31 23:59']
+
+If the timestamp string is treated as a slice, it can be used to index ``DataFrame`` with ``[]`` as well.
+
+.. ipython:: python
+
+    dft_minute = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]},
+                               index=series_minute.index)
+    dft_minute['2011-12-31 23']
+
+
+:: warning::
+
+   However if the string is treated as an exact match the selection in ``DataFrame``'s ``[]`` will be column-wise and not row-wise, see :ref:`Indexing Basics <indexing.basics>`. For example ``dft_minute['2011-12-31 23:59']`` will raise ``KeyError`` as ``'2012-12-31 23:59'`` has the same resolution as index and there is no column with such name:
+
+   To select a single row, use ``.loc``.
+
+   .. ipython:: python
+
+     dft_minute.loc['2011-12-31 23:59']
+
+Note also that ``DatetimeIndex`` resolution cannot be less precise than day.
 
-These ``datetime`` objects  are specific ``hours, minutes,`` and ``seconds`` even though they were not explicitly specified (they are ``0``).
+.. ipython:: python
+
+    series_monthly = pd.Series([1, 2, 3],
+                              pd.DatetimeIndex(['2011-12',
+                                                '2012-01',
+                                                '2012-02']))
+    series_monthly.index.resolution
+    series_monthly['2011-12'] # returns Series
+
+
+Exact Indexing
+~~~~~~~~~~~~~~
+
+As discussed in previous section, indexing a ``DateTimeIndex`` with a partial string depends on the "accuracy" of the period, in other words how specific the interval is in relation to the resolution of the index. In contrast, indexing with ``Timestamp`` or ``datetime`` objects is exact, because the objects have exact meaning. These also follow the semantics of *including both endpoints*.
+
+These ``Timestamp`` and ``datetime`` objects have exact ``hours, minutes,`` and ``seconds``, even though they were not explicitly specified (they are ``0``).
 
 .. ipython:: python
 
@@ -525,10 +583,10 @@ regularity will result in a ``DatetimeIndex`` (but frequency is lost):
 
    ts[[0, 2, 6]].index
 
-.. _timeseries.offsets:
+.. _timeseries.components:
 
 Time/Date Components
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+--------------------
 
 There are several time/date properties that one can access from ``Timestamp`` or a collection of timestamps like a ``DateTimeIndex``.
 
@@ -564,6 +622,8 @@ There are several time/date properties that one can access from ``Timestamp`` or
 
 Furthermore, if you have a ``Series`` with datetimelike values, then you can access these properties via the ``.dt`` accessor, see the :ref:`docs <basics.dt_accessors>`
 
+.. _timeseries.offsets:
+
 DateOffset objects
 ------------------
 
@@ -628,12 +688,12 @@ We could have done the same thing with ``DateOffset``:
 
 The key features of a ``DateOffset`` object are:
 
-  - it can be added / subtracted to/from a datetime object to obtain a
-    shifted date
-  - it can be multiplied by an integer (positive or negative) so that the
-    increment will be applied multiple times
-  - it has ``rollforward`` and ``rollback`` methods for moving a date forward
-    or backward to the next or previous "offset date"
+- it can be added / subtracted to/from a datetime object to obtain a
+  shifted date
+- it can be multiplied by an integer (positive or negative) so that the
+  increment will be applied multiple times
+- it has ``rollforward`` and ``rollback`` methods for moving a date forward
+  or backward to the next or previous "offset date"
 
 Subclasses of ``DateOffset`` define the ``apply`` function which dictates
 custom date increment logic, such as adding business days:
@@ -745,7 +805,7 @@ used exactly like a ``Timedelta`` - see the
 
 Note that some offsets (such as ``BQuarterEnd``) do not have a
 vectorized implementation.  They can still be used but may
-calculate significantly slower and will raise a ``PerformanceWarning``
+calculate significantly slower and will show a ``PerformanceWarning``
 
 .. ipython:: python
    :okwarning:
@@ -755,8 +815,8 @@ calculate significantly slower and will raise a ``PerformanceWarning``
 
 .. _timeseries.custombusinessdays:
 
-Custom Business Days (Experimental)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Custom Business Days
+~~~~~~~~~~~~~~~~~~~~
 
 The ``CDay`` or ``CustomBusinessDay`` class provides a parametric
 ``BusinessDay`` class which can be used to create customized business day
@@ -785,7 +845,7 @@ Let's map to the weekday names
 
     pd.Series(dts.weekday, dts).map(pd.Series('Mon Tue Wed Thu Fri Sat Sun'.split()))
 
-As of v0.14 holiday calendars can be used to provide the list of holidays.  See the
+Holiday calendars can be used to provide the list of holidays.  See the
 :ref:`holiday calendar<timeseries.holiday>` section for more information.
 
 .. ipython:: python
@@ -1289,12 +1349,15 @@ limited to, financial applications.
 See some :ref:`cookbook examples <cookbook.resample>` for some advanced strategies
 
 Starting in version 0.18.1, the ``resample()`` function can be used directly from
-DataFrameGroupBy objects, see the :ref:`groupby docs <groupby.transform.window_resample>`.
+``DataFrameGroupBy`` objects, see the :ref:`groupby docs <groupby.transform.window_resample>`.
 
 .. note::
 
    ``.resample()`` is similar to using a ``.rolling()`` operation with a time-based offset, see a discussion :ref:`here <stats.moments.ts-versus-resampling>`
 
+Basics
+~~~~~~
+
 .. ipython:: python
 
    rng = pd.date_range('1/1/2012', periods=100, freq='S')

doc/source/whatsnew/v0.19.2.txt

@@ -22,6 +22,7 @@ Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Improved performance of ``.replace()`` (:issue:`12745`)
+- Improved performance of ``PeriodIndex`` (:issue:`14822`)
 - Improved performance ``Series`` creation with a datetime index and dictionary data (:issue:`14894`)
 
 .. _whatsnew_0192.enhancements.other:

doc/source/whatsnew/v0.20.0.txt

@@ -193,14 +193,42 @@ in prior versions of pandas) (:issue:`11915`).
 
 .. _whatsnew_0200.api:
 
+Other API Changes
+^^^^^^^^^^^^^^^^^
+
 - ``CParserError`` has been renamed to ``ParserError`` in ``pd.read_csv`` and will be removed in the future (:issue:`12665`)
 - ``SparseArray.cumsum()`` and ``SparseSeries.cumsum()`` will now always return ``SparseArray`` and ``SparseSeries`` respectively (:issue:`12855`)
+- :ref:`DatetimeIndex Partial String Indexing <timeseries.partialindexing>` now works as exact match provided that string resolution coincides with index resolution, including a case when both are seconds (:issue:`14826`). See :ref:`Slice vs. Exact Match <timeseries.slice_vs_exact_match>` for details.
 
+  .. ipython:: python
 
+    df = DataFrame({'a': [1, 2, 3]}, DatetimeIndex(['2011-12-31 23:59:59',
+                                                    '2012-01-01 00:00:00',
+                                                    '2012-01-01 00:00:01']))
+  Previous Behavior:
 
+  .. code-block:: ipython
 
-Other API Changes
-^^^^^^^^^^^^^^^^^
+    In [4]: df['2011-12-31 23:59:59']
+    Out[4]:
+                         a
+    2011-12-31 23:59:59  1
+
+    In [5]: df['a']['2011-12-31 23:59:59']
+    Out[5]:
+    2011-12-31 23:59:59    1
+    Name: a, dtype: int64
+
+
+  New Behavior:
+
+  .. code-block:: ipython
+
+    In [4]: df['2011-12-31 23:59:59']
+    KeyError: '2011-12-31 23:59:59'
+
+    In [5]: df['a']['2011-12-31 23:59:59']
+    Out[5]: 1
 
 .. _whatsnew_0200.deprecations:
 
@@ -253,7 +281,7 @@ Bug Fixes
 
 
 
-
+- Bug in ``Series`` construction with a datetimetz (:issue:`14928`)
 
 
 

pandas/core/base.py

@@ -814,7 +814,7 @@ def transpose(self, *args, **kwargs):
     @property
     def shape(self):
         """ return a tuple of the shape of the underlying data """
-        return self.values.shape
+        return self._values.shape
 
     @property
     def ndim(self):
@@ -842,22 +842,22 @@ def data(self):
     @property
     def itemsize(self):
         """ return the size of the dtype of the item of the underlying data """
-        return self.values.itemsize
+        return self._values.itemsize
 
     @property
     def nbytes(self):
         """ return the number of bytes in the underlying data """
-        return self.values.nbytes
+        return self._values.nbytes
 
     @property
     def strides(self):
         """ return the strides of the underlying data """
-        return self.values.strides
+        return self._values.strides
 
     @property
     def size(self):
         """ return the number of elements in the underlying data """
-        return self.values.size
+        return self._values.size
 
     @property
     def flags(self):

pandas/core/ops.py

@@ -545,9 +545,9 @@ def _offset(lvalues, rvalues):
 
                 # with tz, convert to UTC
                 if self.is_datetime64tz_lhs:
-                    lvalues = lvalues.tz_localize(None)
+                    lvalues = lvalues.tz_convert('UTC').tz_localize(None)
                 if self.is_datetime64tz_rhs:
-                    rvalues = rvalues.tz_localize(None)
+                    rvalues = rvalues.tz_convert('UTC').tz_localize(None)
 
                 lvalues = lvalues.view(np.int64)
                 rvalues = rvalues.view(np.int64)

pandas/tests/groupby/test_filters.py

@@ -596,6 +596,19 @@ def test_filter_non_bool_raises(self):
         with tm.assertRaisesRegexp(TypeError, 'filter function returned a.*'):
             df.groupby('a').filter(lambda g: g.c.mean())
 
+    def test_filter_dropna_with_empty_groups(self):
+        # GH 10780
+        data = pd.Series(np.random.rand(9), index=np.repeat([1, 2, 3], 3))
+        groupped = data.groupby(level=0)
+        result_false = groupped.filter(lambda x: x.mean() > 1, dropna=False)
+        expected_false = pd.Series([np.nan] * 9,
+                                   index=np.repeat([1, 2, 3], 3))
+        tm.assert_series_equal(result_false, expected_false)
+
+        result_true = groupped.filter(lambda x: x.mean() > 1, dropna=True)
+        expected_true = pd.Series(index=pd.Index([], dtype=int))
+        tm.assert_series_equal(result_true, expected_true)
+
 
 def assert_fp_equal(a, b):
     assert (np.abs(a - b) < 1e-12).all()