Merge remote-tracking branch 'upstream/master' into hist_legend

Author: rhshadrach

ci/deps/azure-37-numpydev.yaml

@@ -14,7 +14,8 @@ dependencies:
   - pytz
   - pip
   - pip:
-    - cython>=0.29.16
+    - cython==0.29.16
+    # GH#33507 cython 3.0a1 is causing TypeErrors 2020-04-13
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"
     - "--pre"

doc/source/getting_started/intro_tutorials/03_subset_data.rst

@@ -23,7 +23,7 @@
                     <div class="card-body">
                         <p class="card-text">
 
-This tutorial uses the titanic data set, stored as CSV. The data
+This tutorial uses the Titanic data set, stored as CSV. The data
 consists of the following data columns:
 
 -  PassengerId: Id of every passenger.
@@ -72,7 +72,7 @@ How do I select specific columns from a ``DataFrame``?
     <ul class="task-bullet">
         <li>
 
-I’m interested in the age of the titanic passengers.
+I’m interested in the age of the Titanic passengers.
 
 .. ipython:: python
 
@@ -111,7 +111,7 @@ the number of rows is returned.
     <ul class="task-bullet">
         <li>
 
-I’m interested in the age and sex of the titanic passengers.
+I’m interested in the age and sex of the Titanic passengers.
 
 .. ipython:: python
 
@@ -198,7 +198,7 @@ can be used to filter the ``DataFrame`` by putting it in between the
 selection brackets ``[]``. Only rows for which the value is ``True``
 will be selected.
 
-We now from before that the original titanic ``DataFrame`` consists of
+We know from before that the original Titanic ``DataFrame`` consists of
 891 rows. Let’s have a look at the amount of rows which satisfy the
 condition by checking the ``shape`` attribute of the resulting
 ``DataFrame`` ``above_35``:
@@ -212,7 +212,7 @@ condition by checking the ``shape`` attribute of the resulting
     <ul class="task-bullet">
         <li>
 
-I’m interested in the titanic passengers from cabin class 2 and 3.
+I’m interested in the Titanic passengers from cabin class 2 and 3.
 
 .. ipython:: python
 

doc/source/user_guide/timeseries.rst

@@ -786,6 +786,15 @@ Furthermore, if you have a ``Series`` with datetimelike values, then you can
 access these properties via the ``.dt`` accessor, as detailed in the section
 on :ref:`.dt accessors<basics.dt_accessors>`.
 
+.. versionadded:: 1.1.0
+
+You may obtain the year, week and day components of the ISO year from the ISO 8601 standard:
+
+.. ipython:: python
+
+   idx = pd.date_range(start='2019-12-29', freq='D', periods=4)
+   idx.to_series().dt.isocalendar()
+
 .. _timeseries.offsets:
 
 DateOffset objects

doc/source/whatsnew/v1.1.0.rst

@@ -88,6 +88,7 @@ Other enhancements
 - :class:`Series.str` now has a `fullmatch` method that matches a regular expression against the entire string in each row of the series, similar to `re.fullmatch` (:issue:`32806`).
 - :meth:`DataFrame.sample` will now also allow array-like and BitGenerator objects to be passed to ``random_state`` as seeds (:issue:`32503`)
 - :meth:`MultiIndex.union` will now raise `RuntimeWarning` if the object inside are unsortable, pass `sort=False` to suppress this warning (:issue:`33015`)
+- :class:`Series.dt` and :class:`DatatimeIndex` now have an `isocalendar` method that returns a :class:`DataFrame` with year, week, and day calculated according to the ISO 8601 calendar (:issue:`33206`).
 - The :meth:`DataFrame.to_feather` method now supports additional keyword
   arguments (e.g. to set the compression) that are added in pyarrow 0.17
   (:issue:`33422`).
@@ -377,7 +378,7 @@ Performance improvements
   sparse values from ``scipy.sparse`` matrices using the
   :meth:`DataFrame.sparse.from_spmatrix` constructor (:issue:`32821`,
   :issue:`32825`,  :issue:`32826`, :issue:`32856`, :issue:`32858`).
-- Performance improvement in reductions (sum, min, max) for nullable (integer and boolean) dtypes (:issue:`30982`, :issue:`33261`).
+- Performance improvement in reductions (sum, prod, min, max) for nullable (integer and boolean) dtypes (:issue:`30982`, :issue:`33261`, :issue:`33442`).
 
 
 .. ---------------------------------------------------------------------------
@@ -396,6 +397,7 @@ Categorical
 - Bug where :class:`Categorical` comparison operator ``__ne__`` would incorrectly evaluate to ``False`` when either element was missing (:issue:`32276`)
 - :meth:`Categorical.fillna` now accepts :class:`Categorical` ``other`` argument (:issue:`32420`)
 - Bug where :meth:`Categorical.replace` would replace with ``NaN`` whenever the new value and replacement value were equal (:issue:`33288`)
+- Bug where an ordered :class:`Categorical` containing only ``NaN`` values would raise rather than returning ``NaN`` when taking the minimum or maximum  (:issue:`33450`)
 
 Datetimelike
 ^^^^^^^^^^^^
@@ -409,6 +411,8 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.searchsorted` not accepting a ``list`` or :class:`Series` as its argument (:issue:`32762`)
 - Bug where :meth:`PeriodIndex` raised when passed a :class:`Series` of strings (:issue:`26109`)
 - Bug in :class:`Timestamp` arithmetic when adding or subtracting a ``np.ndarray`` with ``timedelta64`` dtype (:issue:`33296`)
+- Bug in :meth:`DatetimeIndex.to_period` not infering the frequency when called with no arguments (:issue:`33358`)
+
 
 Timedelta
 ^^^^^^^^^
@@ -463,6 +467,7 @@ Indexing
 - Bug in :meth:`DatetimeIndex.get_loc` raising ``KeyError`` with converted-integer key instead of the user-passed key (:issue:`31425`)
 - Bug in :meth:`Series.xs` incorrectly returning ``Timestamp`` instead of ``datetime64`` in some object-dtype cases (:issue:`31630`)
 - Bug in :meth:`DataFrame.iat` incorrectly returning ``Timestamp`` instead of ``datetime`` in some object-dtype cases (:issue:`32809`)
+- Bug in :meth:`DataFrame.at` when either columns or index is non-unique (:issue:`33041`)
 - Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` when indexing with an integer key on a object-dtype :class:`Index` that is not all-integers (:issue:`31905`)
 - Bug in :meth:`DataFrame.iloc.__setitem__` on a :class:`DataFrame` with duplicate columns incorrectly setting values for all matching columns (:issue:`15686`, :issue:`22036`)
 - Bug in :meth:`DataFrame.loc:` and :meth:`Series.loc` with a :class:`DatetimeIndex`, :class:`TimedeltaIndex`, or :class:`PeriodIndex` incorrectly allowing lookups of non-matching datetime-like dtypes (:issue:`32650`)

pandas/_libs/lib.pyx

@@ -1,9 +1,6 @@
 from collections import abc
 from decimal import Decimal
-from fractions import Fraction
-from numbers import Number
 
-import sys
 import warnings
 
 import cython

pandas/_libs/reshape.pyx

@@ -36,7 +36,7 @@ ctypedef fused reshape_t:
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def unstack(reshape_t[:, :] values, uint8_t[:] mask,
+def unstack(reshape_t[:, :] values, const uint8_t[:] mask,
             Py_ssize_t stride, Py_ssize_t length, Py_ssize_t width,
             reshape_t[:, :] new_values, uint8_t[:, :] new_mask):
     """

pandas/_libs/tslibs/ccalendar.pxd

@@ -2,9 +2,11 @@ from cython cimport Py_ssize_t
 
 from numpy cimport int64_t, int32_t
 
+ctypedef (int32_t, int32_t, int32_t) iso_calendar_t
 
 cdef int dayofweek(int y, int m, int d) nogil
 cdef bint is_leapyear(int64_t year) nogil
 cpdef int32_t get_days_in_month(int year, Py_ssize_t month) nogil
 cpdef int32_t get_week_of_year(int year, int month, int day) nogil
+cpdef iso_calendar_t get_iso_calendar(int year, int month, int day) nogil
 cpdef int32_t get_day_of_year(int year, int month, int day) nogil

pandas/_libs/tslibs/ccalendar.pyx

@@ -150,33 +150,65 @@ cpdef int32_t get_week_of_year(int year, int month, int day) nogil:
     -------
     week_of_year : int32_t
 
+    Notes
+    -----
+    Assumes the inputs describe a valid date.
+    """
+    return get_iso_calendar(year, month, day)[1]
+
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+cpdef iso_calendar_t get_iso_calendar(int year, int month, int day) nogil:
+    """
+    Return the year, week, and day of year corresponding to ISO 8601
+
+    Parameters
+    ----------
+    year : int
+    month : int
+    day : int
+
+    Returns
+    -------
+    year : int32_t
+    week : int32_t
+    day : int32_t
+
     Notes
     -----
     Assumes the inputs describe a valid date.
     """
     cdef:
         int32_t doy, dow
-        int woy
+        int32_t iso_year, iso_week
 
     doy = get_day_of_year(year, month, day)
     dow = dayofweek(year, month, day)
 
     # estimate
-    woy = (doy - 1) - dow + 3
-    if woy >= 0:
-        woy = woy // 7 + 1
+    iso_week = (doy - 1) - dow + 3
+    if iso_week >= 0:
+        iso_week = iso_week // 7 + 1
 
     # verify
-    if woy < 0:
-        if (woy > -2) or (woy == -2 and is_leapyear(year - 1)):
-            woy = 53
+    if iso_week < 0:
+        if (iso_week > -2) or (iso_week == -2 and is_leapyear(year - 1)):
+            iso_week = 53
         else:
-            woy = 52
-    elif woy == 53:
+            iso_week = 52
+    elif iso_week == 53:
         if 31 - day + dow < 3:
-            woy = 1
+            iso_week = 1
+
+    iso_year = year
+    if iso_week == 1 and doy > 7:
+        iso_year += 1
+
+    elif iso_week >= 52 and doy < 7:
+        iso_year -= 1
 
-    return woy
+    return iso_year, iso_week, dow + 1
 
 
 @cython.wraparound(False)

pandas/_libs/tslibs/fields.pyx

@@ -8,14 +8,14 @@ from cython import Py_ssize_t
 
 import numpy as np
 cimport numpy as cnp
-from numpy cimport ndarray, int64_t, int32_t, int8_t
+from numpy cimport ndarray, int64_t, int32_t, int8_t, uint32_t
 cnp.import_array()
 
 from pandas._libs.tslibs.ccalendar import (
     get_locale_names, MONTHS_FULL, DAYS_FULL, DAY_SECONDS)
 from pandas._libs.tslibs.ccalendar cimport (
     get_days_in_month, is_leapyear, dayofweek, get_week_of_year,
-    get_day_of_year)
+    get_day_of_year, get_iso_calendar, iso_calendar_t)
 from pandas._libs.tslibs.np_datetime cimport (
     npy_datetimestruct, pandas_timedeltastruct, dt64_to_dtstruct,
     td64_to_tdstruct)
@@ -670,3 +670,42 @@ cpdef isleapyear_arr(ndarray years):
                       np.logical_and(years % 4 == 0,
                                      years % 100 > 0))] = 1
     return out.view(bool)
+
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def build_isocalendar_sarray(const int64_t[:] dtindex):
+    """
+    Given a int64-based datetime array, return the ISO 8601 year, week, and day
+    as a structured array.
+    """
+    cdef:
+        Py_ssize_t i, count = len(dtindex)
+        npy_datetimestruct dts
+        ndarray[uint32_t] iso_years, iso_weeks, days
+        iso_calendar_t ret_val
+
+    sa_dtype = [
+        ("year", "u4"),
+        ("week", "u4"),
+        ("day", "u4"),
+    ]
+
+    out = np.empty(count, dtype=sa_dtype)
+
+    iso_years = out["year"]
+    iso_weeks = out["week"]
+    days = out["day"]
+
+    with nogil:
+        for i in range(count):
+            if dtindex[i] == NPY_NAT:
+                ret_val = 0, 0, 0
+            else:
+                dt64_to_dtstruct(dtindex[i], &dts)
+                ret_val = get_iso_calendar(dts.year, dts.month, dts.day)
+
+            iso_years[i] = ret_val[0]
+            iso_weeks[i] = ret_val[1]
+            days[i] = ret_val[2]
+    return out