Merge branch 'master' of https://github.com/pandas-dev/pandas into issues2

Author: jbrockmendel

doc/source/whatsnew/v0.24.0.txt

@@ -16,7 +16,7 @@ New features
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 A ``Series`` based on an ``ExtensionArray`` now supports arithmetic and comparison
-operators. (:issue:`19577`). There are two approaches for providing operator support for an ``ExtensionArray``:
+operators (:issue:`19577`). There are two approaches for providing operator support for an ``ExtensionArray``:
 
 1. Define each of the operators on your ``ExtensionArray`` subclass.
 2. Use an operator implementation from pandas that depends on operators that are already defined
@@ -110,7 +110,7 @@ Other Enhancements
   <https://pandas-gbq.readthedocs.io/en/latest/changelog.html#changelog-0-5-0>`__.
   (:issue:`21627`)
 - New method :meth:`HDFStore.walk` will recursively walk the group hierarchy of an HDF5 file (:issue:`10932`)
-- :func:`read_html` copies cell data across ``colspan``s and ``rowspan``s, and it treats all-``th`` table rows as headers if ``header`` kwarg is not given and there is no ``thead`` (:issue:`17054`)
+- :func:`read_html` copies cell data across ``colspan`` and ``rowspan``, and it treats all-``th`` table rows as headers if ``header`` kwarg is not given and there is no ``thead`` (:issue:`17054`)
 - :meth:`Series.nlargest`, :meth:`Series.nsmallest`, :meth:`DataFrame.nlargest`, and :meth:`DataFrame.nsmallest` now accept the value ``"all"`` for the ``keep`` argument. This keeps all ties for the nth largest/smallest value (:issue:`16818`)
 - :class:`IntervalIndex` has gained the :meth:`~IntervalIndex.set_closed` method to change the existing ``closed`` value (:issue:`21670`)
 - :func:`~DataFrame.to_csv` and :func:`~DataFrame.to_json` now support ``compression='infer'`` to infer compression based on filename (:issue:`15008`)
@@ -389,7 +389,7 @@ Timezones
 - Bug in :class:`Timestamp` when passing different string date formats with a timezone offset would produce different timezone offsets (:issue:`12064`)
 - Bug when comparing a tz-naive :class:`Timestamp` to a tz-aware :class:`DatetimeIndex` which would coerce the :class:`DatetimeIndex` to tz-naive (:issue:`12601`)
 - Bug in :meth:`Series.truncate` with a tz-aware :class:`DatetimeIndex` which would cause a core dump (:issue:`9243`)
-- Bug in :class:`Series` constructor which would coerce tz-aware and tz-naive :class:`Timestamp`s to tz-aware (:issue:`13051`)
+- Bug in :class:`Series` constructor which would coerce tz-aware and tz-naive :class:`Timestamp` to tz-aware (:issue:`13051`)
 - Bug in :class:`Index` with ``datetime64[ns, tz]`` dtype that did not localize integer data correctly (:issue:`20964`)
 - Bug in :class:`DatetimeIndex` where constructing with an integer and tz would not localize correctly (:issue:`12619`)
 - Fixed bug where :meth:`DataFrame.describe` and :meth:`Series.describe` on tz-aware datetimes did not show `first` and `last` result (:issue:`21328`)
@@ -433,8 +433,7 @@ Indexing
 - ``DataFrame.__getitem__`` now accepts dictionaries and dictionary keys as list-likes of labels, consistently with ``Series.__getitem__`` (:issue:`21294`)
 - Fixed ``DataFrame[np.nan]`` when columns are non-unique (:issue:`21428`)
 - Bug when indexing :class:`DatetimeIndex` with nanosecond resolution dates and timezones (:issue:`11679`)
-
--
+- Bug where indexing with a Numpy array containing negative values would mutate the indexer (:issue:`21867`)
 
 Missing
 ^^^^^^^

pandas/_libs/groupby.pyx

@@ -1,21 +1,20 @@
 # -*- coding: utf-8 -*-
 # cython: profile=False
 
-cimport numpy as cnp
-import numpy as np
-
 cimport cython
+from cython cimport Py_ssize_t
 
-cnp.import_array()
+from libc.stdlib cimport malloc, free
 
+import numpy as np
 from numpy cimport (ndarray,
                     double_t,
                     int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t,
                     uint32_t, uint64_t, float32_t, float64_t)
 
-from libc.stdlib cimport malloc, free
 
 from util cimport numeric, get_nat
+
 from algos cimport (swap, TiebreakEnumType, TIEBREAK_AVERAGE, TIEBREAK_MIN,
                     TIEBREAK_MAX, TIEBREAK_FIRST, TIEBREAK_DENSE)
 from algos import take_2d_axis1_float64_float64, groupsort_indexer, tiebreakers
@@ -74,8 +73,8 @@ cdef inline float64_t kth_smallest_c(float64_t* a,
         double_t x, t
 
     l = 0
-    m = n -1
-    while (l<m):
+    m = n - 1
+    while l < m:
         x = a[k]
         i = l
         j = m

pandas/_libs/index.pyx

@@ -9,14 +9,12 @@ from cpython.slice cimport PySlice_Check
 import numpy as np
 cimport numpy as cnp
 from numpy cimport (ndarray, float64_t, int32_t,
-                    int64_t, uint8_t, uint64_t, intp_t)
+                    int64_t, uint8_t, uint64_t, intp_t,
+                    # Note: NPY_DATETIME, NPY_TIMEDELTA are only available
+                    # for cimport in cython>=0.27.3
+                    NPY_DATETIME, NPY_TIMEDELTA)
 cnp.import_array()
 
-cdef extern from "numpy/arrayobject.h":
-    # These can be cimported directly from numpy in cython>=0.27.3
-    cdef enum NPY_TYPES:
-        NPY_DATETIME
-        NPY_TIMEDELTA
 
 cimport util
 

pandas/_libs/interval.pyx

@@ -1,17 +1,22 @@
-cimport numpy as cnp
-import numpy as np
+# -*- coding: utf-8 -*-
+import numbers
+
+from cpython.object cimport (Py_EQ, Py_NE, Py_GT, Py_LT, Py_GE, Py_LE,
+                             PyObject_RichCompare)
 
-cimport util
 cimport cython
-import cython
+from cython cimport Py_ssize_t
+
+import numpy as np
 from numpy cimport ndarray
+
+
+cimport util
+
 from tslibs import Timestamp
 from tslibs.timezones cimport tz_compare
 
-from cpython.object cimport (Py_EQ, Py_NE, Py_GT, Py_LT, Py_GE, Py_LE,
-                             PyObject_RichCompare)
 
-import numbers
 _VALID_CLOSED = frozenset(['left', 'right', 'both', 'neither'])
 
 

pandas/_libs/lib.pyx

@@ -753,4 +753,4 @@ def indices_fast(object index, ndarray[int64_t] labels, list keys,
     return result
 
 
-include "inference.pyx"
+include "src/inference.pyx"

pandas/_libs/src/datetime/np_datetime.c

@@ -28,9 +28,9 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt
 #define PyInt_AsLong PyLong_AsLong
 #endif
 
-const pandas_datetimestruct _NS_MIN_DTS = {
+const npy_datetimestruct _NS_MIN_DTS = {
     1677, 9, 21, 0, 12, 43, 145225, 0, 0};
-const pandas_datetimestruct _NS_MAX_DTS = {
+const npy_datetimestruct _NS_MAX_DTS = {
     2262, 4, 11, 23, 47, 16, 854775, 807000, 0};
 
 
@@ -62,7 +62,7 @@ int dayofweek(int y, int m, int d) {
  * Adjusts a datetimestruct based on a minutes offset. Assumes
  * the current values are valid.g
  */
-void add_minutes_to_datetimestruct(pandas_datetimestruct *dts, int minutes) {
+void add_minutes_to_datetimestruct(npy_datetimestruct *dts, int minutes) {
     int isleap;
 
     /* MINUTES */
@@ -111,7 +111,7 @@ void add_minutes_to_datetimestruct(pandas_datetimestruct *dts, int minutes) {
 /*
  * Calculates the days offset from the 1970 epoch.
  */
-npy_int64 get_datetimestruct_days(const pandas_datetimestruct *dts) {
+npy_int64 get_datetimestruct_days(const npy_datetimestruct *dts) {
     int i, month;
     npy_int64 year, days = 0;
     const int *month_lengths;
@@ -211,7 +211,7 @@ static npy_int64 days_to_yearsdays(npy_int64 *days_) {
  * Adjusts a datetimestruct based on a seconds offset. Assumes
  * the current values are valid.
  */
-NPY_NO_EXPORT void add_seconds_to_datetimestruct(pandas_datetimestruct *dts,
+NPY_NO_EXPORT void add_seconds_to_datetimestruct(npy_datetimestruct *dts,
                                                  int seconds) {
     int minutes;
 
@@ -236,7 +236,7 @@ NPY_NO_EXPORT void add_seconds_to_datetimestruct(pandas_datetimestruct *dts,
  * offset from 1970.
  */
 static void set_datetimestruct_days(npy_int64 days,
-                                    pandas_datetimestruct *dts) {
+                                    npy_datetimestruct *dts) {
     const int *month_lengths;
     int i;
 
@@ -255,10 +255,10 @@ static void set_datetimestruct_days(npy_int64 days,
 }
 
 /*
- * Compares two pandas_datetimestruct objects chronologically
+ * Compares two npy_datetimestruct objects chronologically
  */
-int cmp_pandas_datetimestruct(const pandas_datetimestruct *a,
-                              const pandas_datetimestruct *b) {
+int cmp_npy_datetimestruct(const npy_datetimestruct *a,
+                           const npy_datetimestruct *b) {
     if (a->year > b->year) {
         return 1;
     } else if (a->year < b->year) {
@@ -319,7 +319,7 @@ int cmp_pandas_datetimestruct(const pandas_datetimestruct *a,
 /*
  *
  * Tests for and converts a Python datetime.datetime or datetime.date
- * object into a NumPy pandas_datetimestruct.  Uses tzinfo (if present)
+ * object into a NumPy npy_datetimestruct.  Uses tzinfo (if present)
  * to convert to UTC time.
  *
  * While the C API has PyDate_* and PyDateTime_* functions, the following
@@ -331,12 +331,12 @@ int cmp_pandas_datetimestruct(const pandas_datetimestruct *a,
  * if obj doesn't have the needed date or datetime attributes.
  */
 int convert_pydatetime_to_datetimestruct(PyObject *obj,
-                                         pandas_datetimestruct *out) {
+                                         npy_datetimestruct *out) {
     PyObject *tmp;
     int isleap;
 
     /* Initialize the output to all zeros */
-    memset(out, 0, sizeof(pandas_datetimestruct));
+    memset(out, 0, sizeof(npy_datetimestruct));
     out->month = 1;
     out->day = 1;
 
@@ -512,16 +512,16 @@ int convert_pydatetime_to_datetimestruct(PyObject *obj,
     return -1;
 }
 
-npy_datetime pandas_datetimestruct_to_datetime(NPY_DATETIMEUNIT fr,
-                                               pandas_datetimestruct *d) {
+npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT fr,
+                                            npy_datetimestruct *d) {
     npy_datetime result = NPY_DATETIME_NAT;
 
     convert_datetimestruct_to_datetime(fr, d, &result);
     return result;
 }
 
 void pandas_datetime_to_datetimestruct(npy_datetime val, NPY_DATETIMEUNIT fr,
-                                       pandas_datetimestruct *result) {
+                                       npy_datetimestruct *result) {
     convert_datetime_to_datetimestruct(fr, val, result);
 }
 
@@ -539,7 +539,7 @@ void pandas_timedelta_to_timedeltastruct(npy_timedelta val,
  * Returns 0 on success, -1 on failure.
  */
 int convert_datetimestruct_to_datetime(NPY_DATETIMEUNIT base,
-                                       const pandas_datetimestruct *dts,
+                                       const npy_datetimestruct *dts,
                                        npy_datetime *out) {
     npy_datetime ret;
 
@@ -643,11 +643,11 @@ int convert_datetimestruct_to_datetime(NPY_DATETIMEUNIT base,
  */
 int convert_datetime_to_datetimestruct(NPY_DATETIMEUNIT base,
                                        npy_datetime dt,
-                                       pandas_datetimestruct *out) {
+                                       npy_datetimestruct *out) {
     npy_int64 perday;
 
     /* Initialize the output to all zeros */
-    memset(out, 0, sizeof(pandas_datetimestruct));
+    memset(out, 0, sizeof(npy_datetimestruct));
     out->year = 1970;
     out->month = 1;
     out->day = 1;

pandas/_libs/src/datetime/np_datetime.h

@@ -19,30 +19,25 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt
 
 #include <numpy/ndarraytypes.h>
 
-typedef struct {
-        npy_int64 year;
-        npy_int32 month, day, hour, min, sec, us, ps, as;
-} pandas_datetimestruct;
-
 typedef struct {
         npy_int64 days;
         npy_int32 hrs, min, sec, ms, us, ns, seconds, microseconds, nanoseconds;
 } pandas_timedeltastruct;
 
-extern const pandas_datetimestruct _NS_MIN_DTS;
-extern const pandas_datetimestruct _NS_MAX_DTS;
+extern const npy_datetimestruct _NS_MIN_DTS;
+extern const npy_datetimestruct _NS_MAX_DTS;
 
 // stuff pandas needs
 // ----------------------------------------------------------------------------
 
 int convert_pydatetime_to_datetimestruct(PyObject *obj,
-                                         pandas_datetimestruct *out);
+                                         npy_datetimestruct *out);
 
-npy_datetime pandas_datetimestruct_to_datetime(NPY_DATETIMEUNIT fr,
-                                               pandas_datetimestruct *d);
+npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT fr,
+                                            npy_datetimestruct *d);
 
 void pandas_datetime_to_datetimestruct(npy_datetime val, NPY_DATETIMEUNIT fr,
-                                       pandas_datetimestruct *result);
+                                       npy_datetimestruct *result);
 
 void pandas_timedelta_to_timedeltastruct(npy_timedelta val,
                                          NPY_DATETIMEUNIT fr,
@@ -61,27 +56,27 @@ int is_leapyear(npy_int64 year);
  * Calculates the days offset from the 1970 epoch.
  */
 npy_int64
-get_datetimestruct_days(const pandas_datetimestruct *dts);
+get_datetimestruct_days(const npy_datetimestruct *dts);
 
 
 /*
- * Compares two pandas_datetimestruct objects chronologically
+ * Compares two npy_datetimestruct objects chronologically
  */
-int cmp_pandas_datetimestruct(const pandas_datetimestruct *a,
-                              const pandas_datetimestruct *b);
+int cmp_npy_datetimestruct(const npy_datetimestruct *a,
+                           const npy_datetimestruct *b);
 
 
 /*
  * Adjusts a datetimestruct based on a minutes offset. Assumes
  * the current values are valid.
  */
 void
-add_minutes_to_datetimestruct(pandas_datetimestruct *dts, int minutes);
+add_minutes_to_datetimestruct(npy_datetimestruct *dts, int minutes);
 
 
 int
 convert_datetime_to_datetimestruct(NPY_DATETIMEUNIT base,
                                    npy_datetime dt,
-                                   pandas_datetimestruct *out);
+                                   npy_datetimestruct *out);
 
 #endif  // PANDAS__LIBS_SRC_DATETIME_NP_DATETIME_H_

pandas/_libs/src/datetime/np_datetime_strings.c

@@ -63,7 +63,7 @@ This file implements string parsing and creation for NumPy datetime.
  * Returns 0 on success, -1 on failure.
  */
 int parse_iso_8601_datetime(char *str, int len,
-                            pandas_datetimestruct *out,
+                            npy_datetimestruct *out,
                             int *out_local, int *out_tzoffset) {
     int year_leap = 0;
     int i, numdigits;
@@ -86,7 +86,7 @@ int parse_iso_8601_datetime(char *str, int len,
     int hour_was_2_digits = 0;
 
     /* Initialize the output to all zeros */
-    memset(out, 0, sizeof(pandas_datetimestruct));
+    memset(out, 0, sizeof(npy_datetimestruct));
     out->month = 1;
     out->day = 1;
 
@@ -567,7 +567,7 @@ int get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base) {
 
 
 /*
- * Converts an pandas_datetimestruct to an (almost) ISO 8601
+ * Converts an npy_datetimestruct to an (almost) ISO 8601
  * NULL-terminated string using timezone Z (UTC). If the string fits in
  * the space exactly, it leaves out the NULL terminator and returns success.
  *
@@ -580,7 +580,7 @@ int get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base) {
  *  Returns 0 on success, -1 on failure (for example if the output
  *  string was too short).
  */
-int make_iso_8601_datetime(pandas_datetimestruct *dts, char *outstr, int outlen,
+int make_iso_8601_datetime(npy_datetimestruct *dts, char *outstr, int outlen,
                            NPY_DATETIMEUNIT base) {
     char *substr = outstr, sublen = outlen;
     int tmplen;

pandas/_libs/src/datetime/np_datetime_strings.h

@@ -51,9 +51,9 @@ This file implements string parsing and creation for NumPy datetime.
  */
 int
 parse_iso_8601_datetime(char *str, int len,
-                    pandas_datetimestruct *out,
-                    int *out_local,
-                    int *out_tzoffset);
+                        npy_datetimestruct *out,
+                        int *out_local,
+                        int *out_tzoffset);
 
 /*
  * Provides a string length to use for converting datetime
@@ -63,7 +63,7 @@ int
 get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base);
 
 /*
- * Converts an pandas_datetimestruct to an (almost) ISO 8601
+ * Converts an npy_datetimestruct to an (almost) ISO 8601
  * NULL-terminated string using timezone Z (UTC).
  *
  * 'base' restricts the output to that unit. Set 'base' to
@@ -73,7 +73,7 @@ get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base);
  *  string was too short).
  */
 int
-make_iso_8601_datetime(pandas_datetimestruct *dts, char *outstr, int outlen,
+make_iso_8601_datetime(npy_datetimestruct *dts, char *outstr, int outlen,
                        NPY_DATETIMEUNIT base);
 
 #endif  // PANDAS__LIBS_SRC_DATETIME_NP_DATETIME_STRINGS_H_