Merge branch 'master' into format - fix CI fail pandas-dev#34835

Author: Matteo Felici

asv_bench/benchmarks/pandas_vb_common.py

@@ -33,7 +33,7 @@
     np.uint8,
 ]
 datetime_dtypes = [np.datetime64, np.timedelta64]
-string_dtypes = [np.object]
+string_dtypes = [object]
 try:
     extension_dtypes = [
         pd.Int8Dtype,

asv_bench/benchmarks/series_methods.py

@@ -58,17 +58,15 @@ def time_isin_nan_values(self):
 
 class IsInForObjects:
     def setup(self):
-        self.s_nans = Series(np.full(10 ** 4, np.nan)).astype(np.object)
-        self.vals_nans = np.full(10 ** 4, np.nan).astype(np.object)
-        self.s_short = Series(np.arange(2)).astype(np.object)
-        self.s_long = Series(np.arange(10 ** 5)).astype(np.object)
-        self.vals_short = np.arange(2).astype(np.object)
-        self.vals_long = np.arange(10 ** 5).astype(np.object)
+        self.s_nans = Series(np.full(10 ** 4, np.nan)).astype(object)
+        self.vals_nans = np.full(10 ** 4, np.nan).astype(object)
+        self.s_short = Series(np.arange(2)).astype(object)
+        self.s_long = Series(np.arange(10 ** 5)).astype(object)
+        self.vals_short = np.arange(2).astype(object)
+        self.vals_long = np.arange(10 ** 5).astype(object)
         # because of nans floats are special:
-        self.s_long_floats = Series(np.arange(10 ** 5, dtype=np.float)).astype(
-            np.object
-        )
-        self.vals_long_floats = np.arange(10 ** 5, dtype=np.float).astype(np.object)
+        self.s_long_floats = Series(np.arange(10 ** 5, dtype=np.float)).astype(object)
+        self.vals_long_floats = np.arange(10 ** 5, dtype=np.float).astype(object)
 
     def time_isin_nans(self):
         # if nan-objects are different objects,

asv_bench/benchmarks/sparse.py

@@ -32,7 +32,7 @@ def time_series_to_frame(self):
 
 class SparseArrayConstructor:
 
-    params = ([0.1, 0.01], [0, np.nan], [np.int64, np.float64, np.object])
+    params = ([0.1, 0.01], [0, np.nan], [np.int64, np.float64, object])
     param_names = ["dense_proportion", "fill_value", "dtype"]
 
     def setup(self, dense_proportion, fill_value, dtype):

doc/source/user_guide/io.rst

@@ -1884,7 +1884,7 @@ Fallback behavior
 If the JSON serializer cannot handle the container contents directly it will
 fall back in the following manner:
 
-* if the dtype is unsupported (e.g. ``np.complex``) then the ``default_handler``, if provided, will be called
+* if the dtype is unsupported (e.g. ``np.complex_``) then the ``default_handler``, if provided, will be called
   for each value, otherwise an exception is raised.
 
 * if an object is unsupported it will attempt the following:

doc/source/whatsnew/v1.0.5.rst

@@ -1,7 +1,7 @@
 
 .. _whatsnew_105:
 
-What's new in 1.0.5 (June XX, 2020)
+What's new in 1.0.5 (June 17, 2020)
 -----------------------------------
 
 These are the changes in pandas 1.0.5. See :ref:`release` for a full changelog

doc/source/whatsnew/v1.1.0.rst

@@ -292,7 +292,7 @@ Other enhancements
 - :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` now accept an ``errors`` argument (:issue:`22610`)
 - :meth:`groupby.transform` now allows ``func`` to be ``pad``, ``backfill`` and ``cumcount`` (:issue:`31269`).
 - :meth:`~pandas.io.json.read_json` now accepts `nrows` parameter. (:issue:`33916`).
-- :meth `~pandas.io.gbq.read_gbq` now allows to disable progress bar (:issue:`33360`).
+- :meth:`~pandas.io.gbq.read_gbq` now allows to disable progress bar (:issue:`33360`).
 - :meth:`~pandas.io.gbq.read_gbq` now supports the ``max_results`` kwarg from ``pandas-gbq`` (:issue:`34639`).
 
 .. ---------------------------------------------------------------------------
@@ -843,6 +843,9 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.intersection` and :meth:`TimedeltaIndex.intersection` with results not having the correct ``name`` attribute (:issue:`33904`)
 - Bug in :meth:`DatetimeArray.__setitem__`, :meth:`TimedeltaArray.__setitem__`, :meth:`PeriodArray.__setitem__` incorrectly allowing values with ``int64`` dtype to be silently cast (:issue:`33717`)
 - Bug in subtracting :class:`TimedeltaIndex` from :class:`Period` incorrectly raising ``TypeError`` in some cases where it should succeed and ``IncompatibleFrequency`` in some cases where it should raise ``TypeError`` (:issue:`33883`)
+- Bug in constructing a Series or Index from a read-only NumPy array with non-ns
+  resolution which converted to object dtype instead of coercing to ``datetime64[ns]``
+  dtype when within the timestamp bounds (:issue:`34843`).
 - The ``freq`` keyword in :class:`Period`, :func:`date_range`, :func:`period_range`, :func:`pd.tseries.frequencies.to_offset` no longer allows tuples, pass as string instead (:issue:`34703`)
 
 Timedelta

pandas/_libs/hashtable_class_helper.pxi.in

@@ -178,7 +178,7 @@ cdef class StringVector:
             Py_ssize_t n
             object val
 
-        ao = np.empty(self.data.n, dtype=np.object)
+        ao = np.empty(self.data.n, dtype=object)
         for i in range(self.data.n):
             val = self.data.data[i]
             ao[i] = val

pandas/_libs/hashtable_func_helper.pxi.in

@@ -94,7 +94,7 @@ cpdef value_count_{{dtype}}({{c_type}}[:] values, bint dropna):
     build_count_table_{{dtype}}(values, table, dropna)
     {{endif}}
 
-    result_keys = np.empty(table.n_occupied, dtype=np.{{dtype}})
+    result_keys = np.empty(table.n_occupied, '{{dtype}}')
     result_counts = np.zeros(table.n_occupied, dtype=np.int64)
 
     {{if dtype == 'object'}}

pandas/_libs/parsers.pyx

@@ -2037,7 +2037,7 @@ def _concatenate_chunks(list chunks):
         numpy_dtypes = {x for x in dtypes if not is_categorical_dtype(x)}
         if len(numpy_dtypes) > 1:
             common_type = np.find_common_type(numpy_dtypes, [])
-            if common_type == np.object:
+            if common_type == object:
                 warning_columns.append(str(name))
 
         dtype = dtypes.pop()

pandas/_libs/sparse.pyx

@@ -791,4 +791,4 @@ def make_mask_object_ndarray(ndarray[object, ndim=1] arr, object fill_value):
         if value == fill_value and type(value) == type(fill_value):
             mask[i] = 0
 
-    return mask.view(dtype=np.bool)
+    return mask.view(dtype=bool)

pandas/_libs/testing.pyx

@@ -11,7 +11,7 @@ cdef NUMERIC_TYPES = (
     bool,
     int,
     float,
-    np.bool,
+    np.bool_,
     np.int8,
     np.int16,
     np.int32,

pandas/_libs/tslibs/conversion.pyx

@@ -167,7 +167,8 @@ def ensure_datetime64ns(arr: ndarray, copy: bool=True):
     """
     cdef:
         Py_ssize_t i, n = arr.size
-        int64_t[:] ivalues, iresult
+        const int64_t[:] ivalues
+        int64_t[:] iresult
         NPY_DATETIMEUNIT unit
         npy_datetimestruct dts
 

pandas/core/algorithms.py

@@ -171,7 +171,7 @@ def _ensure_data(
         return values, dtype
 
     # we have failed, return object
-    values = np.asarray(values, dtype=np.object)
+    values = np.asarray(values, dtype=object)
     return ensure_object(values), np.dtype("object")
 
 

pandas/core/arrays/sparse/array.py

@@ -150,7 +150,7 @@ def _sparse_array_op(
             # to make template simple, cast here
             left_sp_values = left.sp_values.view(np.uint8)
             right_sp_values = right.sp_values.view(np.uint8)
-            result_dtype = np.bool
+            result_dtype = bool
         else:
             opname = f"sparse_{name}_{dtype}"
             left_sp_values = left.sp_values
@@ -183,7 +183,7 @@ def _wrap_result(name, data, sparse_index, fill_value, dtype=None):
         name = name[2:-2]
 
     if name in ("eq", "ne", "lt", "gt", "le", "ge"):
-        dtype = np.bool
+        dtype = bool
 
     fill_value = lib.item_from_zerodim(fill_value)
 

pandas/core/base.py

@@ -1520,7 +1520,7 @@ def drop_duplicates(self, keep="first"):
     def duplicated(self, keep="first"):
         if isinstance(self, ABCIndexClass):
             if self.is_unique:
-                return np.zeros(len(self), dtype=np.bool)
+                return np.zeros(len(self), dtype=bool)
             return duplicated(self, keep=keep)
         else:
             return self._constructor(

pandas/core/dtypes/cast.py

@@ -225,7 +225,7 @@ def trans(x):
             # if we have any nulls, then we are done
             return result
 
-        elif not isinstance(r[0], (np.integer, np.floating, np.bool, int, float, bool)):
+        elif not isinstance(r[0], (np.integer, np.floating, int, float, bool)):
             # a comparable, e.g. a Decimal may slip in here
             return result
 
@@ -315,7 +315,7 @@ def maybe_cast_result_dtype(dtype: DtypeObj, how: str) -> DtypeObj:
     from pandas.core.arrays.boolean import BooleanDtype
     from pandas.core.arrays.integer import Int64Dtype
 
-    if how in ["add", "cumsum", "sum"] and (dtype == np.dtype(np.bool)):
+    if how in ["add", "cumsum", "sum"] and (dtype == np.dtype(bool)):
         return np.dtype(np.int64)
     elif how in ["add", "cumsum", "sum"] and isinstance(dtype, BooleanDtype):
         return Int64Dtype()
@@ -597,7 +597,7 @@ def _ensure_dtype_type(value, dtype):
     """
     Ensure that the given value is an instance of the given dtype.
 
-    e.g. if out dtype is np.complex64, we should have an instance of that
+    e.g. if out dtype is np.complex64_, we should have an instance of that
     as opposed to a python complex object.
 
     Parameters
@@ -1483,7 +1483,7 @@ def find_common_type(types: List[DtypeObj]) -> DtypeObj:
     if has_bools:
         for t in types:
             if is_integer_dtype(t) or is_float_dtype(t) or is_complex_dtype(t):
-                return np.object
+                return object
 
     return np.find_common_type(types, [])
 
@@ -1742,7 +1742,7 @@ def validate_numeric_casting(dtype: np.dtype, value):
         if is_float(value) and np.isnan(value):
             raise ValueError("Cannot assign nan to integer series")
 
-    if issubclass(dtype.type, (np.integer, np.floating, np.complex)) and not issubclass(
+    if issubclass(dtype.type, (np.integer, np.floating, complex)) and not issubclass(
         dtype.type, np.bool_
     ):
         if is_bool(value):

pandas/core/dtypes/common.py

@@ -1354,7 +1354,7 @@ def is_bool_dtype(arr_or_dtype) -> bool:
     False
     >>> is_bool_dtype(bool)
     True
-    >>> is_bool_dtype(np.bool)
+    >>> is_bool_dtype(np.bool_)
     True
     >>> is_bool_dtype(np.array(['a', 'b']))
     False
@@ -1526,7 +1526,7 @@ def is_complex_dtype(arr_or_dtype) -> bool:
     False
     >>> is_complex_dtype(int)
     False
-    >>> is_complex_dtype(np.complex)
+    >>> is_complex_dtype(np.complex_)
     True
     >>> is_complex_dtype(np.array(['a', 'b']))
     False

pandas/core/generic.py

@@ -10024,7 +10024,7 @@ def describe(
 
         Including only string columns in a ``DataFrame`` description.
 
-        >>> df.describe(include=[np.object])  # doctest: +SKIP
+        >>> df.describe(include=[object])  # doctest: +SKIP
                object
         count       3
         unique      3
@@ -10051,7 +10051,7 @@ def describe(
 
         Excluding object columns from a ``DataFrame`` description.
 
-        >>> df.describe(exclude=[np.object])  # doctest: +SKIP
+        >>> df.describe(exclude=[object])  # doctest: +SKIP
                categorical  numeric
         count            3      3.0
         unique           3      NaN

pandas/core/groupby/groupby.py

@@ -1267,9 +1267,9 @@ def objs_to_bool(vals: np.ndarray) -> Tuple[np.ndarray, Type]:
             if is_object_dtype(vals):
                 vals = np.array([bool(x) for x in vals])
             else:
-                vals = vals.astype(np.bool)
+                vals = vals.astype(bool)
 
-            return vals.view(np.uint8), np.bool
+            return vals.view(np.uint8), bool
 
         def result_to_bool(result: np.ndarray, inference: Type) -> np.ndarray:
             return result.astype(inference, copy=False)
@@ -2059,7 +2059,7 @@ def pre_processor(vals: np.ndarray) -> Tuple[np.ndarray, Optional[Type]]:
                 vals = vals.to_numpy(dtype=float, na_value=np.nan)
             elif is_datetime64_dtype(vals.dtype):
                 inference = "datetime64[ns]"
-                vals = np.asarray(vals).astype(np.float)
+                vals = np.asarray(vals).astype(float)
 
             return vals, inference
 

pandas/core/indexes/base.py

@@ -374,7 +374,7 @@ def __new__(
                 return UInt64Index(data, copy=copy, dtype=dtype, name=name)
             elif is_float_dtype(data.dtype):
                 return Float64Index(data, copy=copy, dtype=dtype, name=name)
-            elif issubclass(data.dtype.type, np.bool) or is_bool_dtype(data):
+            elif issubclass(data.dtype.type, bool) or is_bool_dtype(data):
                 subarr = data.astype("object")
             else:
                 subarr = com.asarray_tuplesafe(data, dtype=object)

pandas/core/internals/managers.py

@@ -1951,7 +1951,7 @@ def _check_comparison_types(
     if isinstance(result, np.ndarray):
         # The shape of the mask can differ to that of the result
         # since we may compare only a subset of a's or b's elements
-        tmp = np.zeros(mask.shape, dtype=np.bool)
+        tmp = np.zeros(mask.shape, dtype=np.bool_)
         tmp[mask] = result
         result = tmp
 

pandas/core/util/hashing.py

@@ -264,7 +264,7 @@ def hash_array(
 
     # First, turn whatever array this is into unsigned 64-bit ints, if we can
     # manage it.
-    elif isinstance(dtype, np.bool):
+    elif isinstance(dtype, bool):
         vals = vals.astype("u8")
     elif issubclass(dtype.type, (np.datetime64, np.timedelta64)):
         vals = vals.view("i8").astype("u8", copy=False)

pandas/core/window/rolling.py

@@ -922,16 +922,19 @@ class Window(_Window):
     * ``blackmanharris``
     * ``nuttall``
     * ``barthann``
-    * ``kaiser`` (needs beta)
-    * ``gaussian`` (needs std)
-    * ``general_gaussian`` (needs power, width)
-    * ``slepian`` (needs width)
-    * ``exponential`` (needs tau), center is set to None.
+    * ``kaiser`` (needs parameter: beta)
+    * ``gaussian`` (needs parameter: std)
+    * ``general_gaussian`` (needs parameters: power, width)
+    * ``slepian`` (needs parameter: width)
+    * ``exponential`` (needs parameter: tau), center is set to None.
 
     If ``win_type=None`` all points are evenly weighted. To learn more about
     different window types see `scipy.signal window functions
     <https://docs.scipy.org/doc/scipy/reference/signal.html#window-functions>`__.
 
+    Certain window types require additional parameters to be passed. Please see
+    the third example below on how to add the additional parameters.
+
     Examples
     --------
     >>> df = pd.DataFrame({'B': [0, 1, 2, np.nan, 4]})

pandas/io/parsers.py

@@ -3476,13 +3476,13 @@ def _get_empty_meta(columns, index_col, index_names, dtype=None):
     # This will enable us to write `dtype[col_name]`
     # without worrying about KeyError issues later on.
     if not isinstance(dtype, dict):
-        # if dtype == None, default will be np.object.
-        default_dtype = dtype or np.object
+        # if dtype == None, default will be object.
+        default_dtype = dtype or object
         dtype = defaultdict(lambda: default_dtype)
     else:
         # Save a copy of the dictionary.
         _dtype = dtype.copy()
-        dtype = defaultdict(lambda: np.object)
+        dtype = defaultdict(lambda: object)
 
         # Convert column indexes to column names.
         for k, v in _dtype.items():

pandas/io/sas/sas7bdat.py

@@ -685,7 +685,7 @@ def read(self, nrows=None):
         nd = self._column_types.count(b"d")
         ns = self._column_types.count(b"s")
 
-        self._string_chunk = np.empty((ns, nrows), dtype=np.object)
+        self._string_chunk = np.empty((ns, nrows), dtype=object)
         self._byte_chunk = np.zeros((nd, 8 * nrows), dtype=np.uint8)
 
         self._current_row_in_chunk_index = 0

pandas/io/stata.py

@@ -322,7 +322,7 @@ def convert_delta_safe(base, deltas, unit) -> Series:
     elif fmt.startswith(("%tC", "tC")):
 
         warnings.warn("Encountered %tC format. Leaving in Stata Internal Format.")
-        conv_dates = Series(dates, dtype=np.object)
+        conv_dates = Series(dates, dtype=object)
         if has_bad_values:
             conv_dates[bad_locs] = NaT
         return conv_dates
@@ -451,7 +451,7 @@ def g(x: datetime.datetime) -> int:
         conv_dates = 4 * (d.year - stata_epoch.year) + (d.month - 1) // 3
     elif fmt in ["%th", "th"]:
         d = parse_dates_safe(dates, year=True)
-        conv_dates = 2 * (d.year - stata_epoch.year) + (d.month > 6).astype(np.int)
+        conv_dates = 2 * (d.year - stata_epoch.year) + (d.month > 6).astype(int)
     elif fmt in ["%ty", "ty"]:
         d = parse_dates_safe(dates, year=True)
         conv_dates = d.year
@@ -553,7 +553,7 @@ def _cast_to_stata_types(data: DataFrame) -> DataFrame:
     ws = ""
     #                  original, if small, if large
     conversion_data = (
-        (np.bool, np.int8, np.int8),
+        (np.bool_, np.int8, np.int8),
         (np.uint8, np.int8, np.int16),
         (np.uint16, np.int16, np.int32),
         (np.uint32, np.int32, np.int64),
@@ -1725,7 +1725,7 @@ def _do_convert_missing(self, data: DataFrame, convert_missing: bool) -> DataFra
             if convert_missing:  # Replacement follows Stata notation
                 missing_loc = np.nonzero(np.asarray(missing))[0]
                 umissing, umissing_loc = np.unique(series[missing], return_inverse=True)
-                replacement = Series(series, dtype=np.object)
+                replacement = Series(series, dtype=object)
                 for j, um in enumerate(umissing):
                     missing_value = StataMissingValue(um)
 

pandas/plotting/_matplotlib/tools.py

@@ -301,7 +301,7 @@ def _handle_shared_axes(axarr, nplots, naxes, nrows, ncols, sharex, sharey):
             try:
                 # first find out the ax layout,
                 # so that we can correctly handle 'gaps"
-                layout = np.zeros((nrows + 1, ncols + 1), dtype=np.bool)
+                layout = np.zeros((nrows + 1, ncols + 1), dtype=np.bool_)
                 for ax in axarr:
                     layout[row_num(ax), col_num(ax)] = ax.get_visible()