Merge remote-tracking branch 'upstream/master' into 32bit-ci

Author: fangchenli

.travis.yml

@@ -1,6 +1,15 @@
 language: python
 python: 3.7
 
+addons:
+  apt:
+    update: true
+    packages:
+      - xvfb
+
+services:
+  - xvfb
+
 # To turn off cached cython files and compiler cache
 # set NOCACHE-true
 # To delete caches go to https://travis-ci.org/OWNER/REPOSITORY/caches or run
@@ -10,30 +19,27 @@ cache:
   ccache: true
   directories:
     - $HOME/.cache # cython cache
-    - $HOME/.ccache # compiler cache
 
 env:
   global:
-    # Variable for test workers
     - PYTEST_WORKERS="auto"
     # create a github personal access token
     # cd pandas-dev/pandas
     # travis encrypt 'PANDAS_GH_TOKEN=personal_access_token' -r pandas-dev/pandas
     - secure: "EkWLZhbrp/mXJOx38CHjs7BnjXafsqHtwxPQrqWy457VDFWhIY1DMnIR/lOWG+a20Qv52sCsFtiZEmMfUjf0pLGXOqurdxbYBGJ7/ikFLk9yV2rDwiArUlVM9bWFnFxHvdz9zewBH55WurrY4ShZWyV+x2dWjjceWG5VpWeI6sA="
 
 git:
-  # for cloning
   depth: false
 
 matrix:
   fast_finish: true
 
   include:
-    # In allowed failures
     - dist: bionic
       python: 3.9-dev
       env:
         - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
+
     - env:
         - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)"
 
@@ -42,7 +48,7 @@ matrix:
 
     - arch: arm64
       env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
+        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
     - env:
         - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
@@ -71,12 +77,6 @@ before_install:
   - uname -a
   - git --version
   - ./ci/check_git_tags.sh
-  # Because travis runs on Google Cloud and has a /etc/boto.cfg,
-  # it breaks moto import, see:
-  # https://github.com/spulec/moto/issues/1771
-  # https://github.com/boto/boto/issues/3741
-  # This overrides travis and tells it to look nowhere.
-  - export BOTO_CONFIG=/dev/null
 
 install:
   - echo "install start"

ci/build39.sh

@@ -1,7 +1,6 @@
 #!/bin/bash -e
 # Special build for python3.9 until numpy puts its own wheels up
 
-sudo apt-get install build-essential gcc xvfb
 pip install --no-deps -U pip wheel setuptools
 pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis
 

ci/setup_env.sh

@@ -42,9 +42,7 @@ else
 fi
 
 if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
-  sudo apt-get update
-  sudo apt-get -y install xvfb
-  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-0/Miniforge3-4.8.5-0-Linux-aarch64.sh"
+  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh"
 else
   CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh"
 fi
@@ -100,8 +98,6 @@ echo "conda list (root environment)"
 conda list
 
 # Clean up any left-over from a previous build
-# (note workaround for https://github.com/conda/conda/issues/2679:
-#  `conda env remove` issue)
 conda remove --all -q -y -n pandas-dev
 
 echo
@@ -142,12 +138,6 @@ conda list pandas
 echo "[Build extensions]"
 python setup.py build_ext -q -i -j2
 
-# TODO: Some of our environments end up with old versions of pip (10.x)
-# Adding a new enough version of pip to the requirements explodes the
-# solve time. Just using pip to update itself.
-# - py35_macos
-# - py35_compat
-# - py36_32bit
 echo "[Updating pip]"
 python -m pip install --no-deps -U pip wheel setuptools
 

doc/source/user_guide/timeseries.rst

@@ -579,7 +579,12 @@ This type of slicing will work on a ``DataFrame`` with a ``DatetimeIndex`` as we
 partial string selection is a form of label slicing, the endpoints **will be** included. This
 would include matching times on an included date:
 
+.. warning::
+
+   Indexing ``DataFrame`` rows with strings is deprecated in pandas 1.2.0 and will be removed in a future version.  Use ``frame.loc[dtstring]`` instead.
+
 .. ipython:: python
+   :okwarning:
 
    dft = pd.DataFrame(np.random.randn(100000, 1), columns=['A'],
                       index=pd.date_range('20130101', periods=100000, freq='T'))
@@ -590,24 +595,28 @@ This starts on the very first time in the month, and includes the last date and
 time for the month:
 
 .. ipython:: python
+   :okwarning:
 
    dft['2013-1':'2013-2']
 
 This specifies a stop time **that includes all of the times on the last day**:
 
 .. ipython:: python
+   :okwarning:
 
    dft['2013-1':'2013-2-28']
 
 This specifies an **exact** stop time (and is not the same as the above):
 
 .. ipython:: python
+   :okwarning:
 
    dft['2013-1':'2013-2-28 00:00:00']
 
 We are stopping on the included end-point as it is part of the index:
 
 .. ipython:: python
+   :okwarning:
 
    dft['2013-1-15':'2013-1-15 12:30:00']
 
@@ -631,6 +640,7 @@ We are stopping on the included end-point as it is part of the index:
 Slicing with string indexing also honors UTC offset.
 
 .. ipython:: python
+   :okwarning:
 
     df = pd.DataFrame([0], index=pd.DatetimeIndex(['2019-01-01'], tz='US/Pacific'))
     df
@@ -681,6 +691,7 @@ If index resolution is second, then the minute-accurate timestamp gives a
 If the timestamp string is treated as a slice, it can be used to index ``DataFrame`` with ``[]`` as well.
 
 .. ipython:: python
+    :okwarning:
 
     dft_minute = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]},
                               index=series_minute.index)
@@ -2027,6 +2038,7 @@ You can pass in dates and strings to ``Series`` and ``DataFrame`` with ``PeriodI
 Passing a string representing a lower frequency than ``PeriodIndex`` returns partial sliced data.
 
 .. ipython:: python
+   :okwarning:
 
    ps['2011']
 

doc/source/whatsnew/v0.11.0.rst

@@ -367,6 +367,7 @@ Enhancements
   - You can now select with a string from a DataFrame with a datelike index, in a similar way to a Series (:issue:`3070`)
 
     .. ipython:: python
+       :okwarning:
 
         idx = pd.date_range("2001-10-1", periods=5, freq='M')
         ts = pd.Series(np.random.rand(len(idx)), index=idx)

doc/source/whatsnew/v1.2.0.rst

@@ -213,6 +213,7 @@ Deprecations
 - Date parser functions :func:`~pandas.io.date_converters.parse_date_time`, :func:`~pandas.io.date_converters.parse_date_fields`, :func:`~pandas.io.date_converters.parse_all_fields` and :func:`~pandas.io.date_converters.generic_parser` from ``pandas.io.date_converters`` are deprecated and will be removed in a future version; use :func:`to_datetime` instead (:issue:`35741`)
 - :meth:`DataFrame.lookup` is deprecated and will be removed in a future version, use :meth:`DataFrame.melt` and :meth:`DataFrame.loc` instead (:issue:`18682`)
 - The :meth:`Index.to_native_types` is deprecated. Use ``.astype(str)`` instead (:issue:`28867`)
+- Deprecated indexing :class:`DataFrame` rows with datetime-like strings ``df[string]``, use ``df.loc[string]`` instead (:issue:`36179`)
 
 .. ---------------------------------------------------------------------------
 
@@ -325,6 +326,7 @@ I/O
 - :meth:`to_picke` and :meth:`read_pickle` did not support compression for file-objects (:issue:`26237`, :issue:`29054`, and :issue:`29570`)
 - Bug in :func:`LongTableBuilder.middle_separator` was duplicating LaTeX longtable entires in the List of Tables of a LaTeX document (:issue:`34360`)
 - Bug in :meth:`read_csv` with `engine='python'` truncating data if multiple items present in first row and first element started with BOM (:issue:`36343`)
+- Removed ``private_key`` and ``verbose`` from :func:`read_gbq` as they are no longer supported in `pandas-gbq` (:issue:`34654` :issue:`30200`)
 
 Plotting
 ^^^^^^^^
@@ -372,6 +374,7 @@ ExtensionArray
 
 Other
 ^^^^^
+
 - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` incorrectly raising ``AssertionError`` instead of ``ValueError`` when invalid parameter combinations are passed (:issue:`36045`)
 - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` with numeric values and string ``to_replace`` (:issue:`34789`)
 - Fixed metadata propagation in the :class:`Series.dt` accessor (:issue:`28283`)

pandas/core/arrays/_mixins.py

@@ -227,3 +227,11 @@ def fillna(self: _T, value=None, method=None, limit=None) -> _T:
         else:
             new_values = self.copy()
         return new_values
+
+    def _reduce(self, name: str, skipna: bool = True, **kwargs):
+        meth = getattr(self, name, None)
+        if meth:
+            return meth(skipna=skipna, **kwargs)
+        else:
+            msg = f"'{type(self).__name__}' does not implement reduction '{name}'"
+            raise TypeError(msg)

pandas/core/arrays/categorical.py

@@ -1963,12 +1963,6 @@ def _reverse_indexer(self) -> Dict[Hashable, np.ndarray]:
     # ------------------------------------------------------------------
     # Reductions
 
-    def _reduce(self, name: str, skipna: bool = True, **kwargs):
-        func = getattr(self, name, None)
-        if func is None:
-            raise TypeError(f"Categorical cannot perform the operation {name}")
-        return func(skipna=skipna, **kwargs)
-
     @deprecate_kwarg(old_arg_name="numeric_only", new_arg_name="skipna")
     def min(self, skipna=True, **kwargs):
         """

pandas/core/arrays/datetimelike.py

@@ -1453,13 +1453,6 @@ def __isub__(self, other):
     # --------------------------------------------------------------
     # Reductions
 
-    def _reduce(self, name: str, skipna: bool = True, **kwargs):
-        op = getattr(self, name, None)
-        if op:
-            return op(skipna=skipna, **kwargs)
-        else:
-            return super()._reduce(name, skipna, **kwargs)
-
     def min(self, axis=None, skipna=True, *args, **kwargs):
         """
         Return the minimum value of the Array or minimum along

pandas/core/arrays/datetimes.py

@@ -76,9 +76,7 @@ def tz_to_dtype(tz):
 
 def _field_accessor(name, field, docstring=None):
     def f(self):
-        values = self.asi8
-        if self.tz is not None and not timezones.is_utc(self.tz):
-            values = self._local_timestamps()
+        values = self._local_timestamps()
 
         if field in self._bool_ops:
             if field.endswith(("start", "end")):
@@ -731,6 +729,8 @@ def _local_timestamps(self):
         This is used to calculate time-of-day information as if the timestamps
         were timezone-naive.
         """
+        if self.tz is None or timezones.is_utc(self.tz):
+            return self.asi8
         return tzconversion.tz_convert_from_utc(self.asi8, self.tz)
 
     def tz_convert(self, tz):
@@ -1167,10 +1167,7 @@ def month_name(self, locale=None):
         >>> idx.month_name()
         Index(['January', 'February', 'March'], dtype='object')
         """
-        if self.tz is not None and not timezones.is_utc(self.tz):
-            values = self._local_timestamps()
-        else:
-            values = self.asi8
+        values = self._local_timestamps()
 
         result = fields.get_date_name_field(values, "month_name", locale=locale)
         result = self._maybe_mask_results(result, fill_value=None)
@@ -1200,10 +1197,7 @@ def day_name(self, locale=None):
         >>> idx.day_name()
         Index(['Monday', 'Tuesday', 'Wednesday'], dtype='object')
         """
-        if self.tz is not None and not timezones.is_utc(self.tz):
-            values = self._local_timestamps()
-        else:
-            values = self.asi8
+        values = self._local_timestamps()
 
         result = fields.get_date_name_field(values, "day_name", locale=locale)
         result = self._maybe_mask_results(result, fill_value=None)
@@ -1217,10 +1211,7 @@ def time(self):
         # If the Timestamps have a timezone that is not UTC,
         # convert them into their i8 representation while
         # keeping their timezone and not using UTC
-        if self.tz is not None and not timezones.is_utc(self.tz):
-            timestamps = self._local_timestamps()
-        else:
-            timestamps = self.asi8
+        timestamps = self._local_timestamps()
 
         return ints_to_pydatetime(timestamps, box="time")
 
@@ -1241,10 +1232,7 @@ def date(self):
         # If the Timestamps have a timezone that is not UTC,
         # convert them into their i8 representation while
         # keeping their timezone and not using UTC
-        if self.tz is not None and not timezones.is_utc(self.tz):
-            timestamps = self._local_timestamps()
-        else:
-            timestamps = self.asi8
+        timestamps = self._local_timestamps()
 
         return ints_to_pydatetime(timestamps, box="date")
 
@@ -1283,10 +1271,7 @@ def isocalendar(self):
         """
         from pandas import DataFrame
 
-        if self.tz is not None and not timezones.is_utc(self.tz):
-            values = self._local_timestamps()
-        else:
-            values = self.asi8
+        values = self._local_timestamps()
         sarray = fields.build_isocalendar_sarray(values)
         iso_calendar_df = DataFrame(
             sarray, columns=["year", "week", "day"], dtype="UInt32"

pandas/core/arrays/numpy_.py

@@ -272,14 +272,6 @@ def _values_for_factorize(self) -> Tuple[np.ndarray, int]:
     # ------------------------------------------------------------------------
     # Reductions
 
-    def _reduce(self, name, skipna=True, **kwargs):
-        meth = getattr(self, name, None)
-        if meth:
-            return meth(skipna=skipna, **kwargs)
-        else:
-            msg = f"'{type(self).__name__}' does not implement reduction '{name}'"
-            raise TypeError(msg)
-
     def any(self, axis=None, out=None, keepdims=False, skipna=True):
         nv.validate_any((), dict(out=out, keepdims=keepdims))
         return nanops.nanany(self._ndarray, axis=axis, skipna=skipna)

pandas/core/arrays/sparse/array.py

@@ -452,7 +452,7 @@ def from_spmatrix(cls, data):
 
         return cls._simple_new(arr, index, dtype)
 
-    def __array__(self, dtype=None, copy=True) -> np.ndarray:
+    def __array__(self, dtype=None) -> np.ndarray:
         fill_value = self.fill_value
 
         if self.sp_index.ngaps == 0:
@@ -1515,7 +1515,7 @@ def _formatter(self, boxed=False):
 SparseArray._add_unary_ops()
 
 
-def make_sparse(arr: np.ndarray, kind="block", fill_value=None, dtype=None, copy=False):
+def make_sparse(arr: np.ndarray, kind="block", fill_value=None, dtype=None):
     """
     Convert ndarray to sparse format
 

pandas/core/frame.py

@@ -5891,14 +5891,23 @@ def _construct_result(self, result) -> DataFrame:
     The resulting index will be a MultiIndex with 'self' and 'other'
     stacked alternately at the inner level.
 
+Raises
+------
+ValueError
+    When the two DataFrames don't have identical labels or shape.
+
 See Also
 --------
 Series.compare : Compare with another Series and show differences.
+DataFrame.equals : Test whether two objects contain the same elements.
 
 Notes
 -----
 Matching NaNs will not appear as a difference.
 
+Can only compare identically-labeled
+(i.e. same shape, identical row and column labels) DataFrames
+
 Examples
 --------
 >>> df = pd.DataFrame(