Merge branch 'master' into bug-drop-multiindex

Author: proost

asv_bench/benchmarks/dtypes.py

@@ -5,6 +5,7 @@
 from .pandas_vb_common import (
     datetime_dtypes,
     extension_dtypes,
+    lib,
     numeric_dtypes,
     string_dtypes,
 )
@@ -40,4 +41,25 @@ def time_pandas_dtype_invalid(self, dtype):
             pass
 
 
+class InferDtypes:
+    param_names = ["dtype"]
+    data_dict = {
+        "np-object": np.array([1] * 100000, dtype="O"),
+        "py-object": [1] * 100000,
+        "np-null": np.array([1] * 50000 + [np.nan] * 50000),
+        "py-null": [1] * 50000 + [None] * 50000,
+        "np-int": np.array([1] * 100000, dtype=int),
+        "np-floating": np.array([1.0] * 100000, dtype=float),
+        "empty": [],
+        "bytes": [b"a"] * 100000,
+    }
+    params = list(data_dict.keys())
+
+    def time_infer_skipna(self, dtype):
+        lib.infer_dtype(self.data_dict[dtype], skipna=True)
+
+    def time_infer(self, dtype):
+        lib.infer_dtype(self.data_dict[dtype], skipna=False)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

ci/azure/posix.yml

@@ -69,20 +69,13 @@ jobs:
       displayName: 'Build versions'
 
     - task: PublishTestResults@2
+      condition: succeededOrFailed()
       inputs:
+        failTaskOnFailedTests: true
         testResultsFiles: 'test-data.xml'
         testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }}
       displayName: 'Publish test results'
 
-    - powershell: |
-        $(Get-Content "test-data.xml" | Out-String) -match 'failures="(.*?)"'
-        if ($matches[1] -eq 0) {
-          Write-Host "No test failures in test-data"
-        } else {
-          Write-Error "$($matches[1]) tests failed"  # will produce $LASTEXITCODE=1
-        }
-      displayName: 'Check for test failures'
-
     - script: |
         source activate pandas-dev
         python ci/print_skipped.py

ci/azure/windows.yml

@@ -23,32 +23,34 @@ jobs:
         Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
         Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin"
       displayName: 'Add conda to PATH'
+
     - script: conda update -q -n base conda
       displayName: 'Update conda'
+
     - bash: |
         conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
       displayName: 'Create anaconda environment'
+
     - bash: |
         source activate pandas-dev
         conda list
-        ci\\incremental\\build.cmd
+        python setup.py build_ext -q -i
+        python -m pip install --no-build-isolation -e .
       displayName: 'Build'
+
     - bash: |
         source activate pandas-dev
         ci/run_tests.sh
       displayName: 'Test'
+
     - task: PublishTestResults@2
+      condition: succeededOrFailed()
       inputs:
+        failTaskOnFailedTests: true
         testResultsFiles: 'test-data.xml'
-        testRunTitle: 'Windows-$(CONDA_PY)'
-    - powershell: |
-        $(Get-Content "test-data.xml" | Out-String) -match 'failures="(.*?)"'
-        if ($matches[1] -eq 0) {
-          Write-Host "No test failures in test-data"
-        } else {
-          Write-Error "$($matches[1]) tests failed"  # will produce $LASTEXITCODE=1
-        }
-      displayName: 'Check for test failures'
+        testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }}
+      displayName: 'Publish test results'
+
     - bash: |
         source activate pandas-dev
         python ci/print_skipped.py

ci/code_checks.sh

@@ -52,7 +52,7 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     black --version
 
     MSG='Checking black formatting' ; echo $MSG
-	black . --check
+    black . --check
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # `setup.cfg` contains the list of error codes that are being ignored in flake8
@@ -104,7 +104,7 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     isort --version-number
 
     # Imports - Check formatting using isort see setup.cfg for settings
-    MSG='Check import format using isort ' ; echo $MSG
+    MSG='Check import format using isort' ; echo $MSG
     ISORT_CMD="isort --recursive --check-only pandas asv_bench"
     if [[ "$GITHUB_ACTIONS" == "true" ]]; then
         eval $ISORT_CMD | awk '{print "##[error]" $0}'; RET=$(($RET + ${PIPESTATUS[0]}))
@@ -122,13 +122,18 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     # Check for imports from collections.abc instead of `from collections import abc`
     MSG='Check for non-standard imports' ; echo $MSG
     invgrep -R --include="*.py*" -E "from pandas.core.common import" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
     invgrep -R --include="*.py*" -E "from pandas.core import common" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
     invgrep -R --include="*.py*" -E "from collections.abc import" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
     invgrep -R --include="*.py*" -E "from numpy import nan" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # Checks for test suite
     # Check for imports from pandas.util.testing instead of `import pandas.util.testing as tm`
     invgrep -R --include="*.py*" -E "from pandas.util.testing import" pandas/tests
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
     invgrep -R --include="*.py*" -E "from pandas.util import testing as tm" pandas/tests
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
@@ -195,6 +200,10 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' .
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Check for use of {foo!r} instead of {repr(foo)}' ; echo $MSG
+    invgrep -R --include=*.{py,pyx} '!r}' pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Check for use of comment-based annotation syntax' ; echo $MSG
     invgrep -R --include="*.py" -P '# type: (?!ignore)' pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
@@ -203,6 +212,10 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include=*.{py,pyx} '\.__class__' pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Check for use of xrange instead of range' ; echo $MSG
+    invgrep -R --include=*.{py,pyx} 'xrange' pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Check that no file in the repo contains trailing whitespaces' ; echo $MSG
     INVGREP_APPEND=" <- trailing whitespaces found"
     invgrep -RI --exclude=\*.{svg,c,cpp,html,js} --exclude-dir=env "\s$" *

ci/incremental/build.cmd

@@ -38,6 +38,6 @@ sh -c "$PYTEST_CMD"
 
 if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then
     echo "uploading coverage"
-    echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
-          bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
+    echo "bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME"
+          bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME
 fi

ci/run_tests.sh

@@ -204,7 +204,11 @@
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-# html_theme_options = {}
+html_theme_options = {
+    "external_links": [],
+    "github_url": "https://github.com/pandas-dev/pandas",
+    "twitter_url": "https://twitter.com/pandas_dev",
+}
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = ["themes"]
@@ -228,7 +232,7 @@
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-html_favicon = os.path.join(html_static_path[0], "favicon.ico")
+html_favicon = "../../web/pandas/static/img/favicon.ico"
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

doc/source/_static/favicon.ico

@@ -569,6 +569,7 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
 - A tuple passed to :meth:`DataFrame.groupby` is now exclusively treated as a single key (:issue:`18314`)
 - Removed the previously deprecated :meth:`Index.contains`, use ``key in index`` instead (:issue:`30103`)
 - Addition and subtraction of ``int`` or integer-arrays is no longer allowed in :class:`Timestamp`, :class:`DatetimeIndex`, :class:`TimedeltaIndex`, use ``obj + n * obj.freq`` instead of ``obj + n`` (:issue:`22535`)
+- Removed :meth:`Series.ptp` (:issue:`21614`)
 - Removed :meth:`Series.from_array` (:issue:`18258`)
 - Removed :meth:`DataFrame.from_items` (:issue:`18458`)
 - Removed :meth:`DataFrame.as_matrix`, :meth:`Series.as_matrix` (:issue:`18458`)
@@ -667,6 +668,7 @@ Performance improvements
 - Performance improvement when checking if values in a :class:`Categorical` are equal, equal or larger or larger than a given scalar.
   The improvement is not present if checking if the :class:`Categorical` is less than or less than or equal than the scalar (:issue:`29820`)
 - Performance improvement in :meth:`Index.equals` and  :meth:`MultiIndex.equals` (:issue:`29134`)
+- Performance improvement in :func:`~pandas.api.types.infer_dtype` when ``skipna`` is ``True`` (:issue:`28814`)
 
 .. _whatsnew_1000.bug_fixes:
 
@@ -712,7 +714,10 @@ Datetimelike
 - Bug in :func:`pandas.to_datetime` when called with ``None`` raising ``TypeError`` instead of returning ``NaT`` (:issue:`30011`)
 - Bug in :func:`pandas.to_datetime` failing for `deques` when using ``cache=True`` (the default) (:issue:`29403`)
 - Bug in :meth:`Series.item` with ``datetime64`` or ``timedelta64`` dtype, :meth:`DatetimeIndex.item`, and :meth:`TimedeltaIndex.item` returning an integer instead of a :class:`Timestamp` or :class:`Timedelta` (:issue:`30175`)
--
+- Bug in :class:`DatetimeIndex` addition when adding a non-optimized :class:`DateOffset` incorrectly dropping timezone information (:issue:`30336`)
+- Bug in :meth:`DataFrame.drop` where attempting to drop non-existent values from a DatetimeIndex would yield a confusing error message (:issue:`30399`)
+- Bug in :meth:`DataFrame.append` would remove the timezone-awareness of new data (:issue:`30238`)
+
 
 Timedelta
 ^^^^^^^^^

doc/source/conf.py

@@ -33,7 +33,8 @@ dependencies:
   - nbconvert>=5.4.1
   - nbsphinx
   - pandoc
-  # Dask and its dependencies
+
+  # Dask and its dependencies (that dont install with dask)
   - dask-core
   - toolz>=0.7.3
   - fsspec>=0.5.1
@@ -54,6 +55,8 @@ dependencies:
   - pytest>=5.0.1
   - pytest-cov
   - pytest-xdist>=1.21
+
+  # downstream tests
   - seaborn
   - statsmodels
 
@@ -74,22 +77,28 @@ dependencies:
   - scipy>=1.1
 
   # optional for io
-  - beautifulsoup4>=4.6.0  # pandas.read_html
+  # ---------------
+  # pd.read_html
+  - beautifulsoup4>=4.6.0
+  - html5lib
+  - lxml
+
+  # pd.read_excel, DataFrame.to_excel, pd.ExcelWriter, pd.ExcelFile
+  - openpyxl<=3.0.1
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  - odfpy
+
   - fastparquet>=0.3.2  # pandas.read_parquet, DataFrame.to_parquet
-  - html5lib  # pandas.read_html
-  - lxml  # pandas.read_html
-  - openpyxl<=3.0.1  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
   - pyarrow>=0.13.1  # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather
+  - python-snappy  # required by pyarrow
+
   - pyqt>=5.9.2  # pandas.read_clipboard
   - pytables>=3.4.2  # pandas.read_hdf, DataFrame.to_hdf
-  - python-snappy  # required by pyarrow
   - s3fs  # pandas.read_csv... when using 's3://...' path
   - sqlalchemy  # pandas.read_sql, DataFrame.to_sql
   - xarray  # DataFrame.to_xarray
-  - xlrd  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
-  - xlsxwriter  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
-  - xlwt  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
-  - odfpy  # pandas.read_excel
   - pyreadstat  # pandas.read_spss
   - pip:
     - git+https://github.com/pandas-dev/pandas-sphinx-theme.git@master