Merge branch 'master' into stable-dupe-sort

Author: AlexKirko

.github/workflows/ci.yml

@@ -37,12 +37,6 @@ jobs:
         ci/code_checks.sh lint
       if: always()
 
-    - name: Dependencies consistency
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh dependencies
-      if: always()
-
     - name: Checks on imported code
       run: |
         source activate pandas-dev

ci/code_checks.sh

@@ -15,11 +15,10 @@
 #   $ ./ci/code_checks.sh code          # checks on imported code
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
-#   $ ./ci/code_checks.sh dependencies  # check that dependencies are consistent
 #   $ ./ci/code_checks.sh typing	# run static type analysis
 
-[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependencies" || "$1" == "typing" ]] || \
-    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|dependencies|typing]"; exit 9999; }
+[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "typing" ]] || \
+    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|typing]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
 RET=0
@@ -38,6 +37,12 @@ function invgrep {
     return $((! $EXIT_STATUS))
 }
 
+function check_namespace {
+    local -r CLASS="${1}"
+    grep -R -l --include "*.py" " ${CLASS}(" pandas/tests | xargs grep -n "pd\.${CLASS}("
+    test $? -gt 0
+}
+
 if [[ "$GITHUB_ACTIONS" == "true" ]]; then
     FLAKE8_FORMAT="##[error]%(path)s:%(row)s:%(col)s:%(code)s:%(text)s"
     INVGREP_PREPEND="##[error]"
@@ -48,31 +53,6 @@ fi
 ### LINTING ###
 if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
 
-    echo "black --version"
-    black --version
-
-    MSG='Checking black formatting' ; echo $MSG
-    black . --check
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    # `setup.cfg` contains the list of error codes that are being ignored in flake8
-
-    echo "flake8 --version"
-    flake8 --version
-
-    # pandas/_libs/src is C code, so no need to search there.
-    MSG='Linting .py code' ; echo $MSG
-    flake8 --format="$FLAKE8_FORMAT" .
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Linting .pyx and .pxd code' ; echo $MSG
-    flake8 --format="$FLAKE8_FORMAT" pandas --append-config=flake8/cython.cfg
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Linting .pxi.in' ; echo $MSG
-    flake8 --format="$FLAKE8_FORMAT" pandas/_libs --append-config=flake8/cython-template.cfg
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
     # Check that cython casting is of the form `<type>obj` as opposed to `<type> obj`;
     # it doesn't make a difference, but we want to be internally consistent.
     # Note: this grep pattern is (intended to be) equivalent to the python
@@ -125,19 +105,6 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    echo "isort --version-number"
-    isort --version-number
-
-    # Imports - Check formatting using isort see setup.cfg for settings
-    MSG='Check import format using isort' ; echo $MSG
-    ISORT_CMD="isort --quiet --check-only pandas asv_bench scripts web"
-    if [[ "$GITHUB_ACTIONS" == "true" ]]; then
-        eval $ISORT_CMD | awk '{print "##[error]" $0}'; RET=$(($RET + ${PIPESTATUS[0]}))
-    else
-        eval $ISORT_CMD
-    fi
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
 fi
 
 ### PATTERNS ###
@@ -234,6 +201,11 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     MSG='Check code for instances of os.remove' ; echo $MSG
     invgrep -R --include="*.py*" --exclude "common.py" --exclude "test_writers.py" --exclude "test_store.py" -E "os\.remove" pandas/tests/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for inconsistent use of pandas namespace in tests' ; echo $MSG
+    check_namespace "Series"
+    RET=$(($RET + $?))
+    echo $MSG "DONE"
 fi
 
 ### CODE ###
@@ -354,15 +326,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
 fi
 
-### DEPENDENCIES ###
-if [[ -z "$CHECK" || "$CHECK" == "dependencies" ]]; then
-
-    MSG='Check that requirements-dev.txt has been generated from environment.yml' ; echo $MSG
-    $BASE_DIR/scripts/generate_pip_deps_from_conda.py --compare --azure
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-fi
-
 ### TYPING ###
 if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then
 
@@ -374,5 +337,4 @@ if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi
 
-
 exit $RET

doc/source/getting_started/index.rst

@@ -533,7 +533,7 @@ pandas has great support for time series and has an extensive set of tools for w
             <div id="collapseTen" class="collapse" data-parent="#accordion">
                 <div class="card-body">
 
-Data sets do not only contain numerical data. pandas provides a wide range of functions to cleaning textual data and extract useful information from it.
+Data sets do not only contain numerical data. pandas provides a wide range of functions to clean textual data and extract useful information from it.
 
 .. raw:: html
 

doc/source/index.rst.template

@@ -17,7 +17,7 @@ pandas documentation
 `Source Repository <https://github.com/pandas-dev/pandas>`__ |
 `Issues & Ideas <https://github.com/pandas-dev/pandas/issues>`__ |
 `Q&A Support <https://stackoverflow.com/questions/tagged/pandas>`__ |
-`Mailing List <https://groups.google.com/forum/#!forum/pydata>`__
+`Mailing List <https://groups.google.com/g/pydata>`__
 
 :mod:`pandas` is an open source, BSD-licensed library providing high-performance,
 easy-to-use data structures and data analysis tools for the `Python <https://www.python.org/>`__

doc/source/whatsnew/v1.1.4.rst

@@ -21,6 +21,7 @@ Fixed regressions
 - Fixed regression in :meth:`Series.astype` converting ``None`` to ``"nan"`` when casting to string (:issue:`36904`)
 - Fixed regression in :class:`RollingGroupby` causing a segmentation fault with Index of dtype object (:issue:`36727`)
 - Fixed regression in :meth:`DataFrame.resample(...).apply(...)` raised ``AttributeError`` when input was a :class:`DataFrame` and only a :class:`Series` was evaluated (:issue:`36951`)
+- Fixed regression in :class:`PeriodDtype` comparing both equal and unequal to its string representation (:issue:`37265`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.2.0.rst

@@ -409,7 +409,7 @@ Conversion
 Strings
 ^^^^^^^
 - Bug in :meth:`Series.to_string`, :meth:`DataFrame.to_string`, and :meth:`DataFrame.to_latex` adding a leading space when ``index=False`` (:issue:`24980`)
--
+- Bug in :func:`to_numeric` raising a ``TypeError`` when attempting to convert a string dtype :class:`Series` containing only numeric strings and ``NA`` (:issue:`37262`)
 -
 
 
@@ -524,7 +524,7 @@ Other
 - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` incorrectly raising ``AssertionError`` instead of ``ValueError`` when invalid parameter combinations are passed (:issue:`36045`)
 - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` with numeric values and string ``to_replace`` (:issue:`34789`)
 - Fixed bug in metadata propagation incorrectly copying DataFrame columns as metadata when the column name overlaps with the metadata name (:issue:`37037`)
-- Fixed metadata propagation in the :class:`Series.dt` and :class:`Series.str` accessors (:issue:`28283`)
+- Fixed metadata propagation in the :class:`Series.dt` and :class:`Series.str` accessors and :class:`DataFrame.duplicated` and ::class:`DataFrame.stack` methods (:issue:`28283`)
 - Bug in :meth:`Index.union` behaving differently depending on whether operand is a :class:`Index` or other list-like (:issue:`36384`)
 - Passing an array with 2 or more dimensions to the :class:`Series` constructor now raises the more specific ``ValueError``, from a bare ``Exception`` previously (:issue:`35744`)
 

pandas/_libs/lib.pyx

@@ -2019,7 +2019,7 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
         elif util.is_bool_object(val):
             floats[i] = uints[i] = ints[i] = bools[i] = val
             seen.bool_ = True
-        elif val is None:
+        elif val is None or val is C_NA:
             seen.saw_null()
             floats[i] = complexes[i] = NaN
         elif hasattr(val, '__len__') and len(val) == 0:

pandas/_testing.py

@@ -6,6 +6,7 @@
 import gzip
 import operator
 import os
+import re
 from shutil import rmtree
 import string
 import tempfile
@@ -2546,10 +2547,11 @@ def wrapper(*args, **kwargs):
 
 @contextmanager
 def assert_produces_warning(
-    expected_warning=Warning,
+    expected_warning: Optional[Union[Type[Warning], bool]] = Warning,
     filter_level="always",
-    check_stacklevel=True,
-    raise_on_extra_warnings=True,
+    check_stacklevel: bool = True,
+    raise_on_extra_warnings: bool = True,
+    match: Optional[str] = None,
 ):
     """
     Context manager for running code expected to either raise a specific
@@ -2584,6 +2586,8 @@ class for all warnings. To check that no warning is returned,
     raise_on_extra_warnings : bool, default True
         Whether extra warnings not of the type `expected_warning` should
         cause the test to fail.
+    match : str, optional
+        Match warning message.
 
     Examples
     --------
@@ -2610,28 +2614,28 @@ class for all warnings. To check that no warning is returned,
     with warnings.catch_warnings(record=True) as w:
 
         saw_warning = False
+        matched_message = False
+
         warnings.simplefilter(filter_level)
         yield w
         extra_warnings = []
 
         for actual_warning in w:
-            if expected_warning and issubclass(
-                actual_warning.category, expected_warning
-            ):
+            if not expected_warning:
+                continue
+
+            expected_warning = cast(Type[Warning], expected_warning)
+            if issubclass(actual_warning.category, expected_warning):
                 saw_warning = True
 
                 if check_stacklevel and issubclass(
                     actual_warning.category, (FutureWarning, DeprecationWarning)
                 ):
-                    from inspect import getframeinfo, stack
+                    _assert_raised_with_correct_stacklevel(actual_warning)
+
+                if match is not None and re.search(match, str(actual_warning.message)):
+                    matched_message = True
 
-                    caller = getframeinfo(stack()[2][0])
-                    msg = (
-                        "Warning not set with correct stacklevel. "
-                        f"File where warning is raised: {actual_warning.filename} != "
-                        f"{caller.filename}. Warning message: {actual_warning.message}"
-                    )
-                    assert actual_warning.filename == caller.filename, msg
             else:
                 extra_warnings.append(
                     (
@@ -2641,18 +2645,41 @@ class for all warnings. To check that no warning is returned,
                         actual_warning.lineno,
                     )
                 )
+
         if expected_warning:
-            msg = (
-                f"Did not see expected warning of class "
-                f"{repr(expected_warning.__name__)}"
-            )
-            assert saw_warning, msg
+            expected_warning = cast(Type[Warning], expected_warning)
+            if not saw_warning:
+                raise AssertionError(
+                    f"Did not see expected warning of class "
+                    f"{repr(expected_warning.__name__)}"
+                )
+
+            if match and not matched_message:
+                raise AssertionError(
+                    f"Did not see warning {repr(expected_warning.__name__)} "
+                    f"matching {match}"
+                )
+
         if raise_on_extra_warnings and extra_warnings:
             raise AssertionError(
                 f"Caused unexpected warning(s): {repr(extra_warnings)}"
             )
 
 
+def _assert_raised_with_correct_stacklevel(
+    actual_warning: warnings.WarningMessage,
+) -> None:
+    from inspect import getframeinfo, stack
+
+    caller = getframeinfo(stack()[3][0])
+    msg = (
+        "Warning not set with correct stacklevel. "
+        f"File where warning is raised: {actual_warning.filename} != "
+        f"{caller.filename}. Warning message: {actual_warning.message}"
+    )
+    assert actual_warning.filename == caller.filename, msg
+
+
 class RNGContext:
     """
     Context manager to set the numpy random number generator speed. Returns

pandas/_typing.py

@@ -38,6 +38,8 @@
     from pandas.core.indexes.base import Index
     from pandas.core.series import Series
 
+    from pandas.io.formats.format import EngFormatter
+
 # array-like
 
 AnyArrayLike = TypeVar("AnyArrayLike", "ExtensionArray", "Index", "Series", np.ndarray)
@@ -127,6 +129,10 @@
 EncodingVar = TypeVar("EncodingVar", str, None, Optional[str])
 
 
+# type of float formatter in DataFrameFormatter
+FloatFormatType = Union[str, Callable, "EngFormatter"]
+
+
 @dataclass
 class IOargs(Generic[ModeVar, EncodingVar]):
     """

pandas/core/arrays/datetimelike.py

@@ -444,7 +444,7 @@ def _validate_comparison_value(self, other, opname: str):
 
         else:
             try:
-                other = self._validate_listlike(other, opname, allow_object=True)
+                other = self._validate_listlike(other, allow_object=True)
                 self._check_compatible_with(other)
             except TypeError as err:
                 if is_object_dtype(getattr(other, "dtype", None)):
@@ -548,7 +548,7 @@ def _validate_scalar(self, value, msg: Optional[str] = None):
 
         return value
 
-    def _validate_listlike(self, value, opname: str, allow_object: bool = False):
+    def _validate_listlike(self, value, allow_object: bool = False):
         if isinstance(value, type(self)):
             return value
 
@@ -578,18 +578,17 @@ def _validate_listlike(self, value, opname: str, allow_object: bool = False):
 
         elif not type(self)._is_recognized_dtype(value.dtype):
             raise TypeError(
-                f"{opname} requires compatible dtype or scalar, "
-                f"not {type(value).__name__}"
+                f"value should be a '{self._scalar_type.__name__}', 'NaT', "
+                f"or array of those. Got '{type(value).__name__}' instead."
             )
-
         return value
 
     def _validate_searchsorted_value(self, value):
         msg = "searchsorted requires compatible dtype or scalar"
         if not is_list_like(value):
             value = self._validate_scalar(value, msg)
         else:
-            value = self._validate_listlike(value, "searchsorted")
+            value = self._validate_listlike(value)
 
         rv = self._unbox(value)
         return self._rebox_native(rv)
@@ -600,7 +599,7 @@ def _validate_setitem_value(self, value):
             f"or array of those. Got '{type(value).__name__}' instead."
         )
         if is_list_like(value):
-            value = self._validate_listlike(value, "setitem")
+            value = self._validate_listlike(value)
         else:
             value = self._validate_scalar(value, msg)
 
@@ -622,7 +621,7 @@ def _validate_where_value(self, other):
         if not is_list_like(other):
             other = self._validate_scalar(other, msg)
         else:
-            other = self._validate_listlike(other, "where")
+            other = self._validate_listlike(other)
 
         return self._unbox(other, setitem=True)
 

pandas/core/dtypes/dtypes.py

@@ -907,6 +907,9 @@ def __eq__(self, other: Any) -> bool:
 
         return isinstance(other, PeriodDtype) and self.freq == other.freq
 
+    def __ne__(self, other: Any) -> bool:
+        return not self.__eq__(other)
+
     def __setstate__(self, state):
         # for pickle compat. __getstate__ is defined in the
         # PandasExtensionDtype superclass and uses the public properties to