Merge remote-tracking branch 'upstream/master' into pandas.io.sas.sasreader

Author: simonjayhawkins

.pre-commit-config.yaml

@@ -16,3 +16,15 @@ repos:
     -   id: isort
         language: python_venv
         exclude: ^pandas/__init__\.py$|^pandas/core/api\.py$
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.730
+    hooks:
+     -  id: mypy
+        # We run mypy over all files because of:
+        #  * changes in type definitions may affect non-touched files.
+        #  * Running it with `mypy pandas` and the filenames will lead to
+        #    spurious duplicate module errors,
+        #    see also https://github.com/pre-commit/mirrors-mypy/issues/5
+        pass_filenames: false
+        args:
+        - pandas

ci/deps/azure-36-locale.yaml

@@ -9,6 +9,7 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
+  - pytest-asyncio
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/azure-37-locale.yaml

@@ -8,6 +8,7 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
+  - pytest-asyncio
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

doc/source/development/contributing.rst

@@ -1363,6 +1363,7 @@ some common prefixes along with general guidelines for when to use them:
 * TST: Additions/updates to tests
 * BLD: Updates to the build process/scripts
 * PERF: Performance improvement
+* TYP: Type annotations
 * CLN: Code cleanup
 
 The following defines how a commit message should be structured.  Please reference the

doc/source/whatsnew/v1.0.0.rst

@@ -1016,6 +1016,7 @@ Other
 - Bug in :meth:`Series.diff` where a boolean series would incorrectly raise a ``TypeError`` (:issue:`17294`)
 - :meth:`Series.append` will no longer raise a ``TypeError`` when passed a tuple of ``Series`` (:issue:`28410`)
 - Fix corrupted error message when calling ``pandas.libs._json.encode()`` on a 0d array (:issue:`18878`)
+- Backtick quoting in :meth:`DataFrame.query` and :meth:`DataFrame.eval` can now also be used to use invalid identifiers like names that start with a digit, are python keywords, or are using single character operators. (:issue:`27017`)
 - Bug in ``pd.core.util.hashing.hash_pandas_object`` where arrays containing tuples were incorrectly treated as non-hashable (:issue:`28969`)
 - Bug in :meth:`DataFrame.append` that raised ``IndexError`` when appending with empty list (:issue:`28769`)
 - Fix :class:`AbstractHolidayCalendar` to return correct results for

environment.yml

@@ -55,6 +55,7 @@ dependencies:
   - pytest>=5.0.1
   - pytest-cov
   - pytest-xdist>=1.21
+  - pytest-asyncio
 
   # downstream tests
   - seaborn
@@ -70,7 +71,7 @@ dependencies:
   - blosc
   - bottleneck>=1.2.1
   - ipykernel
-  - ipython>=5.6.0,<=7.10.1 # see gh-30527
+  - ipython>=7.11.1
   - jinja2  # pandas.Styler
   - matplotlib>=2.2.2  # pandas.plotting, Series.plot, DataFrame.plot
   - numexpr>=2.6.8

pandas/_libs/src/ujson/python/objToJSON.c

@@ -241,65 +241,39 @@ static int scaleNanosecToUnit(npy_int64 *value, NPY_DATETIMEUNIT unit) {
 static PyObject *get_values(PyObject *obj) {
     PyObject *values = NULL;
 
-    values = PyObject_GetAttrString(obj, "values");
     PRINTMARK();
 
-    if (values && !PyArray_CheckExact(values)) {
-
-        if (PyObject_HasAttrString(values, "to_numpy")) {
-            values = PyObject_CallMethod(values, "to_numpy", NULL);
-        }
-
-        if (PyObject_HasAttrString(values, "values")) {
-            PyObject *subvals = get_values(values);
-            PyErr_Clear();
-            PRINTMARK();
-            // subvals are sometimes missing a dimension
-            if (subvals) {
-                PyArrayObject *reshape = (PyArrayObject *)subvals;
-                PyObject *shape = PyObject_GetAttrString(obj, "shape");
-                PyArray_Dims dims;
-                PRINTMARK();
-
-                if (!shape || !PyArray_IntpConverter(shape, &dims)) {
-                    subvals = NULL;
-                } else {
-                    subvals = PyArray_Newshape(reshape, &dims, NPY_ANYORDER);
-                    PyDimMem_FREE(dims.ptr);
-                }
-                Py_DECREF(reshape);
-                Py_XDECREF(shape);
-            }
-            Py_DECREF(values);
-            values = subvals;
-        } else {
-            PRINTMARK();
-            Py_DECREF(values);
-            values = NULL;
-        }
-    }
-
-    if (!values && PyObject_HasAttrString(obj, "_internal_get_values")) {
+    if (PyObject_HasAttrString(obj, "_internal_get_values")) {
         PRINTMARK();
         values = PyObject_CallMethod(obj, "_internal_get_values", NULL);
-        if (values && !PyArray_CheckExact(values)) {
+
+        if (values == NULL) {
+            // Clear so we can subsequently try another method
+            PyErr_Clear();
+        } else if (!PyArray_CheckExact(values)) {
+            // Didn't get a numpy array, so keep trying
             PRINTMARK();
             Py_DECREF(values);
             values = NULL;
         }
     }
 
-    if (!values && PyObject_HasAttrString(obj, "get_block_values")) {
+    if ((values == NULL) && PyObject_HasAttrString(obj, "get_block_values")) {
         PRINTMARK();
         values = PyObject_CallMethod(obj, "get_block_values", NULL);
-        if (values && !PyArray_CheckExact(values)) {
+
+        if (values == NULL) {
+            // Clear so we can subsequently try another method
+            PyErr_Clear();
+        } else if (!PyArray_CheckExact(values)) {
+            // Didn't get a numpy array, so keep trying
             PRINTMARK();
             Py_DECREF(values);
             values = NULL;
         }
     }
 
-    if (!values) {
+    if (values == NULL) {
         PyObject *typeRepr = PyObject_Repr((PyObject *)Py_TYPE(obj));
         PyObject *repr;
         PRINTMARK();
@@ -435,8 +409,8 @@ static char *int64ToIso(int64_t value, NPY_DATETIMEUNIT base, size_t *len) {
 }
 
 /* JSON callback. returns a char* and mutates the pointer to *len */
-static char *NpyDateTimeToIsoCallback(JSOBJ Py_UNUSED(unused), JSONTypeContext *tc,
-                                      size_t *len) {
+static char *NpyDateTimeToIsoCallback(JSOBJ Py_UNUSED(unused),
+                                      JSONTypeContext *tc, size_t *len) {
     NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit;
     return int64ToIso(GET_TC(tc)->longValue, base, len);
 }

pandas/core/arrays/boolean.py

@@ -62,6 +62,8 @@ class BooleanDtype(ExtensionDtype):
     BooleanDtype
     """
 
+    name = "boolean"
+
     @property
     def na_value(self) -> "Scalar":
         """
@@ -81,19 +83,6 @@ def type(self) -> Type:
     def kind(self) -> str:
         return "b"
 
-    @property
-    def name(self) -> str:
-        """
-        The alias for BooleanDtype is ``'boolean'``.
-        """
-        return "boolean"
-
-    @classmethod
-    def construct_from_string(cls, string: str) -> ExtensionDtype:
-        if string == "boolean":
-            return cls()
-        return super().construct_from_string(string)
-
     @classmethod
     def construct_array_type(cls) -> "Type[BooleanArray]":
         return BooleanArray

pandas/core/arrays/period.py

@@ -29,6 +29,7 @@
     is_datetime64_dtype,
     is_float_dtype,
     is_list_like,
+    is_object_dtype,
     is_period_dtype,
     pandas_dtype,
 )
@@ -41,6 +42,7 @@
 )
 from pandas.core.dtypes.missing import isna, notna
 
+from pandas.core import ops
 import pandas.core.algorithms as algos
 from pandas.core.arrays import datetimelike as dtl
 import pandas.core.common as com
@@ -92,22 +94,44 @@ def wrapper(self, other):
             self._check_compatible_with(other)
 
             result = ordinal_op(other.ordinal)
-        elif isinstance(other, cls):
-            self._check_compatible_with(other)
-
-            result = ordinal_op(other.asi8)
-
-            mask = self._isnan | other._isnan
-            if mask.any():
-                result[mask] = nat_result
 
-            return result
         elif other is NaT:
             result = np.empty(len(self.asi8), dtype=bool)
             result.fill(nat_result)
-        else:
+
+        elif not is_list_like(other):
             return invalid_comparison(self, other, op)
 
+        else:
+            if isinstance(other, list):
+                # TODO: could use pd.Index to do inference?
+                other = np.array(other)
+
+            if not isinstance(other, (np.ndarray, cls)):
+                return invalid_comparison(self, other, op)
+
+            if is_object_dtype(other):
+                with np.errstate(all="ignore"):
+                    result = ops.comp_method_OBJECT_ARRAY(
+                        op, self.astype(object), other
+                    )
+                o_mask = isna(other)
+
+            elif not is_period_dtype(other):
+                # e.g. is_timedelta64_dtype(other)
+                return invalid_comparison(self, other, op)
+
+            else:
+                assert isinstance(other, cls), type(other)
+
+                self._check_compatible_with(other)
+
+                result = ordinal_op(other.asi8)
+                o_mask = other._isnan
+
+            if o_mask.any():
+                result[o_mask] = nat_result
+
         if self._hasnans:
             result[self._isnan] = nat_result
 
@@ -215,12 +239,7 @@ def __init__(self, values, freq=None, dtype=None, copy=False):
 
         if isinstance(values, type(self)):
             if freq is not None and freq != values.freq:
-                msg = DIFFERENT_FREQ.format(
-                    cls=type(self).__name__,
-                    own_freq=values.freq.freqstr,
-                    other_freq=freq.freqstr,
-                )
-                raise IncompatibleFrequency(msg)
+                raise raise_on_incompatible(values, freq)
             values, freq = values._data, values.freq
 
         values = np.array(values, dtype="int64", copy=copy)
@@ -323,7 +342,7 @@ def _check_compatible_with(self, other):
         if other is NaT:
             return
         if self.freqstr != other.freqstr:
-            _raise_on_incompatible(self, other)
+            raise raise_on_incompatible(self, other)
 
     # --------------------------------------------------------------------
     # Data / Attributes
@@ -332,7 +351,7 @@ def _check_compatible_with(self, other):
     def dtype(self):
         return self._dtype
 
-    # read-only property overwriting read/write
+    # error: Read-only property cannot override read-write property  [misc]
     @property  # type: ignore
     def freq(self):
         """
@@ -654,7 +673,7 @@ def _sub_period(self, other):
         return new_data
 
     def _addsub_int_array(
-        self, other: np.ndarray, op: Callable[[Any], Any],
+        self, other: np.ndarray, op: Callable[[Any, Any], Any],
     ) -> "PeriodArray":
         """
         Add or subtract array of integers; equivalent to applying
@@ -682,7 +701,7 @@ def _add_offset(self, other):
         assert not isinstance(other, Tick)
         base = libfrequencies.get_base_alias(other.rule_code)
         if base != self.freq.rule_code:
-            _raise_on_incompatible(self, other)
+            raise raise_on_incompatible(self, other)
 
         # Note: when calling parent class's _add_timedeltalike_scalar,
         #  it will call delta_to_nanoseconds(delta).  Because delta here
@@ -750,7 +769,7 @@ def _add_delta(self, other):
         """
         if not isinstance(self.freq, Tick):
             # We cannot add timedelta-like to non-tick PeriodArray
-            _raise_on_incompatible(self, other)
+            raise raise_on_incompatible(self, other)
 
         new_ordinals = super()._add_delta(other)
         return type(self)(new_ordinals, freq=self.freq)
@@ -802,28 +821,29 @@ def _check_timedeltalike_freq_compat(self, other):
             # by which will be added to self.
             return delta
 
-        _raise_on_incompatible(self, other)
+        raise raise_on_incompatible(self, other)
 
 
 PeriodArray._add_comparison_ops()
 
 
-def _raise_on_incompatible(left, right):
+def raise_on_incompatible(left, right):
     """
     Helper function to render a consistent error message when raising
     IncompatibleFrequency.
 
     Parameters
     ----------
     left : PeriodArray
-    right : DateOffset, Period, ndarray, or timedelta-like
+    right : None, DateOffset, Period, ndarray, or timedelta-like
 
-    Raises
+    Returns
     ------
     IncompatibleFrequency
+        Exception to be raised by the caller.
     """
     # GH#24283 error message format depends on whether right is scalar
-    if isinstance(right, np.ndarray):
+    if isinstance(right, np.ndarray) or right is None:
         other_freq = None
     elif isinstance(right, (ABCPeriodIndex, PeriodArray, Period, DateOffset)):
         other_freq = right.freqstr
@@ -833,7 +853,7 @@ def _raise_on_incompatible(left, right):
     msg = DIFFERENT_FREQ.format(
         cls=type(left).__name__, own_freq=left.freqstr, other_freq=other_freq
     )
-    raise IncompatibleFrequency(msg)
+    return IncompatibleFrequency(msg)
 
 
 # -------------------------------------------------------------------

pandas/core/arrays/string_.py

@@ -47,26 +47,15 @@ class StringDtype(ExtensionDtype):
     StringDtype
     """
 
+    name = "string"
+
     #: StringDtype.na_value uses pandas.NA
     na_value = libmissing.NA
 
     @property
     def type(self) -> Type:
         return str
 
-    @property
-    def name(self) -> str:
-        """
-        The alias for StringDtype is ``'string'``.
-        """
-        return "string"
-
-    @classmethod
-    def construct_from_string(cls, string: str) -> ExtensionDtype:
-        if string == "string":
-            return cls()
-        return super().construct_from_string(string)
-
     @classmethod
     def construct_array_type(cls) -> "Type[StringArray]":
         return StringArray