Merge branch 'master' of https://github.com/pandas-dev/pandas into collect

Author: jbrockmendel

ci/run_tests.sh

@@ -37,7 +37,8 @@ echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"
 
 if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then
+    SHA=`git rev-parse HEAD`
     echo "uploading coverage"
-    echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
-          bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
+    echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME -C $SHA"
+          bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME -C `git rev-parse HEAD`
 fi

doc/source/whatsnew/v1.0.0.rst

@@ -712,7 +712,7 @@ Datetimelike
 - Bug in :func:`pandas.to_datetime` when called with ``None`` raising ``TypeError`` instead of returning ``NaT`` (:issue:`30011`)
 - Bug in :func:`pandas.to_datetime` failing for `deques` when using ``cache=True`` (the default) (:issue:`29403`)
 - Bug in :meth:`Series.item` with ``datetime64`` or ``timedelta64`` dtype, :meth:`DatetimeIndex.item`, and :meth:`TimedeltaIndex.item` returning an integer instead of a :class:`Timestamp` or :class:`Timedelta` (:issue:`30175`)
--
+- Bug in :class:`DatetimeIndex` addition when adding a non-optimized :class:`DateOffset` incorrectly dropping timezone information (:issue:`30336`)
 
 Timedelta
 ^^^^^^^^^

pandas/_libs/parsers.pyx

@@ -1367,7 +1367,26 @@ def _ensure_encoded(list lst):
 # common NA values
 # no longer excluding inf representations
 # '1.#INF','-1.#INF', '1.#INF000000',
-_NA_VALUES = _ensure_encoded(list(icom._NA_VALUES))
+STR_NA_VALUES = {
+    "-1.#IND",
+    "1.#QNAN",
+    "1.#IND",
+    "-1.#QNAN",
+    "#N/A N/A",
+    "#N/A",
+    "N/A",
+    "n/a",
+    "NA",
+    "#NA",
+    "NULL",
+    "null",
+    "NaN",
+    "-NaN",
+    "nan",
+    "-nan",
+    "",
+}
+_NA_VALUES = _ensure_encoded(list(STR_NA_VALUES))
 
 
 def _maybe_upcast(arr):

pandas/core/arrays/datetimes.py

@@ -794,16 +794,17 @@ def _add_offset(self, offset):
                 values = self.tz_localize(None)
             else:
                 values = self
-            result = offset.apply_index(values)
-            if self.tz is not None:
-                result = result.tz_localize(self.tz)
+            result = offset.apply_index(values).tz_localize(self.tz)
 
         except NotImplementedError:
             warnings.warn(
                 "Non-vectorized DateOffset being applied to Series or DatetimeIndex",
                 PerformanceWarning,
             )
             result = self.astype("O") + offset
+            if len(self) == 0:
+                # _from_sequence won't be able to infer self.tz
+                return type(self)._from_sequence(result).tz_localize(self.tz)
 
         return type(self)._from_sequence(result, freq="infer")
 

pandas/core/indexing.py

@@ -232,7 +232,7 @@ def _has_valid_tuple(self, key: Tuple):
             except ValueError:
                 raise ValueError(
                     "Location based indexing can only have "
-                    "[{types}] types".format(types=self._valid_types)
+                    f"[{self._valid_types}] types"
                 )
 
     def _is_nested_tuple_indexer(self, tup: Tuple) -> bool:
@@ -286,7 +286,7 @@ def _has_valid_positional_setitem_indexer(self, indexer) -> bool:
         bool
         """
         if isinstance(indexer, dict):
-            raise IndexError("{0} cannot enlarge its target object".format(self.name))
+            raise IndexError(f"{self.name} cannot enlarge its target object")
         else:
             if not isinstance(indexer, tuple):
                 indexer = _tuplify(self.ndim, indexer)
@@ -300,13 +300,10 @@ def _has_valid_positional_setitem_indexer(self, indexer) -> bool:
                 elif is_integer(i):
                     if i >= len(ax):
                         raise IndexError(
-                            "{name} cannot enlarge its target "
-                            "object".format(name=self.name)
+                            f"{self.name} cannot enlarge its target object"
                         )
                 elif isinstance(i, dict):
-                    raise IndexError(
-                        "{name} cannot enlarge its target object".format(name=self.name)
-                    )
+                    raise IndexError(f"{self.name} cannot enlarge its target object")
 
         return True
 
@@ -1166,17 +1163,14 @@ def _validate_read_indexer(
 
         if missing:
             if missing == len(indexer):
-                raise KeyError(
-                    "None of [{key}] are in the [{axis}]".format(
-                        key=key, axis=self.obj._get_axis_name(axis)
-                    )
-                )
+                axis_name = self.obj._get_axis_name(axis)
+                raise KeyError(f"None of [{key}] are in the [{axis_name}]")
 
             # We (temporarily) allow for some missing keys with .loc, except in
             # some cases (e.g. setting) in which "raise_missing" will be False
             if not (self.name == "loc" and not raise_missing):
                 not_found = list(set(key) - set(ax))
-                raise KeyError("{} not in index".format(not_found))
+                raise KeyError(f"{not_found} not in index")
 
             # we skip the warning on Categorical/Interval
             # as this check is actually done (check for
@@ -1905,18 +1899,13 @@ def _validate_key(self, key, axis: int):
 
             # check that the key has a numeric dtype
             if not is_numeric_dtype(arr.dtype):
-                raise IndexError(
-                    ".iloc requires numeric indexers, got {arr}".format(arr=arr)
-                )
+                raise IndexError(f".iloc requires numeric indexers, got {arr}")
 
             # check that the key does not exceed the maximum size of the index
             if len(arr) and (arr.max() >= len_axis or arr.min() < -len_axis):
                 raise IndexError("positional indexers are out-of-bounds")
         else:
-            raise ValueError(
-                "Can only index by location with "
-                "a [{types}]".format(types=self._valid_types)
-            )
+            raise ValueError(f"Can only index by location with a [{self._valid_types}]")
 
     def _has_valid_setitem_indexer(self, indexer):
         self._has_valid_positional_setitem_indexer(indexer)
@@ -2063,10 +2052,7 @@ def _convert_to_indexer(self, obj, axis: int, raise_missing: bool = False):
             self._validate_key(obj, axis)
             return obj
         except ValueError:
-            raise ValueError(
-                "Can only index by location with "
-                "a [{types}]".format(types=self._valid_types)
-            )
+            raise ValueError(f"Can only index by location with a [{self._valid_types}]")
 
 
 class _ScalarAccessIndexer(_NDFrameIndexerBase):
@@ -2327,7 +2313,7 @@ def check_bool_indexer(index: Index, key) -> np.ndarray:
         # GH26658
         if len(result) != len(index):
             raise IndexError(
-                "Item wrong length {} instead of {}.".format(len(result), len(index))
+                f"Item wrong length {len(result)} instead of {len(index)}."
             )
 
     return result

pandas/core/internals/blocks.py

@@ -115,8 +115,8 @@ def __init__(self, values, placement, ndim=None):
 
         if self._validate_ndim and self.ndim and len(self.mgr_locs) != len(self.values):
             raise ValueError(
-                "Wrong number of items passed {val}, placement implies "
-                "{mgr}".format(val=len(self.values), mgr=len(self.mgr_locs))
+                f"Wrong number of items passed {len(self.values)}, "
+                f"placement implies {len(self.mgr_locs)}"
             )
 
     def _check_ndim(self, values, ndim):
@@ -144,9 +144,10 @@ def _check_ndim(self, values, ndim):
             ndim = values.ndim
 
         if self._validate_ndim and values.ndim != ndim:
-            msg = "Wrong number of dimensions. values.ndim != ndim [{} != {}]"
-            raise ValueError(msg.format(values.ndim, ndim))
-
+            raise ValueError(
+                "Wrong number of dimensions. "
+                f"values.ndim != ndim [{values.ndim} != {ndim}]"
+            )
         return ndim
 
     @property
@@ -184,7 +185,7 @@ def is_categorical_astype(self, dtype):
         if dtype is Categorical or dtype is CategoricalDtype:
             # this is a pd.Categorical, but is not
             # a valid type for astypeing
-            raise TypeError("invalid type {0} for astype".format(dtype))
+            raise TypeError(f"invalid type {dtype} for astype")
 
         elif is_categorical_dtype(dtype):
             return True
@@ -264,18 +265,14 @@ def __repr__(self) -> str:
         name = type(self).__name__
         if self._is_single_block:
 
-            result = "{name}: {len} dtype: {dtype}".format(
-                name=name, len=len(self), dtype=self.dtype
-            )
+            result = f"{name}: {len(self)} dtype: {self.dtype}"
 
         else:
 
             shape = " x ".join(pprint_thing(s) for s in self.shape)
-            result = "{name}: {index}, {shape}, dtype: {dtype}".format(
-                name=name,
-                index=pprint_thing(self.mgr_locs.indexer),
-                shape=shape,
-                dtype=self.dtype,
+            result = (
+                f"{name}: {pprint_thing(self.mgr_locs.indexer)}, "
+                f"{shape}, dtype: {self.dtype}"
             )
 
         return result
@@ -329,7 +326,7 @@ def ftype(self):
             dtype = self.dtype.subtype
         else:
             dtype = self.dtype
-        return "{dtype}:{ftype}".format(dtype=dtype, ftype=self._ftype)
+        return f"{dtype}:{self._ftype}"
 
     def merge(self, other):
         return _merge_blocks([self, other])
@@ -544,15 +541,15 @@ def astype(self, dtype, copy: bool = False, errors: str = "raise"):
 
         if errors not in errors_legal_values:
             invalid_arg = (
-                "Expected value of kwarg 'errors' to be one of {}. "
-                "Supplied value is '{}'".format(list(errors_legal_values), errors)
+                "Expected value of kwarg 'errors' to be one of "
+                f"{list(errors_legal_values)}. Supplied value is '{errors}'"
             )
             raise ValueError(invalid_arg)
 
         if inspect.isclass(dtype) and issubclass(dtype, ExtensionDtype):
             msg = (
-                "Expected an instance of {}, but got the class instead. "
-                "Try instantiating 'dtype'.".format(dtype.__name__)
+                f"Expected an instance of {dtype.__name__}, "
+                "but got the class instead. Try instantiating 'dtype'."
             )
             raise TypeError(msg)
 
@@ -613,15 +610,9 @@ def astype(self, dtype, copy: bool = False, errors: str = "raise"):
         if newb.is_numeric and self.is_numeric:
             if newb.shape != self.shape:
                 raise TypeError(
-                    "cannot set astype for copy = [{copy}] for dtype "
-                    "({dtype} [{shape}]) to different shape "
-                    "({newb_dtype} [{newb_shape}])".format(
-                        copy=copy,
-                        dtype=self.dtype.name,
-                        shape=self.shape,
-                        newb_dtype=newb.dtype.name,
-                        newb_shape=newb.shape,
-                    )
+                    f"cannot set astype for copy = [{copy}] for dtype "
+                    f"({self.dtype.name} [{self.shape}]) to different shape "
+                    f"({newb.dtype.name} [{newb.shape}])"
                 )
         return newb
 
@@ -658,7 +649,7 @@ def to_native_types(self, slicer=None, na_rep="nan", quoting=None, **kwargs):
 
         if not self.is_object and not quoting:
             itemsize = writers.word_len(na_rep)
-            values = values.astype("<U{size}".format(size=itemsize))
+            values = values.astype(f"<U{itemsize}")
         else:
             values = np.array(values, dtype="object")
 
@@ -1045,8 +1036,7 @@ def coerce_to_target_dtype(self, other):
                 return self.astype(object)
 
             raise AssertionError(
-                "possible recursion in "
-                "coerce_to_target_dtype: {} {}".format(self, other)
+                f"possible recursion in coerce_to_target_dtype: {self} {other}"
             )
 
         elif self.is_timedelta or is_timedelta64_dtype(dtype):
@@ -1056,8 +1046,7 @@ def coerce_to_target_dtype(self, other):
                 return self.astype(object)
 
             raise AssertionError(
-                "possible recursion in "
-                "coerce_to_target_dtype: {} {}".format(self, other)
+                f"possible recursion in coerce_to_target_dtype: {self} {other}"
             )
 
         try:
@@ -1202,8 +1191,7 @@ def _interpolate(
         if method in ("krogh", "piecewise_polynomial", "pchip"):
             if not index.is_monotonic:
                 raise ValueError(
-                    "{0} interpolation requires that the "
-                    "index be monotonic.".format(method)
+                    f"{method} interpolation requires that the index be monotonic."
                 )
         # process 1-d slices in the axis direction
 
@@ -1585,15 +1573,15 @@ def iget(self, col):
         if self.ndim == 2 and isinstance(col, tuple):
             col, loc = col
             if not com.is_null_slice(col) and col != 0:
-                raise IndexError("{0} only contains one item".format(self))
+                raise IndexError(f"{self} only contains one item")
             elif isinstance(col, slice):
                 if col != slice(None):
                     raise NotImplementedError(col)
                 return self.values[[loc]]
             return self.values[loc]
         else:
             if col != 0:
-                raise IndexError("{0} only contains one item".format(self))
+                raise IndexError(f"{self} only contains one item")
             return self.values
 
     def should_store(self, value):
@@ -2312,7 +2300,7 @@ def _slice(self, slicer):
         if isinstance(slicer, tuple):
             col, loc = slicer
             if not com.is_null_slice(col) and col != 0:
-                raise IndexError("{0} only contains one item".format(self))
+                raise IndexError(f"{self} only contains one item")
             return self.values[loc]
         return self.values[slicer]
 

pandas/core/internals/construction.py

@@ -167,8 +167,7 @@ def init_ndarray(values, index, columns, dtype=None, copy=False):
             except Exception as orig:
                 # e.g. ValueError when trying to cast object dtype to float64
                 raise ValueError(
-                    "failed to cast to '{dtype}' (Exception "
-                    "was: {orig})".format(dtype=dtype, orig=orig)
+                    f"failed to cast to '{dtype}' (Exception was: {orig})"
                 ) from orig
 
     index, columns = _get_axes(*values.shape, index=index, columns=columns)
@@ -365,8 +364,8 @@ def extract_index(data):
             if have_series:
                 if lengths[0] != len(index):
                     msg = (
-                        "array length {length} does not match index "
-                        "length {idx_len}".format(length=lengths[0], idx_len=len(index))
+                        f"array length {lengths[0]} does not match index "
+                        f"length {len(index)}"
                     )
                     raise ValueError(msg)
             else:
@@ -401,7 +400,7 @@ def get_names_from_index(data):
         if n is not None:
             index[i] = n
         else:
-            index[i] = "Unnamed {count}".format(count=count)
+            index[i] = f"Unnamed {count}"
             count += 1
 
     return index
@@ -571,8 +570,8 @@ def _convert_object_array(content, columns, coerce_float=False, dtype=None):
         if len(columns) != len(content):  # pragma: no cover
             # caller's responsibility to check for this...
             raise AssertionError(
-                "{col:d} columns passed, passed data had "
-                "{con} columns".format(col=len(columns), con=len(content))
+                f"{len(columns)} columns passed, passed data had "
+                f"{len(content)} columns"
             )
 
     # provide soft conversion of object dtypes