Merge branch 'pandas-dev:main' into issue2

Author: ahmedibrhm

.github/workflows/32-bit-linux.yml

@@ -12,6 +12,9 @@ on:
     paths-ignore:
       - "doc/**"
 
+permissions:
+  contents: read
+
 jobs:
   pytest:
     runs-on: ubuntu-latest

.github/workflows/assign.yml

@@ -3,8 +3,14 @@ on:
   issue_comment:
     types: created
 
+permissions:
+  contents: read
+
 jobs:
   issue_assign:
+    permissions:
+      issues: write
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
     - if: github.event.comment.body == 'take'

.github/workflows/asv-bot.yml

@@ -9,8 +9,15 @@ env:
   ENV_FILE: environment.yml
   COMMENT: ${{github.event.comment.body}}
 
+permissions:
+  contents: read
+
 jobs:
   autotune:
+    permissions:
+      contents: read
+      issues: write
+      pull-requests: write
     name: "Run benchmarks"
     # TODO: Support more benchmarking options later, against different branches, against self, etc
     if: startsWith(github.event.comment.body, '@github-actions benchmark')

.github/workflows/autoupdate-pre-commit-config.yml

@@ -5,8 +5,14 @@ on:
     - cron: "0 7 1 * *" # At 07:00 on 1st of every month.
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   update-pre-commit:
+    permissions:
+      contents: write  # for technote-space/create-pr-action to push code
+      pull-requests: write  # for technote-space/create-pr-action to create a PR
     if: github.repository_owner == 'pandas-dev'
     name: Autoupdate pre-commit config
     runs-on: ubuntu-latest

.github/workflows/code-checks.yml

@@ -14,6 +14,9 @@ env:
   ENV_FILE: environment.yml
   PANDAS_CI: 1
 
+permissions:
+  contents: read
+
 jobs:
   pre_commit:
     name: pre-commit

.github/workflows/docbuild-and-upload.yml

@@ -14,6 +14,9 @@ env:
   ENV_FILE: environment.yml
   PANDAS_CI: 1
 
+permissions:
+  contents: read
+
 jobs:
   web_and_docs:
     name: Doc Build and Upload

.github/workflows/macos-windows.yml

@@ -18,6 +18,9 @@ env:
   PATTERN: "not slow and not db and not network and not single_cpu"
 
 
+permissions:
+  contents: read
+
 jobs:
   pytest:
     defaults:

.github/workflows/python-dev.yml

@@ -27,6 +27,9 @@ env:
   COVERAGE: true
   PYTEST_TARGET: pandas
 
+permissions:
+  contents: read
+
 jobs:
   build:
     if: false # Comment this line out to "unfreeze"

.github/workflows/sdist.yml

@@ -13,6 +13,9 @@ on:
     paths-ignore:
       - "doc/**"
 
+permissions:
+  contents: read
+
 jobs:
   build:
     if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}

.github/workflows/stale-pr.yml

@@ -4,8 +4,13 @@ on:
   # * is a special character in YAML so you have to quote this string
   - cron: "0 0 * * *"
 
+permissions:
+  contents: read
+
 jobs:
   stale:
+    permissions:
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
     - uses: actions/stale@v4

.github/workflows/ubuntu.yml

@@ -15,6 +15,9 @@ on:
 env:
   PANDAS_CI: 1
 
+permissions:
+  contents: read
+
 jobs:
   pytest:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -93,7 +93,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - [email protected].253
+        - [email protected].258
 -   repo: local
     hooks:
     -   id: pyright_reportGeneralTypeIssues

README.md

@@ -169,4 +169,4 @@ Or maybe through using pandas you have an idea of your own or are looking for so
 
 Feel free to ask questions on the [mailing list](https://groups.google.com/forum/?fromgroups#!forum/pydata) or on [Gitter](https://gitter.im/pydata/pandas).
 
-As contributors and maintainers to this project, you are expected to abide by pandas' code of conduct. More information can be found at: [Contributor Code of Conduct](https://github.com/pandas-dev/pandas/blob/main/.github/CODE_OF_CONDUCT.md)
+As contributors and maintainers to this project, you are expected to abide by pandas' code of conduct. More information can be found at: [Contributor Code of Conduct](https://github.com/pandas-dev/.github/blob/master/CODE_OF_CONDUCT.md)

asv_bench/benchmarks/tslibs/strftime.py renamed to asv_bench/benchmarks/strftime.py

@@ -26,11 +26,14 @@ Exceptions and warnings
 
    errors.AbstractMethodError
    errors.AccessorRegistrationWarning
+   errors.AttributeConflictWarning
+   errors.ClosedFileError
    errors.CSSWarning
    errors.DataError
    errors.DtypeWarning
    errors.DuplicateLabelError
    errors.EmptyDataError
+   errors.IncompatibilityWarning
    errors.IndexingError
    errors.InvalidIndexError
    errors.IntCastingNaNError
@@ -44,6 +47,7 @@ Exceptions and warnings
    errors.ParserError
    errors.ParserWarning
    errors.PerformanceWarning
+   errors.PossibleDataLossError
    errors.PyperclipException
    errors.PyperclipWindowsException
    errors.SettingWithCopyError

doc/source/reference/testing.rst

@@ -1001,6 +1001,8 @@ Groupby/resample/rolling
 - Bug in :meth:`.DataFrameGroupBy.describe` and :meth:`.SeriesGroupBy.describe` produces inconsistent results for empty datasets (:issue:`41575`)
 - Bug in :meth:`DataFrame.resample` reduction methods when used with ``on`` would attempt to aggregate the provided column (:issue:`47079`)
 - Bug in :meth:`DataFrame.groupby` and :meth:`Series.groupby` would not respect ``dropna=False`` when the input DataFrame/Series had a NaN values in a :class:`MultiIndex` (:issue:`46783`)
+- Bug in :meth:`DataFrameGroupBy.resample` raises ``KeyError`` when getting the result from a key list which misses the resample key (:issue:`47362`)
+-
 
 Reshaping
 ^^^^^^^^^

doc/source/whatsnew/v1.5.0.rst

@@ -152,7 +152,7 @@ def format_array_from_datetime(
         # a format based on precision
         basic_format = format is None
         if basic_format:
-            reso_obj = get_resolution(values, reso=reso)
+            reso_obj = get_resolution(values, tz=tz, reso=reso)
             show_ns = reso_obj == Resolution.RESO_NS
             show_us = reso_obj == Resolution.RESO_US
             show_ms = reso_obj == Resolution.RESO_MS

pandas/_libs/tslib.pyx

@@ -144,9 +144,13 @@ cpdef inline (int64_t, int) precision_from_unit(str unit):
         NPY_DATETIMEUNIT reso = abbrev_to_npy_unit(unit)
 
     if reso == NPY_DATETIMEUNIT.NPY_FR_Y:
+        # each 400 years we have 97 leap years, for an average of 97/400=.2425
+        #  extra days each year. We get 31556952 by writing
+        #  3600*24*365.2425=31556952
         m = 1_000_000_000 * 31556952
         p = 9
     elif reso == NPY_DATETIMEUNIT.NPY_FR_M:
+        # 2629746 comes from dividing the "Y" case by 12.
         m = 1_000_000_000 * 2629746
         p = 9
     elif reso == NPY_DATETIMEUNIT.NPY_FR_W:

pandas/_libs/tslibs/conversion.pyx

@@ -42,5 +42,5 @@ def ints_to_pydatetime(
 def tz_convert_from_utc(
     stamps: npt.NDArray[np.int64],
     tz: tzinfo | None,
-    reso: int = ...,
+    reso: int = ...,  # NPY_DATETIMEUNIT
 ) -> npt.NDArray[np.int64]: ...

pandas/_libs/tslibs/vectorized.pyi

@@ -73,7 +73,6 @@
 from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import (
-    DT64NS_DTYPE,
     is_all_strings,
     is_categorical_dtype,
     is_datetime64_any_dtype,
@@ -1103,6 +1102,7 @@ def _add_datetimelike_scalar(self, other) -> DatetimeArray:
         self = cast("TimedeltaArray", self)
 
         from pandas.core.arrays import DatetimeArray
+        from pandas.core.arrays.datetimes import tz_to_dtype
 
         assert other is not NaT
         other = Timestamp(other)
@@ -1113,10 +1113,17 @@ def _add_datetimelike_scalar(self, other) -> DatetimeArray:
             # Preserve our resolution
             return DatetimeArray._simple_new(result, dtype=result.dtype)
 
+        if self._reso != other._reso:
+            raise NotImplementedError(
+                "Addition between TimedeltaArray and Timestamp with mis-matched "
+                "resolutions is not yet supported."
+            )
+
         i8 = self.asi8
         result = checked_add_with_arr(i8, other.value, arr_mask=self._isnan)
-        dtype = DatetimeTZDtype(tz=other.tz) if other.tz else DT64NS_DTYPE
-        return DatetimeArray(result, dtype=dtype, freq=self.freq)
+        dtype = tz_to_dtype(tz=other.tz, unit=self._unit)
+        res_values = result.view(f"M8[{self._unit}]")
+        return DatetimeArray._simple_new(res_values, dtype=dtype, freq=self.freq)
 
     @final
     def _add_datetime_arraylike(self, other) -> DatetimeArray:

pandas/core/arrays/datetimelike.py

@@ -91,22 +91,23 @@
 _midnight = time(0, 0)
 
 
-def tz_to_dtype(tz):
+def tz_to_dtype(tz: tzinfo | None, unit: str = "ns"):
     """
     Return a datetime64[ns] dtype appropriate for the given timezone.
 
     Parameters
     ----------
     tz : tzinfo or None
+    unit : str, default "ns"
 
     Returns
     -------
     np.dtype or Datetime64TZDType
     """
     if tz is None:
-        return DT64NS_DTYPE
+        return np.dtype(f"M8[{unit}]")
     else:
-        return DatetimeTZDtype(tz=tz)
+        return DatetimeTZDtype(tz=tz, unit=unit)
 
 
 def _field_accessor(name: str, field: str, docstring=None):
@@ -800,7 +801,7 @@ def tz_convert(self, tz) -> DatetimeArray:
             )
 
         # No conversion since timestamps are all UTC to begin with
-        dtype = tz_to_dtype(tz)
+        dtype = tz_to_dtype(tz, unit=self._unit)
         return self._simple_new(self._ndarray, dtype=dtype, freq=self.freq)
 
     @dtl.ravel_compat
@@ -965,10 +966,14 @@ def tz_localize(self, tz, ambiguous="raise", nonexistent="raise") -> DatetimeArr
             # Convert to UTC
 
             new_dates = tzconversion.tz_localize_to_utc(
-                self.asi8, tz, ambiguous=ambiguous, nonexistent=nonexistent
+                self.asi8,
+                tz,
+                ambiguous=ambiguous,
+                nonexistent=nonexistent,
+                reso=self._reso,
             )
-        new_dates = new_dates.view(DT64NS_DTYPE)
-        dtype = tz_to_dtype(tz)
+        new_dates = new_dates.view(f"M8[{self._unit}]")
+        dtype = tz_to_dtype(tz, unit=self._unit)
 
         freq = None
         if timezones.is_utc(tz) or (len(self) == 1 and not isna(new_dates[0])):

pandas/core/arrays/datetimes.py

@@ -90,8 +90,11 @@ class StringDtype(StorageExtensionDtype):
 
     name = "string"
 
-    #: StringDtype.na_value uses pandas.NA
-    na_value = libmissing.NA
+    #: StringDtype().na_value uses pandas.NA
+    @property
+    def na_value(self) -> libmissing.NAType:
+        return libmissing.NA
+
     _metadata = ("storage",)
 
     def __init__(self, storage=None) -> None:
@@ -335,13 +338,11 @@ def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy=False):
             na_values = scalars._mask
             result = scalars._data
             result = lib.ensure_string_array(result, copy=copy, convert_na_value=False)
-            result[na_values] = StringDtype.na_value
+            result[na_values] = libmissing.NA
 
         else:
-            # convert non-na-likes to str, and nan-likes to StringDtype.na_value
-            result = lib.ensure_string_array(
-                scalars, na_value=StringDtype.na_value, copy=copy
-            )
+            # convert non-na-likes to str, and nan-likes to StringDtype().na_value
+            result = lib.ensure_string_array(scalars, na_value=libmissing.NA, copy=copy)
 
         # Manually creating new array avoids the validation step in the __init__, so is
         # faster. Refactor need for validation?
@@ -396,7 +397,7 @@ def __setitem__(self, key, value):
         # validate new items
         if scalar_value:
             if isna(value):
-                value = StringDtype.na_value
+                value = libmissing.NA
             elif not isinstance(value, str):
                 raise ValueError(
                     f"Cannot set non-string value '{value}' into a StringArray."
@@ -497,7 +498,7 @@ def _cmp_method(self, other, op):
 
         if op.__name__ in ops.ARITHMETIC_BINOPS:
             result = np.empty_like(self._ndarray, dtype="object")
-            result[mask] = StringDtype.na_value
+            result[mask] = libmissing.NA
             result[valid] = op(self._ndarray[valid], other)
             return StringArray(result)
         else:
@@ -512,7 +513,7 @@ def _cmp_method(self, other, op):
     # String methods interface
     # error: Incompatible types in assignment (expression has type "NAType",
     # base class "PandasArray" defined the type as "float")
-    _str_na_value = StringDtype.na_value  # type: ignore[assignment]
+    _str_na_value = libmissing.NA  # type: ignore[assignment]
 
     def _str_map(
         self, f, na_value=None, dtype: Dtype | None = None, convert: bool = True

pandas/core/arrays/string_.py

@@ -242,8 +242,9 @@ def astype(self, dtype, copy: bool = True):
     # ------------------------------------------------------------------------
     # String methods interface
 
-    # error: Cannot determine type of 'na_value'
-    _str_na_value = StringDtype.na_value  # type: ignore[has-type]
+    # error: Incompatible types in assignment (expression has type "NAType",
+    # base class "ObjectStringArrayMixin" defined the type as "float")
+    _str_na_value = libmissing.NA  # type: ignore[assignment]
 
     def _str_map(
         self, f, na_value=None, dtype: Dtype | None = None, convert: bool = True