Merge remote-tracking branch 'upstream/main' into tst/ref/test_sql

Author: mroeschke

doc/source/user_guide/timeseries.rst

@@ -890,7 +890,7 @@ into ``freq`` keyword arguments. The available date offsets and associated frequ
     :class:`~pandas.tseries.offsets.CBMonthBegin` or :class:`~pandas.tseries.offsets.CustomBusinessMonthBegin`, ``'CBMS'``, "custom business month begin"
     :class:`~pandas.tseries.offsets.SemiMonthEnd`, ``'SM'``, "15th (or other day_of_month) and calendar month end"
     :class:`~pandas.tseries.offsets.SemiMonthBegin`, ``'SMS'``, "15th (or other day_of_month) and calendar month begin"
-    :class:`~pandas.tseries.offsets.QuarterEnd`, ``'Q'``, "calendar quarter end"
+    :class:`~pandas.tseries.offsets.QuarterEnd`, ``'QE'``, "calendar quarter end"
     :class:`~pandas.tseries.offsets.QuarterBegin`, ``'QS'``, "calendar quarter begin"
     :class:`~pandas.tseries.offsets.BQuarterEnd`, ``'BQ``, "business quarter end"
     :class:`~pandas.tseries.offsets.BQuarterBegin`, ``'BQS'``, "business quarter begin"
@@ -1254,7 +1254,7 @@ frequencies. We will refer to these aliases as *offset aliases*.
     "SMS", "semi-month start frequency (1st and 15th)"
     "BMS", "business month start frequency"
     "CBMS", "custom business month start frequency"
-    "Q", "quarter end frequency"
+    "QE", "quarter end frequency"
     "BQ", "business quarter end frequency"
     "QS", "quarter start frequency"
     "BQS", "business quarter start frequency"
@@ -1373,18 +1373,18 @@ For some frequencies you can specify an anchoring suffix:
     "W\-THU", "weekly frequency (Thursdays)"
     "W\-FRI", "weekly frequency (Fridays)"
     "W\-SAT", "weekly frequency (Saturdays)"
-    "(B)Q(S)\-DEC", "quarterly frequency, year ends in December. Same as 'Q'"
-    "(B)Q(S)\-JAN", "quarterly frequency, year ends in January"
-    "(B)Q(S)\-FEB", "quarterly frequency, year ends in February"
-    "(B)Q(S)\-MAR", "quarterly frequency, year ends in March"
-    "(B)Q(S)\-APR", "quarterly frequency, year ends in April"
-    "(B)Q(S)\-MAY", "quarterly frequency, year ends in May"
-    "(B)Q(S)\-JUN", "quarterly frequency, year ends in June"
-    "(B)Q(S)\-JUL", "quarterly frequency, year ends in July"
-    "(B)Q(S)\-AUG", "quarterly frequency, year ends in August"
-    "(B)Q(S)\-SEP", "quarterly frequency, year ends in September"
-    "(B)Q(S)\-OCT", "quarterly frequency, year ends in October"
-    "(B)Q(S)\-NOV", "quarterly frequency, year ends in November"
+    "(B)Q(E)(S)\-DEC", "quarterly frequency, year ends in December. Same as 'QE'"
+    "(B)Q(E)(S)\-JAN", "quarterly frequency, year ends in January"
+    "(B)Q(E)(S)\-FEB", "quarterly frequency, year ends in February"
+    "(B)Q(E)(S)\-MAR", "quarterly frequency, year ends in March"
+    "(B)Q(E)(S)\-APR", "quarterly frequency, year ends in April"
+    "(B)Q(E)(S)\-MAY", "quarterly frequency, year ends in May"
+    "(B)Q(E)(S)\-JUN", "quarterly frequency, year ends in June"
+    "(B)Q(E)(S)\-JUL", "quarterly frequency, year ends in July"
+    "(B)Q(E)(S)\-AUG", "quarterly frequency, year ends in August"
+    "(B)Q(E)(S)\-SEP", "quarterly frequency, year ends in September"
+    "(B)Q(E)(S)\-OCT", "quarterly frequency, year ends in October"
+    "(B)Q(E)(S)\-NOV", "quarterly frequency, year ends in November"
     "(B)Y(S)\-DEC", "annual frequency, anchored end of December. Same as 'Y'"
     "(B)Y(S)\-JAN", "annual frequency, anchored end of January"
     "(B)Y(S)\-FEB", "annual frequency, anchored end of February"
@@ -1692,7 +1692,7 @@ the end of the interval.
 .. warning::
 
     The default values for ``label`` and ``closed`` is '**left**' for all
-    frequency offsets except for 'ME', 'Y', 'Q', 'BME', 'BY', 'BQ', and 'W'
+    frequency offsets except for 'ME', 'Y', 'QE', 'BME', 'BY', 'BQ', and 'W'
     which all have a default of 'right'.
 
     This might unintendedly lead to looking ahead, where the value for a later

doc/source/whatsnew/v0.18.0.rst

@@ -808,11 +808,19 @@ Upsampling operations take you from a lower frequency to a higher frequency. The
 performed with the ``Resampler`` objects with :meth:`~Resampler.backfill`,
 :meth:`~Resampler.ffill`, :meth:`~Resampler.fillna` and :meth:`~Resampler.asfreq` methods.
 
-.. ipython:: python
+.. code-block:: ipython
 
-   s = pd.Series(np.arange(5, dtype='int64'),
+   In [89]: s = pd.Series(np.arange(5, dtype='int64'),
                  index=pd.date_range('2010-01-01', periods=5, freq='Q'))
-   s
+
+   In [90]: s
+   Out[90]:
+   2010-03-31    0
+   2010-06-30    1
+   2010-09-30    2
+   2010-12-31    3
+   2011-03-31    4
+   Freq: Q-DEC, Length: 5, dtype: int64
 
 Previously
 

doc/source/whatsnew/v2.1.2.rst

@@ -35,6 +35,7 @@ Bug fixes
 - Fixed bug in :meth:`Series.str.extractall` for :class:`ArrowDtype` dtype being converted to object (:issue:`53846`)
 - Fixed bug where PDEP-6 warning about setting an item of an incompatible dtype was being shown when creating a new conditional column (:issue:`55025`)
 - Silence ``Period[B]`` warnings introduced by :issue:`53446` during normal plotting activity (:issue:`55138`)
+- Fixed bug in :class:`Series` constructor not inferring string dtype when ``NA`` is the first value and ``infer_string`` is set (:issue:` 55655`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_212.other:

doc/source/whatsnew/v2.2.0.rst

@@ -210,8 +210,8 @@ Other API changes
 Deprecations
 ~~~~~~~~~~~~
 
-Deprecate alias ``M`` in favour of ``ME`` for offsets
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Deprecate aliases ``M`` and ``Q`` in favour of ``ME`` and ``QE`` for offsets
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The alias ``M`` is deprecated in favour of ``ME`` for offsets, please use ``ME`` for "month end" instead of ``M`` (:issue:`9586`)
 
@@ -232,6 +232,25 @@ For example:
 
     pd.date_range('2020-01-01', periods=3, freq='ME')
 
+The alias ``Q`` is deprecated in favour of ``QE`` for offsets, please use ``QE`` for "quarter end" instead of ``Q`` (:issue:`9586`)
+
+For example:
+
+*Previous behavior*:
+
+.. code-block:: ipython
+
+    In [8]: pd.date_range('2020-01-01', periods=3, freq='Q-NOV')
+    Out[8]:
+    DatetimeIndex(['2020-02-29', '2020-05-31', '2020-08-31'],
+                  dtype='datetime64[ns]', freq='Q-NOV')
+
+*Future behavior*:
+
+.. ipython:: python
+
+    pd.date_range('2020-01-01', periods=3, freq='QE-NOV')
+
 Other Deprecations
 ^^^^^^^^^^^^^^^^^^
 - Changed :meth:`Timedelta.resolution_string` to return ``h``, ``min``, ``s``, ``ms``, ``us``, and ``ns`` instead of ``H``, ``T``, ``S``, ``L``, ``U``, and ``N``, for compatibility with respective deprecations in frequency aliases (:issue:`52536`)

pandas/_libs/lib.pyx

@@ -2665,6 +2665,9 @@ def maybe_convert_objects(ndarray[object] objects,
             else:
                 seen.object_ = True
                 break
+        elif val is C_NA:
+            seen.object_ = True
+            continue
         else:
             seen.object_ = True
             break

pandas/_libs/parsers.pyx

@@ -228,7 +228,7 @@ cdef extern from "pandas/parser/tokenizer.h":
         # pick one, depending on whether the converter requires GIL
         double (*double_converter)(const char *, char **,
                                    char, char, char,
-                                   int, int *, int *) nogil
+                                   int, int *, int *) noexcept nogil
 
         #  error handling
         char *warn_msg
@@ -1605,7 +1605,7 @@ cdef _categorical_convert(parser_t *parser, int64_t col,
 
 # -> ndarray[f'|S{width}']
 cdef _to_fw_string(parser_t *parser, int64_t col, int64_t line_start,
-                   int64_t line_end, int64_t width):
+                   int64_t line_end, int64_t width) noexcept:
     cdef:
         char *data
         ndarray result
@@ -1621,7 +1621,7 @@ cdef _to_fw_string(parser_t *parser, int64_t col, int64_t line_start,
 
 cdef void _to_fw_string_nogil(parser_t *parser, int64_t col,
                               int64_t line_start, int64_t line_end,
-                              size_t width, char *data) nogil:
+                              size_t width, char *data) noexcept nogil:
     cdef:
         int64_t i
         coliter_t it
@@ -1677,7 +1677,7 @@ cdef _try_double(parser_t *parser, int64_t col,
 cdef int _try_double_nogil(parser_t *parser,
                            float64_t (*double_converter)(
                                const char *, char **, char,
-                               char, char, int, int *, int *) nogil,
+                               char, char, int, int *, int *) noexcept nogil,
                            int64_t col, int64_t line_start, int64_t line_end,
                            bint na_filter, kh_str_starts_t *na_hashset,
                            bint use_na_flist,

pandas/_libs/tslibs/dtypes.pyx

@@ -201,7 +201,19 @@ OFFSET_TO_PERIOD_FREQSTR: dict = {
     "us": "us",
     "ns": "ns",
     "h": "h",
-    "Q": "Q",
+    "QE": "Q",
+    "QE-DEC": "Q-DEC",
+    "QE-JAN": "Q-JAN",
+    "QE-FEB": "Q-FEB",
+    "QE-MAR": "Q-MAR",
+    "QE-APR": "Q-APR",
+    "QE-MAY": "Q-MAY",
+    "QE-JUN": "Q-JUN",
+    "QE-JUL": "Q-JUL",
+    "QE-AUG": "Q-AUG",
+    "QE-SEP": "Q-SEP",
+    "QE-OCT": "Q-OCT",
+    "QE-NOV": "Q-NOV",
     "W": "W",
     "ME": "M",
     "Y": "Y",
@@ -211,6 +223,19 @@ OFFSET_TO_PERIOD_FREQSTR: dict = {
 }
 OFFSET_DEPR_FREQSTR: dict[str, str]= {
     "M": "ME",
+    "Q": "QE",
+    "Q-DEC": "QE-DEC",
+    "Q-JAN": "QE-JAN",
+    "Q-FEB": "QE-FEB",
+    "Q-MAR": "QE-MAR",
+    "Q-APR": "QE-APR",
+    "Q-MAY": "QE-MAY",
+    "Q-JUN": "QE-JUN",
+    "Q-JUL": "QE-JUL",
+    "Q-AUG": "QE-AUG",
+    "Q-SEP": "QE-SEP",
+    "Q-OCT": "QE-OCT",
+    "Q-NOV": "QE-NOV",
 }
 cdef dict c_OFFSET_TO_PERIOD_FREQSTR = OFFSET_TO_PERIOD_FREQSTR
 cdef dict c_OFFSET_DEPR_FREQSTR = OFFSET_DEPR_FREQSTR

pandas/_libs/tslibs/offsets.pyx

@@ -2767,7 +2767,7 @@ cdef class QuarterEnd(QuarterOffset):
     Timestamp('2022-03-31 00:00:00')
     """
     _default_starting_month = 3
-    _prefix = "Q"
+    _prefix = "QE"
     _day_opt = "end"
 
     cdef readonly:
@@ -4585,7 +4585,7 @@ prefix_mapping = {
         Second,  # 's'
         Minute,  # 'min'
         Micro,  # 'us'
-        QuarterEnd,  # 'Q'
+        QuarterEnd,  # 'QE'
         QuarterBegin,  # 'QS'
         Milli,  # 'ms'
         Hour,  # 'h'
@@ -4603,7 +4603,7 @@ opattern = re.compile(
 
 _lite_rule_alias = {
     "W": "W-SUN",
-    "Q": "Q-DEC",
+    "QE": "QE-DEC",
 
     "Y": "Y-DEC",      # YearEnd(month=12),
     "YS": "YS-JAN",    # YearBegin(month=1),
@@ -4617,7 +4617,7 @@ _lite_rule_alias = {
     "ns": "ns",
 }
 
-_dont_uppercase = {"h", "bh", "cbh", "MS", "ms", "s", "me"}
+_dont_uppercase = {"h", "bh", "cbh", "MS", "ms", "s", "me", "qe"}
 
 
 INVALID_FREQ_ERR_MSG = "Invalid frequency: {0}"

pandas/core/indexes/accessors.py

@@ -284,7 +284,7 @@ class DatetimeProperties(Properties):
     2    2
     dtype: int32
 
-    >>> quarters_series = pd.Series(pd.date_range("2000-01-01", periods=3, freq="q"))
+    >>> quarters_series = pd.Series(pd.date_range("2000-01-01", periods=3, freq="QE"))
     >>> quarters_series
     0   2000-03-31
     1   2000-06-30

pandas/core/resample.py

@@ -2130,7 +2130,7 @@ def __init__(
         else:
             freq = to_offset(freq)
 
-        end_types = {"ME", "Y", "Q", "BME", "BY", "BQ", "W"}
+        end_types = {"ME", "Y", "QE", "BME", "BY", "BQ", "W"}
         rule = freq.rule_code
         if rule in end_types or ("-" in rule and rule[: rule.find("-")] in end_types):
             if closed is None:
@@ -2329,7 +2329,7 @@ def _adjust_bin_edges(
         if self.freq.name in ("BME", "ME", "W") or self.freq.name.split("-")[0] in (
             "BQ",
             "BY",
-            "Q",
+            "QE",
             "Y",
             "W",
         ):

pandas/io/json/_table_schema.py

@@ -15,6 +15,7 @@
 from pandas._libs import lib
 from pandas._libs.json import ujson_loads
 from pandas._libs.tslibs import timezones
+from pandas._libs.tslibs.dtypes import freq_to_period_freqstr
 from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.base import _registry as registry
@@ -34,6 +35,8 @@
 from pandas import DataFrame
 import pandas.core.common as com
 
+from pandas.tseries.frequencies import to_offset
+
 if TYPE_CHECKING:
     from pandas._typing import (
         DtypeObj,
@@ -207,8 +210,12 @@ def convert_json_field_to_pandas_type(field) -> str | CategoricalDtype:
         if field.get("tz"):
             return f"datetime64[ns, {field['tz']}]"
         elif field.get("freq"):
+            # GH#9586 rename frequency M to ME for offsets
+            offset = to_offset(field["freq"])
+            freq_n, freq_name = offset.n, offset.name
+            freq = freq_to_period_freqstr(freq_n, freq_name)
             # GH#47747 using datetime over period to minimize the change surface
-            return f"period[{field['freq']}]"
+            return f"period[{freq}]"
         else:
             return "datetime64[ns]"
     elif typ == "any":

pandas/tests/arithmetic/test_datetime64.py

@@ -1076,7 +1076,7 @@ def test_dt64arr_add_dtlike_raises(self, tz_naive_fixture, box_with_array):
     # Note: freq here includes both Tick and non-Tick offsets; this is
     #  relevant because historically integer-addition was allowed if we had
     #  a freq.
-    @pytest.mark.parametrize("freq", ["h", "D", "W", "2ME", "MS", "Q", "B", None])
+    @pytest.mark.parametrize("freq", ["h", "D", "W", "2ME", "MS", "QE", "B", None])
     @pytest.mark.parametrize("dtype", [None, "uint8"])
     def test_dt64arr_addsub_intlike(
         self, request, dtype, box_with_array, freq, tz_naive_fixture

pandas/tests/arrays/test_datetimelike.py

@@ -30,7 +30,7 @@
 
 
 # TODO: more freq variants
-@pytest.fixture(params=["D", "B", "W", "ME", "Q", "Y"])
+@pytest.fixture(params=["D", "B", "W", "ME", "QE", "Y"])
 def freqstr(request):
     """Fixture returning parametrized frequency in string format."""
     return request.param

pandas/tests/arrays/test_datetimes.py

@@ -746,6 +746,25 @@ def test_iter_zoneinfo_fold(self, tz):
         assert str(left) == str(right2)
         assert left.utcoffset() == right2.utcoffset()
 
+    @pytest.mark.parametrize(
+        "freq, freq_depr",
+        [
+            ("2ME", "2M"),
+            ("2QE", "2Q"),
+            ("2QE-SEP", "2Q-SEP"),
+        ],
+    )
+    def test_date_range_frequency_M_Q_deprecated(self, freq, freq_depr):
+        # GH#9586
+        depr_msg = (
+            f"'{freq_depr[1:]}' will be deprecated, please use '{freq[1:]}' instead."
+        )
+
+        expected = pd.date_range("1/1/2000", periods=4, freq=freq)
+        with tm.assert_produces_warning(FutureWarning, match=depr_msg):
+            result = pd.date_range("1/1/2000", periods=4, freq=freq_depr)
+        tm.assert_index_equal(result, expected)
+
 
 def test_factorize_sort_without_freq():
     dta = DatetimeArray._from_sequence([0, 2, 1])

pandas/tests/frame/methods/test_asfreq.py

@@ -234,12 +234,23 @@ def test_asfreq_2ME(self, freq, freq_half):
         result = DataFrame({"s": Series([0.0, 2.0, 4.0], index=index)})
         tm.assert_frame_equal(result, expected)
 
-    def test_asfreq_frequency_M_deprecated(self):
-        depr_msg = "'M' will be deprecated, please use 'ME' instead."
+    @pytest.mark.parametrize(
+        "freq, freq_depr",
+        [
+            ("2ME", "2M"),
+            ("2QE", "2Q"),
+            ("2QE-SEP", "2Q-SEP"),
+        ],
+    )
+    def test_asfreq_frequency_M_Q_deprecated(self, freq, freq_depr):
+        # GH#9586
+        depr_msg = (
+            f"'{freq_depr[1:]}' will be deprecated, please use '{freq[1:]}' instead."
+        )
 
-        index = date_range("1/1/2000", periods=4, freq="ME")
+        index = date_range("1/1/2000", periods=4, freq=f"{freq[1:]}")
         df = DataFrame({"s": Series([0.0, 1.0, 2.0, 3.0], index=index)})
-        expected = df.asfreq(freq="5ME")
+        expected = df.asfreq(freq=freq)
         with tm.assert_produces_warning(FutureWarning, match=depr_msg):
-            result = df.asfreq(freq="5M")
-            tm.assert_frame_equal(result, expected)
+            result = df.asfreq(freq=freq_depr)
+        tm.assert_frame_equal(result, expected)

pandas/tests/frame/test_nonunique_indexes.py

@@ -29,7 +29,7 @@ def test_setattr_columns_vs_construct_with_columns(self):
         check(df, expected)
 
     def test_setattr_columns_vs_construct_with_columns_datetimeindx(self):
-        idx = date_range("20130101", periods=4, freq="Q-NOV")
+        idx = date_range("20130101", periods=4, freq="QE-NOV")
         df = DataFrame(
             [[1, 1, 1, 5], [1, 1, 2, 5], [2, 1, 3, 5]], columns=["a", "a", "a", "a"]
         )

pandas/tests/groupby/test_timegrouper.py

@@ -336,7 +336,7 @@ def test_timegrouper_with_reg_groups(self):
             )
             tm.assert_frame_equal(result, expected)
 
-    @pytest.mark.parametrize("freq", ["D", "ME", "Y", "Q-APR"])
+    @pytest.mark.parametrize("freq", ["D", "ME", "Y", "QE-APR"])
     def test_timegrouper_with_reg_groups_freq(self, freq):
         # GH 6764 multiple grouping with/without sort
         df = DataFrame(