Merge remote-tracking branch 'upstream/main' into pandas-asan

Author: WillAyd

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
         # TODO: remove autofixe-only rules when they are checked by ruff
         name: ruff-selected-autofixes
         alias: ruff-selected-autofixes
-        args: [--select, "ANN001,ANN204", --fix-only, --exit-non-zero-on-fix]
+        args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix]
 -   repo: https://github.com/jendrikseipp/vulture
     rev: 'v2.10'
     hooks:

doc/make.py

@@ -102,7 +102,7 @@ def _process_single_doc(self, single_doc):
             )
 
     @staticmethod
-    def _run_os(*args):
+    def _run_os(*args) -> None:
         """
         Execute a command as a OS terminal.
 
@@ -149,7 +149,7 @@ def _sphinx_build(self, kind: str):
         ]
         return subprocess.call(cmd)
 
-    def _open_browser(self, single_doc_html):
+    def _open_browser(self, single_doc_html) -> None:
         """
         Open a browser tab showing single
         """
@@ -183,7 +183,7 @@ def _get_page_title(self, page):
 
         return title.astext()
 
-    def _add_redirects(self):
+    def _add_redirects(self) -> None:
         """
         Create in the build directory an html file with a redirect,
         for every row in REDIRECTS_FILE.
@@ -272,14 +272,14 @@ def latex_forced(self):
         return self.latex(force=True)
 
     @staticmethod
-    def clean():
+    def clean() -> None:
         """
         Clean documentation generated files.
         """
         shutil.rmtree(BUILD_PATH, ignore_errors=True)
         shutil.rmtree(os.path.join(SOURCE_PATH, "reference", "api"), ignore_errors=True)
 
-    def zip_html(self):
+    def zip_html(self) -> None:
         """
         Compress HTML documentation into a zip file.
         """

doc/scripts/eval_performance.py

@@ -64,7 +64,7 @@ def bench(mn=3, mx=7, num=100, engines=("python", "numexpr"), verbose=False):
     return ev, qu
 
 
-def plot_perf(df, engines, title, filename=None):
+def plot_perf(df, engines, title, filename=None) -> None:
     from matplotlib.pyplot import figure
 
     sns.set()

doc/source/conf.py

@@ -502,7 +502,7 @@ class AccessorDocumenter(MethodDocumenter):
     # lower than MethodDocumenter so this is not chosen for normal methods
     priority = 0.6
 
-    def format_signature(self):
+    def format_signature(self) -> str:
         # this method gives an error/warning for the accessors, therefore
         # overriding it (accessor has no arguments)
         return ""
@@ -632,7 +632,7 @@ def get_items(self, names):
 
 
 # based on numpy doc/source/conf.py
-def linkcode_resolve(domain, info):
+def linkcode_resolve(domain, info) -> str | None:
     """
     Determine the URL corresponding to Python object
     """
@@ -694,12 +694,12 @@ def linkcode_resolve(domain, info):
 
 # remove the docstring of the flags attribute (inherited from numpy ndarray)
 # because these give doc build errors (see GH issue 5331)
-def remove_flags_docstring(app, what, name, obj, options, lines):
+def remove_flags_docstring(app, what, name, obj, options, lines) -> None:
     if what == "attribute" and name.endswith(".flags"):
         del lines[:]
 
 
-def process_class_docstrings(app, what, name, obj, options, lines):
+def process_class_docstrings(app, what, name, obj, options, lines) -> None:
     """
     For those classes for which we use ::
 
@@ -751,7 +751,7 @@ def process_class_docstrings(app, what, name, obj, options, lines):
 ]
 
 
-def process_business_alias_docstrings(app, what, name, obj, options, lines):
+def process_business_alias_docstrings(app, what, name, obj, options, lines) -> None:
     """
     Starting with sphinx 3.4, the "autodoc-process-docstring" event also
     gets called for alias classes. This results in numpydoc adding the
@@ -774,7 +774,7 @@ def process_business_alias_docstrings(app, what, name, obj, options, lines):
     suppress_warnings.append("ref.ref")
 
 
-def rstjinja(app, docname, source):
+def rstjinja(app, docname, source) -> None:
     """
     Render our pages as a jinja template for fancy templating goodness.
     """
@@ -787,7 +787,7 @@ def rstjinja(app, docname, source):
     source[0] = rendered
 
 
-def setup(app):
+def setup(app) -> None:
     app.connect("source-read", rstjinja)
     app.connect("autodoc-process-docstring", remove_flags_docstring)
     app.connect("autodoc-process-docstring", process_class_docstrings)

doc/source/user_guide/io.rst

@@ -836,6 +836,7 @@ order) and the new column names will be the concatenation of the component
 column names:
 
 .. ipython:: python
+   :okwarning:
 
    data = (
        "KORD,19990127, 19:00:00, 18:56:00, 0.8100\n"
@@ -856,6 +857,7 @@ By default the parser removes the component date columns, but you can choose
 to retain them via the ``keep_date_col`` keyword:
 
 .. ipython:: python
+   :okwarning:
 
    df = pd.read_csv(
        "tmp.csv", header=None, parse_dates=[[1, 2], [1, 3]], keep_date_col=True
@@ -871,6 +873,7 @@ single column.
 You can also use a dict to specify custom name columns:
 
 .. ipython:: python
+   :okwarning:
 
    date_spec = {"nominal": [1, 2], "actual": [1, 3]}
    df = pd.read_csv("tmp.csv", header=None, parse_dates=date_spec)
@@ -883,6 +886,7 @@ data columns:
 
 
 .. ipython:: python
+   :okwarning:
 
    date_spec = {"nominal": [1, 2], "actual": [1, 3]}
    df = pd.read_csv(
@@ -902,6 +906,10 @@ data columns:
    for your data to store datetimes in this format, load times will be
    significantly faster, ~20x has been observed.
 
+.. deprecated:: 2.2.0
+   Combining date columns inside read_csv is deprecated. Use ``pd.to_datetime``
+   on the relevant result columns instead.
+
 
 Date parsing functions
 ++++++++++++++++++++++

doc/source/user_guide/timeseries.rst

@@ -1770,7 +1770,8 @@ We can instead only resample those groups where we have points as follows:
     def round(t, freq):
         # round a Timestamp to a specified freq
         freq = to_offset(freq)
-        return pd.Timestamp((t.value // freq.delta.value) * freq.delta.value)
+        td = pd.Timedelta(freq)
+        return pd.Timestamp((t.value // td.value) * td.value)
 
     ts.groupby(partial(round, freq="3min")).sum()
 

doc/source/whatsnew/v2.2.0.rst

@@ -438,6 +438,7 @@ Set the following option to opt into the future behavior:
 Other Deprecations
 ^^^^^^^^^^^^^^^^^^
 - Changed :meth:`Timedelta.resolution_string` to return ``h``, ``min``, ``s``, ``ms``, ``us``, and ``ns`` instead of ``H``, ``T``, ``S``, ``L``, ``U``, and ``N``, for compatibility with respective deprecations in frequency aliases (:issue:`52536`)
+- Deprecated :attr:`offsets.Day.delta`, :attr:`offsets.Hour.delta`, :attr:`offsets.Minute.delta`, :attr:`offsets.Second.delta`, :attr:`offsets.Milli.delta`, :attr:`offsets.Micro.delta`, :attr:`offsets.Nano.delta`, use ``pd.Timedelta(obj)`` instead (:issue:`55498`)
 - Deprecated :func:`pandas.api.types.is_interval` and :func:`pandas.api.types.is_period`, use ``isinstance(obj, pd.Interval)`` and ``isinstance(obj, pd.Period)`` instead (:issue:`55264`)
 - Deprecated :func:`pd.core.internals.api.make_block`, use public APIs instead (:issue:`40226`)
 - Deprecated :func:`read_gbq` and :meth:`DataFrame.to_gbq`. Use ``pandas_gbq.read_gbq`` and ``pandas_gbq.to_gbq`` instead https://pandas-gbq.readthedocs.io/en/latest/api.html (:issue:`55525`)
@@ -480,13 +481,15 @@ Other Deprecations
 - Deprecated strings ``H``, ``S``, ``U``, and ``N`` denoting units in :func:`to_timedelta` (:issue:`52536`)
 - Deprecated strings ``H``, ``T``, ``S``, ``L``, ``U``, and ``N`` denoting units in :class:`Timedelta` (:issue:`52536`)
 - Deprecated strings ``T``, ``S``, ``L``, ``U``, and ``N`` denoting frequencies in :class:`Minute`, :class:`Second`, :class:`Milli`, :class:`Micro`, :class:`Nano` (:issue:`52536`)
+- Deprecated support for combining parsed datetime columns in :func:`read_csv` along with the ``keep_date_col`` keyword (:issue:`55569`)
 - Deprecated the :attr:`.DataFrameGroupBy.grouper` and :attr:`SeriesGroupBy.grouper`; these attributes will be removed in a future version of pandas (:issue:`56521`)
 - Deprecated the :class:`.Grouping` attributes ``group_index``, ``result_index``, and ``group_arraylike``; these will be removed in a future version of pandas (:issue:`56148`)
 - Deprecated the ``errors="ignore"`` option in :func:`to_datetime`, :func:`to_timedelta`, and :func:`to_numeric`; explicitly catch exceptions instead (:issue:`54467`)
 - Deprecated the ``fastpath`` keyword in the :class:`Series` constructor (:issue:`20110`)
 - Deprecated the ``kind`` keyword in :meth:`Series.resample` and :meth:`DataFrame.resample`, explicitly cast the object's ``index`` instead (:issue:`55895`)
 - Deprecated the ``ordinal`` keyword in :class:`PeriodIndex`, use :meth:`PeriodIndex.from_ordinals` instead (:issue:`55960`)
 - Deprecated the ``unit`` keyword in :class:`TimedeltaIndex` construction, use :func:`to_timedelta` instead (:issue:`55499`)
+- Deprecated the ``verbose`` keyword in :func:`read_csv` and :func:`read_table` (:issue:`55569`)
 - Deprecated the behavior of :meth:`DataFrame.replace` and :meth:`Series.replace` with :class:`CategoricalDtype`; in a future version replace will change the values while preserving the categories. To change the categories, use ``ser.cat.rename_categories`` instead (:issue:`55147`)
 - Deprecated the behavior of :meth:`Series.value_counts` and :meth:`Index.value_counts` with object dtype; in a future version these will not perform dtype inference on the resulting :class:`Index`, do ``result.index = result.index.infer_objects()`` to retain the old behavior (:issue:`56161`)
 - Deprecated the default of ``observed=False`` in :meth:`DataFrame.pivot_table`; will be ``True`` in a future version (:issue:`56236`)
@@ -615,6 +618,7 @@ Indexing
 - Bug in :meth:`DataFrame.loc` when setting :class:`Series` with extension dtype into NumPy dtype (:issue:`55604`)
 - Bug in :meth:`Index.difference` not returning a unique set of values when ``other`` is empty or ``other`` is considered non-comparable (:issue:`55113`)
 - Bug in setting :class:`Categorical` values into a :class:`DataFrame` with numpy dtypes raising ``RecursionError`` (:issue:`52927`)
+- Fixed bug when creating new column with missing values when setting a single string value (:issue:`56204`)
 
 Missing
 ^^^^^^^

generate_pxi.py

@@ -4,7 +4,7 @@
 from Cython import Tempita
 
 
-def process_tempita(pxifile, outfile):
+def process_tempita(pxifile, outfile) -> None:
     with open(pxifile, encoding="utf-8") as f:
         tmpl = f.read()
     pyxcontent = Tempita.sub(tmpl)
@@ -13,7 +13,7 @@ def process_tempita(pxifile, outfile):
         f.write(pyxcontent)
 
 
-def main():
+def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("infile", type=str, help="Path to the input file")
     parser.add_argument("-o", "--outdir", type=str, help="Path to the output directory")

generate_version.py

@@ -10,7 +10,7 @@
 sys.path.insert(0, "")
 
 
-def write_version_info(path):
+def write_version_info(path) -> None:
     version = None
     git_version = None
 
@@ -29,7 +29,7 @@ def write_version_info(path):
         file.write(f'__git_version__="{git_version}"\n')
 
 
-def main():
+def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "-o",

pandas/_libs/tslibs/offsets.pyx

@@ -913,8 +913,19 @@ cdef class Tick(SingleConstructorOffset):
         # Since cdef classes have no __dict__, we need to override
         return ""
 
+    @cache_readonly
+    def _as_pd_timedelta(self):
+        return Timedelta(self)
+
     @property
     def delta(self):
+        warnings.warn(
+            # GH#55498
+            f"{type(self).__name__}.delta is deprecated and will be removed in "
+            "a future version. Use pd.Timedelta(obj) instead",
+            FutureWarning,
+            stacklevel=find_stack_level(),
+        )
         try:
             return self.n * Timedelta(self._nanos_inc)
         except OverflowError as err:
@@ -962,22 +973,22 @@ cdef class Tick(SingleConstructorOffset):
             except ValueError:
                 # e.g. "infer"
                 return False
-        return self.delta == other
+        return self._as_pd_timedelta == other
 
     def __ne__(self, other):
         return not (self == other)
 
     def __le__(self, other):
-        return self.delta.__le__(other)
+        return self._as_pd_timedelta.__le__(other)
 
     def __lt__(self, other):
-        return self.delta.__lt__(other)
+        return self._as_pd_timedelta.__lt__(other)
 
     def __ge__(self, other):
-        return self.delta.__ge__(other)
+        return self._as_pd_timedelta.__ge__(other)
 
     def __gt__(self, other):
-        return self.delta.__gt__(other)
+        return self._as_pd_timedelta.__gt__(other)
 
     def __mul__(self, other):
         if is_float_object(other):
@@ -997,21 +1008,21 @@ cdef class Tick(SingleConstructorOffset):
     def __truediv__(self, other):
         if not isinstance(self, Tick):
             # cython semantics mean the args are sometimes swapped
-            result = other.delta.__rtruediv__(self)
+            result = other._as_pd_timedelta.__rtruediv__(self)
         else:
-            result = self.delta.__truediv__(other)
+            result = self._as_pd_timedelta.__truediv__(other)
         return _wrap_timedelta_result(result)
 
     def __rtruediv__(self, other):
-        result = self.delta.__rtruediv__(other)
+        result = self._as_pd_timedelta.__rtruediv__(other)
         return _wrap_timedelta_result(result)
 
     def __add__(self, other):
         if isinstance(other, Tick):
             if type(self) is type(other):
                 return type(self)(self.n + other.n)
             else:
-                return delta_to_tick(self.delta + other.delta)
+                return delta_to_tick(self._as_pd_timedelta + other._as_pd_timedelta)
         try:
             return self._apply(other)
         except ApplyTypeError:
@@ -1029,15 +1040,15 @@ cdef class Tick(SingleConstructorOffset):
         # Timestamp can handle tz and nano sec, thus no need to use apply_wraps
         if isinstance(other, _Timestamp):
             # GH#15126
-            return other + self.delta
+            return other + self._as_pd_timedelta
         elif other is NaT:
             return NaT
         elif cnp.is_datetime64_object(other) or PyDate_Check(other):
             # PyDate_Check includes date, datetime
             return Timestamp(other) + self
 
         if cnp.is_timedelta64_object(other) or PyDelta_Check(other):
-            return other + self.delta
+            return other + self._as_pd_timedelta
 
         raise ApplyTypeError(f"Unhandled type: {type(other).__name__}")
 

pandas/conftest.py

@@ -188,19 +188,15 @@ def pytest_collection_modifyitems(items, config) -> None:
         ("read_parquet", "Passing a BlockManager to DataFrame is deprecated"),
     ]
 
-    for item in items:
-        if is_doctest:
+    if is_doctest:
+        for item in items:
             # autouse=True for the add_doctest_imports can lead to expensive teardowns
             # since doctest_namespace is a session fixture
             item.add_marker(pytest.mark.usefixtures("add_doctest_imports"))
 
             for path, message in ignored_doctest_warnings:
                 ignore_doctest_warning(item, path, message)
 
-        # mark all tests in the pandas/tests/frame directory with "arraymanager"
-        if "/frame/" in item.nodeid:
-            item.add_marker(pytest.mark.arraymanager)
-
 
 hypothesis_health_checks = [hypothesis.HealthCheck.too_slow]
 if Version(hypothesis.__version__) >= Version("6.83.2"):

pandas/core/arrays/arrow/array.py

@@ -693,22 +693,31 @@ def _evaluate_op_method(self, other, op, arrow_funcs):
         other = self._box_pa(other)
 
         if pa.types.is_string(pa_type) or pa.types.is_binary(pa_type):
-            if op in [operator.add, roperator.radd, operator.mul, roperator.rmul]:
+            if op in [operator.add, roperator.radd]:
                 sep = pa.scalar("", type=pa_type)
                 if op is operator.add:
                     result = pc.binary_join_element_wise(self._pa_array, other, sep)
                 elif op is roperator.radd:
                     result = pc.binary_join_element_wise(other, self._pa_array, sep)
-                else:
-                    if not (
-                        isinstance(other, pa.Scalar) and pa.types.is_integer(other.type)
-                    ):
-                        raise TypeError("Can only string multiply by an integer.")
-                    result = pc.binary_join_element_wise(
-                        *([self._pa_array] * other.as_py()), sep
-                    )
                 return type(self)(result)
-
+            elif op in [operator.mul, roperator.rmul]:
+                binary = self._pa_array
+                integral = other
+                if not pa.types.is_integer(integral.type):
+                    raise TypeError("Can only string multiply by an integer.")
+                pa_integral = pc.if_else(pc.less(integral, 0), 0, integral)
+                result = pc.binary_repeat(binary, pa_integral)
+                return type(self)(result)
+        elif (
+            pa.types.is_string(other.type) or pa.types.is_binary(other.type)
+        ) and op in [operator.mul, roperator.rmul]:
+            binary = other
+            integral = self._pa_array
+            if not pa.types.is_integer(integral.type):
+                raise TypeError("Can only string multiply by an integer.")
+            pa_integral = pc.if_else(pc.less(integral, 0), 0, integral)
+            result = pc.binary_repeat(binary, pa_integral)
+            return type(self)(result)
         if (
             isinstance(other, pa.Scalar)
             and pc.is_null(other).as_py()