Merge branch 'master' into reduction_dtypes_II

Author: topper-123

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         language: python
         require_serial: true
         types_or: [python, pyi]
-        additional_dependencies: [black==23.1.0]
+        additional_dependencies: [black==23.3.0]
 -   repo: https://github.com/charliermarsh/ruff-pre-commit
     rev: v0.0.264
     hooks:
@@ -83,9 +83,6 @@ repos:
     hooks:
     -   id: pylint
         stages: [manual]
--   repo: https://github.com/pycqa/pylint
-    rev: v2.16.2
-    hooks:
     -   id: pylint
         alias: redefined-outer-name
         name: Redefining name from outer scope
@@ -99,6 +96,11 @@ repos:
             |^pandas/conftest\.py  # keep excluded
         args: [--disable=all, --enable=redefined-outer-name]
         stages: [manual]
+    -   id: pylint
+        alias: unspecified-encoding
+        name: Using open without explicitly specifying an encoding
+        args: [--disable=all, --enable=unspecified-encoding]
+        stages: [manual]
 -   repo: https://github.com/PyCQA/isort
     rev: 5.12.0
     hooks:

asv_bench/benchmarks/dtypes.py

@@ -24,7 +24,7 @@
 
 
 class Dtypes:
-    params = _dtypes + list(map(lambda dt: dt.name, _dtypes))
+    params = _dtypes + [dt.name for dt in _dtypes]
     param_names = ["dtype"]
 
     def time_pandas_dtype(self, dtype):

asv_bench/benchmarks/io/csv.py

@@ -12,6 +12,7 @@
     DataFrame,
     concat,
     date_range,
+    period_range,
     read_csv,
     to_datetime,
 )
@@ -98,24 +99,76 @@ def time_frame_date_no_format_index(self):
         self.data.to_csv(self.fname)
 
 
+class ToCSVPeriod(BaseIO):
+    fname = "__test__.csv"
+
+    params = ([1000, 10000], ["D", "H"])
+    param_names = ["nobs", "freq"]
+
+    def setup(self, nobs, freq):
+        rng = period_range(start="2000-01-01", periods=nobs, freq=freq)
+        self.data = DataFrame(rng)
+        if freq == "D":
+            self.default_fmt = "%Y-%m-%d"
+        elif freq == "H":
+            self.default_fmt = "%Y-%m-%d %H:00"
+
+    def time_frame_period_formatting_default(self, nobs, freq):
+        self.data.to_csv(self.fname)
+
+    def time_frame_period_formatting_default_explicit(self, nobs, freq):
+        self.data.to_csv(self.fname, date_format=self.default_fmt)
+
+    def time_frame_period_formatting(self, nobs, freq):
+        # Nb: `date_format` is not actually taken into account here today, so the
+        # performance is currently identical to `time_frame_period_formatting_default`
+        # above. This timer is therefore expected to degrade when GH#51621 is fixed.
+        # (Remove this comment when GH#51621 is fixed.)
+        self.data.to_csv(self.fname, date_format="%Y-%m-%d___%H:%M:%S")
+
+
+class ToCSVPeriodIndex(BaseIO):
+    fname = "__test__.csv"
+
+    params = ([1000, 10000], ["D", "H"])
+    param_names = ["nobs", "freq"]
+
+    def setup(self, nobs, freq):
+        rng = period_range(start="2000-01-01", periods=nobs, freq=freq)
+        self.data = DataFrame({"a": 1}, index=rng)
+        if freq == "D":
+            self.default_fmt = "%Y-%m-%d"
+        elif freq == "H":
+            self.default_fmt = "%Y-%m-%d %H:00"
+
+    def time_frame_period_formatting_index(self, nobs, freq):
+        self.data.to_csv(self.fname, date_format="%Y-%m-%d___%H:%M:%S")
+
+    def time_frame_period_formatting_index_default(self, nobs, freq):
+        self.data.to_csv(self.fname)
+
+    def time_frame_period_formatting_index_default_explicit(self, nobs, freq):
+        self.data.to_csv(self.fname, date_format=self.default_fmt)
+
+
 class ToCSVDatetimeBig(BaseIO):
     fname = "__test__.csv"
     timeout = 1500
     params = [1000, 10000, 100000]
-    param_names = ["obs"]
+    param_names = ["nobs"]
 
-    def setup(self, obs):
+    def setup(self, nobs):
         d = "2018-11-29"
         dt = "2018-11-26 11:18:27.0"
         self.data = DataFrame(
             {
-                "dt": [np.datetime64(dt)] * obs,
-                "d": [np.datetime64(d)] * obs,
-                "r": [np.random.uniform()] * obs,
+                "dt": [np.datetime64(dt)] * nobs,
+                "d": [np.datetime64(d)] * nobs,
+                "r": [np.random.uniform()] * nobs,
             }
         )
 
-    def time_frame(self, obs):
+    def time_frame(self, nobs):
         self.data.to_csv(self.fname)
 
 
@@ -444,7 +497,7 @@ class ReadCSVMemoryGrowth(BaseIO):
     param_names = ["engine"]
 
     def setup(self, engine):
-        with open(self.fname, "w") as f:
+        with open(self.fname, "w", encoding="utf-8") as f:
             for i in range(self.num_rows):
                 f.write(f"{i}\n")
 

asv_bench/benchmarks/pandas_vb_common.py

@@ -17,7 +17,7 @@
 try:
     import pandas._testing as tm
 except ImportError:
-    import pandas.util.testing as tm  # noqa:F401
+    import pandas.util.testing as tm  # noqa: F401
 
 
 numeric_dtypes = [

asv_bench/benchmarks/strftime.py

@@ -7,58 +7,109 @@
 class DatetimeStrftime:
     timeout = 1500
     params = [1000, 10000]
-    param_names = ["obs"]
+    param_names = ["nobs"]
 
-    def setup(self, obs):
+    def setup(self, nobs):
         d = "2018-11-29"
         dt = "2018-11-26 11:18:27.0"
         self.data = pd.DataFrame(
             {
-                "dt": [np.datetime64(dt)] * obs,
-                "d": [np.datetime64(d)] * obs,
-                "r": [np.random.uniform()] * obs,
+                "dt": [np.datetime64(dt)] * nobs,
+                "d": [np.datetime64(d)] * nobs,
+                "r": [np.random.uniform()] * nobs,
             }
         )
 
-    def time_frame_date_to_str(self, obs):
+    def time_frame_date_to_str(self, nobs):
         self.data["d"].astype(str)
 
-    def time_frame_date_formatting_default(self, obs):
+    def time_frame_date_formatting_default(self, nobs):
+        self.data["d"].dt.strftime(date_format=None)
+
+    def time_frame_date_formatting_default_explicit(self, nobs):
         self.data["d"].dt.strftime(date_format="%Y-%m-%d")
 
-    def time_frame_date_formatting_custom(self, obs):
+    def time_frame_date_formatting_custom(self, nobs):
         self.data["d"].dt.strftime(date_format="%Y---%m---%d")
 
-    def time_frame_datetime_to_str(self, obs):
+    def time_frame_datetime_to_str(self, nobs):
         self.data["dt"].astype(str)
 
-    def time_frame_datetime_formatting_default_date_only(self, obs):
+    def time_frame_datetime_formatting_default(self, nobs):
+        self.data["dt"].dt.strftime(date_format=None)
+
+    def time_frame_datetime_formatting_default_explicit_date_only(self, nobs):
         self.data["dt"].dt.strftime(date_format="%Y-%m-%d")
 
-    def time_frame_datetime_formatting_default(self, obs):
+    def time_frame_datetime_formatting_default_explicit(self, nobs):
         self.data["dt"].dt.strftime(date_format="%Y-%m-%d %H:%M:%S")
 
-    def time_frame_datetime_formatting_default_with_float(self, obs):
+    def time_frame_datetime_formatting_default_with_float(self, nobs):
         self.data["dt"].dt.strftime(date_format="%Y-%m-%d %H:%M:%S.%f")
 
-    def time_frame_datetime_formatting_custom(self, obs):
+    def time_frame_datetime_formatting_custom(self, nobs):
         self.data["dt"].dt.strftime(date_format="%Y-%m-%d --- %H:%M:%S")
 
 
+class PeriodStrftime:
+    timeout = 1500
+    params = ([1000, 10000], ["D", "H"])
+    param_names = ["nobs", "freq"]
+
+    def setup(self, nobs, freq):
+        self.data = pd.DataFrame(
+            {
+                "p": pd.period_range(start="2000-01-01", periods=nobs, freq=freq),
+                "r": [np.random.uniform()] * nobs,
+            }
+        )
+        self.data["i"] = self.data["p"]
+        self.data.set_index("i", inplace=True)
+        if freq == "D":
+            self.default_fmt = "%Y-%m-%d"
+        elif freq == "H":
+            self.default_fmt = "%Y-%m-%d %H:00"
+
+    def time_frame_period_to_str(self, nobs, freq):
+        self.data["p"].astype(str)
+
+    def time_frame_period_formatting_default(self, nobs, freq):
+        self.data["p"].dt.strftime(date_format=None)
+
+    def time_frame_period_formatting_default_explicit(self, nobs, freq):
+        self.data["p"].dt.strftime(date_format=self.default_fmt)
+
+    def time_frame_period_formatting_index_default(self, nobs, freq):
+        self.data.index.format()
+
+    def time_frame_period_formatting_index_default_explicit(self, nobs, freq):
+        self.data.index.format(self.default_fmt)
+
+    def time_frame_period_formatting_custom(self, nobs, freq):
+        self.data["p"].dt.strftime(date_format="%Y-%m-%d --- %H:%M:%S")
+
+    def time_frame_period_formatting_iso8601_strftime_Z(self, nobs, freq):
+        self.data["p"].dt.strftime(date_format="%Y-%m-%dT%H:%M:%SZ")
+
+    def time_frame_period_formatting_iso8601_strftime_offset(self, nobs, freq):
+        """Not optimized yet as %z is not supported by `convert_strftime_format`"""
+        self.data["p"].dt.strftime(date_format="%Y-%m-%dT%H:%M:%S%z")
+
+
 class BusinessHourStrftime:
     timeout = 1500
     params = [1000, 10000]
-    param_names = ["obs"]
+    param_names = ["nobs"]
 
-    def setup(self, obs):
+    def setup(self, nobs):
         self.data = pd.DataFrame(
             {
-                "off": [offsets.BusinessHour()] * obs,
+                "off": [offsets.BusinessHour()] * nobs,
             }
         )
 
-    def time_frame_offset_str(self, obs):
+    def time_frame_offset_str(self, nobs):
         self.data["off"].apply(str)
 
-    def time_frame_offset_repr(self, obs):
+    def time_frame_offset_repr(self, nobs):
         self.data["off"].apply(repr)

asv_bench/benchmarks/tslibs/period.py

@@ -60,6 +60,10 @@ class PeriodUnaryMethods:
 
     def setup(self, freq):
         self.per = Period("2012-06-01", freq=freq)
+        if freq == "M":
+            self.default_fmt = "%Y-%m"
+        elif freq == "min":
+            self.default_fmt = "%Y-%m-%d %H:%M"
 
     def time_to_timestamp(self, freq):
         self.per.to_timestamp()
@@ -70,6 +74,21 @@ def time_now(self, freq):
     def time_asfreq(self, freq):
         self.per.asfreq("A")
 
+    def time_str(self, freq):
+        str(self.per)
+
+    def time_repr(self, freq):
+        repr(self.per)
+
+    def time_strftime_default(self, freq):
+        self.per.strftime(None)
+
+    def time_strftime_default_explicit(self, freq):
+        self.per.strftime(self.default_fmt)
+
+    def time_strftime_custom(self, freq):
+        self.per.strftime("%b. %d, %Y was a %A")
+
 
 class PeriodConstructor:
     params = [["D"], [True, False]]

doc/make.py

@@ -163,12 +163,12 @@ def _get_page_title(self, page):
             components=(docutils.parsers.rst.Parser,)
         )
         doc = docutils.utils.new_document("<doc>", option_parser.get_default_values())
-        with open(fname) as f:
+        with open(fname, encoding="utf-8") as f:
             data = f.read()
 
         parser = docutils.parsers.rst.Parser()
         # do not generate any warning when parsing the rst
-        with open(os.devnull, "a") as f:
+        with open(os.devnull, "a", encoding="utf-8") as f:
             doc.reporter.stream = f
             parser.parse(data, doc)
 
@@ -186,7 +186,7 @@ def _add_redirects(self):
         Create in the build directory an html file with a redirect,
         for every row in REDIRECTS_FILE.
         """
-        with open(REDIRECTS_FILE) as mapping_fd:
+        with open(REDIRECTS_FILE, encoding="utf-8") as mapping_fd:
             reader = csv.reader(mapping_fd)
             for row in reader:
                 if not row or row[0].strip().startswith("#"):
@@ -209,7 +209,7 @@ def _add_redirects(self):
                     # sphinx specific stuff
                     title = "this page"
 
-                with open(path, "w") as moved_page_fd:
+                with open(path, "w", encoding="utf-8") as moved_page_fd:
                     html = f"""\
 <html>
     <head>

doc/source/conf.py

@@ -117,9 +117,9 @@
                 elif single_doc and rel_fname != pattern:
                     exclude_patterns.append(rel_fname)
 
-with open(os.path.join(source_path, "index.rst.template")) as f:
+with open(os.path.join(source_path, "index.rst.template"), encoding="utf-8") as f:
     t = jinja2.Template(f.read())
-with open(os.path.join(source_path, "index.rst"), "w") as f:
+with open(os.path.join(source_path, "index.rst"), "w", encoding="utf-8") as f:
     f.write(
         t.render(
             include_api=include_api,

doc/source/user_guide/window.rst

@@ -96,7 +96,7 @@ be calculated with :meth:`~Rolling.apply` by specifying a separate column of wei
        return arr
 
    df = pd.DataFrame([[1, 2, 0.6], [2, 3, 0.4], [3, 4, 0.2], [4, 5, 0.7]])
-   df.rolling(2, method="table", min_periods=0).apply(weighted_mean, raw=True, engine="numba")  # noqa:E501
+   df.rolling(2, method="table", min_periods=0).apply(weighted_mean, raw=True, engine="numba")  # noqa: E501
 
 .. versionadded:: 1.3
 

doc/source/whatsnew/v2.0.2.rst

@@ -13,19 +13,24 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
+- Fixed regression in :meth:`DataFrame.loc` losing :class:`MultiIndex` name when enlarging object (:issue:`53053`)
+- Fixed regression in :meth:`DataFrame.to_string` printing a backslash at the end of the first row of data, instead of headers, when the DataFrame doesn't fit the line width (:issue:`53054`)
+- Fixed regression in :meth:`MultiIndex.join` returning levels in wrong order (:issue:`53093`)
 -
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_202.bug_fixes:
 
 Bug fixes
 ~~~~~~~~~
+- Bug in :func:`api.interchange.from_dataframe` was raising ``IndexError`` on empty categorical data (:issue:`53077`)
 - Bug in :func:`api.interchange.from_dataframe` was returning :class:`DataFrame`'s of incorrect sizes when called on slices (:issue:`52824`)
 - Bug in :func:`api.interchange.from_dataframe` was unnecessarily raising on bitmasks (:issue:`49888`)
+- Bug in :func:`to_timedelta` was raising ``ValueError`` with ``pandas.NA`` (:issue:`52909`)
 - Bug in :meth:`DataFrame.convert_dtypes` ignores ``convert_*`` keywords when set to False ``dtype_backend="pyarrow"`` (:issue:`52872`)
 - Bug in :meth:`Series.describe` treating pyarrow-backed timestamps and timedeltas as categorical data (:issue:`53001`)
 - Bug in :meth:`pd.array` raising for ``NumPy`` array and ``pa.large_string`` or ``pa.large_binary`` (:issue:`52590`)
--
+
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_202.other: