BUG: Fixed Unicode decoding error in Period.strftime when a locale-specific directive is used

.github/workflows/posix.yml

@@ -46,9 +46,13 @@ jobs:
           - env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             extra_apt: "language-pack-zh-hans"
+            # Use the utf8 version as the default, it has no bad side-effect.
             lang: "zh_CN.utf8"
             lc_all: "zh_CN.utf8"
-            name: "Locale: zh_CN.utf8"
+            # Also install zh_CN (its encoding is gb2312) but do not activate it.
+            # It will be temporarily activated during tests with locale.setlocale
+            extra_loc: "zh_CN"
+            name: "Locale: zh_CN.utf8 + extra zh_CN (gb2312)"
           - env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_data_manager: "array"
@@ -68,6 +72,7 @@ jobs:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
       EXTRA_APT: ${{ matrix.extra_apt || '' }}
+      EXTRA_LOC: ${{ matrix.extra_loc || '' }}
       LANG: ${{ matrix.lang || '' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode || '' }}
@@ -138,6 +143,12 @@ jobs:
       # xsel for clipboard tests
       run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }}
 
+    - name: Generate extra locales
+      # These extra locales will be available for locale.setlocale() calls in tests
+      run: |
+        sudo locale-gen ${{ env.EXTRA_LOC }}
+      if: ${{ env.EXTRA_LOC != '' }}
+
     - uses: conda-incubator/setup-miniconda@v2
       with:
         mamba-version: "*"

doc/source/whatsnew/v1.5.0.rst

@@ -443,6 +443,7 @@ Period
 ^^^^^^
 - Bug in subtraction of :class:`Period` from :class:`PeriodArray` returning wrong results (:issue:`45999`)
 - Bug in :meth:`Period.strftime` and :meth:`PeriodIndex.strftime`, directives ``%l`` and ``%u`` were giving wrong results (:issue:`46252`)
+- Bug in :meth:`Period.strftime` and :meth:`PeriodIndex.strftime`, Unicode decoding error when a locale-specific directive is used (:issue:`46319`)
 -
 
 Plotting

pandas/_libs/tslibs/period.pyx

@@ -1236,7 +1236,8 @@ cdef str _period_strftime(int64_t value, int freq, bytes fmt):
     # Execute c_strftime to process the usual datetime directives
     formatted = c_strftime(&dts, <char*>fmt)
 
-    result = util.char_to_string(formatted)
+    # Decode result according to current locale
+    result = util.char_to_string_locale(formatted)
     free(formatted)
 
     # Now we will fill the placeholders corresponding to our additional directives

pandas/_libs/tslibs/util.pxd

@@ -27,6 +27,9 @@ cdef extern from "Python.h":
     const char* PyUnicode_AsUTF8AndSize(object obj,
                                         Py_ssize_t* length) except NULL
 
+    object PyUnicode_DecodeLocale(const char *str, const char *errors) nogil
+
+
 from numpy cimport (
     float64_t,
     int64_t,
@@ -220,3 +223,8 @@ cdef inline const char* get_c_string_buf_and_size(str py_string,
 
 cdef inline const char* get_c_string(str py_string) except NULL:
     return get_c_string_buf_and_size(py_string, NULL)
+
+
+cdef inline object char_to_string_locale(const char* data):
+    """As opposed to PyUnicode_FromString, use current system locale to decode."""
+    return PyUnicode_DecodeLocale(data, NULL)

pandas/conftest.py

@@ -28,6 +28,7 @@
     timezone,
 )
 from decimal import Decimal
+import locale
 import operator
 import os
 
@@ -1202,6 +1203,39 @@ def utc_fixture(request):
 utc_fixture2 = utc_fixture
 
 
+@pytest.fixture(
+    params=[
+        pytest.param(None, id=str(locale.getlocale())),
+        "it_IT.utf8",
+        "zh_CN",
+    ]
+)
+def overridden_locale(request):
+    """
+    Fixture to temporarily change the locale.
+
+    If a locale cannot be set (because it is not available on the host)
+    the test is skipped.
+    """
+    old = locale.setlocale(locale.LC_ALL)
+    target = request.param
+    if target is None:
+        # Current locale - don't change
+        yield old
+    else:
+        try:
+            # Try changing the locale.
+            locale.setlocale(locale.LC_ALL, target)
+        except locale.Error as e:
+            # Not available on this host. Skip test.
+            pytest.skip(f"Skipping as locale cannot be set. {type(e).__name__}: {e}")
+        else:
+            # Run test with the temporary local
+            yield target
+            # Set back to normal
+            locale.setlocale(locale.LC_ALL, old)
+
+
 # ----------------------------------------------------------------
 # Dtypes
 # ----------------------------------------------------------------

pandas/tests/io/formats/test_format.py

@@ -1,7 +1,10 @@
 """
 Test output formatting for Series/DataFrame, including to_string & reprs
 """
-from datetime import datetime
+from datetime import (
+    datetime,
+    time,
+)
 from io import StringIO
 import itertools
 from operator import methodcaller
@@ -46,6 +49,13 @@
 use_32bit_repr = is_platform_windows() or not IS64
 
 
+def get_local_am_pm():
+    """Return the AM and PM strings returned by strftime in current locale."""
+    am_local = time(1).strftime("%p")
+    pm_local = time(13).strftime("%p")
+    return am_local, pm_local
+
+
 @pytest.fixture(params=["string", "pathlike", "buffer"])
 def filepath_or_buffer_id(request):
     """
@@ -3224,6 +3234,22 @@ def test_period_tz(self):
             per = dt.to_period(freq="H")
         assert per.format()[0] == "2013-01-01 00:00"
 
+    def test_period_custom_locale(self, overridden_locale):
+        # GH#46319 locale-specific formatting directive leads to non-utf8 str result
+
+        # Get locale-specific reference
+        am_local, pm_local = get_local_am_pm()
+
+        # Scalar
+        per = pd.Period("2018-03-11 13:00", freq="H")
+        assert per.strftime("%p") == pm_local
+
+        # Index
+        per = pd.period_range("2003-01-01 01:00:00", periods=2, freq="12h")
+        formatted = per.format(date_format="%y %I:%M:%S%p")
+        assert formatted[0] == f"03 01:00:00{am_local}"
+        assert formatted[1] == f"03 01:00:00{pm_local}"
+
 
 class TestDatetimeIndexFormat:
     def test_datetime(self):