BUG: Fixed Unicode decoding error in Period.strftime when a locale-specific directive is used

.github/workflows/posix.yml

@@ -40,15 +40,23 @@ jobs:
           - env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             extra_apt: "language-pack-it"
+            # Use the utf8 version as the default, it has no bad side-effect.
             lang: "it_IT.utf8"
             lc_all: "it_IT.utf8"
-            name: "Locale: it_IT.utf8"
+            # Also install it_IT (its encoding is ISO8859-1) but do not activate it.
+            # It will be temporarily activated during tests with locale.setlocale
+            extra_loc: "it_IT"
+            name: "Locale: it_IT"
           - env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             extra_apt: "language-pack-zh-hans"
+            # Use the utf8 version as the default, it has no bad side-effect.
             lang: "zh_CN.utf8"
             lc_all: "zh_CN.utf8"
-            name: "Locale: zh_CN.utf8"
+            # Also install zh_CN (its encoding is gb2312) but do not activate it.
+            # It will be temporarily activated during tests with locale.setlocale
+            extra_loc: "zh_CN"
+            name: "Locale: zh_CN"
           - env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_data_manager: "array"
@@ -68,6 +76,7 @@ jobs:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
       EXTRA_APT: ${{ matrix.extra_apt || '' }}
+      EXTRA_LOC: ${{ matrix.extra_loc || '' }}
       LANG: ${{ matrix.lang || '' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode || '' }}
@@ -138,6 +147,12 @@ jobs:
       # xsel for clipboard tests
       run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }}
 
+    - name: Generate extra locales
+      # These extra locales will be available for locale.setlocale() calls in tests
+      run: |
+        sudo locale-gen ${{ env.EXTRA_LOC }}
+      if: ${{ env.EXTRA_LOC != '' }}
+
     - uses: conda-incubator/setup-miniconda@v2
       with:
         mamba-version: "*"

doc/source/whatsnew/v1.5.0.rst

@@ -489,6 +489,7 @@ Period
 ^^^^^^
 - Bug in subtraction of :class:`Period` from :class:`PeriodArray` returning wrong results (:issue:`45999`)
 - Bug in :meth:`Period.strftime` and :meth:`PeriodIndex.strftime`, directives ``%l`` and ``%u`` were giving wrong results (:issue:`46252`)
+- Bug in :meth:`Period.strftime` and :meth:`PeriodIndex.strftime`, Unicode decoding error when a locale-specific directive is used (:issue:`46319`)
 -
 
 Plotting

pandas/_libs/tslibs/period.pyx

@@ -1165,7 +1165,9 @@ cdef str period_format(int64_t value, int freq, object fmt=None):
         return "NaT"
 
     if isinstance(fmt, str):
-        fmt = fmt.encode("utf-8")
+        # Encode using current locale, in case fmt contains non-utf8 chars
+        # fmt = fmt.encode("utf-8")
+        fmt = <bytes>util.string_encode_locale(fmt)
 
     if fmt is None:
         freq_group = get_freq_group(freq)
@@ -1234,7 +1236,8 @@ cdef str _period_strftime(int64_t value, int freq, bytes fmt):
     # Execute c_strftime to process the usual datetime directives
     formatted = c_strftime(&dts, <char*>fmt)
 
-    result = util.char_to_string(formatted)
+    # Decode result according to current locale
+    result = util.char_to_string_locale(formatted)
     free(formatted)
 
     # Now we will fill the placeholders corresponding to our additional directives

pandas/_libs/tslibs/util.pxd

@@ -26,6 +26,10 @@ cdef extern from "Python.h":
     const char* PyUnicode_AsUTF8AndSize(object obj,
                                         Py_ssize_t* length) except NULL
 
+    object PyUnicode_EncodeLocale(object obj, const char *errors) nogil
+    object PyUnicode_DecodeLocale(const char *str, const char *errors) nogil
+
+
 from numpy cimport (
     float64_t,
     int64_t,
@@ -219,3 +223,13 @@ cdef inline const char* get_c_string_buf_and_size(str py_string,
 
 cdef inline const char* get_c_string(str py_string) except NULL:
     return get_c_string_buf_and_size(py_string, NULL)
+
+
+cdef inline bytes string_encode_locale(str py_string):
+    """As opposed to PyUnicode_Encode, use current system locale to encode."""
+    return PyUnicode_EncodeLocale(py_string, NULL)
+
+
+cdef inline object char_to_string_locale(const char* data):
+    """As opposed to PyUnicode_FromString, use current system locale to decode."""
+    return PyUnicode_DecodeLocale(data, NULL)

pandas/conftest.py

@@ -28,6 +28,7 @@
     timezone,
 )
 from decimal import Decimal
+import locale
 import operator
 import os
 
@@ -1202,6 +1203,36 @@ def utc_fixture(request):
 utc_fixture2 = utc_fixture
 
 
+@pytest.fixture(
+    params=[
+        pytest.param(None, id=str(locale.getlocale())),
+        "it_IT.utf8",
+        "it_IT",  # Note: encoding will be 'ISO8859-1'
+        "zh_CN.utf8",
+        "zh_CN",  # Note: encoding will be 'gb2312'
+    ]
+)
+def overridden_locale(request):
+    """
+    Fixture to temporarily change the locale.
+
+    If a locale cannot be set (because it is not available on the host)
+    the test is skipped.
+    """
+    target = request.param
+    if target is None:
+        # Use current locale for this test.
+        yield locale.setlocale(locale.LC_ALL)
+    else:
+        if tm.can_set_locale(target, locale.LC_ALL):
+            # Change locale temporarily for this test.
+            with tm.set_locale(target, locale.LC_ALL):
+                yield target
+        else:
+            # Not available on this host. Skip test.
+            pytest.skip(f"Skipping as locale {repr(locale)} cannot be set on host.")
+
+
 # ----------------------------------------------------------------
 # Dtypes
 # ----------------------------------------------------------------

pandas/tests/io/formats/test_format.py

@@ -1,7 +1,10 @@
 """
 Test output formatting for Series/DataFrame, including to_string & reprs
 """
-from datetime import datetime
+from datetime import (
+    datetime,
+    time,
+)
 from io import StringIO
 import itertools
 from operator import methodcaller
@@ -46,6 +49,13 @@
 use_32bit_repr = is_platform_windows() or not IS64
 
 
+def get_local_am_pm():
+    """Return the AM and PM strings returned by strftime in current locale."""
+    am_local = time(1).strftime("%p")
+    pm_local = time(13).strftime("%p")
+    return am_local, pm_local
+
+
 @pytest.fixture(params=["string", "pathlike", "buffer"])
 def filepath_or_buffer_id(request):
     """
@@ -3224,6 +3234,35 @@ def test_period_tz(self):
             per = dt.to_period(freq="H")
         assert per.format()[0] == "2013-01-01 00:00"
 
+    def test_period_non_ascii_fmt(self, overridden_locale):
+        # GH#46468 non-ascii char in input format string leads to wrong output
+
+        # Scalar
+        per = pd.Period("2018-03-11 13:00", freq="H")
+        assert per.strftime("%y é") == "18 é"
+
+        # Index
+        per = pd.period_range("2003-01-01 01:00:00", periods=2, freq="12h")
+        formatted = per.format(date_format="%y é")
+        assert formatted[0] == "03 é"
+        assert formatted[1] == "03 é"
+
+    def test_period_custom_locale_directive(self, overridden_locale):
+        # GH#46319 locale-specific directive leads to non-utf8 c strftime char* result
+
+        # Get locale-specific reference
+        am_local, pm_local = get_local_am_pm()
+
+        # Scalar
+        per = pd.Period("2018-03-11 13:00", freq="H")
+        assert per.strftime("%p") == pm_local
+
+        # Index
+        per = pd.period_range("2003-01-01 01:00:00", periods=2, freq="12h")
+        formatted = per.format(date_format="%y %I:%M:%S%p")
+        assert formatted[0] == f"03 01:00:00{am_local}"
+        assert formatted[1] == f"03 01:00:00{pm_local}"
+
 
 class TestDatetimeIndexFormat:
     def test_datetime(self):