WIP: CI: Pameterize locale tests specifically for Linux

.github/workflows/posix.yml

@@ -25,23 +25,20 @@ jobs:
     strategy:
       matrix:
         settings: [
-          [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
-          [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-38.yaml, "not slow and not clipboard", "", "", "", "", ""],
-          [actions-38-slow.yaml, "slow", "", "", "", "", ""],
-          [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
-          [actions-39-slow.yaml, "slow", "", "", "", "", ""],
-          [actions-pypy-38.yaml, "not slow and not clipboard", "", "", "", "", "--max-worker-restart 0"],
-          [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
-          [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""]
+          [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", ""],
+          [actions-38-locale_clipboard.yaml, "not slow and not network", "language-pack-zh-hans language-pack-it xsel", "", ""],
+          [actions-38-slow.yaml, "slow", "", "", ""],
+          [actions-39-slow.yaml, "slow", "", "", ""],
+          [actions-38.yaml, "not slow and not clipboard", "", "", ""],
+          [actions-39.yaml, "not slow and not clipboard", "", "", ""],
+          [actions-pypy-38.yaml, "not slow and not clipboard", "", "", "--max-worker-restart 0"],
+          [actions-39-numpydev.yaml, "not slow and not network", "xsel", "deprecate", "-W error"],
         ]
       fail-fast: false
     env:
       ENV_FILE: ci/deps/${{ matrix.settings[0] }}
       PATTERN: ${{ matrix.settings[1] }}
       EXTRA_APT: ${{ matrix.settings[2] }}
-      LANG: ${{ matrix.settings[3] }}
-      LC_ALL: ${{ matrix.settings[4] }}
       PANDAS_TESTING_MODE: ${{ matrix.settings[5] }}
       TEST_ARGS: ${{ matrix.settings[6] }}
       PYTEST_TARGET:  pandas

ci/setup_env.sh

@@ -1,18 +1,5 @@
 #!/bin/bash -e
 
-# edit the locale file if needed
-if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
-    echo "Adding locale to the first line of pandas/__init__.py"
-    rm -f pandas/__init__.pyc
-    SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n"
-    sed -i "$SEDC" pandas/__init__.py
-
-    echo "[head -4 pandas/__init__.py]"
-    head -4 pandas/__init__.py
-    echo
-fi
-
-
 echo "Install Miniconda"
 DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
 if [[ "$(uname -m)" == 'aarch64' ]]; then

pandas/_libs/tslibs/nattype.pyx

@@ -530,7 +530,7 @@ class NaTType(_NaT):
         Examples
         --------
         >>> ts = pd.Timestamp('2020-03-14T15:32:52.192548651')
-        >>> ts.strftime('%Y-%m-%d %X')
+        >>> ts.strftime('%Y-%m-%d %H:%M:%S')
         '2020-03-14 15:32:52'
         """,
     )

pandas/_libs/tslibs/timestamps.pyx

@@ -1199,7 +1199,7 @@ class Timestamp(_Timestamp):
         Examples
         --------
         >>> ts = pd.Timestamp('2020-03-14T15:32:52.192548651')
-        >>> ts.strftime('%Y-%m-%d %X')
+        >>> ts.strftime('%Y-%m-%d %H:%M:%S')
         '2020-03-14 15:32:52'
         """
         return datetime.strftime(self, format)

pandas/_testing/__init__.py

@@ -26,6 +26,7 @@
 )
 
 from pandas._typing import Dtype
+from pandas.compat import is_platform_linux
 
 from pandas.core.dtypes.common import (
     is_float_dtype,
@@ -190,6 +191,13 @@
 
 EMPTY_STRING_PATTERN = re.compile("^$")
 
+# Our Linux CI environments install the associated language packs
+if os.environ.get("PANDAS_CI", "0") == "1" and is_platform_linux():
+    ci_locales = get_locales() or []
+else:
+    ci_locales = []
+TESTING_LOCALES = [loc for loc in ci_locales if loc in {"it_IT.UTF-8", "zh_CN.UTF-8"}]
+
 # set testing_mode
 _testing_mode_warnings = (DeprecationWarning, ResourceWarning)
 

pandas/tests/config/test_localization.py

@@ -13,6 +13,7 @@
 from pandas.compat import is_platform_windows
 
 import pandas as pd
+import pandas._testing as tm
 
 _all_locales = get_locales() or []
 _current_locale = locale.getlocale()
@@ -62,21 +63,9 @@ def test_get_locales_prefix():
 
 
 @_skip_if_only_one_locale
-@pytest.mark.parametrize(
-    "lang,enc",
-    [
-        ("it_CH", "UTF-8"),
-        ("en_US", "ascii"),
-        ("zh_CN", "GB2312"),
-        ("it_IT", "ISO-8859-1"),
-    ],
-)
-def test_set_locale(lang, enc):
-    if all(x is None for x in _current_locale):
-        # Not sure why, but on some Travis runs with pytest,
-        #  getlocale() returned (None, None).
-        pytest.skip("Current locale is not set.")
-
+@pytest.mark.parametrize("test_local", tm.TESTING_LOCALES)
+def test_set_locale(test_local):
+    lang, enc = test_local.split(".")
     enc = codecs.lookup(enc).name
     new_locale = lang, enc
 

pandas/tests/frame/conftest.py

@@ -9,6 +9,12 @@
 import pandas._testing as tm
 
 
+@pytest.fixture(params=tm.TESTING_LOCALES, autouse=True)
+def with_locale(request):
+    with tm.set_locale(request.param):
+        yield
+
+
 @pytest.fixture
 def float_frame_with_na():
     """

pandas/tests/frame/methods/test_between_time.py

@@ -2,12 +2,12 @@
     datetime,
     time,
 )
+import locale
 
 import numpy as np
 import pytest
 
 from pandas._libs.tslibs import timezones
-import pandas.util._test_decorators as td
 
 from pandas import (
     DataFrame,
@@ -18,14 +18,9 @@
 
 
 class TestBetweenTime:
-    @td.skip_if_has_locale
-    def test_between_time_formats(self, frame_or_series):
-        # GH#11818
-        rng = date_range("1/1/2000", "1/5/2000", freq="5min")
-        ts = DataFrame(np.random.randn(len(rng), 2), index=rng)
-        ts = tm.get_obj(ts, frame_or_series)
-
-        strings = [
+    @pytest.mark.parametrize(
+        "time_string",
+        [
             ("2:00", "2:30"),
             ("0200", "0230"),
             ("2:00am", "2:30am"),
@@ -34,11 +29,23 @@ def test_between_time_formats(self, frame_or_series):
             ("020000", "023000"),
             ("2:00:00am", "2:30:00am"),
             ("020000am", "023000am"),
-        ]
-        expected_length = 28
+        ],
+    )
+    def test_between_time_formats(self, request, time_string, frame_or_series):
+        # GH#11818
+        request.node.add_marker(
+            pytest.mark.xfail(
+                "am" in time_string[0] and locale.getlocale()[0] == "zh_CN",
+                reason="Does not pass with LC_ALL=zh_CN.utf8",
+            )
+        )
+        rng = date_range("1/1/2000", "1/5/2000", freq="5min")
+        ts = DataFrame(np.random.randn(len(rng), 2), index=rng)
+        if frame_or_series is Series:
+            ts = ts[0]
 
-        for time_string in strings:
-            assert len(ts.between_time(*time_string)) == expected_length
+        expected_length = 28
+        assert len(ts.between_time(*time_string)) == expected_length
 
     @pytest.mark.parametrize("tzstr", ["US/Eastern", "dateutil/US/Eastern"])
     def test_localized_between_time(self, tzstr, frame_or_series):

pandas/tests/io/conftest.py

@@ -17,6 +17,12 @@
 from pandas.io.parsers import read_csv
 
 
+@pytest.fixture(params=tm.TESTING_LOCALES, autouse=True)
+def with_locale(request):
+    with tm.set_locale(request.param):
+        yield
+
+
 @pytest.fixture
 def tips_file(datapath):
     """Path to the tips dataset"""

pandas/tests/io/excel/test_readers.py

@@ -738,9 +738,10 @@ def test_bad_sheetname_raises(self, read_ext, sheet_name):
 
     def test_missing_file_raises(self, read_ext):
         bad_file = f"foo{read_ext}"
-        # CI tests with zh_CN.utf8, translates to "No such file or directory"
+        # Error message matches CI locales: "No such file or directory"
         with pytest.raises(
-            FileNotFoundError, match=r"(No such file or directory|没有那个文件或目录)"
+            FileNotFoundError,
+            match=r"(No such file or directory|没有那个文件或目录|File o directory)",
         ):
             pd.read_excel(bad_file)
 

pandas/tests/io/parser/common/test_index.py

@@ -4,6 +4,7 @@
 """
 from datetime import datetime
 from io import StringIO
+import locale
 import os
 
 import pytest
@@ -85,14 +86,20 @@ def test_pass_names_with_index(all_parsers, data, kwargs, expected):
 
 
 @pytest.mark.parametrize("index_col", [[0, 1], [1, 0]])
-def test_multi_index_no_level_names(all_parsers, index_col):
+def test_multi_index_no_level_names(request, all_parsers, index_col):
     data = """index1,index2,A,B,C,D
 foo,one,2,3,4,5
 foo,two,7,8,9,10
 foo,three,12,13,14,15
 bar,one,12,13,14,15
 bar,two,12,13,14,15
 """
+    if locale.getlocale()[0] != "zh_CN":
+        request.node.add_marker(
+            pytest.mark.xfail(
+                reason="Only passes with LC_ALL=zh_CN.utf8",
+            )
+        )
     headless_data = "\n".join(data.split("\n")[1:])
 
     names = ["A", "B", "C", "D"]

pandas/tests/io/parser/test_parse_dates.py

@@ -8,6 +8,7 @@
     datetime,
 )
 from io import StringIO
+import locale
 import warnings
 
 from dateutil.parser import parse as du_parse
@@ -862,7 +863,7 @@ def test_parse_dates_column_list(all_parsers, parse_dates):
 
 @xfail_pyarrow
 @pytest.mark.parametrize("index_col", [[0, 1], [1, 0]])
-def test_multi_index_parse_dates(all_parsers, index_col):
+def test_multi_index_parse_dates(request, all_parsers, index_col):
     data = """index1,index2,A,B,C
 20090101,one,a,1,2
 20090101,two,b,3,4
@@ -874,6 +875,12 @@ def test_multi_index_parse_dates(all_parsers, index_col):
 20090103,two,b,3,4
 20090103,three,c,4,5
 """
+    request.node.add_marker(
+        pytest.mark.xfail(
+            locale.getlocale()[0] == "it_IT" and index_col == ["index2", "index1"],
+            reason="Only passes with LC_ALL=zh_CN.utf8",
+        )
+    )
     parser = all_parsers
     index = MultiIndex.from_product(
         [
@@ -1841,7 +1848,7 @@ def test_parse_dates_and_keep_orgin_column(all_parsers):
     tm.assert_frame_equal(result, expected)
 
 
-def test_dayfirst_warnings():
+def test_dayfirst_warnings(request):
     # GH 12585
     warning_msg_day_first = (
         "Parsing '31/12/2014' in DD/MM/YYYY format. Provide "
@@ -1883,6 +1890,12 @@ def test_dayfirst_warnings():
 
     # D. infer_datetime_format=True overrides dayfirst default
     # no warning + correct result
+    request.node.add_marker(
+        pytest.mark.xfail(
+            locale.getlocale()[0] == "zh_CN",
+            reason="Does not pass with LC_ALL=zh_CN.utf8",
+        )
+    )
     res4 = read_csv(
         StringIO(input),
         parse_dates=["date"],

pandas/tests/strings/conftest.py

@@ -2,8 +2,16 @@
 import pytest
 
 from pandas import Series
+import pandas._testing as tm
 from pandas.core import strings as strings
 
+
+@pytest.fixture(params=tm.TESTING_LOCALES, autouse=True)
+def with_locale(request):
+    with tm.set_locale(request.param):
+        yield
+
+
 _any_string_method = [
     ("cat", (), {"sep": ","}),
     ("cat", (Series(list("zyx")),), {"sep": ",", "join": "left"}),

pandas/tests/tools/conftest.py

@@ -0,0 +1,9 @@
+import pytest
+
+import pandas._testing as tm
+
+
+@pytest.fixture(params=tm.TESTING_LOCALES, autouse=True)
+def with_locale(request):
+    with tm.set_locale(request.param):
+        yield