CI/TST: Run tests in development mode with EncodingWarnings (#54472)

Author: mroeschke

ci/run_tests.sh

@@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED
 
 COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml"
 
-PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
 
 if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""

pandas/_testing/contexts.py

@@ -128,6 +128,8 @@ def ensure_clean(
     encoding = kwargs.pop("encoding", None)
     if return_filelike:
         kwargs.setdefault("mode", "w+b")
+        if encoding is None and "b" not in kwargs["mode"]:
+            encoding = "utf-8"
         handle_or_str = open(path, encoding=encoding, **kwargs)
 
     try:

pandas/io/common.py

@@ -935,6 +935,8 @@ class _BufferedWriter(BytesIO, ABC):  # type: ignore[misc]
     This wrapper writes to the underlying buffer on close.
     """
 
+    buffer = BytesIO()
+
     @abstractmethod
     def write_to_buffer(self) -> None:
         ...
@@ -943,15 +945,13 @@ def close(self) -> None:
         if self.closed:
             # already closed
             return
-        if self.getvalue():
+        if self.getbuffer().nbytes:
             # write to buffer
             self.seek(0)
-            # error: "_BufferedWriter" has no attribute "buffer"
-            with self.buffer:  # type: ignore[attr-defined]
+            with self.buffer:
                 self.write_to_buffer()
         else:
-            # error: "_BufferedWriter" has no attribute "buffer"
-            self.buffer.close()  # type: ignore[attr-defined]
+            self.buffer.close()
         super().close()
 
 
@@ -967,13 +967,12 @@ def __init__(
         super().__init__()
         self.archive_name = archive_name
         self.name = name
-        # error: Argument "fileobj" to "open" of "TarFile" has incompatible
-        # type "Union[ReadBuffer[bytes], WriteBuffer[bytes], None]"; expected
-        # "Optional[IO[bytes]]"
-        self.buffer = tarfile.TarFile.open(
+        # error: Incompatible types in assignment (expression has type "TarFile",
+        # base class "_BufferedWriter" defined the type as "BytesIO")
+        self.buffer: tarfile.TarFile = tarfile.TarFile.open(  # type: ignore[assignment]
             name=name,
             mode=self.extend_mode(mode),
-            fileobj=fileobj,  # type: ignore[arg-type]
+            fileobj=fileobj,
             **kwargs,
         )
 
@@ -1023,10 +1022,9 @@ def __init__(
         self.archive_name = archive_name
 
         kwargs.setdefault("compression", zipfile.ZIP_DEFLATED)
-        # error: No overload variant of "ZipFile" matches argument types "str |
-        # PathLike[str] | ReadBuffer[bytes] | WriteBuffer[bytes]", "str", "dict[str,
-        # Any]"
-        self.buffer = zipfile.ZipFile(  # type: ignore[call-overload]
+        # error: Incompatible types in assignment (expression has type "ZipFile",
+        # base class "_BufferedWriter" defined the type as "BytesIO")
+        self.buffer: zipfile.ZipFile = zipfile.ZipFile(  # type: ignore[assignment]
             file, mode, **kwargs
         )
 

pandas/tests/arrays/categorical/test_warnings.py

@@ -17,6 +17,6 @@ async def test_tab_complete_warning(self, ip):
 
         # GH 31324 newer jedi version raises Deprecation warning;
         #  appears resolved 2021-02-02
-        with tm.assert_produces_warning(None):
+        with tm.assert_produces_warning(None, raise_on_extra_warnings=False):
             with provisionalcompleter("ignore"):
                 list(ip.Completer.completions("c.", 1))

pandas/tests/frame/test_api.py

@@ -303,7 +303,7 @@ async def test_tab_complete_warning(self, ip, frame_or_series):
 
         # GH 31324 newer jedi version raises Deprecation warning;
         #  appears resolved 2021-02-02
-        with tm.assert_produces_warning(None):
+        with tm.assert_produces_warning(None, raise_on_extra_warnings=False):
             with provisionalcompleter("ignore"):
                 list(ip.Completer.completions("obj.", 1))
 

pandas/tests/indexes/test_base.py

@@ -1226,7 +1226,7 @@ async def test_tab_complete_warning(self, ip):
 
         # GH 31324 newer jedi version raises Deprecation warning;
         #  appears resolved 2021-02-02
-        with tm.assert_produces_warning(None):
+        with tm.assert_produces_warning(None, raise_on_extra_warnings=False):
             with provisionalcompleter("ignore"):
                 list(ip.Completer.completions("idx.", 4))
 

pandas/tests/io/formats/test_format.py

@@ -95,6 +95,8 @@ def assert_filepath_or_buffer_equals(
     """
     Assertion helper for checking filepath_or_buffer.
     """
+    if encoding is None:
+        encoding = "utf-8"
 
     def _assert_filepath_or_buffer_equals(expected):
         if filepath_or_buffer_id == "string":

pandas/tests/io/parser/common/test_common_basic.py

@@ -856,7 +856,7 @@ def test_read_seek(all_parsers):
     prefix = "### DATA\n"
     content = "nkey,value\ntables,rectangular\n"
     with tm.ensure_clean() as path:
-        Path(path).write_text(prefix + content)
+        Path(path).write_text(prefix + content, encoding="utf-8")
         with open(path, encoding="utf-8") as file:
             file.readline()
             actual = parser.read_csv(file)

pandas/tests/io/parser/test_encoding.py

@@ -310,7 +310,7 @@ def test_not_readable(all_parsers, mode):
     content = b"abcd"
     if "t" in mode:
         content = "abcd"
-    with tempfile.SpooledTemporaryFile(mode=mode) as handle:
+    with tempfile.SpooledTemporaryFile(mode=mode, encoding="utf-8") as handle:
         handle.write(content)
         handle.seek(0)
         df = parser.read_csv(handle)

pandas/tests/io/parser/test_read_fwf.py

@@ -702,7 +702,7 @@ def test_binary_mode():
         [["bba", "bab", "b a"]], columns=["aaa", "aaa.1", "aaa.2"], index=[0]
     )
     with tm.ensure_clean() as path:
-        Path(path).write_text(data)
+        Path(path).write_text(data, encoding="utf-8")
         with open(path, "rb") as file:
             df = read_fwf(file)
             file.seek(0)

pandas/tests/io/parser/test_unsupported.py

@@ -182,7 +182,7 @@ def test_close_file_handle_on_invalid_usecols(all_parsers):
             pytest.skip("GH#45547 causing timeouts on windows/mac builds 2022-01-22")
 
     with tm.ensure_clean("test.csv") as fname:
-        Path(fname).write_text("col1,col2\na,b\n1,2")
+        Path(fname).write_text("col1,col2\na,b\n1,2", encoding="utf-8")
         with tm.assert_produces_warning(False):
             with pytest.raises(error, match="col3"):
                 parser.read_csv(fname, usecols=["col1", "col2", "col3"])

pandas/tests/io/test_sql.py

@@ -140,7 +140,7 @@ def create_and_load_iris_sqlite3(conn: sqlite3.Connection, iris_file: Path):
             "Name" TEXT
         )"""
     cur.execute(stmt)
-    with iris_file.open(newline=None) as csvfile:
+    with iris_file.open(newline=None, encoding="utf-8") as csvfile:
         reader = csv.reader(csvfile)
         next(reader)
         stmt = "INSERT INTO iris VALUES(?, ?, ?, ?, ?)"
@@ -153,7 +153,7 @@ def create_and_load_iris(conn, iris_file: Path, dialect: str):
 
     iris = iris_table_metadata(dialect)
 
-    with iris_file.open(newline=None) as csvfile:
+    with iris_file.open(newline=None, encoding="utf-8") as csvfile:
         reader = csv.reader(csvfile)
         header = next(reader)
         params = [dict(zip(header, row)) for row in reader]

pandas/tests/resample/test_resampler_grouper.py

@@ -41,7 +41,7 @@ async def test_tab_complete_ipython6_warning(ip):
 
     # GH 31324 newer jedi version raises Deprecation warning;
     #  appears resolved 2021-02-02
-    with tm.assert_produces_warning(None):
+    with tm.assert_produces_warning(None, raise_on_extra_warnings=False):
         with provisionalcompleter("ignore"):
             list(ip.Completer.completions("rs.", 1))