test(fix): use os.device_encoding(0) to figure out stdin encoding in run_command() (Fix #1777) (#1780)

* Use os.device_encoding(0) to figure out stdin encoding in run_command().

* Use the encoding of stdout (fd 1) to decode the output of subprocess.Popen.

Co-authored-by: Ned Batchelder <[email protected]>

* refactor to remove a helper used in only one place

---------

Co-authored-by: Ned Batchelder <[email protected]>

Author: devdanzin

coverage/misc.py

@@ -13,7 +13,6 @@
 import importlib
 import importlib.util
 import inspect
-import locale
 import os
 import os.path
 import re
@@ -22,7 +21,7 @@
 
 from types import ModuleType
 from typing import (
-    Any, IO, Iterable, Iterator, Mapping, NoReturn, Sequence, TypeVar,
+    Any, Iterable, Iterator, Mapping, NoReturn, Sequence, TypeVar,
 )
 
 from coverage.exceptions import CoverageException
@@ -156,18 +155,6 @@ def ensure_dir_for_file(path: str) -> None:
     ensure_dir(os.path.dirname(path))
 
 
-def output_encoding(outfile: IO[str] | None = None) -> str:
-    """Determine the encoding to use for output written to `outfile` or stdout."""
-    if outfile is None:
-        outfile = sys.stdout
-    encoding = (
-        getattr(outfile, "encoding", None) or
-        getattr(sys.__stdout__, "encoding", None) or
-        locale.getpreferredencoding()
-    )
-    return encoding
-
-
 class Hasher:
     """Hashes Python data for fingerprinting."""
     def __init__(self) -> None:

tests/helpers.py

@@ -9,6 +9,7 @@
 import contextlib
 import dis
 import io
+import locale
 import os
 import os.path
 import re
@@ -28,7 +29,6 @@
 from coverage import env
 from coverage.debug import DebugControl
 from coverage.exceptions import CoverageWarning
-from coverage.misc import output_encoding
 from coverage.types import TArc, TLineNo
 
 
@@ -44,11 +44,13 @@ def run_command(cmd: str) -> tuple[int, str]:
         with open("/tmp/processes.txt", "a") as proctxt:  # type: ignore[unreachable]
             print(os.getenv("PYTEST_CURRENT_TEST", "unknown"), file=proctxt, flush=True)
 
+    encoding = os.device_encoding(1) or locale.getpreferredencoding()
+
     # In some strange cases (PyPy3 in a virtualenv!?) the stdout encoding of
     # the subprocess is set incorrectly to ascii.  Use an environment variable
     # to force the encoding to be the same as ours.
     sub_env = dict(os.environ)
-    sub_env['PYTHONIOENCODING'] = output_encoding()
+    sub_env['PYTHONIOENCODING'] = encoding
 
     proc = subprocess.Popen(
         cmd,
@@ -62,7 +64,7 @@ def run_command(cmd: str) -> tuple[int, str]:
     status = proc.returncode
 
     # Get the output, and canonicalize it to strings with newlines.
-    output_str = output.decode(output_encoding()).replace("\r", "")
+    output_str = output.decode(encoding).replace("\r", "")
     return status, output_str