BUG: read_csv interpreting NA value as comment when NA contains comment string

doc/source/whatsnew/v1.3.0.rst

@@ -148,6 +148,7 @@ MultiIndex
 I/O
 ^^^
 
+- Bug in :func:`read_csv` interpreting ``NA`` value as comment, when ``NA`` does contain the comment string fixed for ``engine="python"`` (:issue:`34002`)
 - Bug in :func:`read_csv` raising ``TypeError`` when ``names`` and ``parse_dates`` is specified for ``engine="c"`` (:issue:`33699`)
 -
 

pandas/io/parsers.py

@@ -2991,7 +2991,11 @@ def _check_comments(self, lines):
         for line in lines:
             rl = []
             for x in line:
-                if not isinstance(x, str) or self.comment not in x:
+                if (
+                    not isinstance(x, str)
+                    or self.comment not in x
+                    or x in self.na_values
+                ):
                     rl.append(x)
                 else:
                     x = x[: x.find(self.comment)]

pandas/tests/io/parser/test_python_parser_only.py

@@ -8,6 +8,7 @@
 import csv
 from io import BytesIO, StringIO
 
+import numpy as np
 import pytest
 
 from pandas.errors import ParserError
@@ -314,3 +315,26 @@ def test_malformed_skipfooter(python_parser_only):
     msg = "Expected 3 fields in line 4, saw 5"
     with pytest.raises(ParserError, match=msg):
         parser.read_csv(StringIO(data), header=1, comment="#", skipfooter=1)
+
+
+def test_comment_char_in_default_value(python_parser_only):
+    # GH#34002
+    from io import StringIO
+
+    data = (
+        "# this is a comment\n"
+        "col1,col2,col3,col4\n"
+        "1,2,3,4#inline comment\n"
+        "4,5#,6,10\n"
+        "7,8,#N/A,11\n"
+    )
+    result = python_parser_only.read_csv(StringIO(data), comment="#", na_values="#N/A")
+    expected = DataFrame(
+        {
+            "col1": [1, 4, 7],
+            "col2": [2, 5, 8],
+            "col3": [3.0, np.nan, np.nan],
+            "col4": [4.0, np.nan, 11.0],
+        }
+    )
+    tm.assert_frame_equal(result, expected)