Merge remote-tracking branch 'upstream/main' into stata-read-dta110

Author: cmjcharlton

asv_bench/asv.conf.json

@@ -41,6 +41,7 @@
     // pip (with all the conda available packages installed first,
     // followed by the pip installed packages).
     "matrix": {
+        "pip+build": [],
         "Cython": ["3.0"],
         "matplotlib": [],
         "sqlalchemy": [],

doc/source/whatsnew/v3.0.0.rst

@@ -350,6 +350,7 @@ Bug fixes
 - Fixed bug in :meth:`Series.diff` allowing non-integer values for the ``periods`` argument. (:issue:`56607`)
 - Fixed bug in :meth:`Series.rank` that doesn't preserve missing values for nullable integers when ``na_option='keep'``. (:issue:`56976`)
 - Fixed bug in :meth:`Series.replace` and :meth:`DataFrame.replace` inconsistently replacing matching instances when ``regex=True`` and missing values are present. (:issue:`56599`)
+- Fixed bug in :meth:`read_csv` raising ``TypeError`` when ``index_col`` is specified and ``na_values`` is a dict containing the key ``None``. (:issue:`57547`)
 
 Categorical
 ^^^^^^^^^^^

pandas/_libs/tslib.pyx

@@ -70,7 +70,6 @@ from pandas._libs.tslibs.conversion cimport (
 from pandas._libs.tslibs.dtypes cimport npy_unit_to_abbrev
 from pandas._libs.tslibs.nattype cimport (
     NPY_NAT,
-    c_NaT as NaT,
     c_nat_strings as nat_strings,
 )
 from pandas._libs.tslibs.timestamps cimport _Timestamp
@@ -346,39 +345,6 @@ def array_with_unit_to_datetime(
     return result, tz
 
 
-cdef _array_with_unit_to_datetime_object_fallback(ndarray[object] values, str unit):
-    cdef:
-        Py_ssize_t i, n = len(values)
-        ndarray[object] oresult
-        tzinfo tz = None
-
-    # TODO: fix subtle differences between this and no-unit code
-    oresult = cnp.PyArray_EMPTY(values.ndim, values.shape, cnp.NPY_OBJECT, 0)
-    for i in range(n):
-        val = values[i]
-
-        if checknull_with_nat_and_na(val):
-            oresult[i] = <object>NaT
-        elif is_integer_object(val) or is_float_object(val):
-
-            if val != val or val == NPY_NAT:
-                oresult[i] = <object>NaT
-            else:
-                try:
-                    oresult[i] = Timestamp(val, unit=unit)
-                except OutOfBoundsDatetime:
-                    oresult[i] = val
-
-        elif isinstance(val, str):
-            if len(val) == 0 or val in nat_strings:
-                oresult[i] = <object>NaT
-
-            else:
-                oresult[i] = val
-
-    return oresult, tz
-
-
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def first_non_null(values: ndarray) -> int:

pandas/core/apply.py

@@ -1710,9 +1710,9 @@ def normalize_keyword_aggregation(
     # TODO: aggspec type: typing.Dict[str, List[AggScalar]]
     aggspec = defaultdict(list)
     order = []
-    columns, pairs = list(zip(*kwargs.items()))
+    columns = tuple(kwargs.keys())
 
-    for column, aggfunc in pairs:
+    for column, aggfunc in kwargs.values():
         aggspec[column].append(aggfunc)
         order.append((column, com.get_callable_name(aggfunc) or aggfunc))
 

pandas/core/frame.py

@@ -6168,12 +6168,13 @@ class    max    type
             names = self.index._get_default_index_names(names, default)
 
             if isinstance(self.index, MultiIndex):
-                to_insert = zip(self.index.levels, self.index.codes)
+                to_insert = zip(reversed(self.index.levels), reversed(self.index.codes))
             else:
                 to_insert = ((self.index, None),)
 
             multi_col = isinstance(self.columns, MultiIndex)
-            for i, (lev, lab) in reversed(list(enumerate(to_insert))):
+            for j, (lev, lab) in enumerate(to_insert, start=1):
+                i = self.index.nlevels - j
                 if level is not None and i not in level:
                     continue
                 name = names[i]

pandas/core/groupby/ops.py

@@ -706,7 +706,7 @@ def groups(self) -> dict[Hashable, Index]:
             return self.groupings[0].groups
         result_index, ids = self.result_index_and_ids
         values = result_index._values
-        categories = Categorical(ids, categories=np.arange(len(result_index)))
+        categories = Categorical(ids, categories=range(len(result_index)))
         result = {
             # mypy is not aware that group has to be an integer
             values[group]: self.axis.take(axis_ilocs)  # type: ignore[call-overload]

pandas/core/indexing.py

@@ -899,7 +899,7 @@ def __setitem__(self, key, value) -> None:
 
         check_dict_or_set_indexers(key)
         if isinstance(key, tuple):
-            key = tuple(list(x) if is_iterator(x) else x for x in key)
+            key = (list(x) if is_iterator(x) else x for x in key)
             key = tuple(com.apply_if_callable(x, self.obj) for x in key)
         else:
             maybe_callable = com.apply_if_callable(key, self.obj)
@@ -1177,7 +1177,7 @@ def _check_deprecated_callable_usage(self, key: Any, maybe_callable: T) -> T:
     def __getitem__(self, key):
         check_dict_or_set_indexers(key)
         if type(key) is tuple:
-            key = tuple(list(x) if is_iterator(x) else x for x in key)
+            key = (list(x) if is_iterator(x) else x for x in key)
             key = tuple(com.apply_if_callable(x, self.obj) for x in key)
             if self._is_scalar_access(key):
                 return self.obj._get_value(*key, takeable=self._takeable)

pandas/core/sorting.py

@@ -172,8 +172,6 @@ def maybe_lift(lab, size: int) -> tuple[np.ndarray, int]:
         for i, (lab, size) in enumerate(zip(labels, shape)):
             labels[i], lshape[i] = maybe_lift(lab, size)
 
-    labels = list(labels)
-
     # Iteratively process all the labels in chunks sized so less
     # than lib.i8max unique int ids will be required for each chunk
     while True:

pandas/io/parsers/base_parser.py

@@ -487,6 +487,8 @@ def _agg_index(self, index, try_parse_dates: bool = True) -> Index:
                     col_na_values, col_na_fvalues = _get_na_values(
                         col_name, self.na_values, self.na_fvalues, self.keep_default_na
                     )
+                else:
+                    col_na_values, col_na_fvalues = set(), set()
 
             clean_dtypes = self._clean_mapping(self.dtype)
 

pandas/io/parsers/python_parser.py

@@ -354,14 +354,15 @@ def _convert_data(
 
         if isinstance(self.na_values, dict):
             for col in self.na_values:
-                na_value = self.na_values[col]
-                na_fvalue = self.na_fvalues[col]
+                if col is not None:
+                    na_value = self.na_values[col]
+                    na_fvalue = self.na_fvalues[col]
 
-                if isinstance(col, int) and col not in self.orig_names:
-                    col = self.orig_names[col]
+                    if isinstance(col, int) and col not in self.orig_names:
+                        col = self.orig_names[col]
 
-                clean_na_values[col] = na_value
-                clean_na_fvalues[col] = na_fvalue
+                    clean_na_values[col] = na_value
+                    clean_na_fvalues[col] = na_fvalue
         else:
             clean_na_values = self.na_values
             clean_na_fvalues = self.na_fvalues

pandas/tests/io/parser/test_na_values.py

@@ -532,6 +532,47 @@ def test_na_values_dict_aliasing(all_parsers):
     tm.assert_dict_equal(na_values, na_values_copy)
 
 
+def test_na_values_dict_null_column_name(all_parsers):
+    # see gh-57547
+    parser = all_parsers
+    data = ",x,y\n\nMA,1,2\nNA,2,1\nOA,,3"
+    names = [None, "x", "y"]
+    na_values = {name: STR_NA_VALUES for name in names}
+    dtype = {None: "object", "x": "float64", "y": "float64"}
+
+    if parser.engine == "pyarrow":
+        msg = "The pyarrow engine doesn't support passing a dict for na_values"
+        with pytest.raises(ValueError, match=msg):
+            parser.read_csv(
+                StringIO(data),
+                index_col=0,
+                header=0,
+                dtype=dtype,
+                names=names,
+                na_values=na_values,
+                keep_default_na=False,
+            )
+        return
+
+    expected = DataFrame(
+        {None: ["MA", "NA", "OA"], "x": [1.0, 2.0, np.nan], "y": [2.0, 1.0, 3.0]}
+    )
+
+    expected = expected.set_index(None)
+
+    result = parser.read_csv(
+        StringIO(data),
+        index_col=0,
+        header=0,
+        dtype=dtype,
+        names=names,
+        na_values=na_values,
+        keep_default_na=False,
+    )
+
+    tm.assert_frame_equal(result, expected)
+
+
 def test_na_values_dict_col_index(all_parsers):
     # see gh-14203
     data = "a\nfoo\n1"