Skip to content

Commit 5d26773

Browse files
author
Khor Chean Wei
authored
Merge branch 'main' into numeric_to_df_cum
2 parents ad521c6 + d460abd commit 5d26773

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+212
-367
lines changed

.github/workflows/code-checks.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ jobs:
8585
echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV
8686
if: ${{ steps.build.outcome == 'success' && always() }}
8787

88-
- name: Typing + pylint
88+
- name: Typing
8989
uses: pre-commit/[email protected]
9090
with:
9191
extra_args: --verbose --hook-stage manual --all-files

.pre-commit-config.yaml

+1-26
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ ci:
1616
autofix_prs: false
1717
autoupdate_schedule: monthly
1818
# manual stage hooks
19-
skip: [pylint, pyright, mypy]
19+
skip: [pyright, mypy]
2020
repos:
2121
- repo: https://github.com/astral-sh/ruff-pre-commit
2222
rev: v0.3.4
@@ -30,12 +30,6 @@ repos:
3030
files: ^pandas
3131
exclude: ^pandas/tests
3232
args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix]
33-
- id: ruff
34-
name: ruff-use-pd_array-in-core
35-
alias: ruff-use-pd_array-in-core
36-
files: ^pandas/core/
37-
exclude: ^pandas/core/api\.py$
38-
args: [--select, "ICN001", --exit-non-zero-on-fix]
3933
- id: ruff-format
4034
exclude: ^scripts
4135
- repo: https://github.com/jendrikseipp/vulture
@@ -73,25 +67,6 @@ repos:
7367
- id: fix-encoding-pragma
7468
args: [--remove]
7569
- id: trailing-whitespace
76-
- repo: https://github.com/pylint-dev/pylint
77-
rev: v3.0.1
78-
hooks:
79-
- id: pylint
80-
stages: [manual]
81-
args: [--load-plugins=pylint.extensions.redefined_loop_name, --fail-on=I0021]
82-
- id: pylint
83-
alias: redefined-outer-name
84-
name: Redefining name from outer scope
85-
files: ^pandas/
86-
exclude: |
87-
(?x)
88-
^pandas/tests # keep excluded
89-
|/_testing/ # keep excluded
90-
|^pandas/util/_test_decorators\.py # keep excluded
91-
|^pandas/_version\.py # keep excluded
92-
|^pandas/conftest\.py # keep excluded
93-
args: [--disable=all, --enable=redefined-outer-name]
94-
stages: [manual]
9570
- repo: https://github.com/PyCQA/isort
9671
rev: 5.13.2
9772
hooks:

doc/source/whatsnew/v3.0.0.rst

+1
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,7 @@ Bug fixes
338338
- Fixed bug in :class:`SparseDtype` for equal comparison with na fill value. (:issue:`54770`)
339339
- Fixed bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`)
340340
- Fixed bug in :meth:`DataFrame.cumsum` which was raising ``IndexError`` if dtype is ``timedelta64[ns]`` (:issue:`57956`)
341+
- Fixed bug in :meth:`DataFrame.eval` and :meth:`DataFrame.query` which caused an exception when using NumPy attributes via ``@`` notation, e.g., ``df.eval("@np.floor(a)")``. (:issue:`58041`)
341342
- Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`)
342343
- Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`)
343344
- Fixed bug in :meth:`DataFrame.transform` that was returning the wrong order unless the index was monotonically increasing. (:issue:`57069`)

pandas/_libs/tslib.pyx

+4-43
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,6 @@ from pandas._libs.tslibs.nattype cimport (
7474
c_nat_strings as nat_strings,
7575
)
7676
from pandas._libs.tslibs.timestamps cimport _Timestamp
77-
from pandas._libs.tslibs.timezones cimport tz_compare
7877

7978
from pandas._libs.tslibs import (
8079
Resolution,
@@ -452,13 +451,9 @@ cpdef array_to_datetime(
452451
ndarray[int64_t] iresult
453452
npy_datetimestruct dts
454453
bint utc_convert = bool(utc)
455-
bint seen_datetime_offset = False
456454
bint is_raise = errors == "raise"
457455
bint is_coerce = errors == "coerce"
458-
bint is_same_offsets
459456
_TSObject tsobj
460-
float tz_offset
461-
set out_tzoffset_vals = set()
462457
tzinfo tz, tz_out = None
463458
cnp.flatiter it = cnp.PyArray_IterNew(values)
464459
NPY_DATETIMEUNIT item_reso
@@ -568,12 +563,12 @@ cpdef array_to_datetime(
568563
# dateutil timezone objects cannot be hashed, so
569564
# store the UTC offsets in seconds instead
570565
nsecs = tz.utcoffset(None).total_seconds()
571-
out_tzoffset_vals.add(nsecs)
572-
seen_datetime_offset = True
566+
state.out_tzoffset_vals.add(nsecs)
567+
state.found_aware_str = True
573568
else:
574569
# Add a marker for naive string, to track if we are
575570
# parsing mixed naive and aware strings
576-
out_tzoffset_vals.add("naive")
571+
state.out_tzoffset_vals.add("naive")
577572
state.found_naive_str = True
578573

579574
else:
@@ -588,41 +583,7 @@ cpdef array_to_datetime(
588583
raise
589584
return values, None
590585

591-
if seen_datetime_offset and not utc_convert:
592-
# GH#17697, GH#57275
593-
# 1) If all the offsets are equal, return one offset for
594-
# the parsed dates to (maybe) pass to DatetimeIndex
595-
# 2) If the offsets are different, then do not force the parsing
596-
# and raise a ValueError: "cannot parse datetimes with
597-
# mixed time zones unless `utc=True`" instead
598-
is_same_offsets = len(out_tzoffset_vals) == 1
599-
if not is_same_offsets:
600-
raise ValueError(
601-
"Mixed timezones detected. Pass utc=True in to_datetime "
602-
"or tz='UTC' in DatetimeIndex to convert to a common timezone."
603-
)
604-
elif state.found_naive or state.found_other:
605-
# e.g. test_to_datetime_mixed_awareness_mixed_types
606-
raise ValueError("Cannot mix tz-aware with tz-naive values")
607-
elif tz_out is not None:
608-
# GH#55693
609-
tz_offset = out_tzoffset_vals.pop()
610-
tz_out2 = timezone(timedelta(seconds=tz_offset))
611-
if not tz_compare(tz_out, tz_out2):
612-
# e.g. test_to_datetime_mixed_tzs_mixed_types
613-
raise ValueError(
614-
"Mixed timezones detected. Pass utc=True in to_datetime "
615-
"or tz='UTC' in DatetimeIndex to convert to a common timezone."
616-
)
617-
# e.g. test_to_datetime_mixed_types_matching_tzs
618-
else:
619-
tz_offset = out_tzoffset_vals.pop()
620-
tz_out = timezone(timedelta(seconds=tz_offset))
621-
elif not utc_convert:
622-
if tz_out and (state.found_other or state.found_naive_str):
623-
# found_other indicates a tz-naive int, float, dt64, or date
624-
# e.g. test_to_datetime_mixed_awareness_mixed_types
625-
raise ValueError("Cannot mix tz-aware with tz-naive values")
586+
tz_out = state.check_for_mixed_inputs(tz_out, utc)
626587

627588
if infer_reso:
628589
if state.creso_ever_changed:

pandas/_libs/tslibs/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
"is_supported_dtype",
3737
]
3838

39-
from pandas._libs.tslibs import dtypes # pylint: disable=import-self
39+
from pandas._libs.tslibs import dtypes
4040
from pandas._libs.tslibs.conversion import localize_pydatetime
4141
from pandas._libs.tslibs.dtypes import (
4242
Resolution,

pandas/_libs/tslibs/strptime.pxd

+3
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,12 @@ cdef class DatetimeParseState:
1818
bint found_tz
1919
bint found_naive
2020
bint found_naive_str
21+
bint found_aware_str
2122
bint found_other
2223
bint creso_ever_changed
2324
NPY_DATETIMEUNIT creso
25+
set out_tzoffset_vals
2426

2527
cdef tzinfo process_datetime(self, datetime dt, tzinfo tz, bint utc_convert)
2628
cdef bint update_creso(self, NPY_DATETIMEUNIT item_reso) noexcept
29+
cdef tzinfo check_for_mixed_inputs(self, tzinfo tz_out, bint utc)

pandas/_libs/tslibs/strptime.pyx

+62-38
Original file line numberDiff line numberDiff line change
@@ -252,8 +252,11 @@ cdef class DatetimeParseState:
252252
# found_naive_str refers to a string that was parsed to a timezone-naive
253253
# datetime.
254254
self.found_naive_str = False
255+
self.found_aware_str = False
255256
self.found_other = False
256257

258+
self.out_tzoffset_vals = set()
259+
257260
self.creso = creso
258261
self.creso_ever_changed = False
259262

@@ -292,6 +295,58 @@ cdef class DatetimeParseState:
292295
"tz-naive values")
293296
return tz
294297

298+
cdef tzinfo check_for_mixed_inputs(
299+
self,
300+
tzinfo tz_out,
301+
bint utc,
302+
):
303+
cdef:
304+
bint is_same_offsets
305+
float tz_offset
306+
307+
if self.found_aware_str and not utc:
308+
# GH#17697, GH#57275
309+
# 1) If all the offsets are equal, return one offset for
310+
# the parsed dates to (maybe) pass to DatetimeIndex
311+
# 2) If the offsets are different, then do not force the parsing
312+
# and raise a ValueError: "cannot parse datetimes with
313+
# mixed time zones unless `utc=True`" instead
314+
is_same_offsets = len(self.out_tzoffset_vals) == 1
315+
if not is_same_offsets or (self.found_naive or self.found_other):
316+
# e.g. test_to_datetime_mixed_awareness_mixed_types (array_to_datetime)
317+
raise ValueError(
318+
"Mixed timezones detected. Pass utc=True in to_datetime "
319+
"or tz='UTC' in DatetimeIndex to convert to a common timezone."
320+
)
321+
elif tz_out is not None:
322+
# GH#55693
323+
tz_offset = self.out_tzoffset_vals.pop()
324+
tz_out2 = timezone(timedelta(seconds=tz_offset))
325+
if not tz_compare(tz_out, tz_out2):
326+
# e.g. (array_strptime)
327+
# test_to_datetime_mixed_offsets_with_utc_false_removed
328+
# e.g. test_to_datetime_mixed_tzs_mixed_types (array_to_datetime)
329+
raise ValueError(
330+
"Mixed timezones detected. Pass utc=True in to_datetime "
331+
"or tz='UTC' in DatetimeIndex to convert to a common timezone."
332+
)
333+
# e.g. (array_strptime)
334+
# test_guess_datetime_format_with_parseable_formats
335+
# e.g. test_to_datetime_mixed_types_matching_tzs (array_to_datetime)
336+
else:
337+
# e.g. test_to_datetime_iso8601_with_timezone_valid (array_strptime)
338+
tz_offset = self.out_tzoffset_vals.pop()
339+
tz_out = timezone(timedelta(seconds=tz_offset))
340+
elif not utc:
341+
if tz_out and (self.found_other or self.found_naive_str):
342+
# found_other indicates a tz-naive int, float, dt64, or date
343+
# e.g. test_to_datetime_mixed_awareness_mixed_types (array_to_datetime)
344+
raise ValueError(
345+
"Mixed timezones detected. Pass utc=True in to_datetime "
346+
"or tz='UTC' in DatetimeIndex to convert to a common timezone."
347+
)
348+
return tz_out
349+
295350

296351
def array_strptime(
297352
ndarray[object] values,
@@ -319,11 +374,8 @@ def array_strptime(
319374
npy_datetimestruct dts
320375
int64_t[::1] iresult
321376
object val
322-
bint seen_datetime_offset = False
323377
bint is_raise = errors=="raise"
324378
bint is_coerce = errors=="coerce"
325-
bint is_same_offsets
326-
set out_tzoffset_vals = set()
327379
tzinfo tz, tz_out = None
328380
bint iso_format = format_is_iso(fmt)
329381
NPY_DATETIMEUNIT out_bestunit, item_reso
@@ -418,15 +470,15 @@ def array_strptime(
418470
) from err
419471
if out_local == 1:
420472
nsecs = out_tzoffset * 60
421-
out_tzoffset_vals.add(nsecs)
422-
seen_datetime_offset = True
473+
state.out_tzoffset_vals.add(nsecs)
474+
state.found_aware_str = True
423475
tz = timezone(timedelta(minutes=out_tzoffset))
424476
value = tz_localize_to_utc_single(
425477
value, tz, ambiguous="raise", nonexistent=None, creso=creso
426478
)
427479
else:
428480
tz = None
429-
out_tzoffset_vals.add("naive")
481+
state.out_tzoffset_vals.add("naive")
430482
state.found_naive_str = True
431483
iresult[i] = value
432484
continue
@@ -475,12 +527,12 @@ def array_strptime(
475527
elif creso == NPY_DATETIMEUNIT.NPY_FR_ms:
476528
nsecs = nsecs // 10**3
477529

478-
out_tzoffset_vals.add(nsecs)
479-
seen_datetime_offset = True
530+
state.out_tzoffset_vals.add(nsecs)
531+
state.found_aware_str = True
480532
else:
481533
state.found_naive_str = True
482534
tz = None
483-
out_tzoffset_vals.add("naive")
535+
state.out_tzoffset_vals.add("naive")
484536

485537
except ValueError as ex:
486538
ex.args = (
@@ -499,35 +551,7 @@ def array_strptime(
499551
raise
500552
return values, None
501553

502-
if seen_datetime_offset and not utc:
503-
is_same_offsets = len(out_tzoffset_vals) == 1
504-
if not is_same_offsets or (state.found_naive or state.found_other):
505-
raise ValueError(
506-
"Mixed timezones detected. Pass utc=True in to_datetime "
507-
"or tz='UTC' in DatetimeIndex to convert to a common timezone."
508-
)
509-
elif tz_out is not None:
510-
# GH#55693
511-
tz_offset = out_tzoffset_vals.pop()
512-
tz_out2 = timezone(timedelta(seconds=tz_offset))
513-
if not tz_compare(tz_out, tz_out2):
514-
# e.g. test_to_datetime_mixed_offsets_with_utc_false_removed
515-
raise ValueError(
516-
"Mixed timezones detected. Pass utc=True in to_datetime "
517-
"or tz='UTC' in DatetimeIndex to convert to a common timezone."
518-
)
519-
# e.g. test_guess_datetime_format_with_parseable_formats
520-
else:
521-
# e.g. test_to_datetime_iso8601_with_timezone_valid
522-
tz_offset = out_tzoffset_vals.pop()
523-
tz_out = timezone(timedelta(seconds=tz_offset))
524-
elif not utc:
525-
if tz_out and (state.found_other or state.found_naive_str):
526-
# found_other indicates a tz-naive int, float, dt64, or date
527-
raise ValueError(
528-
"Mixed timezones detected. Pass utc=True in to_datetime "
529-
"or tz='UTC' in DatetimeIndex to convert to a common timezone."
530-
)
554+
tz_out = state.check_for_mixed_inputs(tz_out, utc)
531555

532556
if infer_reso:
533557
if state.creso_ever_changed:

pandas/core/api.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
UInt64Dtype,
4242
)
4343
from pandas.core.arrays.string_ import StringDtype
44-
from pandas.core.construction import array
44+
from pandas.core.construction import array # noqa: ICN001
4545
from pandas.core.flags import Flags
4646
from pandas.core.groupby import (
4747
Grouper,

pandas/core/arrays/categorical.py

-1
Original file line numberDiff line numberDiff line change
@@ -2482,7 +2482,6 @@ def unique(self) -> Self:
24822482
['b', 'a']
24832483
Categories (3, object): ['a' < 'b' < 'c']
24842484
"""
2485-
# pylint: disable=useless-parent-delegation
24862485
return super().unique()
24872486

24882487
def _cast_quantile_result(self, res_values: np.ndarray) -> np.ndarray:

pandas/core/arrays/datetimelike.py

-1
Original file line numberDiff line numberDiff line change
@@ -506,7 +506,6 @@ def view(self, dtype: Literal["m8[ns]"]) -> TimedeltaArray: ...
506506
@overload
507507
def view(self, dtype: Dtype | None = ...) -> ArrayLike: ...
508508

509-
# pylint: disable-next=useless-parent-delegation
510509
def view(self, dtype: Dtype | None = None) -> ArrayLike:
511510
# we need to explicitly call super() method as long as the `@overload`s
512511
# are present in this file.

pandas/core/base.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ def __sizeof__(self) -> int:
127127
"""
128128
memory_usage = getattr(self, "memory_usage", None)
129129
if memory_usage:
130-
mem = memory_usage(deep=True) # pylint: disable=not-callable
130+
mem = memory_usage(deep=True)
131131
return int(mem if is_scalar(mem) else mem.sum())
132132

133133
# no memory_usage attribute, so fall back to object's 'sizeof'

pandas/core/computation/ops.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ def _resolve_name(self):
115115
res = self.env.resolve(local_name, is_local=is_local)
116116
self.update(res)
117117

118-
if hasattr(res, "ndim") and res.ndim > 2:
118+
if hasattr(res, "ndim") and isinstance(res.ndim, int) and res.ndim > 2:
119119
raise NotImplementedError(
120120
"N-dimensional objects, where N > 2, are not supported with eval"
121121
)

pandas/core/dtypes/common.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ def is_scipy_sparse(arr) -> bool:
250250
"""
251251
global _is_scipy_sparse
252252

253-
if _is_scipy_sparse is None: # pylint: disable=used-before-assignment
253+
if _is_scipy_sparse is None:
254254
try:
255255
from scipy.sparse import issparse as _is_scipy_sparse
256256
except ImportError:

pandas/core/generic.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6729,7 +6729,7 @@ def _pad_or_backfill(
67296729

67306730
if axis == 1:
67316731
if not self._mgr.is_single_block and inplace:
6732-
raise NotImplementedError()
6732+
raise NotImplementedError
67336733
# e.g. test_align_fill_method
67346734
result = self.T._pad_or_backfill(
67356735
method=method, limit=limit, limit_area=limit_area

pandas/core/groupby/groupby.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -5014,7 +5014,7 @@ def shift(
50145014
period = cast(int, period)
50155015
if freq is not None:
50165016
f = lambda x: x.shift(
5017-
period, # pylint: disable=cell-var-from-loop
5017+
period,
50185018
freq,
50195019
0, # axis
50205020
fill_value,

0 commit comments

Comments
 (0)