Skip to content

Support numpy 2.0 in tests #943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 26, 2024
Merged

Support numpy 2.0 in tests #943

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c0a593c
change np.float_ to np.double
Dr-Irv Jun 25, 2024
12de93e
changes to support numpy 2.0
Dr-Irv Jun 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions pandas-stubs/_typing.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ BooleanDtypeArg: TypeAlias = (
# Numpy bool type
# https://numpy.org/doc/stable/reference/arrays.scalars.html#numpy.bool_
| type[np.bool_]
| Literal["?", "b1", "bool8", "bool_"]
| Literal["?", "b1", "bool_"]
# PyArrow boolean type and its string alias
| Literal["bool[pyarrow]", "boolean[pyarrow]"]
)
Expand Down Expand Up @@ -147,7 +147,7 @@ IntDtypeArg: TypeAlias = (
| Literal["q", "longlong"] # NOTE: int128 not assigned
# https://numpy.org/doc/stable/reference/arrays.scalars.html#numpy.intp
| type[np.intp] # signed pointer (=`intptr_t`, platform dependent)
| Literal["p", "intp", "int0"]
| Literal["p", "intp"]
# PyArrow integer types and their string aliases
| Literal["int8[pyarrow]", "int16[pyarrow]", "int32[pyarrow]", "int64[pyarrow]"]
)
Expand Down Expand Up @@ -176,7 +176,7 @@ UIntDtypeArg: TypeAlias = (
| Literal["Q", "ulonglong"] # NOTE: uint128 not assigned
# https://numpy.org/doc/stable/reference/arrays.scalars.html#numpy.uintp
| type[np.uintp] # unsigned pointer (=`uintptr_t`, platform dependent)
| Literal["P", "uintp", "uint0"]
| Literal["P", "uintp"]
# PyArrow unsigned integer types and their string aliases
| Literal["uint8[pyarrow]", "uint16[pyarrow]", "uint32[pyarrow]", "uint64[pyarrow]"]
)
Expand Down Expand Up @@ -361,7 +361,7 @@ BytesDtypeArg: TypeAlias = (
# Numpy bytes type and its string alias
# https://numpy.org/doc/stable/reference/arrays.scalars.html#numpy.bytes_
| type[np.bytes_]
| Literal["S", "a", "bytes_", "bytes0", "string_"]
| Literal["S", "bytes_", "bytes0", "string_"]
# PyArrow binary type and its string alias
| Literal["binary[pyarrow]"]
)
Expand Down
2 changes: 1 addition & 1 deletion pandas-stubs/core/resample.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ class Resampler(BaseGroupBy[NDFrameT]):
def count(self: Resampler[DataFrame]) -> DataFrame: ...
def quantile(
self,
q: float | list[float] | npt.NDArray[np.float_] | Series[float] = ...,
q: float | list[float] | npt.NDArray[np.double] | Series[float] = ...,
**kwargs,
) -> NDFrameT: ...

Expand Down
6 changes: 3 additions & 3 deletions pandas-stubs/core/reshape/tile.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ def qcut(
retbins: Literal[True],
precision: int = ...,
duplicates: Literal["raise", "drop"] = ...,
) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.float_]]: ...
) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.double]]: ...
@overload
def qcut(
x: Series,
Expand All @@ -260,7 +260,7 @@ def qcut(
retbins: Literal[True],
precision: int = ...,
duplicates: Literal["raise", "drop"] = ...,
) -> tuple[Series, npt.NDArray[np.float_]]: ...
) -> tuple[Series, npt.NDArray[np.double]]: ...
@overload
def qcut(
x: Index | npt.NDArray | Sequence[int] | Sequence[float],
Expand All @@ -270,4 +270,4 @@ def qcut(
retbins: Literal[True],
precision: int = ...,
duplicates: Literal["raise", "drop"] = ...,
) -> tuple[Categorical, npt.NDArray[np.float_]]: ...
) -> tuple[Categorical, npt.NDArray[np.double]]: ...
10 changes: 10 additions & 0 deletions pandas-stubs/core/series.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,16 @@ _ListLike: TypeAlias = (
class Series(IndexOpsMixin[S1], NDFrame):
__hash__: ClassVar[None]

@overload
def __new__( # type: ignore[overload-overlap]
cls,
data: npt.NDArray[np.float64],
index: Axes | None = ...,
*,
dtype: Dtype = ...,
name: Hashable = ...,
copy: bool = ...,
) -> Series[float]: ...
@overload
def __new__( # type: ignore[overload-overlap]
cls,
Expand Down
4 changes: 2 additions & 2 deletions pandas-stubs/plotting/_core.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,7 @@ class PlotAccessor:
| Callable[[gaussian_kde], float]
| None
) = ...,
ind: npt.NDArray[np.float_] | int | None = ...,
ind: npt.NDArray[np.double] | int | None = ...,
*,
subplots: Literal[False] | None = ...,
**kwargs,
Expand All @@ -354,7 +354,7 @@ class PlotAccessor:
| Callable[[gaussian_kde], float]
| None
) = ...,
ind: npt.NDArray[np.float_] | int | None = ...,
ind: npt.NDArray[np.double] | int | None = ...,
*,
subplots: Literal[True],
**kwargs,
Expand Down
98 changes: 60 additions & 38 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,20 @@ readme = "README.md"
homepage = "https://pandas.pydata.org"
repository = "https://github.com/pandas-dev/pandas-stubs"
classifiers = [
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: BSD License",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Topic :: Scientific/Engineering"
]
packages = [
{ "include" = "pandas-stubs"}
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: BSD License",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Topic :: Scientific/Engineering",
]
packages = [{ "include" = "pandas-stubs" }]

[tool.poetry.urls]
"Bug Tracker" = "https://github.com/pandas-dev/pandas-stubs/issues"
Expand All @@ -33,26 +31,26 @@ packages = [
python = ">=3.9"
types-pytz = ">= 2022.1.1"
numpy = [
{ version = ">=1.23.5", python = ">=3.9,<3.12" },
{ version = ">=1.26.0", python = ">=3.12,<3.13" }
{ version = ">=1.23.5,<2.0.0", python = ">=3.9,<3.12" },
{ version = ">=2.0.0", python = ">=3.12,<3.13" },
]

[tool.poetry.group.dev.dependencies]
mypy = "1.10.0"
mypy = "1.10.1"
pandas = "2.2.2"
pyarrow = ">=10.0.1"
pytest = ">=7.1.2"
pyright = ">=1.1.365"
pyright = ">=1.1.369"
poethepoet = ">=0.16.5"
loguru = ">=0.6.0"
typing-extensions = ">=4.4.0"
matplotlib = ">=3.5.1,<3.9.0" # TODO https://github.com/pandas-dev/pandas/issues/58851
matplotlib = ">=3.5.1,<3.9.0" # TODO https://github.com/pandas-dev/pandas/issues/58851
pre-commit = ">=2.19.0"
black = ">=23.3.0"
isort = ">=5.12.0"
openpyxl = ">=3.0.10"
# for tables, MacOS gives random CI failures on 3.9.2
tables = { version = "==3.9.2", python = "<4"} # 3.8.0 depends on blosc2 which caps python to <4
tables = { version = "==3.9.2", python = "<4" } # 3.8.0 depends on blosc2 which caps python to <4
lxml = ">=4.9.1"
pyreadstat = ">=1.2.0"
xlrd = ">=2.0.1"
Expand Down Expand Up @@ -88,15 +86,19 @@ script = "scripts.test:test(dist=True)"
[tool.poe.tasks.pytest]
help = "Run pytest"
script = "scripts.test:pytest(nightly)"
args = [{name = "nightly", positional = false, default = false, type = "boolean", required = false, help= "Use pandas nightly (off by default)"}]
args = [
{ name = "nightly", positional = false, default = false, type = "boolean", required = false, help = "Use pandas nightly (off by default)" },
]

[tool.poe.tasks.style]
help = "Run pre-commit"
script = "scripts.test.run:style"

[tool.poe.tasks.mypy]
help = "Run mypy on 'tests' (using the local stubs) and on the local stubs"
args = [{name = "mypy_nightly", positional = false, default = false, type = "boolean", required = false, help= "Use mypy nightly (off by default)"}]
args = [
{ name = "mypy_nightly", positional = false, default = false, type = "boolean", required = false, help = "Use mypy nightly (off by default)" },
]
script = "scripts.test:mypy_src(mypy_nightly)"

[tool.poe.tasks.mypy_dist]
Expand All @@ -114,18 +116,38 @@ script = "scripts.test:test(dist=True, type_checker='pyright')"
[tool.poe.tasks.stubtest]
script = "scripts.test:stubtest(allowlist, check_missing, nightly)"
help = "Run stubtest to compare the installed stubs against pandas"
args = [{ name = "allowlist", positional = true, default = "", required = false, help= "Path to an allowlist (optional)" }, {name = "check_missing", positional = false, default = false, type = "boolean", required = false, help= "Report errors when the stubs are incomplete (off by default)"}, {name = "nightly", positional = false, default = false, type = "boolean", required = false, help= "Compare against pandas nightly (off by default)"}]
args = [
{ name = "allowlist", positional = true, default = "", required = false, help = "Path to an allowlist (optional)" },
{ name = "check_missing", positional = false, default = false, type = "boolean", required = false, help = "Report errors when the stubs are incomplete (off by default)" },
{ name = "nightly", positional = false, default = false, type = "boolean", required = false, help = "Compare against pandas nightly (off by default)" },
]


[tool.black]
target-version = ['py39']

[tool.isort]
known_pre_libs = "pandas._config"
known_pre_core = ["pandas._libs", "pandas._typing", "pandas.util._*", "pandas.compat", "pandas.errors"]
known_pre_core = [
"pandas._libs",
"pandas._typing",
"pandas.util._*",
"pandas.compat",
"pandas.errors",
]
known_dtypes = "pandas.core.dtypes"
known_post_core = ["pandas.tseries", "pandas.io", "pandas.plotting"]
sections = ["FUTURE", "STDLIB", "THIRDPARTY" ,"PRE_LIBS" , "PRE_CORE", "DTYPES", "FIRSTPARTY", "POST_CORE", "LOCALFOLDER"]
sections = [
"FUTURE",
"STDLIB",
"THIRDPARTY",
"PRE_LIBS",
"PRE_CORE",
"DTYPES",
"FIRSTPARTY",
"POST_CORE",
"LOCALFOLDER",
]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you use a formatting tool for these changes?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I used VSCode to do the editing, and I have a setting to tell it to auto-format files after saving, so I'm guessing that's what happened.

Will you merge?

profile = "black"
combine_as_imports = true
force_grid_wrap = 2
Expand All @@ -142,16 +164,16 @@ follow_imports_for_stubs = false
no_site_packages = false
no_silence_site_packages = false
# Disallow dynamic typing
disallow_any_unimported = false # TODO
disallow_any_expr = false # TODO
disallow_any_decorated = false # TODO
disallow_any_explicit = false # TODO
disallow_any_generics = false # TODO
disallow_any_unimported = false # TODO
disallow_any_expr = false # TODO
disallow_any_decorated = false # TODO
disallow_any_explicit = false # TODO
disallow_any_generics = false # TODO
disallow_subclassing_any = false # TODO
# Untyped definitions and calls
disallow_untyped_calls = false # TODO
disallow_untyped_defs = false # TODO
disallow_incomplete_defs = false # TODO
disallow_untyped_calls = false # TODO
disallow_untyped_defs = false # TODO
disallow_incomplete_defs = false # TODO
check_untyped_defs = true
disallow_untyped_decorators = true
# None and Optional handling
Expand All @@ -161,16 +183,16 @@ strict_optional = true
warn_redundant_casts = true
warn_unused_ignores = true
warn_no_return = true
warn_return_any = false # TODO
warn_unreachable = false # GH#27396
warn_return_any = false # TODO
warn_unreachable = false # GH#27396
# Suppressing errors
ignore_errors = false
enable_error_code = "ignore-without-code" # same as in pandas
# Miscellaneous strictness flags
allow_untyped_globals = false
allow_redefinition = false
local_partial_types = false
implicit_reexport = false # pyright behaves the same
implicit_reexport = false # pyright behaves the same
strict_equality = true
# Configuring error messages
show_error_context = false
Expand All @@ -181,7 +203,7 @@ show_error_codes = true
typeCheckingMode = "strict"
stubPath = "."
include = ["tests", "pandas-stubs"]
enableTypeIgnoreComments = false # use pyright-specific ignores
enableTypeIgnoreComments = false # use pyright-specific ignores
# disable subset of strict
reportMissingParameterType = false
reportMissingTypeArgument = false
Expand Down
4 changes: 3 additions & 1 deletion tests/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
Literal,
)

import numpy as np
import pandas as pd
from pandas.core.groupby.groupby import BaseGroupBy
from pandas.util.version import Version
Expand All @@ -23,6 +24,7 @@
TYPE_CHECKING_INVALID_USAGE: Final = TYPE_CHECKING
WINDOWS = os.name == "nt" or "cygwin" in platform.system().lower()
PD_LTE_22 = Version(pd.__version__) < Version("2.2.999")
NUMPY20 = np.lib.NumpyVersion(np.__version__) >= "2.0.0"


def check(
Expand All @@ -40,7 +42,7 @@ def check(
if isinstance(actual, pd.Series):
value = actual.iloc[index_to_check_for_type]
elif isinstance(actual, pd.Index):
value = actual[index_to_check_for_type] # type: ignore[assignment]
value = actual[index_to_check_for_type]
elif isinstance(actual, BaseGroupBy):
value = actual.obj
elif hasattr(actual, "__iter__"):
Expand Down
6 changes: 6 additions & 0 deletions tests/test_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
WINDOWS,
check,
)
from tests import NUMPY20 # See https://github.com/PyTables/PyTables/issues/1172

from pandas.io.api import to_pickle
from pandas.io.json._json import JsonReader
Expand Down Expand Up @@ -347,12 +348,14 @@ def test_sas_xport() -> None:
pass


@pytest.mark.skipif(NUMPY20, reason="numpy 2.0 not compatible with Pytables")
def test_hdf():
with ensure_clean() as path:
check(assert_type(DF.to_hdf(path, key="df"), None), type(None))
check(assert_type(read_hdf(path), Union[DataFrame, Series]), DataFrame)


@pytest.mark.skipif(NUMPY20, reason="numpy 2.0 not compatible with Pytables")
def test_hdfstore() -> None:
with ensure_clean() as path:
store = HDFStore(path, model="w")
Expand Down Expand Up @@ -396,6 +399,7 @@ def test_hdfstore() -> None:
store.close()


@pytest.mark.skipif(NUMPY20, reason="numpy 2.0 not compatible with Pytables")
def test_read_hdf_iterator() -> None:
with ensure_clean() as path:
check(assert_type(DF.to_hdf(path, key="df", format="table"), None), type(None))
Expand All @@ -410,6 +414,7 @@ def test_read_hdf_iterator() -> None:
ti.close()


@pytest.mark.skipif(NUMPY20, reason="numpy 2.0 not compatible with Pytables")
def test_hdf_context_manager() -> None:
with ensure_clean() as path:
check(assert_type(DF.to_hdf(path, key="df", format="table"), None), type(None))
Expand All @@ -418,6 +423,7 @@ def test_hdf_context_manager() -> None:
check(assert_type(store.get("df"), Union[DataFrame, Series]), DataFrame)


@pytest.mark.skipif(NUMPY20, reason="numpy 2.0 not compatible with Pytables")
def test_hdf_series():
s = DF["a"]
with ensure_clean() as path:
Expand Down
20 changes: 10 additions & 10 deletions tests/test_pandas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1004,10 +1004,10 @@ def test_qcut() -> None:
check(assert_type(c0, pd.Categorical), pd.Categorical)
check(assert_type(d0, pd.Series), pd.Series)

check(assert_type(a1, npt.NDArray[np.float_]), np.ndarray)
check(assert_type(b1, npt.NDArray[np.float_]), np.ndarray)
check(assert_type(c1, npt.NDArray[np.float_]), np.ndarray)
check(assert_type(d1, npt.NDArray[np.float_]), np.ndarray)
check(assert_type(a1, npt.NDArray[np.double]), np.ndarray)
check(assert_type(b1, npt.NDArray[np.double]), np.ndarray)
check(assert_type(c1, npt.NDArray[np.double]), np.ndarray)
check(assert_type(d1, npt.NDArray[np.double]), np.ndarray)

e0, e1 = pd.qcut(val_list, [0.25, 0.5, 0.75], retbins=True)
f0, f1 = pd.qcut(val_arr, np.array([0.25, 0.5, 0.75]), retbins=True)
Expand All @@ -1023,12 +1023,12 @@ def test_qcut() -> None:
check(assert_type(i0, npt.NDArray[np.intp]), np.ndarray)
check(assert_type(j0, npt.NDArray[np.intp]), np.ndarray)

check(assert_type(e1, npt.NDArray[np.float_]), np.ndarray)
check(assert_type(f1, npt.NDArray[np.float_]), np.ndarray)
check(assert_type(g1, npt.NDArray[np.float_]), np.ndarray)
check(assert_type(h1, npt.NDArray[np.float_]), np.ndarray)
check(assert_type(i1, npt.NDArray[np.float_]), np.ndarray)
check(assert_type(j1, npt.NDArray[np.float_]), np.ndarray)
check(assert_type(e1, npt.NDArray[np.double]), np.ndarray)
check(assert_type(f1, npt.NDArray[np.double]), np.ndarray)
check(assert_type(g1, npt.NDArray[np.double]), np.ndarray)
check(assert_type(h1, npt.NDArray[np.double]), np.ndarray)
check(assert_type(i1, npt.NDArray[np.double]), np.ndarray)
check(assert_type(j1, npt.NDArray[np.double]), np.ndarray)


def test_merge() -> None:
Expand Down
Loading
Loading