From d402d0867302024a501fbba82c146c8e02e60616 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Mar 2024 14:17:45 -0400 Subject: [PATCH 01/15] version bump --- .circleci/config.yml | 2 +- .github/workflows/unit-tests.yml | 6 +++--- ci/deps/actions-310.yaml | 4 ++-- ci/deps/actions-311-downstream_compat.yaml | 4 ++-- ci/deps/actions-311-numpydev.yaml | 4 ++-- ci/deps/actions-311-pyarrownightly.yaml | 4 ++-- ci/deps/actions-311-sanitizers.yaml | 4 ++-- ci/deps/actions-311.yaml | 4 ++-- ci/deps/actions-312.yaml | 4 ++-- ci/deps/actions-39-minimum_versions.yaml | 4 ++-- ci/deps/actions-39.yaml | 4 ++-- ci/deps/actions-pypy-39.yaml | 4 ++-- ci/deps/circle-310-arm64.yaml | 4 ++-- environment.yml | 4 ++-- meson.build | 2 +- pyproject.toml | 4 ++-- requirements-dev.txt | 4 ++-- 17 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea93575ac9430..67bb6fd5367e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ jobs: - run: | /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev . ~/virtualenvs/pandas-dev/bin/activate - python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1 + python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.15.0 meson[ninja]==1.4.0 python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1 python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror" python -m pip list --no-cache-dir diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 855973a22886a..0204fa446b5a9 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -240,7 +240,7 @@ jobs: run: | /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev . ~/virtualenvs/pandas-dev/bin/activate - python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1 + python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.4.0 meson-python==0.15.0 python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true" python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1 python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror" @@ -279,7 +279,7 @@ jobs: run: | /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev . ~/virtualenvs/pandas-dev/bin/activate - python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1 + python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.15.0 meson[ninja]==1.4.0 python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1 python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror" python -m pip list --no-cache-dir @@ -351,7 +351,7 @@ jobs: - name: Build Environment run: | python --version - python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1 + python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.4.0 meson-python==0.15.0 python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy python -m pip install versioneer[toml] python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 85ee5230b31be..719b92392f3ac 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -7,8 +7,8 @@ dependencies: # build dependencies - versioneer[toml] - cython>=0.29.33 - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/ci/deps/actions-311-downstream_compat.yaml b/ci/deps/actions-311-downstream_compat.yaml index efd790d77afbb..4ca927bd97ecb 100644 --- a/ci/deps/actions-311-downstream_compat.yaml +++ b/ci/deps/actions-311-downstream_compat.yaml @@ -8,8 +8,8 @@ dependencies: # build dependencies - versioneer[toml] - cython>=0.29.33 - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/ci/deps/actions-311-numpydev.yaml b/ci/deps/actions-311-numpydev.yaml index b62e8630f2059..b219ec0158127 100644 --- a/ci/deps/actions-311-numpydev.yaml +++ b/ci/deps/actions-311-numpydev.yaml @@ -6,8 +6,8 @@ dependencies: # build dependencies - versioneer[toml] - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 - cython>=0.29.33 # test dependencies diff --git a/ci/deps/actions-311-pyarrownightly.yaml b/ci/deps/actions-311-pyarrownightly.yaml index d84063ac2a9ba..b0fae6f0f7f93 100644 --- a/ci/deps/actions-311-pyarrownightly.yaml +++ b/ci/deps/actions-311-pyarrownightly.yaml @@ -6,9 +6,9 @@ dependencies: # build dependencies - versioneer[toml] - - meson[ninja]=1.2.1 + - meson[ninja]=1.4.0 - cython>=0.29.33 - - meson-python=0.13.1 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/ci/deps/actions-311-sanitizers.yaml b/ci/deps/actions-311-sanitizers.yaml index f5f04c90bffad..7205d44063c2d 100644 --- a/ci/deps/actions-311-sanitizers.yaml +++ b/ci/deps/actions-311-sanitizers.yaml @@ -7,8 +7,8 @@ dependencies: # build dependencies - versioneer[toml] - cython>=0.29.33 - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index 535c260582eec..2c9031175fd42 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -7,8 +7,8 @@ dependencies: # build dependencies - versioneer[toml] - cython>=0.29.33 - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/ci/deps/actions-312.yaml b/ci/deps/actions-312.yaml index 8b3f19f55e4b6..9bde872b567f0 100644 --- a/ci/deps/actions-312.yaml +++ b/ci/deps/actions-312.yaml @@ -7,8 +7,8 @@ dependencies: # build dependencies - versioneer[toml] - cython>=0.29.33 - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/ci/deps/actions-39-minimum_versions.yaml b/ci/deps/actions-39-minimum_versions.yaml index 94cb21d1621b6..3213da20cf46f 100644 --- a/ci/deps/actions-39-minimum_versions.yaml +++ b/ci/deps/actions-39-minimum_versions.yaml @@ -9,8 +9,8 @@ dependencies: # build dependencies - versioneer[toml] - cython>=0.29.33 - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 4cc9b1fbe2491..5c6984816e6f5 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -7,8 +7,8 @@ dependencies: # build dependencies - versioneer[toml] - cython>=0.29.33 - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/ci/deps/actions-pypy-39.yaml b/ci/deps/actions-pypy-39.yaml index d9c8dd81b7c33..cab5a9148da75 100644 --- a/ci/deps/actions-pypy-39.yaml +++ b/ci/deps/actions-pypy-39.yaml @@ -10,8 +10,8 @@ dependencies: # build dependencies - versioneer[toml] - cython>=0.29.33 - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/ci/deps/circle-310-arm64.yaml b/ci/deps/circle-310-arm64.yaml index 869aae8596681..3aa6d541bc83b 100644 --- a/ci/deps/circle-310-arm64.yaml +++ b/ci/deps/circle-310-arm64.yaml @@ -7,8 +7,8 @@ dependencies: # build dependencies - versioneer[toml] - cython>=0.29.33 - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/environment.yml b/environment.yml index 3528f12c66a8b..cc5961a5ed345 100644 --- a/environment.yml +++ b/environment.yml @@ -9,8 +9,8 @@ dependencies: # build dependencies - versioneer[toml] - cython~=3.0.5 - - meson[ninja]=1.2.1 - - meson-python=0.13.1 + - meson[ninja]=1.4.0 + - meson-python=0.15.0 # test dependencies - pytest>=7.3.2 diff --git a/meson.build b/meson.build index 06623a305ab54..03dbcd7ef43c8 100644 --- a/meson.build +++ b/meson.build @@ -4,7 +4,7 @@ project( 'c', 'cpp', 'cython', version: run_command(['generate_version.py', '--print'], check: true).stdout().strip(), license: 'BSD-3', - meson_version: '>=1.2.1', + meson_version: '>=1.4.0', default_options: [ 'buildtype=release', 'c_std=c11', diff --git a/pyproject.toml b/pyproject.toml index f96fbee4a5818..3ca922463b53f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,8 @@ # Minimum requirements for the build system to execute. # See https://github.com/scipy/scipy/pull/12940 for the AIX issue. requires = [ - "meson-python==0.13.1", - "meson==1.2.1", + "meson-python==0.15.0", + "meson==1.4.0", "wheel", "Cython~=3.0.5", # Note: sync with setup.py, environment.yml and asv.conf.json # Any NumPy version should be fine for compiling. Users are unlikely diff --git a/requirements-dev.txt b/requirements-dev.txt index 40c7403cb88e8..c57d71c300450 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,8 +4,8 @@ pip versioneer[toml] cython~=3.0.5 -meson[ninja]==1.2.1 -meson-python==0.13.1 +meson[ninja]==1.4.0 +meson-python==0.15.0 pytest>=7.3.2 pytest-cov pytest-xdist>=2.2.0 From 668f74c78d1f00a0ba52dff5e0f6b167a72e7ebf Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Mar 2024 14:34:06 -0400 Subject: [PATCH 02/15] Bump meson to 0.14.0 --- pandas/_libs/meson.build | 2 +- pandas/_libs/tslibs/meson.build | 2 +- pandas/_libs/window/meson.build | 2 ++ pandas/meson.build | 26 +++++++++++++++++--------- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index 7621915ebcfdb..109bddb4d1134 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -129,7 +129,7 @@ foreach ext_name, ext_dict : libs_sources ext_dict.get('sources'), cython_args: cython_args, include_directories: [inc_np, inc_pd], - dependencies: ext_dict.get('deps', ''), + dependencies: ext_dict.get('deps', []) + [np_dep], subdir: 'pandas/_libs', install: true ) diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build index 85410f771233f..9bcdf799659b8 100644 --- a/pandas/_libs/tslibs/meson.build +++ b/pandas/_libs/tslibs/meson.build @@ -34,7 +34,7 @@ foreach ext_name, ext_dict : tslibs_sources ext_dict.get('sources'), cython_args: cython_args, include_directories: [inc_np, inc_pd], - dependencies: ext_dict.get('deps', ''), + dependencies: ext_dict.get('deps', []) + [np_dep], subdir: 'pandas/_libs/tslibs', install: true ) diff --git a/pandas/_libs/window/meson.build b/pandas/_libs/window/meson.build index ad15644f73a0c..f3adabe8e7a36 100644 --- a/pandas/_libs/window/meson.build +++ b/pandas/_libs/window/meson.build @@ -3,6 +3,7 @@ py.extension_module( ['aggregations.pyx'], cython_args: ['-X always_allow_keywords=true'], include_directories: [inc_np, inc_pd], + dependencies: [np_dep], subdir: 'pandas/_libs/window', override_options : ['cython_language=cpp'], install: true @@ -13,6 +14,7 @@ py.extension_module( ['indexers.pyx'], cython_args: ['-X always_allow_keywords=true'], include_directories: [inc_np, inc_pd], + dependencies: [np_dep], subdir: 'pandas/_libs/window', install: true ) diff --git a/pandas/meson.build b/pandas/meson.build index 435103a954d86..98007e5099a24 100644 --- a/pandas/meson.build +++ b/pandas/meson.build @@ -1,7 +1,13 @@ -incdir_numpy = run_command(py, - [ - '-c', - ''' +# NumPy 2.0 can be detected with meson >= 0.14.0 +np_dep = dependency('numpy', required : false) + +if np_dep.found() + inc_np = [] +else + incdir_numpy = run_command(py, + [ + '-c', + ''' import os import numpy as np try: @@ -12,12 +18,14 @@ try: except Exception: incdir = np.get_include() print(incdir) - ''' - ], - check: true -).stdout().strip() + ''' + ], + check: true + ).stdout().strip() + + inc_np = include_directories(incdir_numpy) +endif -inc_np = include_directories(incdir_numpy) inc_pd = include_directories('_libs/include') fs.copyfile('__init__.py') From ea08a124b78a946706445d1d9696fb5ac3320c82 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Mar 2024 14:48:59 -0400 Subject: [PATCH 03/15] Fix build warnings/errors --- pandas/_libs/byteswap.pyx | 10 +++++----- .../_libs/include/pandas/vendored/klib/khash_python.h | 4 +--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pandas/_libs/byteswap.pyx b/pandas/_libs/byteswap.pyx index 67cd7ad58d229..f6ef7ae9e2ef4 100644 --- a/pandas/_libs/byteswap.pyx +++ b/pandas/_libs/byteswap.pyx @@ -15,7 +15,7 @@ from libc.string cimport memcpy def read_float_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap): cdef uint32_t value - assert offset + sizeof(value) < len(data) + assert offset + sizeof(value) < len(data) cdef const void *ptr = (data) + offset memcpy(&value, ptr, sizeof(value)) if byteswap: @@ -28,7 +28,7 @@ def read_float_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap): def read_double_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap): cdef uint64_t value - assert offset + sizeof(value) < len(data) + assert offset + sizeof(value) < len(data) cdef const void *ptr = (data) + offset memcpy(&value, ptr, sizeof(value)) if byteswap: @@ -41,7 +41,7 @@ def read_double_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap): def read_uint16_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap): cdef uint16_t res - assert offset + sizeof(res) < len(data) + assert offset + sizeof(res) < len(data) memcpy(&res, (data) + offset, sizeof(res)) if byteswap: res = _byteswap2(res) @@ -50,7 +50,7 @@ def read_uint16_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap): def read_uint32_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap): cdef uint32_t res - assert offset + sizeof(res) < len(data) + assert offset + sizeof(res) < len(data) memcpy(&res, (data) + offset, sizeof(res)) if byteswap: res = _byteswap4(res) @@ -59,7 +59,7 @@ def read_uint32_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap): def read_uint64_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap): cdef uint64_t res - assert offset + sizeof(res) < len(data) + assert offset + sizeof(res) < len(data) memcpy(&res, (data) + offset, sizeof(res)) if byteswap: res = _byteswap8(res) diff --git a/pandas/_libs/include/pandas/vendored/klib/khash_python.h b/pandas/_libs/include/pandas/vendored/klib/khash_python.h index 811fdd139de2c..a567c029e07f1 100644 --- a/pandas/_libs/include/pandas/vendored/klib/khash_python.h +++ b/pandas/_libs/include/pandas/vendored/klib/khash_python.h @@ -33,11 +33,9 @@ static void *traced_calloc(size_t num, size_t size) { } static void *traced_realloc(void *old_ptr, size_t size) { + PyTraceMalloc_Untrack(KHASH_TRACE_DOMAIN, (uintptr_t)old_ptr); void *ptr = realloc(old_ptr, size); if (ptr != NULL) { - if (old_ptr != ptr) { - PyTraceMalloc_Untrack(KHASH_TRACE_DOMAIN, (uintptr_t)old_ptr); - } PyTraceMalloc_Track(KHASH_TRACE_DOMAIN, (uintptr_t)ptr, size); } return ptr; From bd4b57c15c52293e90cbb8ed33e5f531003620f5 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Mar 2024 15:03:53 -0400 Subject: [PATCH 04/15] more fixes --- pandas/_libs/lib.pyx | 6 ++++-- pandas/_libs/tslib.pyx | 34 ---------------------------------- 2 files changed, 4 insertions(+), 36 deletions(-) diff --git a/pandas/_libs/lib.pyx b/pandas/_libs/lib.pyx index 00668576d5d53..ee6c1854526ba 100644 --- a/pandas/_libs/lib.pyx +++ b/pandas/_libs/lib.pyx @@ -137,7 +137,7 @@ except ImportError: @cython.wraparound(False) @cython.boundscheck(False) -def memory_usage_of_objects(arr: object[:]) -> int64_t: +def memory_usage_of_objects(arr: ndarray[object]) -> int64_t: """ Return the memory usage of an object array in bytes. @@ -3058,7 +3058,9 @@ def to_object_array_tuples(rows: object) -> np.ndarray: @cython.wraparound(False) @cython.boundscheck(False) -def fast_multiget(dict mapping, object[:] keys, default=np.nan) -> "ArrayLike": +def fast_multiget(dict mapping, + ndarray[object, ndim=1] keys, + default=np.nan) -> "ArrayLike": cdef: Py_ssize_t i, n = len(keys) object val diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index d320bfdbe3022..b29252a5b8e51 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -70,7 +70,6 @@ from pandas._libs.tslibs.conversion cimport ( from pandas._libs.tslibs.dtypes cimport npy_unit_to_abbrev from pandas._libs.tslibs.nattype cimport ( NPY_NAT, - c_NaT as NaT, c_nat_strings as nat_strings, ) from pandas._libs.tslibs.timestamps cimport _Timestamp @@ -347,39 +346,6 @@ def array_with_unit_to_datetime( return result, tz -cdef _array_with_unit_to_datetime_object_fallback(ndarray[object] values, str unit): - cdef: - Py_ssize_t i, n = len(values) - ndarray[object] oresult - tzinfo tz = None - - # TODO: fix subtle differences between this and no-unit code - oresult = cnp.PyArray_EMPTY(values.ndim, values.shape, cnp.NPY_OBJECT, 0) - for i in range(n): - val = values[i] - - if checknull_with_nat_and_na(val): - oresult[i] = NaT - elif is_integer_object(val) or is_float_object(val): - - if val != val or val == NPY_NAT: - oresult[i] = NaT - else: - try: - oresult[i] = Timestamp(val, unit=unit) - except OutOfBoundsDatetime: - oresult[i] = val - - elif isinstance(val, str): - if len(val) == 0 or val in nat_strings: - oresult[i] = NaT - - else: - oresult[i] = val - - return oresult, tz - - @cython.wraparound(False) @cython.boundscheck(False) def first_non_null(values: ndarray) -> int: From a279e4abc6b4c4468d48d1646ac8abb8913d474c Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Mar 2024 15:12:16 -0400 Subject: [PATCH 05/15] try fix? --- pandas/_libs/tslibs/conversion.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 3a55f5fa0c003..193d73a31f885 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -148,8 +148,9 @@ def cast_from_unit_vectorized( if p: frac = np.round(frac, p) + rng = range(len(values)) try: - for i in range(len(values)): + for i in rng: if base[i] == NPY_NAT: out[i] = NPY_NAT else: From 5cb0305891db91c3297620d6b709eb7ed3e8d6e9 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Mar 2024 15:32:19 -0400 Subject: [PATCH 06/15] more fixes --- meson.build | 4 ++++ pandas/_libs/tslibs/conversion.pyx | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/meson.build b/meson.build index 03dbcd7ef43c8..60c02d29b4715 100644 --- a/meson.build +++ b/meson.build @@ -27,6 +27,10 @@ add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp') add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'c') add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'cpp') +if build_machine.system() == 'darwin' + # error: pragma diagnostic pop could not pop, no matching push + add_project_arguments('-Wno-unknown-pragmas') +endif if fs.exists('_version_meson.py') py.install_sources('_version_meson.py', subdir: 'pandas') diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 193d73a31f885..aa512dc1f9b00 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -105,7 +105,7 @@ def cast_from_unit_vectorized( int64_t m int p NPY_DATETIMEUNIT in_reso, out_reso - Py_ssize_t i + Py_ssize_t i = 0 assert values.dtype.kind == "f" @@ -148,9 +148,8 @@ def cast_from_unit_vectorized( if p: frac = np.round(frac, p) - rng = range(len(values)) try: - for i in rng: + for i in range(len(values)): if base[i] == NPY_NAT: out[i] = NPY_NAT else: From 97e3f504f05a080fbccdb43da80ecde6662596bc Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Mar 2024 15:36:39 -0400 Subject: [PATCH 07/15] fix macOS meson build --- meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meson.build b/meson.build index 60c02d29b4715..130f00cdeb1f7 100644 --- a/meson.build +++ b/meson.build @@ -29,7 +29,7 @@ add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'c if build_machine.system() == 'darwin' # error: pragma diagnostic pop could not pop, no matching push - add_project_arguments('-Wno-unknown-pragmas') + add_project_arguments('-Wno-unknown-pragmas', language: 'c') endif if fs.exists('_version_meson.py') From 8cfbd7fb464edb89abdf3f8c629d1d44b3c73735 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Mar 2024 21:17:52 -0400 Subject: [PATCH 08/15] Linux fixes --- pandas/_libs/hashtable_func_helper.pxi.in | 10 +++++++++- pandas/_libs/meson.build | 12 ++++++------ pandas/_libs/tslibs/util.pxd | 4 ++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/pandas/_libs/hashtable_func_helper.pxi.in b/pandas/_libs/hashtable_func_helper.pxi.in index ca1b28b9442ca..9b2d50fc8eeb5 100644 --- a/pandas/_libs/hashtable_func_helper.pxi.in +++ b/pandas/_libs/hashtable_func_helper.pxi.in @@ -43,7 +43,13 @@ cdef value_count_{{dtype}}(const {{dtype}}_t[:] values, bint dropna, const uint8 # Don't use Py_ssize_t, since table.n_buckets is unsigned khiter_t k - {{c_type}} val + # without explicit initialization the compiler detects that val could + # be uninitialized towards the end of this function + {{if dtype == 'object'}} + {{c_type}} val = None + {{else}} + {{c_type}} val = {{to_c_type}}(0) + {{endif}} int ret = 0 bint uses_mask = mask is not None @@ -114,6 +120,8 @@ cdef value_count_{{dtype}}(const {{dtype}}_t[:] values, bint dropna, const uint8 result_counts[i] = table.vals[k] if na_counter > 0: + if n <= 0: # in this case val would be uninitialized + raise ValueError("Expected n > 0") result_counts[table.size] = na_counter result_keys.append(val) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index 20f04c3375865..1749dac0c4484 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -61,18 +61,18 @@ subdir('tslibs') libs_sources = { # Dict of extension name -> dict of {sources, include_dirs, and deps} # numpy include dir is implicitly included - 'algos': {'sources': ['algos.pyx', _algos_common_helper, _algos_take_helper], 'deps': _khash_primitive_helper_dep}, + 'algos': {'sources': ['algos.pyx', _algos_common_helper, _algos_take_helper], 'deps': [_khash_primitive_helper_dep]}, 'arrays': {'sources': ['arrays.pyx']}, 'groupby': {'sources': ['groupby.pyx']}, 'hashing': {'sources': ['hashing.pyx']}, - 'hashtable': {'sources': ['hashtable.pyx', _hashtable_class_helper, _hashtable_func_helper], 'deps': _khash_primitive_helper_dep}, - 'index': {'sources': ['index.pyx', _index_class_helper], 'deps': _khash_primitive_helper_dep}, + 'hashtable': {'sources': ['hashtable.pyx', _hashtable_class_helper, _hashtable_func_helper], 'deps': [_khash_primitive_helper_dep]}, + 'index': {'sources': ['index.pyx', _index_class_helper], 'deps': [_khash_primitive_helper_dep]}, 'indexing': {'sources': ['indexing.pyx']}, 'internals': {'sources': ['internals.pyx']}, 'interval': {'sources': ['interval.pyx', _intervaltree_helper], - 'deps': _khash_primitive_helper_dep}, + 'deps': [_khash_primitive_helper_dep]}, 'join': {'sources': ['join.pyx', _khash_primitive_helper], - 'deps': _khash_primitive_helper_dep}, + 'deps': [_khash_primitive_helper_dep]}, 'lib': {'sources': ['lib.pyx', 'src/parser/tokenizer.c']}, 'missing': {'sources': ['missing.pyx']}, 'pandas_datetime': {'sources': ['src/vendored/numpy/datetime/np_datetime.c', @@ -83,7 +83,7 @@ libs_sources = { 'src/parser/io.c', 'src/parser/pd_parser.c']}, 'parsers': {'sources': ['parsers.pyx', 'src/parser/tokenizer.c', 'src/parser/io.c'], - 'deps': _khash_primitive_helper_dep}, + 'deps': [_khash_primitive_helper_dep]}, 'json': {'sources': ['src/vendored/ujson/python/ujson.c', 'src/vendored/ujson/python/objToJSON.c', 'src/vendored/ujson/python/JSONtoObj.c', diff --git a/pandas/_libs/tslibs/util.pxd b/pandas/_libs/tslibs/util.pxd index e4ac3a9e167a3..a5822e57d3fa6 100644 --- a/pandas/_libs/tslibs/util.pxd +++ b/pandas/_libs/tslibs/util.pxd @@ -185,11 +185,11 @@ cdef inline const char* get_c_string(str py_string) except NULL: return get_c_string_buf_and_size(py_string, NULL) -cdef inline bytes string_encode_locale(str py_string) noexcept: +cdef inline bytes string_encode_locale(str py_string): """As opposed to PyUnicode_Encode, use current system locale to encode.""" return PyUnicode_EncodeLocale(py_string, NULL) -cdef inline object char_to_string_locale(const char* data) noexcept: +cdef inline object char_to_string_locale(const char* data): """As opposed to PyUnicode_FromString, use current system locale to decode.""" return PyUnicode_DecodeLocale(data, NULL) From 184ecca955163555e38c4a741c369e9356569faf Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Mar 2024 21:43:51 -0400 Subject: [PATCH 09/15] 32 bit fix --- pandas/_libs/hashtable_func_helper.pxi.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/hashtable_func_helper.pxi.in b/pandas/_libs/hashtable_func_helper.pxi.in index 9b2d50fc8eeb5..3bba114bfe360 100644 --- a/pandas/_libs/hashtable_func_helper.pxi.in +++ b/pandas/_libs/hashtable_func_helper.pxi.in @@ -111,7 +111,7 @@ cdef value_count_{{dtype}}(const {{dtype}}_t[:] values, bint dropna, const uint8 cdef: int64_t[::1] result_counts = np.empty(table.size + na_add, dtype=np.int64) - for i in range(table.size): + for i in range(table.size): # cast required for 32 bit builds {{if dtype == 'object'}} k = kh_get_{{ttype}}(table, result_keys.data[i]) {{else}} From f2a13b6ce43acaecb8b258933f75740fb9cc2dbc Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 19 Mar 2024 13:59:11 -0400 Subject: [PATCH 10/15] Change around warning levels --- meson.build | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/meson.build b/meson.build index 130f00cdeb1f7..77d9c5f39fa89 100644 --- a/meson.build +++ b/meson.build @@ -8,7 +8,6 @@ project( default_options: [ 'buildtype=release', 'c_std=c11', - 'warning_level=2', ] ) @@ -17,7 +16,6 @@ py = import('python').find_installation(pure: false) tempita = files('generate_pxi.py') versioneer = files('generate_version.py') - add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'c') add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp') @@ -27,11 +25,23 @@ add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp') add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'c') add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'cpp') -if build_machine.system() == 'darwin' +if meson.get_compiler('c').get_id() == 'clang' # error: pragma diagnostic pop could not pop, no matching push add_project_arguments('-Wno-unknown-pragmas', language: 'c') endif +if meson.get_compiler('c').get_id() == 'msvc' + # MSVC throws a lot of C4244 warnings around loss of data with warning_level=2 + # On other compilers we would have to add -Wconversion to see these + # these expose a lot of potential issues in our code base + add_project_arguments(['/W2'], language: 'c') + add_project_arguments(['/W2'], language: 'cpp') +else + add_project_arguments(['-Wall', '-Wextra'], language: 'c') + add_project_arguments(['-Wall', '-Wextra'], language: 'cpp') +endif + + if fs.exists('_version_meson.py') py.install_sources('_version_meson.py', subdir: 'pandas') else From a7ae1be422375e64c5cca13443a847e9572fe57d Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 19 Mar 2024 14:08:30 -0400 Subject: [PATCH 11/15] darwin fix --- meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meson.build b/meson.build index 77d9c5f39fa89..02674475bb9d8 100644 --- a/meson.build +++ b/meson.build @@ -25,7 +25,7 @@ add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp') add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'c') add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'cpp') -if meson.get_compiler('c').get_id() == 'clang' +if build_machine.system() == 'darwin' # error: pragma diagnostic pop could not pop, no matching push add_project_arguments('-Wno-unknown-pragmas', language: 'c') endif From 5487fd088ed68190177182d8d9cd7e3ab5f14339 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 19 Mar 2024 14:18:11 -0400 Subject: [PATCH 12/15] Add arguments at once --- meson.build | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/meson.build b/meson.build index 02674475bb9d8..4d2a70e90a7c1 100644 --- a/meson.build +++ b/meson.build @@ -25,11 +25,6 @@ add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp') add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'c') add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'cpp') -if build_machine.system() == 'darwin' - # error: pragma diagnostic pop could not pop, no matching push - add_project_arguments('-Wno-unknown-pragmas', language: 'c') -endif - if meson.get_compiler('c').get_id() == 'msvc' # MSVC throws a lot of C4244 warnings around loss of data with warning_level=2 # On other compilers we would have to add -Wconversion to see these @@ -37,8 +32,13 @@ if meson.get_compiler('c').get_id() == 'msvc' add_project_arguments(['/W2'], language: 'c') add_project_arguments(['/W2'], language: 'cpp') else - add_project_arguments(['-Wall', '-Wextra'], language: 'c') - add_project_arguments(['-Wall', '-Wextra'], language: 'cpp') + args = ['-Wall', '-Wextra'] + if build_machine.system() == 'darwin' + # error: pragma diagnostic pop could not pop, no matching push + args.append('-Wno-unknown-pragmas') + endif + add_project_arguments(args, language: 'c') + add_project_arguments(args, language: 'cpp') endif From 7def60a3f2a68cc1e6a96e24b2ca782810bea939 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 19 Mar 2024 14:20:42 -0400 Subject: [PATCH 13/15] meson fix --- meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meson.build b/meson.build index 4d2a70e90a7c1..6269ae9d09872 100644 --- a/meson.build +++ b/meson.build @@ -35,7 +35,7 @@ else args = ['-Wall', '-Wextra'] if build_machine.system() == 'darwin' # error: pragma diagnostic pop could not pop, no matching push - args.append('-Wno-unknown-pragmas') + args += ['-Wno-unknown-pragmas'] endif add_project_arguments(args, language: 'c') add_project_arguments(args, language: 'cpp') From bc61810bb24e8a611567206367833ec621e2cbb7 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 19 Mar 2024 14:32:34 -0400 Subject: [PATCH 14/15] less windows warn --- meson.build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meson.build b/meson.build index 6269ae9d09872..8fd0bf8368fc2 100644 --- a/meson.build +++ b/meson.build @@ -29,8 +29,8 @@ if meson.get_compiler('c').get_id() == 'msvc' # MSVC throws a lot of C4244 warnings around loss of data with warning_level=2 # On other compilers we would have to add -Wconversion to see these # these expose a lot of potential issues in our code base - add_project_arguments(['/W2'], language: 'c') - add_project_arguments(['/W2'], language: 'cpp') + add_project_arguments(['/W1'], language: 'c') + add_project_arguments(['/W1'], language: 'cpp') else args = ['-Wall', '-Wextra'] if build_machine.system() == 'darwin' From f3a4d71af4f2633c79e70e6c3dbfda68931fe5fc Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 19 Mar 2024 14:42:12 -0400 Subject: [PATCH 15/15] Disable MSVC warnings --- meson.build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meson.build b/meson.build index 8fd0bf8368fc2..d4916f8c92182 100644 --- a/meson.build +++ b/meson.build @@ -29,8 +29,8 @@ if meson.get_compiler('c').get_id() == 'msvc' # MSVC throws a lot of C4244 warnings around loss of data with warning_level=2 # On other compilers we would have to add -Wconversion to see these # these expose a lot of potential issues in our code base - add_project_arguments(['/W1'], language: 'c') - add_project_arguments(['/W1'], language: 'cpp') + add_project_arguments(['/w'], language: 'c') + add_project_arguments(['/w'], language: 'cpp') else args = ['-Wall', '-Wextra'] if build_machine.system() == 'darwin'