diff --git a/.circleci/setup_env.sh b/.circleci/setup_env.sh index c03a7ff4be8b3..c88250d6f48d2 100755 --- a/.circleci/setup_env.sh +++ b/.circleci/setup_env.sh @@ -98,9 +98,10 @@ if [ "$(conda list -f qt --json)" != [] ]; then fi echo "Build extensions" -python setup.py build_ext -q -j3 +cmake . +cmake --build . --config Release --parallel echo "Install pandas" -python -m pip install --no-build-isolation --no-use-pep517 -e . +python -m pip install --no-build-isolation -e . echo "done" diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 23bb988ef4d73..4d2ff17b4541b 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -12,8 +12,9 @@ runs: - name: Build Pandas run: | - python setup.py build_ext -j $N_JOBS - python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index + cmake . + cmake --build . --config Release --parallel + python -m pip install -e . --no-build-isolation --no-index shell: bash -el {0} env: # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index 67e99b4486a12..25eac102ca599 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -38,10 +38,11 @@ jobs: /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ . ~/virtualenvs/pandas-dev/bin/activate && \ python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ - pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ - python setup.py build_ext -q -j1 && \ - python -m pip install --no-build-isolation --no-use-pep517 -e . && \ - python -m pip list && \ + pip install cmake cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ + cmake . && \ + cmake --build . --parallel && \ + python -m pip install --no-build-isolation -e . && \ + python -m pip list && \ export PANDAS_CI=1 && \ pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" diff --git a/.gitignore b/.gitignore index 07b1f056d511b..b0782dc31b09b 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,10 @@ *.so .build_cache_dir MANIFEST +Makefile +CMakeCache.txt +CMakeFiles/ +cmake_install.cmake # Python files # ################ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000000..fcf450e557e39 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,29 @@ +cmake_minimum_required(VERSION 3.18) + +set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9") +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD_REQUIRED True) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +project(pandas) + +if(WIN32) + if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE + Release + CACHE STRING "Build type" FORCE) + endif() + find_package(Python3 REQUIRED COMPONENTS Interpreter Development NumPy) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE "") + link_directories(${Python3_LIBRARY_DIRS}) +else() + # we only choose Development.Module to support virtual environments where + # libpython may not be available see + # https://github.com/pypa/manylinux/issues/484 + find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) +endif() + +add_compile_definitions(NPY_NO_DEPRECATED_API=0) +add_subdirectory("pandas/_libs") +add_subdirectory("pandas/io/sas") diff --git a/Dockerfile b/Dockerfile index 650ba14271092..8fd0bc0b927f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,5 +49,6 @@ RUN . /opt/conda/etc/profile.d/conda.sh \ && conda activate base \ && cd "$pandas_home" \ && export \ - && python setup.py build_ext -j 4 \ + && cmake . \ + && cmake --build . --parallel \ && python -m pip install --no-build-isolation -e . diff --git a/MANIFEST.in b/MANIFEST.in index d2b1b8cb887bc..cd13a469043b2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include RELEASE.md include versioneer.py +include CMakeLists.txt graft doc prune doc/build @@ -7,6 +8,7 @@ prune doc/build graft LICENSES graft pandas +graft build_support global-exclude *.bz2 global-exclude *.csv @@ -28,7 +30,6 @@ global-exclude *.odt global-exclude *.orc global-exclude *.sas7bdat global-exclude *.sav -global-exclude *.so global-exclude *.xls global-exclude *.xlsb global-exclude *.xlsm @@ -42,6 +43,9 @@ global-exclude *~ global-exclude .DS_Store global-exclude .git* global-exclude \#* +recursive-exclude **/CMakeFiles * +global-exclude *.cmake +global-exclude CMakeCache.txt global-exclude *.c global-exclude *.cpp diff --git a/Makefile b/Makefile deleted file mode 100644 index c0aa685ed47ac..0000000000000 --- a/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -.PHONY : develop build clean clean_pyc doc lint-diff black test-scripts - -all: develop - -clean: - -python setup.py clean - -clean_pyc: - -find . -name '*.py[co]' -exec rm {} \; - -build: clean_pyc - python setup.py build_ext - -lint-diff: - git diff upstream/main --name-only -- "*.py" | xargs flake8 - -black: - black . - -develop: build - python -m pip install --no-build-isolation -e . - -doc: - -rm -rf doc/build doc/source/generated - cd doc; \ - python make.py clean; \ - python make.py html - -test-scripts: - pytest scripts diff --git a/build_support/build_backend.py b/build_support/build_backend.py new file mode 100644 index 0000000000000..90fe28fb9b33a --- /dev/null +++ b/build_support/build_backend.py @@ -0,0 +1,21 @@ +import pathlib +import subprocess + +from setuptools import build_meta as _orig + +prepare_metadata_for_build_wheel = _orig.prepare_metadata_for_build_wheel +build_sdist = _orig.build_sdist +get_requires_for_build_wheel = _orig.get_requires_for_build_wheel +get_requires_for_build_sdist = _orig.get_requires_for_build_sdist + + +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + filedir = pathlib.Path(__file__).resolve().parent.parent + subprocess.run(["cmake", "."], cwd=filedir, check=True) + subprocess.run( + ["cmake", "--build", ".", "--config", "Release", "--parallel"], + cwd=filedir, + check=True, + ) + + return _orig.build_wheel(wheel_directory, config_settings, metadata_directory) diff --git a/doc/source/development/contributing_environment.rst b/doc/source/development/contributing_environment.rst index c881770aa7584..32c9ea82f8ddf 100644 --- a/doc/source/development/contributing_environment.rst +++ b/doc/source/development/contributing_environment.rst @@ -42,7 +42,8 @@ Run Container:: If you bind your local repo for the first time, you have to build the C extensions afterwards. Run the following command inside the container:: - python setup.py build_ext -j 4 + cmake . + cmake --build . --config Release --parallel You need to rebuild the C extensions anytime the Cython code in ``pandas/_libs`` changes. This most frequently occurs when changing or merging branches. @@ -169,7 +170,8 @@ We'll now kick off a three-step process: source activate pandas-dev # Build and install pandas - python setup.py build_ext -j 4 + cmake . + cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-use-pep517 At this point you should be able to import pandas from your locally built version:: @@ -216,7 +218,8 @@ You also need to have ``setuptools`` 51.0.0 or later to build pandas. python -m pip install -r requirements-dev.txt # Build and install pandas - python setup.py build_ext -j 4 + cmake . + cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-use-pep517 **Unix**/**macOS with pyenv** @@ -240,7 +243,8 @@ Consult the docs for setting up pyenv `here `__. python -m pip install -r requirements-dev.txt # Build and install pandas - python setup.py build_ext -j 4 + cmake . + cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-use-pep517 **Windows** @@ -266,5 +270,6 @@ should already exist. python -m pip install -r requirements-dev.txt # Build and install pandas - python setup.py build_ext -j 4 + cmake . + cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-use-pep517 diff --git a/doc/source/development/debugging_extensions.rst b/doc/source/development/debugging_extensions.rst index 7ba2091e18853..b2865f97e2613 100644 --- a/doc/source/development/debugging_extensions.rst +++ b/doc/source/development/debugging_extensions.rst @@ -8,11 +8,12 @@ Debugging C extensions Pandas uses select C extensions for high performance IO operations. In case you need to debug segfaults or general issues with those extensions, the following steps may be helpful. -First, be sure to compile the extensions with the appropriate flags to generate debug symbols and remove optimizations. This can be achieved as follows: +First, be sure to compile the extensions with the appropriate flags to generate debug symbols and remove optimizations. This can be achieved on Unix-like systems as follows: .. code-block:: sh - python setup.py build_ext --inplace -j4 --with-debugging-symbols + cmake . -DCMAKE_BUILD_TYPE=Debug + cmake --build . --parallel Using a debugger ================ diff --git a/environment.yml b/environment.yml index f1472f453b935..5dc0dcc477dae 100644 --- a/environment.yml +++ b/environment.yml @@ -79,6 +79,7 @@ dependencies: - asv # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms. + - cmake>=3.18.0 - c-compiler - cxx-compiler diff --git a/pandas/__init__.py b/pandas/__init__.py index 5016bde000c3b..229aa77efdefe 100644 --- a/pandas/__init__.py +++ b/pandas/__init__.py @@ -28,7 +28,8 @@ raise ImportError( f"C extension: {_module} not built. If you want to import " "pandas from the source directory, you may need to run " - "'python setup.py build_ext --force' to build the C extensions first." + "'cmake . && cmake --build . --config Release --parallel' " + "to build the C extensions first." ) from _err else: del _tslib, _lib, _hashtable diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt new file mode 100644 index 0000000000000..3aa1e7a225476 --- /dev/null +++ b/pandas/_libs/CMakeLists.txt @@ -0,0 +1,142 @@ +add_custom_command( + OUTPUT algos_common_helper.pxi + algos_take_helper.pxi + hashtable_class_helper.pxi + hashtable_func_helper.pxi + index_class_helper.pxi + intervaltree.pxi + khash_for_primitive_helper.pxi + sparse_op_helper.pxi + COMMAND ${Python3_EXECUTABLE} generate_templates.py) + +add_custom_command( + OUTPUT algos.c + COMMAND ${Python3_EXECUTABLE} -m cython -3 algos.pyx + DEPENDS algos_common_helper.pxi algos_take_helper.pxi) +python3_add_library(algos MODULE WITH_SOABI algos.c) +target_include_directories( + algos PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib") + +# There is a khash header file in src/klib and a cython generated one in _libs. +# Depending on the build timing one of the other could get picked up, though +# unclear why we need both? If we stick to the non-generated version we can +# remove any DEPENDS khash.h +add_custom_command( + OUTPUT khash.h + COMMAND ${Python3_EXECUTABLE} -m cython -3 khash.pxd + DEPENDS hkash_for_primitive_helper.pxi) + +add_custom_command( + OUTPUT hashtable.c + COMMAND ${Python3_EXECUTABLE} -m cython -3 hashtable.pyx + DEPENDS hashtable_class_helper.pxi hashtable_func_helper.pxi) + +python3_add_library(hashtable MODULE WITH_SOABI hashtable.c) +target_include_directories( + hashtable PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + "src/klib" DEPENDS khash.h) + +add_custom_command( + OUTPUT index.c + COMMAND ${Python3_EXECUTABLE} -m cython -3 index.pyx + DEPENDS index_class_helper.pxi) +python3_add_library(index MODULE WITH_SOABI index.c) +target_include_directories( + index PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" + "./tslibs") + +add_custom_command( + OUTPUT interval.c + COMMAND ${Python3_EXECUTABLE} -m cython -3 interval.pyx + DEPENDS intervaltree.pxi) +python3_add_library(interval MODULE WITH_SOABI interval.c) +target_include_directories( + interval PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + "src/klib" "./tslibs") + +add_custom_command( + OUTPUT sparse.c + COMMAND ${Python3_EXECUTABLE} -m cython -3 sparse.pyx + DEPENDS sparse_op_helper.pxi) +python3_add_library(sparse MODULE WITH_SOABI sparse.c) +target_include_directories( + sparse PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + "src/klib") + +set(BASIC_LIBRARIES + arrays + groupby + hashing + indexing + internals + reduction + ops + ops_dispatch + properties + reshape + testing + writers) +foreach(LIB ${BASIC_LIBRARIES}) + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python3_EXECUTABLE} -m cython -3 + ${LIB}.pyx) + python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) + target_include_directories(${LIB} PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) +endforeach() + +add_subdirectory("tslibs") + +add_custom_command(OUTPUT tslib.c COMMAND ${Python3_EXECUTABLE} -m cython -3 + tslib.pyx) +python3_add_library(tslib ${LIB} MODULE WITH_SOABI tslib.c + tslibs/src/datetime/np_datetime.c) +target_include_directories( + tslib PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "./tslibs") + +add_custom_command(OUTPUT missing.c COMMAND ${Python3_EXECUTABLE} -m cython -3 + missing.pyx) +python3_add_library(missing MODULE WITH_SOABI missing.c) +target_include_directories( + missing PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + "./tslibs") + +add_custom_command(OUTPUT lib.c COMMAND ${Python3_EXECUTABLE} -m cython -3 + lib.pyx) +python3_add_library(lib MODULE WITH_SOABI lib.c src/parser/tokenizer.c) +target_include_directories( + lib PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" + "./tslibs") + +add_custom_command( + OUTPUT join.c + COMMAND ${Python3_EXECUTABLE} -m cython -3 join.pyx + DEPENDS khash_for_primitive_helper.pxi) +python3_add_library(join MODULE WITH_SOABI join.c) +target_include_directories(join PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS} "src/klib") + +add_custom_command(OUTPUT parsers.c COMMAND ${Python3_EXECUTABLE} -m cython -3 + parsers.pyx) +python3_add_library(parsers MODULE WITH_SOABI parsers.c src/parser/tokenizer.c + src/parser/io.c) +target_include_directories( + parsers PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + "src/klib" "src") + +python3_add_library( + ujson + MODULE + WITH_SOABI + src/ujson/python/ujson.c + src/ujson/python/objToJSON.c + src/ujson/python/date_conversions.c + src/ujson/python/JSONtoObj.c + src/ujson/lib/ultrajsonenc.c + src/ujson/lib/ultrajsondec.c + tslibs/src/datetime/np_datetime.c + tslibs/src/datetime/np_datetime_strings.c) +target_include_directories( + ujson PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + src/ujson/python src/ujson/lib src/datetime) + +add_subdirectory("window") diff --git a/pandas/_libs/generate_templates.py b/pandas/_libs/generate_templates.py new file mode 100644 index 0000000000000..013568a2812b8 --- /dev/null +++ b/pandas/_libs/generate_templates.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from Cython import Tempita + +if __name__ == "__main__": + for template in ( + "algos_common_helper.pxi.in", + "algos_take_helper.pxi.in", + "hashtable_class_helper.pxi.in", + "hashtable_func_helper.pxi.in", + "index_class_helper.pxi.in", + "intervaltree.pxi.in", + "khash_for_primitive_helper.pxi.in", + "sparse_op_helper.pxi.in", + ): + pyxcontent = Tempita.sub(open(template).read()) + with open(template.replace(".in", ""), "w") as outfile: + outfile.write(pyxcontent) diff --git a/pandas/_libs/src/ujson/python/ujson.c b/pandas/_libs/src/ujson/python/ujson.c index 5d4a5693c0ff6..c9d5a91da2e86 100644 --- a/pandas/_libs/src/ujson/python/ujson.c +++ b/pandas/_libs/src/ujson/python/ujson.c @@ -74,7 +74,7 @@ static PyModuleDef moduledef = { }; -PyMODINIT_FUNC PyInit_json(void) { +PyMODINIT_FUNC PyInit_ujson(void) { import_array() initObjToJSON(); // TODO(username): clean up, maybe via tp_free? return PyModuleDef_Init(&moduledef); diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt new file mode 100644 index 0000000000000..0f8ebdeb8991c --- /dev/null +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -0,0 +1,43 @@ +set(BASIC_LIBRARIES base ccalendar dtypes nattype strptime timezones) +foreach(LIB ${BASIC_LIBRARIES}) + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python3_EXECUTABLE} -m cython -3 + ${LIB}.pyx) + python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) + target_include_directories(${LIB} PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) +endforeach() + +set(NP_DATETIME_REQUIRING + conversion + fields + offsets + period + timedeltas + timestamps + tzconversion + vectorized) +foreach(LIB ${NP_DATETIME_REQUIRING}) + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python3_EXECUTABLE} -m cython -3 + ${LIB}.pyx) + python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c + src/datetime/np_datetime.c) + target_include_directories( + ${LIB} PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + "src/datetime") +endforeach() + +add_custom_command(OUTPUT np_datetime.c COMMAND ${Python3_EXECUTABLE} -m cython + -3 np_datetime.pyx) +python3_add_library( + np_datetime MODULE WITH_SOABI np_datetime.c src/datetime/np_datetime.c + src/datetime/np_datetime_strings.c) +target_include_directories(np_datetime PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) + +add_custom_command(OUTPUT parsing.c COMMAND ${Python3_EXECUTABLE} -m cython -3 + parsing.pyx) +python3_add_library(parsing MODULE WITH_SOABI parsing.c + "../src/parser/tokenizer.c") +target_include_directories( + parsing PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + "../src/klib") diff --git a/pandas/_libs/json.pyi b/pandas/_libs/ujson.pyi similarity index 100% rename from pandas/_libs/json.pyi rename to pandas/_libs/ujson.pyi diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt new file mode 100644 index 0000000000000..862abf42fd89f --- /dev/null +++ b/pandas/_libs/window/CMakeLists.txt @@ -0,0 +1,12 @@ +add_custom_command(OUTPUT indexers.c COMMAND ${Python3_EXECUTABLE} -m cython -3 + indexers.pyx) +python3_add_library(indexers MODULE WITH_SOABI indexers.c) +target_include_directories(indexers PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) + +add_custom_command( + OUTPUT aggregations.cpp COMMAND ${Python3_EXECUTABLE} -m cython -3 --cplus + aggregations.pyx) +python3_add_library(aggregations MODULE WITH_SOABI aggregations.cpp) +target_include_directories(aggregations PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) diff --git a/pandas/io/excel/_odswriter.py b/pandas/io/excel/_odswriter.py index 185e93591cfe0..694a39ff36d1d 100644 --- a/pandas/io/excel/_odswriter.py +++ b/pandas/io/excel/_odswriter.py @@ -10,7 +10,7 @@ cast, ) -import pandas._libs.json as json +import pandas._libs.ujson as json from pandas._typing import ( FilePath, StorageOptions, diff --git a/pandas/io/excel/_xlsxwriter.py b/pandas/io/excel/_xlsxwriter.py index a3edccd3a5779..4d299c717e23f 100644 --- a/pandas/io/excel/_xlsxwriter.py +++ b/pandas/io/excel/_xlsxwriter.py @@ -2,7 +2,7 @@ from typing import Any -import pandas._libs.json as json +import pandas._libs.ujson as json from pandas._typing import ( FilePath, StorageOptions, diff --git a/pandas/io/excel/_xlwt.py b/pandas/io/excel/_xlwt.py index 234d9e72de10d..538b5f2a2c5e3 100644 --- a/pandas/io/excel/_xlwt.py +++ b/pandas/io/excel/_xlwt.py @@ -7,7 +7,7 @@ cast, ) -import pandas._libs.json as json +import pandas._libs.ujson as json from pandas._typing import ( FilePath, StorageOptions, diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index 02a0b27f82ef8..6d59ca267af7c 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -21,8 +21,8 @@ import numpy as np -import pandas._libs.json as json from pandas._libs.tslibs import iNaT +import pandas._libs.ujson as json from pandas._typing import ( CompressionOptions, DtypeArg, diff --git a/pandas/io/json/_table_schema.py b/pandas/io/json/_table_schema.py index b7a8b5cc82f7a..57a3d5cff3cd7 100644 --- a/pandas/io/json/_table_schema.py +++ b/pandas/io/json/_table_schema.py @@ -12,7 +12,7 @@ ) import warnings -import pandas._libs.json as json +import pandas._libs.ujson as json from pandas._typing import ( DtypeObj, JSONSerializable, diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt new file mode 100644 index 0000000000000..a8742edc7a939 --- /dev/null +++ b/pandas/io/sas/CMakeLists.txt @@ -0,0 +1,5 @@ +add_custom_command(OUTPUT _sas.c COMMAND ${Python3_EXECUTABLE} -m cython -3 + _sas.pyx) +python3_add_library(_sas MODULE WITH_SOABI _sas.c) +target_include_directories(_sas PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) diff --git a/pandas/io/sas/sas.pyx b/pandas/io/sas/_sas.pyx similarity index 100% rename from pandas/io/sas/sas.pyx rename to pandas/io/sas/_sas.pyx diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index b9f568fc9577b..61b9c1d851c0d 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -5,7 +5,6 @@ import pytest from pandas._libs import lib -from pandas.compat import IS64 from pandas.errors import ( PerformanceWarning, SpecificationError, @@ -2572,7 +2571,6 @@ def test_groupby_series_with_tuple_name(): tm.assert_series_equal(result, expected) -@pytest.mark.xfail(not IS64, reason="GH#38778: fail on 32-bit system") @pytest.mark.parametrize( "func, values", [("sum", [97.0, 98.0]), ("mean", [24.25, 24.5])] ) @@ -2585,7 +2583,6 @@ def test_groupby_numerical_stability_sum_mean(func, values): tm.assert_frame_equal(result, expected) -@pytest.mark.xfail(not IS64, reason="GH#38778: fail on 32-bit system") def test_groupby_numerical_stability_cumsum(): # GH#38934 data = [1e16, 1e16, 97, 98, -5e15, -5e15, -5e15, -5e15] diff --git a/pandas/tests/io/json/test_ujson.py b/pandas/tests/io/json/test_ujson.py index ae13d8d5fb180..ff43933be1232 100644 --- a/pandas/tests/io/json/test_ujson.py +++ b/pandas/tests/io/json/test_ujson.py @@ -12,7 +12,7 @@ import pytest import pytz -import pandas._libs.json as ujson +import pandas._libs.ujson as ujson from pandas.compat import ( IS64, is_platform_windows, diff --git a/pandas/tests/io/parser/common/test_float.py b/pandas/tests/io/parser/common/test_float.py index 2ca98de914f9e..e245b2ca019ee 100644 --- a/pandas/tests/io/parser/common/test_float.py +++ b/pandas/tests/io/parser/common/test_float.py @@ -7,7 +7,10 @@ import numpy as np import pytest -from pandas.compat import is_platform_linux +from pandas.compat import ( + is_platform_linux, + is_platform_mac, +) from pandas import DataFrame import pandas._testing as tm @@ -53,8 +56,8 @@ def test_too_many_exponent_digits(all_parsers_all_precisions, exp, request): data = f"data\n10E{exp}" result = parser.read_csv(StringIO(data), float_precision=precision) if precision == "round_trip": - if exp == 999999999999999999 and is_platform_linux(): - mark = pytest.mark.xfail(reason="GH38794, on Linux gives object result") + if exp == 999999999999999999 and (is_platform_linux() or is_platform_mac()): + mark = pytest.mark.xfail(reason="GH38794, on Unix gives object result") request.node.add_marker(mark) value = np.inf if exp > 0 else 0.0 diff --git a/pandas/tests/io/parser/test_c_parser_only.py b/pandas/tests/io/parser/test_c_parser_only.py index 9a81790ca3bb0..6d92225f9458d 100644 --- a/pandas/tests/io/parser/test_c_parser_only.py +++ b/pandas/tests/io/parser/test_c_parser_only.py @@ -17,10 +17,7 @@ import numpy as np import pytest -from pandas.compat import ( - IS64, - is_ci_environment, -) +from pandas.compat import is_ci_environment from pandas.errors import ParserError import pandas.util._test_decorators as td @@ -678,10 +675,7 @@ def test_float_precision_options(c_parser_only): df3 = parser.read_csv(StringIO(s), float_precision="legacy") - if IS64: - assert not df.iloc[0, 0] == df3.iloc[0, 0] - else: - assert df.iloc[0, 0] == df3.iloc[0, 0] + assert not df.iloc[0, 0] == df3.iloc[0, 0] msg = "Unrecognized float_precision option: junk" diff --git a/pandas/tests/window/test_rolling.py b/pandas/tests/window/test_rolling.py index c9ec2985488be..b229d1b8db51e 100644 --- a/pandas/tests/window/test_rolling.py +++ b/pandas/tests/window/test_rolling.py @@ -7,6 +7,7 @@ import pytest from pandas.compat import ( + IS64, is_platform_arm, is_platform_mac, ) @@ -1191,7 +1192,9 @@ def test_rolling_sem(frame_or_series): tm.assert_series_equal(result, expected) -@pytest.mark.xfail(is_platform_arm() and not is_platform_mac(), reason="GH 38921") +@pytest.mark.xfail( + (is_platform_arm() and not is_platform_mac()) or not IS64, reason="GH 38921" +) @pytest.mark.parametrize( ("func", "third_value", "values"), [ diff --git a/pyproject.toml b/pyproject.toml index 67c56123a847c..81aa906bdd2bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,14 +2,15 @@ # Minimum requirements for the build system to execute. # See https://github.com/scipy/scipy/pull/12940 for the AIX issue. requires = [ + "cmake>=3.18.0", "setuptools>=51.0.0", "wheel", "Cython>=0.29.32,<3", # Note: sync with setup.py, environment.yml and asv.conf.json "oldest-supported-numpy>=0.10" ] -# uncomment to enable pep517 after versioneer problem is fixed. -# https://github.com/python-versioneer/python-versioneer/issues/193 -# build-backend = "setuptools.build_meta" + +build-backend = "build_backend" +backend-path = ["build_support"] [tool.black] target-version = ['py38', 'py39'] diff --git a/requirements-dev.txt b/requirements-dev.txt index 60dd738e43ba3..3a59ebc79e315 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -62,6 +62,7 @@ torch moto flask asv +cmake>=3.18.0 black==22.3.0 cpplint flake8==5.0.4 diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 12e8aa36c3794..79e4fea9fdf36 --- a/setup.py +++ b/setup.py @@ -5,662 +5,21 @@ (https://github.com/zeromq/pyzmq) which have been permitted for use under the BSD license. Parts are from lxml (https://github.com/lxml/lxml) """ - -import argparse -import multiprocessing import os -from os.path import join as pjoin -import platform -import shutil import sys -from sysconfig import get_config_vars -import numpy -from pkg_resources import parse_version -from setuptools import ( - Command, - Extension, - setup, -) -from setuptools.command.build_ext import build_ext as _build_ext +from setuptools import setup +# uncomment to enable pep517 after versioneer problem is fixed. +# https://github.com/python-versioneer/python-versioneer/issues/193 +sys.path.insert(0, os.path.dirname(__file__)) import versioneer cmdclass = versioneer.get_cmdclass() -def is_platform_windows(): - return sys.platform == "win32" or sys.platform == "cygwin" - - -def is_platform_mac(): - return sys.platform == "darwin" - - -# note: sync with pyproject.toml, environment.yml and asv.conf.json -min_cython_ver = "0.29.32" - -try: - from Cython import ( - Tempita, - __version__ as _CYTHON_VERSION, - ) - from Cython.Build import cythonize - - _CYTHON_INSTALLED = parse_version(_CYTHON_VERSION) >= parse_version(min_cython_ver) -except ImportError: - _CYTHON_VERSION = None - _CYTHON_INSTALLED = False - cythonize = lambda x, *args, **kwargs: x # dummy func - - -_pxi_dep_template = { - "algos": ["_libs/algos_common_helper.pxi.in", "_libs/algos_take_helper.pxi.in"], - "hashtable": [ - "_libs/hashtable_class_helper.pxi.in", - "_libs/hashtable_func_helper.pxi.in", - "_libs/khash_for_primitive_helper.pxi.in", - ], - "index": ["_libs/index_class_helper.pxi.in"], - "sparse": ["_libs/sparse_op_helper.pxi.in"], - "interval": ["_libs/intervaltree.pxi.in"], -} - -_pxifiles = [] -_pxi_dep = {} -for module, files in _pxi_dep_template.items(): - pxi_files = [pjoin("pandas", x) for x in files] - _pxifiles.extend(pxi_files) - _pxi_dep[module] = pxi_files - - -class build_ext(_build_ext): - @classmethod - def render_templates(cls, pxifiles): - for pxifile in pxifiles: - # build pxifiles first, template extension must be .pxi.in - assert pxifile.endswith(".pxi.in") - outfile = pxifile[:-3] - - if ( - os.path.exists(outfile) - and os.stat(pxifile).st_mtime < os.stat(outfile).st_mtime - ): - # if .pxi.in is not updated, no need to output .pxi - continue - - with open(pxifile) as f: - tmpl = f.read() - pyxcontent = Tempita.sub(tmpl) - - with open(outfile, "w") as f: - f.write(pyxcontent) - - def build_extensions(self): - # if building from c files, don't need to - # generate template output - if _CYTHON_INSTALLED: - self.render_templates(_pxifiles) - - super().build_extensions() - - -class CleanCommand(Command): - """Custom command to clean the .so and .pyc files.""" - - user_options = [("all", "a", "")] - - def initialize_options(self): - self.all = True - self._clean_me = [] - self._clean_trees = [] - - base = pjoin("pandas", "_libs", "src") - tsbase = pjoin("pandas", "_libs", "tslibs", "src") - dt = pjoin(tsbase, "datetime") - util = pjoin("pandas", "util") - parser = pjoin(base, "parser") - ujson_python = pjoin(base, "ujson", "python") - ujson_lib = pjoin(base, "ujson", "lib") - self._clean_exclude = [ - pjoin(dt, "np_datetime.c"), - pjoin(dt, "np_datetime_strings.c"), - pjoin(parser, "tokenizer.c"), - pjoin(parser, "io.c"), - pjoin(ujson_python, "ujson.c"), - pjoin(ujson_python, "objToJSON.c"), - pjoin(ujson_python, "JSONtoObj.c"), - pjoin(ujson_python, "date_conversions.c"), - pjoin(ujson_lib, "ultrajsonenc.c"), - pjoin(ujson_lib, "ultrajsondec.c"), - pjoin(util, "move.c"), - ] - - for root, dirs, files in os.walk("pandas"): - for f in files: - filepath = pjoin(root, f) - if filepath in self._clean_exclude: - continue - - if os.path.splitext(f)[-1] in ( - ".pyc", - ".so", - ".o", - ".pyo", - ".pyd", - ".c", - ".cpp", - ".orig", - ): - self._clean_me.append(filepath) - for d in dirs: - if d == "__pycache__": - self._clean_trees.append(pjoin(root, d)) - - # clean the generated pxi files - for pxifile in _pxifiles: - pxifile = pxifile.replace(".pxi.in", ".pxi") - self._clean_me.append(pxifile) - - for d in ("build", "dist"): - if os.path.exists(d): - self._clean_trees.append(d) - - def finalize_options(self): - pass - - def run(self): - for clean_me in self._clean_me: - try: - os.unlink(clean_me) - except OSError: - pass - for clean_tree in self._clean_trees: - try: - shutil.rmtree(clean_tree) - except OSError: - pass - - -# we need to inherit from the versioneer -# class as it encodes the version info -sdist_class = cmdclass["sdist"] - - -class CheckSDist(sdist_class): - """Custom sdist that ensures Cython has compiled all pyx files to c.""" - - _pyxfiles = [ - "pandas/_libs/arrays.pyx", - "pandas/_libs/lib.pyx", - "pandas/_libs/hashtable.pyx", - "pandas/_libs/tslib.pyx", - "pandas/_libs/index.pyx", - "pandas/_libs/internals.pyx", - "pandas/_libs/algos.pyx", - "pandas/_libs/join.pyx", - "pandas/_libs/indexing.pyx", - "pandas/_libs/interval.pyx", - "pandas/_libs/hashing.pyx", - "pandas/_libs/missing.pyx", - "pandas/_libs/reduction.pyx", - "pandas/_libs/testing.pyx", - "pandas/_libs/sparse.pyx", - "pandas/_libs/ops.pyx", - "pandas/_libs/parsers.pyx", - "pandas/_libs/tslibs/base.pyx", - "pandas/_libs/tslibs/ccalendar.pyx", - "pandas/_libs/tslibs/dtypes.pyx", - "pandas/_libs/tslibs/period.pyx", - "pandas/_libs/tslibs/strptime.pyx", - "pandas/_libs/tslibs/np_datetime.pyx", - "pandas/_libs/tslibs/timedeltas.pyx", - "pandas/_libs/tslibs/timestamps.pyx", - "pandas/_libs/tslibs/timezones.pyx", - "pandas/_libs/tslibs/conversion.pyx", - "pandas/_libs/tslibs/fields.pyx", - "pandas/_libs/tslibs/offsets.pyx", - "pandas/_libs/tslibs/parsing.pyx", - "pandas/_libs/tslibs/tzconversion.pyx", - "pandas/_libs/tslibs/vectorized.pyx", - "pandas/_libs/window/indexers.pyx", - "pandas/_libs/writers.pyx", - "pandas/io/sas/sas.pyx", - ] - - _cpp_pyxfiles = [ - "pandas/_libs/window/aggregations.pyx", - ] - - def initialize_options(self): - sdist_class.initialize_options(self) - - def run(self): - if "cython" in cmdclass: - self.run_command("cython") - else: - # If we are not running cython then - # compile the extensions correctly - pyx_files = [(self._pyxfiles, "c"), (self._cpp_pyxfiles, "cpp")] - - for pyxfiles, extension in pyx_files: - for pyxfile in pyxfiles: - sourcefile = pyxfile[:-3] + extension - msg = ( - f"{extension}-source file '{sourcefile}' not found.\n" - "Run 'setup.py cython' before sdist." - ) - assert os.path.isfile(sourcefile), msg - sdist_class.run(self) - - -class CheckingBuildExt(build_ext): - """ - Subclass build_ext to get clearer report if Cython is necessary. - """ - - def check_cython_extensions(self, extensions): - for ext in extensions: - for src in ext.sources: - if not os.path.exists(src): - print(f"{ext.name}: -> [{ext.sources}]") - raise Exception( - f"""Cython-generated file '{src}' not found. - Cython is required to compile pandas from a development branch. - Please install Cython or download a release package of pandas. - """ - ) - - def build_extensions(self): - self.check_cython_extensions(self.extensions) - build_ext.build_extensions(self) - - -class CythonCommand(build_ext): - """ - Custom command subclassed from Cython.Distutils.build_ext - to compile pyx->c, and stop there. All this does is override the - C-compile method build_extension() with a no-op. - """ - - def build_extension(self, ext): - pass - - -class DummyBuildSrc(Command): - """numpy's build_src command interferes with Cython's build_ext.""" - - user_options = [] - - def initialize_options(self): - self.py_modules_dict = {} - - def finalize_options(self): - pass - - def run(self): - pass - - -cmdclass["clean"] = CleanCommand -cmdclass["build_ext"] = CheckingBuildExt - -if _CYTHON_INSTALLED: - suffix = ".pyx" - cmdclass["cython"] = CythonCommand -else: - suffix = ".c" - cmdclass["build_src"] = DummyBuildSrc - -# ---------------------------------------------------------------------- -# Preparation of compiler arguments - -debugging_symbols_requested = "--with-debugging-symbols" in sys.argv -if debugging_symbols_requested: - sys.argv.remove("--with-debugging-symbols") - - -if sys.byteorder == "big": - endian_macro = [("__BIG_ENDIAN__", "1")] -else: - endian_macro = [("__LITTLE_ENDIAN__", "1")] - - -extra_compile_args = [] -extra_link_args = [] -if is_platform_windows(): - if debugging_symbols_requested: - extra_compile_args.append("/Z7") - extra_link_args.append("/DEBUG") -else: - # PANDAS_CI=1 is set in CI - if os.environ.get("PANDAS_CI", "0") == "1": - extra_compile_args.append("-Werror") - if debugging_symbols_requested: - extra_compile_args.append("-g") - extra_compile_args.append("-UNDEBUG") - extra_compile_args.append("-O0") - -# Build for at least macOS 10.9 when compiling on a 10.9 system or above, -# overriding CPython distuitls behaviour which is to target the version that -# python was built for. This may be overridden by setting -# MACOSX_DEPLOYMENT_TARGET before calling setup.py -if is_platform_mac(): - if "MACOSX_DEPLOYMENT_TARGET" not in os.environ: - current_system = platform.mac_ver()[0] - python_target = get_config_vars().get( - "MACOSX_DEPLOYMENT_TARGET", current_system - ) - target_macos_version = "10.9" - parsed_macos_version = parse_version(target_macos_version) - if ( - parse_version(str(python_target)) < parsed_macos_version - and parse_version(current_system) >= parsed_macos_version - ): - os.environ["MACOSX_DEPLOYMENT_TARGET"] = target_macos_version - - if sys.version_info[:2] == (3, 8): # GH 33239 - extra_compile_args.append("-Wno-error=deprecated-declarations") - - # https://github.com/pandas-dev/pandas/issues/35559 - extra_compile_args.append("-Wno-error=unreachable-code") - -# enable coverage by building cython files by setting the environment variable -# "PANDAS_CYTHON_COVERAGE" (with a Truthy value) or by running build_ext -# with `--with-cython-coverage`enabled -linetrace = os.environ.get("PANDAS_CYTHON_COVERAGE", False) -if "--with-cython-coverage" in sys.argv: - linetrace = True - sys.argv.remove("--with-cython-coverage") - -# Note: if not using `cythonize`, coverage can be enabled by -# pinning `ext.cython_directives = directives` to each ext in extensions. -# github.com/cython/cython/wiki/enhancements-compilerdirectives#in-setuppy -directives = {"linetrace": False, "language_level": 3} -macros = [] -if linetrace: - # https://pypkg.com/pypi/pytest-cython/f/tests/example-project/setup.py - directives["linetrace"] = True - macros = [("CYTHON_TRACE", "1"), ("CYTHON_TRACE_NOGIL", "1")] - -# silence build warnings about deprecated API usage -# we can't do anything about these warnings because they stem from -# cython+numpy version mismatches. -macros.append(("NPY_NO_DEPRECATED_API", "0")) - - -# ---------------------------------------------------------------------- -# Specification of Dependencies - -# TODO(cython#4518): Need to check to see if e.g. `linetrace` has changed and -# possibly re-compile. -def maybe_cythonize(extensions, *args, **kwargs): - """ - Render tempita templates before calling cythonize. This is skipped for - - * clean - * sdist - """ - if "clean" in sys.argv or "sdist" in sys.argv: - # See https://github.com/cython/cython/issues/1495 - return extensions - - elif not _CYTHON_INSTALLED: - # GH#28836 raise a helfpul error message - if _CYTHON_VERSION: - raise RuntimeError( - f"Cannot cythonize with old Cython version ({_CYTHON_VERSION} " - f"installed, needs {min_cython_ver})" - ) - raise RuntimeError("Cannot cythonize without Cython installed.") - - # reuse any parallel arguments provided for compilation to cythonize - parser = argparse.ArgumentParser() - parser.add_argument("--parallel", "-j", type=int, default=1) - parsed, _ = parser.parse_known_args() - - kwargs["nthreads"] = parsed.parallel - build_ext.render_templates(_pxifiles) - return cythonize(extensions, *args, **kwargs) - - -def srcpath(name=None, suffix=".pyx", subdir="src"): - return pjoin("pandas", subdir, name + suffix) - - -lib_depends = ["pandas/_libs/src/parse_helper.h"] - -klib_include = ["pandas/_libs/src/klib"] - -tseries_depends = [ - "pandas/_libs/tslibs/src/datetime/np_datetime.h", - "pandas/_libs/tslibs/src/datetime/np_datetime_strings.h", -] - -ext_data = { - "_libs.algos": { - "pyxfile": "_libs/algos", - "include": klib_include, - "depends": _pxi_dep["algos"], - }, - "_libs.arrays": {"pyxfile": "_libs/arrays"}, - "_libs.groupby": {"pyxfile": "_libs/groupby"}, - "_libs.hashing": {"pyxfile": "_libs/hashing", "depends": []}, - "_libs.hashtable": { - "pyxfile": "_libs/hashtable", - "include": klib_include, - "depends": ( - ["pandas/_libs/src/klib/khash_python.h", "pandas/_libs/src/klib/khash.h"] - + _pxi_dep["hashtable"] - ), - }, - "_libs.index": { - "pyxfile": "_libs/index", - "include": klib_include, - "depends": _pxi_dep["index"], - }, - "_libs.indexing": {"pyxfile": "_libs/indexing"}, - "_libs.internals": {"pyxfile": "_libs/internals"}, - "_libs.interval": { - "pyxfile": "_libs/interval", - "include": klib_include, - "depends": _pxi_dep["interval"], - }, - "_libs.join": {"pyxfile": "_libs/join", "include": klib_include}, - "_libs.lib": { - "pyxfile": "_libs/lib", - "depends": lib_depends + tseries_depends, - "include": klib_include, # due to tokenizer import - "sources": ["pandas/_libs/src/parser/tokenizer.c"], - }, - "_libs.missing": {"pyxfile": "_libs/missing", "depends": tseries_depends}, - "_libs.parsers": { - "pyxfile": "_libs/parsers", - "include": klib_include + ["pandas/_libs/src"], - "depends": [ - "pandas/_libs/src/parser/tokenizer.h", - "pandas/_libs/src/parser/io.h", - ], - "sources": [ - "pandas/_libs/src/parser/tokenizer.c", - "pandas/_libs/src/parser/io.c", - ], - }, - "_libs.reduction": {"pyxfile": "_libs/reduction"}, - "_libs.ops": {"pyxfile": "_libs/ops"}, - "_libs.ops_dispatch": {"pyxfile": "_libs/ops_dispatch"}, - "_libs.properties": {"pyxfile": "_libs/properties"}, - "_libs.reshape": {"pyxfile": "_libs/reshape", "depends": []}, - "_libs.sparse": {"pyxfile": "_libs/sparse", "depends": _pxi_dep["sparse"]}, - "_libs.tslib": { - "pyxfile": "_libs/tslib", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.base": {"pyxfile": "_libs/tslibs/base"}, - "_libs.tslibs.ccalendar": {"pyxfile": "_libs/tslibs/ccalendar"}, - "_libs.tslibs.dtypes": {"pyxfile": "_libs/tslibs/dtypes"}, - "_libs.tslibs.conversion": { - "pyxfile": "_libs/tslibs/conversion", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.fields": { - "pyxfile": "_libs/tslibs/fields", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.nattype": {"pyxfile": "_libs/tslibs/nattype"}, - "_libs.tslibs.np_datetime": { - "pyxfile": "_libs/tslibs/np_datetime", - "depends": tseries_depends, - "sources": [ - "pandas/_libs/tslibs/src/datetime/np_datetime.c", - "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", - ], - }, - "_libs.tslibs.offsets": { - "pyxfile": "_libs/tslibs/offsets", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.parsing": { - "pyxfile": "_libs/tslibs/parsing", - "include": klib_include, - "depends": ["pandas/_libs/src/parser/tokenizer.h"], - "sources": ["pandas/_libs/src/parser/tokenizer.c"], - }, - "_libs.tslibs.period": { - "pyxfile": "_libs/tslibs/period", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.strptime": { - "pyxfile": "_libs/tslibs/strptime", - "depends": tseries_depends, - }, - "_libs.tslibs.timedeltas": { - "pyxfile": "_libs/tslibs/timedeltas", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.timestamps": { - "pyxfile": "_libs/tslibs/timestamps", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.timezones": {"pyxfile": "_libs/tslibs/timezones"}, - "_libs.tslibs.tzconversion": { - "pyxfile": "_libs/tslibs/tzconversion", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.vectorized": { - "pyxfile": "_libs/tslibs/vectorized", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.testing": {"pyxfile": "_libs/testing"}, - "_libs.window.aggregations": { - "pyxfile": "_libs/window/aggregations", - "language": "c++", - "suffix": ".cpp", - "depends": ["pandas/_libs/src/skiplist.h"], - }, - "_libs.window.indexers": {"pyxfile": "_libs/window/indexers"}, - "_libs.writers": {"pyxfile": "_libs/writers"}, - "io.sas._sas": {"pyxfile": "io/sas/sas"}, -} - -extensions = [] - -for name, data in ext_data.items(): - source_suffix = suffix if suffix == ".pyx" else data.get("suffix", ".c") - - sources = [srcpath(data["pyxfile"], suffix=source_suffix, subdir="")] - - sources.extend(data.get("sources", [])) - - include = data.get("include", []) - include.append(numpy.get_include()) - - undef_macros = [] - - if ( - sys.platform == "zos" - and data.get("language") == "c++" - and os.path.basename(os.environ.get("CXX", "/bin/xlc++")) in ("xlc", "xlc++") - ): - data.get("macros", macros).append(("__s390__", "1")) - extra_compile_args.append("-qlanglvl=extended0x:nolibext") - undef_macros.append("_POSIX_THREADS") - - obj = Extension( - f"pandas.{name}", - sources=sources, - depends=data.get("depends", []), - include_dirs=include, - language=data.get("language", "c"), - define_macros=data.get("macros", macros), - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - undef_macros=undef_macros, - ) - - extensions.append(obj) - -# ---------------------------------------------------------------------- -# ujson - -if suffix == ".pyx": - # undo dumb setuptools bug clobbering .pyx sources back to .c - for ext in extensions: - if ext.sources[0].endswith((".c", ".cpp")): - root, _ = os.path.splitext(ext.sources[0]) - ext.sources[0] = root + suffix - -ujson_ext = Extension( - "pandas._libs.json", - depends=[ - "pandas/_libs/src/ujson/lib/ultrajson.h", - "pandas/_libs/src/ujson/python/date_conversions.h", - ], - sources=( - [ - "pandas/_libs/src/ujson/python/ujson.c", - "pandas/_libs/src/ujson/python/objToJSON.c", - "pandas/_libs/src/ujson/python/date_conversions.c", - "pandas/_libs/src/ujson/python/JSONtoObj.c", - "pandas/_libs/src/ujson/lib/ultrajsonenc.c", - "pandas/_libs/src/ujson/lib/ultrajsondec.c", - ] - + [ - "pandas/_libs/tslibs/src/datetime/np_datetime.c", - "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", - ] - ), - include_dirs=[ - "pandas/_libs/src/ujson/python", - "pandas/_libs/src/ujson/lib", - "pandas/_libs/src/datetime", - numpy.get_include(), - ], - extra_compile_args=(["-D_GNU_SOURCE"] + extra_compile_args), - extra_link_args=extra_link_args, - define_macros=macros, -) - - -extensions.append(ujson_ext) - -# ---------------------------------------------------------------------- - - if __name__ == "__main__": - # Freeze to support parallel compilation when using spawn instead of fork - multiprocessing.freeze_support() setup( version=versioneer.get_version(), - ext_modules=maybe_cythonize(extensions, compiler_directives=directives), cmdclass=cmdclass, )