From 67e0d51d6ac2429cb57a7dcd155294b0e40bd1ec Mon Sep 17 00:00:00 2001 From: William Ayd Date: Tue, 14 Jun 2022 19:51:11 -0700 Subject: [PATCH 01/68] initial cmake setup --- CMakeLists.txt | 8 ++++++++ Makefile | 30 ------------------------------ pandas/_libs/CMakeLists.txt | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 30 deletions(-) create mode 100644 CMakeLists.txt delete mode 100644 Makefile create mode 100644 pandas/_libs/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000000..b623b2108fd89 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 3.21) + +project(pandas VERSION 1.4) +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD_REQUIRED True) + +find_package(Python REQUIRED COMPONENTS Development NumPy) +add_subdirectory("pandas/_libs") diff --git a/Makefile b/Makefile deleted file mode 100644 index c0aa685ed47ac..0000000000000 --- a/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -.PHONY : develop build clean clean_pyc doc lint-diff black test-scripts - -all: develop - -clean: - -python setup.py clean - -clean_pyc: - -find . -name '*.py[co]' -exec rm {} \; - -build: clean_pyc - python setup.py build_ext - -lint-diff: - git diff upstream/main --name-only -- "*.py" | xargs flake8 - -black: - black . - -develop: build - python -m pip install --no-build-isolation -e . - -doc: - -rm -rf doc/build doc/source/generated - cd doc; \ - python make.py clean; \ - python make.py html - -test-scripts: - pytest scripts diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt new file mode 100644 index 0000000000000..30086748380e7 --- /dev/null +++ b/pandas/_libs/CMakeLists.txt @@ -0,0 +1,18 @@ +set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops properties reshape) +foreach(LIB ${BASIC_LIBRARIES}) + add_library(${LIB} SHARED ${LIB}.c) + target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) + target_link_libraries(${LIB} ${Python_LIBRARIES}) +endforeach() + +set(KLIB_REQUIRING_LIBRARIES algos hashtable join) +foreach(LIB ${KLIB_REQUIRING_LIBRARIES}) + add_library(${LIB} SHARED ${LIB}.c) + target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") + target_link_libraries(${LIB} ${Python_LIBRARIES}) +endforeach() + +# dispatch comes from ops_dispatch so can't use loop above +add_library(dispatch SHARED ops_dispatch.c) +target_include_directories(dispatch PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_link_libraries(dispatch ${Python_LIBRARIES}) From 08ae8179cdf6e567fe5d0765414d65f233628019 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Tue, 14 Jun 2022 20:16:45 -0700 Subject: [PATCH 02/68] easy conversions --- pandas/_libs/CMakeLists.txt | 33 ++++++++++++++++++++++++------- setup.py | 39 +------------------------------------ 2 files changed, 27 insertions(+), 45 deletions(-) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 30086748380e7..5e1d6fb74c16d 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -1,18 +1,37 @@ -set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops properties reshape) +set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops properties reshape sparse testing writers) foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() -set(KLIB_REQUIRING_LIBRARIES algos hashtable join) -foreach(LIB ${KLIB_REQUIRING_LIBRARIES}) +set(KLIB_REQUIRING algos hashtable join) +foreach(LIB ${KLIB_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() -# dispatch comes from ops_dispatch so can't use loop above -add_library(dispatch SHARED ops_dispatch.c) -target_include_directories(dispatch PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(dispatch ${Python_LIBRARIES}) +set(KLIB_AND_NP_DATETIME_REQUIRING index interval) +foreach(LIB ${KLIB_AND_NP_DATETIME_REQUIRING}) + add_library(${LIB} SHARED ${LIB}.c) + target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") + target_link_libraries(${LIB} ${Python_LIBRARIES}) +endforeach() + +SET(NP_DATETIME_REQUIRING missing tslib) +foreach(LIB ${NP_DATETIME_REQUIRING}) + add_library(${LIB} SHARED ${LIB}.c) + target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") + target_link_libraries(${LIB} ${Python_LIBRARIES}) +endforeach() + +#include ("tslibs") + +# Depends on tokenizer being built +#include ("src/parser") +#set(PARSER_REQUIRING lib parsers) +#foreach(LIB ${PARSER_REQUIRING}) +#add_library(lib SHARED lib.c) +#target_include_directories(lib PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs" "src/parser") +#target_link_libraries(lib ${Python_LIBRARIES}) diff --git a/setup.py b/setup.py index 27e6d8cb10025..97e6bc319bbd8 100755 --- a/setup.py +++ b/setup.py @@ -438,42 +438,12 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): ] ext_data = { - "_libs.algos": { - "pyxfile": "_libs/algos", - "include": klib_include, - "depends": _pxi_dep["algos"], - }, - "_libs.arrays": {"pyxfile": "_libs/arrays"}, - "_libs.groupby": {"pyxfile": "_libs/groupby"}, - "_libs.hashing": {"pyxfile": "_libs/hashing", "depends": []}, - "_libs.hashtable": { - "pyxfile": "_libs/hashtable", - "include": klib_include, - "depends": ( - ["pandas/_libs/src/klib/khash_python.h", "pandas/_libs/src/klib/khash.h"] - + _pxi_dep["hashtable"] - ), - }, - "_libs.index": { - "pyxfile": "_libs/index", - "include": klib_include, - "depends": _pxi_dep["index"], - }, - "_libs.indexing": {"pyxfile": "_libs/indexing"}, - "_libs.internals": {"pyxfile": "_libs/internals"}, - "_libs.interval": { - "pyxfile": "_libs/interval", - "include": klib_include, - "depends": _pxi_dep["interval"], - }, - "_libs.join": {"pyxfile": "_libs/join", "include": klib_include}, "_libs.lib": { "pyxfile": "_libs/lib", "depends": lib_depends + tseries_depends, "include": klib_include, # due to tokenizer import "sources": ["pandas/_libs/src/parser/tokenizer.c"], }, - "_libs.missing": {"pyxfile": "_libs/missing", "depends": tseries_depends}, "_libs.parsers": { "pyxfile": "_libs/parsers", "include": klib_include + ["pandas/_libs/src"], @@ -486,13 +456,8 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "pandas/_libs/src/parser/io.c", ], }, - "_libs.reduction": {"pyxfile": "_libs/reduction"}, - "_libs.ops": {"pyxfile": "_libs/ops"}, "_libs.ops_dispatch": {"pyxfile": "_libs/ops_dispatch"}, - "_libs.properties": {"pyxfile": "_libs/properties"}, - "_libs.reshape": {"pyxfile": "_libs/reshape", "depends": []}, - "_libs.sparse": {"pyxfile": "_libs/sparse", "depends": _pxi_dep["sparse"]}, - "_libs.tslib": {"pyxfile": "_libs/tslib", "depends": tseries_depends}, + "_libs.tslibs.base": {"pyxfile": "_libs/tslibs/base"}, "_libs.tslibs.ccalendar": {"pyxfile": "_libs/tslibs/ccalendar"}, "_libs.tslibs.dtypes": {"pyxfile": "_libs/tslibs/dtypes"}, @@ -556,7 +521,6 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "depends": tseries_depends, "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], }, - "_libs.testing": {"pyxfile": "_libs/testing"}, "_libs.window.aggregations": { "pyxfile": "_libs/window/aggregations", "language": "c++", @@ -564,7 +528,6 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "depends": ["pandas/_libs/src/skiplist.h"], }, "_libs.window.indexers": {"pyxfile": "_libs/window/indexers"}, - "_libs.writers": {"pyxfile": "_libs/writers"}, "io.sas._sas": {"pyxfile": "io/sas/sas"}, } From b483dc6b3a14bda76f4c76bd6dd12afcbb66c480 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Tue, 14 Jun 2022 20:38:28 -0700 Subject: [PATCH 03/68] tslibs --- pandas/_libs/CMakeLists.txt | 10 +++--- pandas/_libs/tslibs/CMakeLists.txt | 19 ++++++++++ setup.py | 57 ------------------------------ 3 files changed, 24 insertions(+), 62 deletions(-) create mode 100644 pandas/_libs/tslibs/CMakeLists.txt diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 5e1d6fb74c16d..43a4f89c8cdee 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -12,21 +12,21 @@ foreach(LIB ${KLIB_REQUIRING}) target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() -set(KLIB_AND_NP_DATETIME_REQUIRING index interval) -foreach(LIB ${KLIB_AND_NP_DATETIME_REQUIRING}) +set(KLIB_AND_TSLIBS_REQUIRING index interval) +foreach(LIB ${KLIB_AND_TSLIBS_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() -SET(NP_DATETIME_REQUIRING missing tslib) -foreach(LIB ${NP_DATETIME_REQUIRING}) +SET(TSLIBS_REQUIRING missing tslib) +foreach(LIB ${TSLIBS_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() -#include ("tslibs") +add_subdirectory("tslibs") # Depends on tokenizer being built #include ("src/parser") diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt new file mode 100644 index 0000000000000..359e33a22abc0 --- /dev/null +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -0,0 +1,19 @@ +set(BASIC_LIBRARIES base ccalendar dtypes nattype strptime timezones) +foreach(LIB ${BASIC_LIBRARIES}) + add_library(${LIB} SHARED ${LIB}.c) + target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) + target_link_libraries(${LIB} ${Python_LIBRARIES}) +endforeach() + +set(NP_DATETIME_REQUIRING conversion fields offsets period timedeltas timestamps tzconversion vectorized) +foreach(LIB ${NP_DATETIME_REQUIRING}) + add_library(${LIB} SHARED ${LIB}.c src/datetime/np_datetime.c) + target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") + target_link_libraries(${LIB} ${Python_LIBRARIES}) +endforeach() + +add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c src/datetime/np_datetime_strings.c) +target_include_directories(np_datetime PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_link_libraries(np_datetime ${Python_LIBRARIES}) + + diff --git a/setup.py b/setup.py index 97e6bc319bbd8..41d5f4d85e490 100755 --- a/setup.py +++ b/setup.py @@ -458,69 +458,12 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): }, "_libs.ops_dispatch": {"pyxfile": "_libs/ops_dispatch"}, - "_libs.tslibs.base": {"pyxfile": "_libs/tslibs/base"}, - "_libs.tslibs.ccalendar": {"pyxfile": "_libs/tslibs/ccalendar"}, - "_libs.tslibs.dtypes": {"pyxfile": "_libs/tslibs/dtypes"}, - "_libs.tslibs.conversion": { - "pyxfile": "_libs/tslibs/conversion", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.fields": { - "pyxfile": "_libs/tslibs/fields", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.nattype": {"pyxfile": "_libs/tslibs/nattype"}, - "_libs.tslibs.np_datetime": { - "pyxfile": "_libs/tslibs/np_datetime", - "depends": tseries_depends, - "sources": [ - "pandas/_libs/tslibs/src/datetime/np_datetime.c", - "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", - ], - }, - "_libs.tslibs.offsets": { - "pyxfile": "_libs/tslibs/offsets", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, "_libs.tslibs.parsing": { "pyxfile": "_libs/tslibs/parsing", "include": klib_include, "depends": ["pandas/_libs/src/parser/tokenizer.h"], "sources": ["pandas/_libs/src/parser/tokenizer.c"], }, - "_libs.tslibs.period": { - "pyxfile": "_libs/tslibs/period", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.strptime": { - "pyxfile": "_libs/tslibs/strptime", - "depends": tseries_depends, - }, - "_libs.tslibs.timedeltas": { - "pyxfile": "_libs/tslibs/timedeltas", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.timestamps": { - "pyxfile": "_libs/tslibs/timestamps", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.timezones": {"pyxfile": "_libs/tslibs/timezones"}, - "_libs.tslibs.tzconversion": { - "pyxfile": "_libs/tslibs/tzconversion", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.vectorized": { - "pyxfile": "_libs/tslibs/vectorized", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, "_libs.window.aggregations": { "pyxfile": "_libs/window/aggregations", "language": "c++", From 5f3673024c7ff6c7d75a8cfa78ba432a69d06d4e Mon Sep 17 00:00:00 2001 From: William Ayd Date: Tue, 14 Jun 2022 20:59:27 -0700 Subject: [PATCH 04/68] full build --- CMakeLists.txt | 1 + pandas/_libs/CMakeLists.txt | 17 +++++++------- pandas/_libs/tslibs/CMakeLists.txt | 4 ++++ pandas/_libs/window/CMakeLists.txt | 7 ++++++ pandas/io/sas/CMakeLists.txt | 3 +++ setup.py | 36 ------------------------------ 6 files changed, 24 insertions(+), 44 deletions(-) create mode 100644 pandas/_libs/window/CMakeLists.txt create mode 100644 pandas/io/sas/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index b623b2108fd89..fbffe63b99762 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,3 +6,4 @@ set(CMAKE_C_STANDARD_REQUIRED True) find_package(Python REQUIRED COMPONENTS Development NumPy) add_subdirectory("pandas/_libs") +add_subdirectory("pandas/io/sas") diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 43a4f89c8cdee..b2ec64a7d71cc 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -1,4 +1,4 @@ -set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops properties reshape sparse testing writers) +set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_dispatch properties reshape sparse testing writers) foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) @@ -27,11 +27,12 @@ foreach(LIB ${TSLIBS_REQUIRING}) endforeach() add_subdirectory("tslibs") +add_subdirectory("window") -# Depends on tokenizer being built -#include ("src/parser") -#set(PARSER_REQUIRING lib parsers) -#foreach(LIB ${PARSER_REQUIRING}) -#add_library(lib SHARED lib.c) -#target_include_directories(lib PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs" "src/parser") -#target_link_libraries(lib ${Python_LIBRARIES}) +add_library(lib SHARED lib.c src/parser/tokenizer.c) +target_include_directories(lib PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") +target_link_libraries(lib ${Python_LIBRARIES}) + +add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) +target_include_directories(parsers PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") +target_link_libraries(parsers ${Python_LIBRARIES}) diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index 359e33a22abc0..1c7e9c11cf1aa 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -16,4 +16,8 @@ add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c src/date target_include_directories(np_datetime PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(np_datetime ${Python_LIBRARIES}) +add_library(parsing SHARED parsing.c "../src/parser/tokenizer.c") +target_include_directories(parsing PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") +target_link_libraries(parsing ${Python_LIBRARIES}) + diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt new file mode 100644 index 0000000000000..0a58995b4358e --- /dev/null +++ b/pandas/_libs/window/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library(indexers SHARED indexers.c) +target_include_directories(indexers PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_link_libraries(indexers ${Python_LIBRARIES}) + +add_library(aggregations SHARED aggregations.cpp) +target_include_directories(aggregations PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_link_libraries(aggregations ${Python_LIBRARIES}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt new file mode 100644 index 0000000000000..1704e9b123cb6 --- /dev/null +++ b/pandas/io/sas/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(_sas SHARED sas.c) +target_include_directories(_sas PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_link_libraries(_sas ${Python_LIBRARIES}) diff --git a/setup.py b/setup.py index 41d5f4d85e490..71d27815bab1f 100755 --- a/setup.py +++ b/setup.py @@ -437,42 +437,6 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "pandas/_libs/tslibs/src/datetime/np_datetime_strings.h", ] -ext_data = { - "_libs.lib": { - "pyxfile": "_libs/lib", - "depends": lib_depends + tseries_depends, - "include": klib_include, # due to tokenizer import - "sources": ["pandas/_libs/src/parser/tokenizer.c"], - }, - "_libs.parsers": { - "pyxfile": "_libs/parsers", - "include": klib_include + ["pandas/_libs/src"], - "depends": [ - "pandas/_libs/src/parser/tokenizer.h", - "pandas/_libs/src/parser/io.h", - ], - "sources": [ - "pandas/_libs/src/parser/tokenizer.c", - "pandas/_libs/src/parser/io.c", - ], - }, - "_libs.ops_dispatch": {"pyxfile": "_libs/ops_dispatch"}, - - "_libs.tslibs.parsing": { - "pyxfile": "_libs/tslibs/parsing", - "include": klib_include, - "depends": ["pandas/_libs/src/parser/tokenizer.h"], - "sources": ["pandas/_libs/src/parser/tokenizer.c"], - }, - "_libs.window.aggregations": { - "pyxfile": "_libs/window/aggregations", - "language": "c++", - "suffix": ".cpp", - "depends": ["pandas/_libs/src/skiplist.h"], - }, - "_libs.window.indexers": {"pyxfile": "_libs/window/indexers"}, - "io.sas._sas": {"pyxfile": "io/sas/sas"}, -} extensions = [] From 8408b07603ad4faaf849073a6154a9ba8545c667 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Tue, 14 Jun 2022 21:10:00 -0700 Subject: [PATCH 05/68] build json --- pandas/_libs/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index b2ec64a7d71cc..606c1f2120981 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -36,3 +36,10 @@ target_link_libraries(lib ${Python_LIBRARIES}) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) target_include_directories(parsers PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") target_link_libraries(parsers ${Python_LIBRARIES}) + +add_library(json SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c + src/ujson/python/date_conversions.c src/ujson/python/JSONtoObj.c + src/ujson/lib/ultrajsonenc.c src/ujson/lib/ultrajsondec.c + tslibs/src/datetime/np_datetime.c tslibs/src/datetime/np_datetime_strings.c) +target_include_directories(json PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) +target_link_libraries(json ${Python_LIBRARIES}) From 3a949d9d6ea617e2802a6b69b081e9465a14fe0a Mon Sep 17 00:00:00 2001 From: William Ayd Date: Wed, 15 Jun 2022 20:58:05 -0700 Subject: [PATCH 06/68] ignore cmake generated files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 87224f1d6060f..e269abeb24496 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,10 @@ *.so .build_cache_dir MANIFEST +Makefile +CMakeCache.txt +CMakeFiles/ +cmake_install.cmake # Python files # ################ From b69775520598665428b67d2b2b2b69a21626be07 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Wed, 15 Jun 2022 21:09:43 -0700 Subject: [PATCH 07/68] public linkage --- CMakeLists.txt | 3 + pandas/_libs/CMakeLists.txt | 28 +++--- pandas/_libs/tslibs/CMakeLists.txt | 8 +- pandas/_libs/window/CMakeLists.txt | 8 +- pandas/io/sas/CMakeLists.txt | 2 +- setup.py | 136 +++++++++++++++++++++++++++-- 6 files changed, 157 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fbffe63b99762..583325973e2af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,9 @@ project(pandas VERSION 1.4) set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED True) +set(CMAKE_SHARED_LIBRARY_PREFIX "") +set(CMAKE_SHARED_LIBRARY_SUFFIX .cpython-38-darwin.so) + find_package(Python REQUIRED COMPONENTS Development NumPy) add_subdirectory("pandas/_libs") add_subdirectory("pandas/io/sas") diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 606c1f2120981..8611ea002de1d 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -1,45 +1,45 @@ set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_dispatch properties reshape sparse testing writers) foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() set(KLIB_REQUIRING algos hashtable join) foreach(LIB ${KLIB_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() set(KLIB_AND_TSLIBS_REQUIRING index interval) foreach(LIB ${KLIB_AND_TSLIBS_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() SET(TSLIBS_REQUIRING missing tslib) foreach(LIB ${TSLIBS_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() add_subdirectory("tslibs") add_subdirectory("window") add_library(lib SHARED lib.c src/parser/tokenizer.c) -target_include_directories(lib PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(lib ${Python_LIBRARIES}) +target_include_directories(lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") +target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) -target_include_directories(parsers PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") -target_link_libraries(parsers ${Python_LIBRARIES}) +target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") +target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) add_library(json SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c src/ujson/python/date_conversions.c src/ujson/python/JSONtoObj.c src/ujson/lib/ultrajsonenc.c src/ujson/lib/ultrajsondec.c tslibs/src/datetime/np_datetime.c tslibs/src/datetime/np_datetime_strings.c) -target_include_directories(json PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) -target_link_libraries(json ${Python_LIBRARIES}) +target_include_directories(json PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) +target_link_libraries(json PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index 1c7e9c11cf1aa..0136e29bc94d3 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -1,23 +1,23 @@ set(BASIC_LIBRARIES base ccalendar dtypes nattype strptime timezones) foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() set(NP_DATETIME_REQUIRING conversion fields offsets period timedeltas timestamps tzconversion vectorized) foreach(LIB ${NP_DATETIME_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c src/datetime/np_datetime.c) - target_include_directories(${LIB} PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c src/datetime/np_datetime_strings.c) -target_include_directories(np_datetime PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(np_datetime ${Python_LIBRARIES}) add_library(parsing SHARED parsing.c "../src/parser/tokenizer.c") -target_include_directories(parsing PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") +target_include_directories(parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") target_link_libraries(parsing ${Python_LIBRARIES}) diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index 0a58995b4358e..9d1dcf6040967 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -1,7 +1,7 @@ add_library(indexers SHARED indexers.c) -target_include_directories(indexers PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(indexers ${Python_LIBRARIES}) +target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_link_libraries(indexers ${Python_LIBRARIES}) add_library(aggregations SHARED aggregations.cpp) -target_include_directories(aggregations PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(aggregations ${Python_LIBRARIES}) +target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_link_libraries(aggregations ${Python_LIBRARIES}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index 1704e9b123cb6..a8a4f75d89f44 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,3 +1,3 @@ add_library(_sas SHARED sas.c) -target_include_directories(_sas PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_include_directories(_sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(_sas ${Python_LIBRARIES}) diff --git a/setup.py b/setup.py index 71d27815bab1f..4e2a38b2d3f62 100755 --- a/setup.py +++ b/setup.py @@ -437,6 +437,136 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "pandas/_libs/tslibs/src/datetime/np_datetime_strings.h", ] +ext_data = { + "_libs.algos": { + "pyxfile": "_libs/algos", + "include": klib_include, + "depends": _pxi_dep["algos"], + }, + "_libs.arrays": {"pyxfile": "_libs/arrays"}, + "_libs.groupby": {"pyxfile": "_libs/groupby"}, + "_libs.hashing": {"pyxfile": "_libs/hashing", "depends": []}, + "_libs.hashtable": { + "pyxfile": "_libs/hashtable", + "include": klib_include, + "depends": ( + ["pandas/_libs/src/klib/khash_python.h", "pandas/_libs/src/klib/khash.h"] + + _pxi_dep["hashtable"] + ), + }, + "_libs.index": { + "pyxfile": "_libs/index", + "include": klib_include, + "depends": _pxi_dep["index"], + }, + "_libs.indexing": {"pyxfile": "_libs/indexing"}, + "_libs.internals": {"pyxfile": "_libs/internals"}, + "_libs.interval": { + "pyxfile": "_libs/interval", + "include": klib_include, + "depends": _pxi_dep["interval"], + }, + "_libs.join": {"pyxfile": "_libs/join", "include": klib_include}, + "_libs.lib": { + "pyxfile": "_libs/lib", + "depends": lib_depends + tseries_depends, + "include": klib_include, # due to tokenizer import + "sources": ["pandas/_libs/src/parser/tokenizer.c"], + }, + "_libs.missing": {"pyxfile": "_libs/missing", "depends": tseries_depends}, + "_libs.parsers": { + "pyxfile": "_libs/parsers", + "include": klib_include + ["pandas/_libs/src"], + "depends": [ + "pandas/_libs/src/parser/tokenizer.h", + "pandas/_libs/src/parser/io.h", + ], + "sources": [ + "pandas/_libs/src/parser/tokenizer.c", + "pandas/_libs/src/parser/io.c", + ], + }, + "_libs.reduction": {"pyxfile": "_libs/reduction"}, + "_libs.ops": {"pyxfile": "_libs/ops"}, + "_libs.ops_dispatch": {"pyxfile": "_libs/ops_dispatch"}, + "_libs.properties": {"pyxfile": "_libs/properties"}, + "_libs.reshape": {"pyxfile": "_libs/reshape", "depends": []}, + "_libs.sparse": {"pyxfile": "_libs/sparse", "depends": _pxi_dep["sparse"]}, + "_libs.tslib": {"pyxfile": "_libs/tslib", "depends": tseries_depends}, + "_libs.tslibs.base": {"pyxfile": "_libs/tslibs/base"}, + "_libs.tslibs.ccalendar": {"pyxfile": "_libs/tslibs/ccalendar"}, + "_libs.tslibs.dtypes": {"pyxfile": "_libs/tslibs/dtypes"}, + "_libs.tslibs.conversion": { + "pyxfile": "_libs/tslibs/conversion", + "depends": tseries_depends, + "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + }, + "_libs.tslibs.fields": { + "pyxfile": "_libs/tslibs/fields", + "depends": tseries_depends, + "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + }, + "_libs.tslibs.nattype": {"pyxfile": "_libs/tslibs/nattype"}, + "_libs.tslibs.np_datetime": { + "pyxfile": "_libs/tslibs/np_datetime", + "depends": tseries_depends, + "sources": [ + "pandas/_libs/tslibs/src/datetime/np_datetime.c", + "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", + ], + }, + "_libs.tslibs.offsets": { + "pyxfile": "_libs/tslibs/offsets", + "depends": tseries_depends, + "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + }, + "_libs.tslibs.parsing": { + "pyxfile": "_libs/tslibs/parsing", + "include": klib_include, + "depends": ["pandas/_libs/src/parser/tokenizer.h"], + "sources": ["pandas/_libs/src/parser/tokenizer.c"], + }, + "_libs.tslibs.period": { + "pyxfile": "_libs/tslibs/period", + "depends": tseries_depends, + "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + }, + "_libs.tslibs.strptime": { + "pyxfile": "_libs/tslibs/strptime", + "depends": tseries_depends, + }, + "_libs.tslibs.timedeltas": { + "pyxfile": "_libs/tslibs/timedeltas", + "depends": tseries_depends, + "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + }, + "_libs.tslibs.timestamps": { + "pyxfile": "_libs/tslibs/timestamps", + "depends": tseries_depends, + "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + }, + "_libs.tslibs.timezones": {"pyxfile": "_libs/tslibs/timezones"}, + "_libs.tslibs.tzconversion": { + "pyxfile": "_libs/tslibs/tzconversion", + "depends": tseries_depends, + "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + }, + "_libs.tslibs.vectorized": { + "pyxfile": "_libs/tslibs/vectorized", + "depends": tseries_depends, + "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + }, + "_libs.testing": {"pyxfile": "_libs/testing"}, + "_libs.window.aggregations": { + "pyxfile": "_libs/window/aggregations", + "language": "c++", + "suffix": ".cpp", + "depends": ["pandas/_libs/src/skiplist.h"], + }, + "_libs.window.indexers": {"pyxfile": "_libs/window/indexers"}, + "_libs.writers": {"pyxfile": "_libs/writers"}, + "io.sas._sas": {"pyxfile": "io/sas/sas"}, +} extensions = [] @@ -525,8 +655,4 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): if __name__ == "__main__": # Freeze to support parallel compilation when using spawn instead of fork multiprocessing.freeze_support() - setup( - version=versioneer.get_version(), - ext_modules=maybe_cythonize(extensions, compiler_directives=directives), - cmdclass=cmdclass, - ) + ext_modules=maybe_cythonize(extensions, compiler_directives=directives) From ad43a3198e6cfd10956922009643511e06c7f103 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Wed, 15 Jun 2022 21:12:03 -0700 Subject: [PATCH 08/68] remove unneeded library links --- pandas/_libs/CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 8611ea002de1d..aa3d09df7f956 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -2,28 +2,28 @@ set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_ foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) + target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() set(KLIB_REQUIRING algos hashtable join) foreach(LIB ${KLIB_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) + target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() set(KLIB_AND_TSLIBS_REQUIRING index interval) foreach(LIB ${KLIB_AND_TSLIBS_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) + target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() SET(TSLIBS_REQUIRING missing tslib) foreach(LIB ${TSLIBS_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) + target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() add_subdirectory("tslibs") @@ -31,15 +31,15 @@ add_subdirectory("window") add_library(lib SHARED lib.c src/parser/tokenizer.c) target_include_directories(lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) +target_link_libraries(lib ${Python_LIBRARIES}) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") -target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) +target_link_libraries(parsers ${Python_LIBRARIES}) add_library(json SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c src/ujson/python/date_conversions.c src/ujson/python/JSONtoObj.c src/ujson/lib/ultrajsonenc.c src/ujson/lib/ultrajsondec.c tslibs/src/datetime/np_datetime.c tslibs/src/datetime/np_datetime_strings.c) target_include_directories(json PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) -target_link_libraries(json PUBLIC ${Python_LIBRARIES}) +target_link_libraries(json ${Python_LIBRARIES}) From d41af1e7022c1b7d44d469c884caba1c4a3d4974 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Wed, 15 Jun 2022 21:29:41 -0700 Subject: [PATCH 09/68] PUBLIC cmake linkage --- pandas/_libs/CMakeLists.txt | 14 +++++++------- pandas/_libs/tslibs/CMakeLists.txt | 8 ++++---- pandas/_libs/window/CMakeLists.txt | 4 ++-- pandas/io/sas/CMakeLists.txt | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index aa3d09df7f956..8611ea002de1d 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -2,28 +2,28 @@ set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_ foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() set(KLIB_REQUIRING algos hashtable join) foreach(LIB ${KLIB_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() set(KLIB_AND_TSLIBS_REQUIRING index interval) foreach(LIB ${KLIB_AND_TSLIBS_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() SET(TSLIBS_REQUIRING missing tslib) foreach(LIB ${TSLIBS_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() add_subdirectory("tslibs") @@ -31,15 +31,15 @@ add_subdirectory("window") add_library(lib SHARED lib.c src/parser/tokenizer.c) target_include_directories(lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(lib ${Python_LIBRARIES}) +target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") -target_link_libraries(parsers ${Python_LIBRARIES}) +target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) add_library(json SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c src/ujson/python/date_conversions.c src/ujson/python/JSONtoObj.c src/ujson/lib/ultrajsonenc.c src/ujson/lib/ultrajsondec.c tslibs/src/datetime/np_datetime.c tslibs/src/datetime/np_datetime_strings.c) target_include_directories(json PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) -target_link_libraries(json ${Python_LIBRARIES}) +target_link_libraries(json PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index 0136e29bc94d3..6add980bb6caf 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -2,22 +2,22 @@ set(BASIC_LIBRARIES base ccalendar dtypes nattype strptime timezones) foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() set(NP_DATETIME_REQUIRING conversion fields offsets period timedeltas timestamps tzconversion vectorized) foreach(LIB ${NP_DATETIME_REQUIRING}) add_library(${LIB} SHARED ${LIB}.c src/datetime/np_datetime.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c src/datetime/np_datetime_strings.c) target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(np_datetime ${Python_LIBRARIES}) +target_link_libraries(np_datetime PUBLIC ${Python_LIBRARIES}) add_library(parsing SHARED parsing.c "../src/parser/tokenizer.c") target_include_directories(parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") -target_link_libraries(parsing ${Python_LIBRARIES}) +target_link_libraries(parsing PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index 9d1dcf6040967..21a5b32c7a358 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -1,7 +1,7 @@ add_library(indexers SHARED indexers.c) target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(indexers ${Python_LIBRARIES}) +target_link_libraries(indexers PUBLIC ${Python_LIBRARIES}) add_library(aggregations SHARED aggregations.cpp) target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(aggregations ${Python_LIBRARIES}) +target_link_libraries(aggregations PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index a8a4f75d89f44..e4ddd51d7f422 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,3 +1,3 @@ add_library(_sas SHARED sas.c) target_include_directories(_sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(_sas ${Python_LIBRARIES}) +target_link_libraries(_sas PUBLIC ${Python_LIBRARIES}) From 4eab4a23e21a804d84b8aaab149b8c2a3c0ee5de Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 15 Jun 2022 22:42:44 -0700 Subject: [PATCH 10/68] build system --- CMakeLists.txt | 2 +- pandas/_libs/CMakeLists.txt | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 583325973e2af..1ca3fe3b3cee7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED True) set(CMAKE_SHARED_LIBRARY_PREFIX "") -set(CMAKE_SHARED_LIBRARY_SUFFIX .cpython-38-darwin.so) +set(CMAKE_SHARED_LIBRARY_SUFFIX .cpython-38-x86_64-linux-gnu.so) find_package(Python REQUIRED COMPONENTS Development NumPy) add_subdirectory("pandas/_libs") diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 8611ea002de1d..86fac6b0a045e 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -19,12 +19,13 @@ foreach(LIB ${KLIB_AND_TSLIBS_REQUIRING}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() -SET(TSLIBS_REQUIRING missing tslib) -foreach(LIB ${TSLIBS_REQUIRING}) - add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) -endforeach() +add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) +target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") +target_link_libraries(tslib PUBLIC ${Python_LIBRARIES}) + +add_library(missing SHARED missing.c) +target_include_directories(missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") +target_link_libraries(missing PUBLIC ${Python_LIBRARIES}) add_subdirectory("tslibs") add_subdirectory("window") From 02b48feacaf98e60ff9437bdac2dbcd474196667 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 15 Jun 2022 22:58:26 -0700 Subject: [PATCH 11/68] cleaned up setup --- setup.py | 658 +------------------------------------------------------ 1 file changed, 7 insertions(+), 651 deletions(-) diff --git a/setup.py b/setup.py index 2f3048df7e509..3988efca75f50 100755 --- a/setup.py +++ b/setup.py @@ -6,657 +6,13 @@ BSD license. Parts are from lxml (https://github.com/lxml/lxml) """ -import argparse -import multiprocessing -import os -from os.path import join as pjoin -import platform -import shutil -import sys -from sysconfig import get_config_vars - -import numpy -from pkg_resources import parse_version -from setuptools import ( - Command, - Extension, - setup, -) -from setuptools.command.build_ext import build_ext as _build_ext - -import versioneer - -cmdclass = versioneer.get_cmdclass() - - -def is_platform_windows(): - return sys.platform == "win32" or sys.platform == "cygwin" - - -def is_platform_mac(): - return sys.platform == "darwin" - - -# note: sync with pyproject.toml, environment.yml and asv.conf.json -min_cython_ver = "0.29.30" - -try: - from Cython import ( - Tempita, - __version__ as _CYTHON_VERSION, - ) - from Cython.Build import cythonize - - _CYTHON_INSTALLED = parse_version(_CYTHON_VERSION) >= parse_version(min_cython_ver) -except ImportError: - _CYTHON_VERSION = None - _CYTHON_INSTALLED = False - cythonize = lambda x, *args, **kwargs: x # dummy func - - -_pxi_dep_template = { - "algos": ["_libs/algos_common_helper.pxi.in", "_libs/algos_take_helper.pxi.in"], - "hashtable": [ - "_libs/hashtable_class_helper.pxi.in", - "_libs/hashtable_func_helper.pxi.in", - "_libs/khash_for_primitive_helper.pxi.in", - ], - "index": ["_libs/index_class_helper.pxi.in"], - "sparse": ["_libs/sparse_op_helper.pxi.in"], - "interval": ["_libs/intervaltree.pxi.in"], -} - -_pxifiles = [] -_pxi_dep = {} -for module, files in _pxi_dep_template.items(): - pxi_files = [pjoin("pandas", x) for x in files] - _pxifiles.extend(pxi_files) - _pxi_dep[module] = pxi_files - - -class build_ext(_build_ext): - @classmethod - def render_templates(cls, pxifiles): - for pxifile in pxifiles: - # build pxifiles first, template extension must be .pxi.in - assert pxifile.endswith(".pxi.in") - outfile = pxifile[:-3] - - if ( - os.path.exists(outfile) - and os.stat(pxifile).st_mtime < os.stat(outfile).st_mtime - ): - # if .pxi.in is not updated, no need to output .pxi - continue - - with open(pxifile) as f: - tmpl = f.read() - pyxcontent = Tempita.sub(tmpl) - - with open(outfile, "w") as f: - f.write(pyxcontent) - - def build_extensions(self): - # if building from c files, don't need to - # generate template output - if _CYTHON_INSTALLED: - self.render_templates(_pxifiles) - - super().build_extensions() - - -class CleanCommand(Command): - """Custom command to clean the .so and .pyc files.""" - - user_options = [("all", "a", "")] - - def initialize_options(self): - self.all = True - self._clean_me = [] - self._clean_trees = [] - - base = pjoin("pandas", "_libs", "src") - tsbase = pjoin("pandas", "_libs", "tslibs", "src") - dt = pjoin(tsbase, "datetime") - util = pjoin("pandas", "util") - parser = pjoin(base, "parser") - ujson_python = pjoin(base, "ujson", "python") - ujson_lib = pjoin(base, "ujson", "lib") - self._clean_exclude = [ - pjoin(dt, "np_datetime.c"), - pjoin(dt, "np_datetime_strings.c"), - pjoin(parser, "tokenizer.c"), - pjoin(parser, "io.c"), - pjoin(ujson_python, "ujson.c"), - pjoin(ujson_python, "objToJSON.c"), - pjoin(ujson_python, "JSONtoObj.c"), - pjoin(ujson_python, "date_conversions.c"), - pjoin(ujson_lib, "ultrajsonenc.c"), - pjoin(ujson_lib, "ultrajsondec.c"), - pjoin(util, "move.c"), - ] - - for root, dirs, files in os.walk("pandas"): - for f in files: - filepath = pjoin(root, f) - if filepath in self._clean_exclude: - continue - - if os.path.splitext(f)[-1] in ( - ".pyc", - ".so", - ".o", - ".pyo", - ".pyd", - ".c", - ".cpp", - ".orig", - ): - self._clean_me.append(filepath) - for d in dirs: - if d == "__pycache__": - self._clean_trees.append(pjoin(root, d)) - - # clean the generated pxi files - for pxifile in _pxifiles: - pxifile = pxifile.replace(".pxi.in", ".pxi") - self._clean_me.append(pxifile) - - for d in ("build", "dist"): - if os.path.exists(d): - self._clean_trees.append(d) - - def finalize_options(self): - pass - - def run(self): - for clean_me in self._clean_me: - try: - os.unlink(clean_me) - except OSError: - pass - for clean_tree in self._clean_trees: - try: - shutil.rmtree(clean_tree) - except OSError: - pass - - -# we need to inherit from the versioneer -# class as it encodes the version info -sdist_class = cmdclass["sdist"] - - -class CheckSDist(sdist_class): - """Custom sdist that ensures Cython has compiled all pyx files to c.""" - - _pyxfiles = [ - "pandas/_libs/arrays.pyx", - "pandas/_libs/lib.pyx", - "pandas/_libs/hashtable.pyx", - "pandas/_libs/tslib.pyx", - "pandas/_libs/index.pyx", - "pandas/_libs/internals.pyx", - "pandas/_libs/algos.pyx", - "pandas/_libs/join.pyx", - "pandas/_libs/indexing.pyx", - "pandas/_libs/interval.pyx", - "pandas/_libs/hashing.pyx", - "pandas/_libs/missing.pyx", - "pandas/_libs/reduction.pyx", - "pandas/_libs/testing.pyx", - "pandas/_libs/sparse.pyx", - "pandas/_libs/ops.pyx", - "pandas/_libs/parsers.pyx", - "pandas/_libs/tslibs/base.pyx", - "pandas/_libs/tslibs/ccalendar.pyx", - "pandas/_libs/tslibs/dtypes.pyx", - "pandas/_libs/tslibs/period.pyx", - "pandas/_libs/tslibs/strptime.pyx", - "pandas/_libs/tslibs/np_datetime.pyx", - "pandas/_libs/tslibs/timedeltas.pyx", - "pandas/_libs/tslibs/timestamps.pyx", - "pandas/_libs/tslibs/timezones.pyx", - "pandas/_libs/tslibs/conversion.pyx", - "pandas/_libs/tslibs/fields.pyx", - "pandas/_libs/tslibs/offsets.pyx", - "pandas/_libs/tslibs/parsing.pyx", - "pandas/_libs/tslibs/tzconversion.pyx", - "pandas/_libs/tslibs/vectorized.pyx", - "pandas/_libs/window/indexers.pyx", - "pandas/_libs/writers.pyx", - "pandas/io/sas/sas.pyx", - ] - - _cpp_pyxfiles = [ - "pandas/_libs/window/aggregations.pyx", - ] - - def initialize_options(self): - sdist_class.initialize_options(self) - - def run(self): - if "cython" in cmdclass: - self.run_command("cython") - else: - # If we are not running cython then - # compile the extensions correctly - pyx_files = [(self._pyxfiles, "c"), (self._cpp_pyxfiles, "cpp")] - - for pyxfiles, extension in pyx_files: - for pyxfile in pyxfiles: - sourcefile = pyxfile[:-3] + extension - msg = ( - f"{extension}-source file '{sourcefile}' not found.\n" - "Run 'setup.py cython' before sdist." - ) - assert os.path.isfile(sourcefile), msg - sdist_class.run(self) - - -class CheckingBuildExt(build_ext): - """ - Subclass build_ext to get clearer report if Cython is necessary. - """ - - def check_cython_extensions(self, extensions): - for ext in extensions: - for src in ext.sources: - if not os.path.exists(src): - print(f"{ext.name}: -> [{ext.sources}]") - raise Exception( - f"""Cython-generated file '{src}' not found. - Cython is required to compile pandas from a development branch. - Please install Cython or download a release package of pandas. - """ - ) - - def build_extensions(self): - self.check_cython_extensions(self.extensions) - build_ext.build_extensions(self) - - -class CythonCommand(build_ext): - """ - Custom command subclassed from Cython.Distutils.build_ext - to compile pyx->c, and stop there. All this does is override the - C-compile method build_extension() with a no-op. - """ - - def build_extension(self, ext): - pass - - -class DummyBuildSrc(Command): - """numpy's build_src command interferes with Cython's build_ext.""" - - user_options = [] - - def initialize_options(self): - self.py_modules_dict = {} - - def finalize_options(self): - pass - - def run(self): - pass - - -cmdclass["clean"] = CleanCommand -cmdclass["build_ext"] = CheckingBuildExt - -if _CYTHON_INSTALLED: - suffix = ".pyx" - cmdclass["cython"] = CythonCommand -else: - suffix = ".c" - cmdclass["build_src"] = DummyBuildSrc - -# ---------------------------------------------------------------------- -# Preparation of compiler arguments - -debugging_symbols_requested = "--with-debugging-symbols" in sys.argv -if debugging_symbols_requested: - sys.argv.remove("--with-debugging-symbols") - - -if sys.byteorder == "big": - endian_macro = [("__BIG_ENDIAN__", "1")] -else: - endian_macro = [("__LITTLE_ENDIAN__", "1")] - - -extra_compile_args = [] -extra_link_args = [] -if is_platform_windows(): - if debugging_symbols_requested: - extra_compile_args.append("/Z7") - extra_link_args.append("/DEBUG") -else: - # PANDAS_CI=1 is set in CI - if os.environ.get("PANDAS_CI", "0") == "1": - extra_compile_args.append("-Werror") - if debugging_symbols_requested: - extra_compile_args.append("-g") - extra_compile_args.append("-UNDEBUG") - extra_compile_args.append("-O0") - -# Build for at least macOS 10.9 when compiling on a 10.9 system or above, -# overriding CPython distuitls behaviour which is to target the version that -# python was built for. This may be overridden by setting -# MACOSX_DEPLOYMENT_TARGET before calling setup.py -if is_platform_mac(): - if "MACOSX_DEPLOYMENT_TARGET" not in os.environ: - current_system = platform.mac_ver()[0] - python_target = get_config_vars().get( - "MACOSX_DEPLOYMENT_TARGET", current_system - ) - target_macos_version = "10.9" - parsed_macos_version = parse_version(target_macos_version) - if ( - parse_version(str(python_target)) < parsed_macos_version - and parse_version(current_system) >= parsed_macos_version - ): - os.environ["MACOSX_DEPLOYMENT_TARGET"] = target_macos_version - - if sys.version_info[:2] == (3, 8): # GH 33239 - extra_compile_args.append("-Wno-error=deprecated-declarations") - - # https://github.com/pandas-dev/pandas/issues/35559 - extra_compile_args.append("-Wno-error=unreachable-code") - -# enable coverage by building cython files by setting the environment variable -# "PANDAS_CYTHON_COVERAGE" (with a Truthy value) or by running build_ext -# with `--with-cython-coverage`enabled -linetrace = os.environ.get("PANDAS_CYTHON_COVERAGE", False) -if "--with-cython-coverage" in sys.argv: - linetrace = True - sys.argv.remove("--with-cython-coverage") - -# Note: if not using `cythonize`, coverage can be enabled by -# pinning `ext.cython_directives = directives` to each ext in extensions. -# github.com/cython/cython/wiki/enhancements-compilerdirectives#in-setuppy -directives = {"linetrace": False, "language_level": 3} -macros = [] -if linetrace: - # https://pypkg.com/pypi/pytest-cython/f/tests/example-project/setup.py - directives["linetrace"] = True - macros = [("CYTHON_TRACE", "1"), ("CYTHON_TRACE_NOGIL", "1")] - -# silence build warnings about deprecated API usage -# we can't do anything about these warnings because they stem from -# cython+numpy version mismatches. -macros.append(("NPY_NO_DEPRECATED_API", "0")) - - -# ---------------------------------------------------------------------- -# Specification of Dependencies - -# TODO(cython#4518): Need to check to see if e.g. `linetrace` has changed and -# possibly re-compile. -def maybe_cythonize(extensions, *args, **kwargs): - """ - Render tempita templates before calling cythonize. This is skipped for - - * clean - * sdist - """ - if "clean" in sys.argv or "sdist" in sys.argv: - # See https://github.com/cython/cython/issues/1495 - return extensions - - elif not _CYTHON_INSTALLED: - # GH#28836 raise a helfpul error message - if _CYTHON_VERSION: - raise RuntimeError( - f"Cannot cythonize with old Cython version ({_CYTHON_VERSION} " - f"installed, needs {min_cython_ver})" - ) - raise RuntimeError("Cannot cythonize without Cython installed.") - - # reuse any parallel arguments provided for compilation to cythonize - parser = argparse.ArgumentParser() - parser.add_argument("--parallel", "-j", type=int, default=1) - parsed, _ = parser.parse_known_args() - - kwargs["nthreads"] = parsed.parallel - build_ext.render_templates(_pxifiles) - return cythonize(extensions, *args, **kwargs) - - -def srcpath(name=None, suffix=".pyx", subdir="src"): - return pjoin("pandas", subdir, name + suffix) - - -lib_depends = ["pandas/_libs/src/parse_helper.h"] - -klib_include = ["pandas/_libs/src/klib"] - -tseries_depends = [ - "pandas/_libs/tslibs/src/datetime/np_datetime.h", - "pandas/_libs/tslibs/src/datetime/np_datetime_strings.h", -] - -ext_data = { - "_libs.algos": { - "pyxfile": "_libs/algos", - "include": klib_include, - "depends": _pxi_dep["algos"], - }, - "_libs.arrays": {"pyxfile": "_libs/arrays"}, - "_libs.groupby": {"pyxfile": "_libs/groupby"}, - "_libs.hashing": {"pyxfile": "_libs/hashing", "depends": []}, - "_libs.hashtable": { - "pyxfile": "_libs/hashtable", - "include": klib_include, - "depends": ( - ["pandas/_libs/src/klib/khash_python.h", "pandas/_libs/src/klib/khash.h"] - + _pxi_dep["hashtable"] - ), - }, - "_libs.index": { - "pyxfile": "_libs/index", - "include": klib_include, - "depends": _pxi_dep["index"], - }, - "_libs.indexing": {"pyxfile": "_libs/indexing"}, - "_libs.internals": {"pyxfile": "_libs/internals"}, - "_libs.interval": { - "pyxfile": "_libs/interval", - "include": klib_include, - "depends": _pxi_dep["interval"], - }, - "_libs.join": {"pyxfile": "_libs/join", "include": klib_include}, - "_libs.lib": { - "pyxfile": "_libs/lib", - "depends": lib_depends + tseries_depends, - "include": klib_include, # due to tokenizer import - "sources": ["pandas/_libs/src/parser/tokenizer.c"], - }, - "_libs.missing": {"pyxfile": "_libs/missing", "depends": tseries_depends}, - "_libs.parsers": { - "pyxfile": "_libs/parsers", - "include": klib_include + ["pandas/_libs/src"], - "depends": [ - "pandas/_libs/src/parser/tokenizer.h", - "pandas/_libs/src/parser/io.h", - ], - "sources": [ - "pandas/_libs/src/parser/tokenizer.c", - "pandas/_libs/src/parser/io.c", - ], - }, - "_libs.reduction": {"pyxfile": "_libs/reduction"}, - "_libs.ops": {"pyxfile": "_libs/ops"}, - "_libs.ops_dispatch": {"pyxfile": "_libs/ops_dispatch"}, - "_libs.properties": {"pyxfile": "_libs/properties"}, - "_libs.reshape": {"pyxfile": "_libs/reshape", "depends": []}, - "_libs.sparse": {"pyxfile": "_libs/sparse", "depends": _pxi_dep["sparse"]}, - "_libs.tslib": { - "pyxfile": "_libs/tslib", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.base": {"pyxfile": "_libs/tslibs/base"}, - "_libs.tslibs.ccalendar": {"pyxfile": "_libs/tslibs/ccalendar"}, - "_libs.tslibs.dtypes": {"pyxfile": "_libs/tslibs/dtypes"}, - "_libs.tslibs.conversion": { - "pyxfile": "_libs/tslibs/conversion", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.fields": { - "pyxfile": "_libs/tslibs/fields", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.nattype": {"pyxfile": "_libs/tslibs/nattype"}, - "_libs.tslibs.np_datetime": { - "pyxfile": "_libs/tslibs/np_datetime", - "depends": tseries_depends, - "sources": [ - "pandas/_libs/tslibs/src/datetime/np_datetime.c", - "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", - ], - }, - "_libs.tslibs.offsets": { - "pyxfile": "_libs/tslibs/offsets", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.parsing": { - "pyxfile": "_libs/tslibs/parsing", - "include": klib_include, - "depends": ["pandas/_libs/src/parser/tokenizer.h"], - "sources": ["pandas/_libs/src/parser/tokenizer.c"], - }, - "_libs.tslibs.period": { - "pyxfile": "_libs/tslibs/period", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.strptime": { - "pyxfile": "_libs/tslibs/strptime", - "depends": tseries_depends, - }, - "_libs.tslibs.timedeltas": { - "pyxfile": "_libs/tslibs/timedeltas", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.timestamps": { - "pyxfile": "_libs/tslibs/timestamps", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.timezones": {"pyxfile": "_libs/tslibs/timezones"}, - "_libs.tslibs.tzconversion": { - "pyxfile": "_libs/tslibs/tzconversion", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.tslibs.vectorized": { - "pyxfile": "_libs/tslibs/vectorized", - "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], - }, - "_libs.testing": {"pyxfile": "_libs/testing"}, - "_libs.window.aggregations": { - "pyxfile": "_libs/window/aggregations", - "language": "c++", - "suffix": ".cpp", - "depends": ["pandas/_libs/src/skiplist.h"], - }, - "_libs.window.indexers": {"pyxfile": "_libs/window/indexers"}, - "_libs.writers": {"pyxfile": "_libs/writers"}, - "io.sas._sas": {"pyxfile": "io/sas/sas"}, -} - -extensions = [] - -for name, data in ext_data.items(): - source_suffix = suffix if suffix == ".pyx" else data.get("suffix", ".c") - - sources = [srcpath(data["pyxfile"], suffix=source_suffix, subdir="")] - - sources.extend(data.get("sources", [])) - - include = data.get("include", []) - include.append(numpy.get_include()) - - undef_macros = [] - - if ( - sys.platform == "zos" - and data.get("language") == "c++" - and os.path.basename(os.environ.get("CXX", "/bin/xlc++")) in ("xlc", "xlc++") - ): - data.get("macros", macros).append(("__s390__", "1")) - extra_compile_args.append("-qlanglvl=extended0x:nolibext") - undef_macros.append("_POSIX_THREADS") - - obj = Extension( - f"pandas.{name}", - sources=sources, - depends=data.get("depends", []), - include_dirs=include, - language=data.get("language", "c"), - define_macros=data.get("macros", macros), - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - undef_macros=undef_macros, - ) - - extensions.append(obj) - -# ---------------------------------------------------------------------- -# ujson - -if suffix == ".pyx": - # undo dumb setuptools bug clobbering .pyx sources back to .c - for ext in extensions: - if ext.sources[0].endswith((".c", ".cpp")): - root, _ = os.path.splitext(ext.sources[0]) - ext.sources[0] = root + suffix - -ujson_ext = Extension( - "pandas._libs.json", - depends=[ - "pandas/_libs/src/ujson/lib/ultrajson.h", - "pandas/_libs/src/ujson/python/date_conversions.h", - ], - sources=( - [ - "pandas/_libs/src/ujson/python/ujson.c", - "pandas/_libs/src/ujson/python/objToJSON.c", - "pandas/_libs/src/ujson/python/date_conversions.c", - "pandas/_libs/src/ujson/python/JSONtoObj.c", - "pandas/_libs/src/ujson/lib/ultrajsonenc.c", - "pandas/_libs/src/ujson/lib/ultrajsondec.c", - ] - + [ - "pandas/_libs/tslibs/src/datetime/np_datetime.c", - "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", - ] - ), - include_dirs=[ - "pandas/_libs/src/ujson/python", - "pandas/_libs/src/ujson/lib", - "pandas/_libs/src/datetime", - numpy.get_include(), - ], - extra_compile_args=(["-D_GNU_SOURCE"] + extra_compile_args), - extra_link_args=extra_link_args, - define_macros=macros, -) - - -extensions.append(ujson_ext) - -# ---------------------------------------------------------------------- +import pathlib +from Cython.Build import cythonize if __name__ == "__main__": - # Freeze to support parallel compilation when using spawn instead of fork - multiprocessing.freeze_support() - ext_modules=maybe_cythonize(extensions, compiler_directives=directives) + # TODO: this creates files but never cleans them up. Also runs serially + # should integrate into CMake process so it is included as part of make / + # make clean + for x in pathlib.Path(".").glob("**/*.pyx"): + cythonize(str(x), language_level=3) From b447b86cc763873477dd0cf28f00171e3fa09d3a Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 15 Jun 2022 23:01:33 -0700 Subject: [PATCH 12/68] restored pandas makefile --- Makefile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000..c0aa685ed47ac --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +.PHONY : develop build clean clean_pyc doc lint-diff black test-scripts + +all: develop + +clean: + -python setup.py clean + +clean_pyc: + -find . -name '*.py[co]' -exec rm {} \; + +build: clean_pyc + python setup.py build_ext + +lint-diff: + git diff upstream/main --name-only -- "*.py" | xargs flake8 + +black: + black . + +develop: build + python -m pip install --no-build-isolation -e . + +doc: + -rm -rf doc/build doc/source/generated + cd doc; \ + python make.py clean; \ + python make.py html + +test-scripts: + pytest scripts From 64a9cbf6442c374974493d8748eca7644daca111 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 16 Jun 2022 16:48:20 -0700 Subject: [PATCH 13/68] inline cythonization --- CMakeLists.txt | 5 +++++ Makefile | 30 ------------------------------ pandas/_libs/CMakeLists.txt | 7 +++++++ pandas/_libs/tslibs/CMakeLists.txt | 4 ++++ pandas/_libs/window/CMakeLists.txt | 2 ++ pandas/io/sas/CMakeLists.txt | 1 + setup.py | 18 ------------------ 7 files changed, 19 insertions(+), 48 deletions(-) delete mode 100644 Makefile delete mode 100755 setup.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ca3fe3b3cee7..b9148539ee3eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,10 +3,15 @@ cmake_minimum_required(VERSION 3.21) project(pandas VERSION 1.4) set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED True) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_SHARED_LIBRARY_PREFIX "") set(CMAKE_SHARED_LIBRARY_SUFFIX .cpython-38-x86_64-linux-gnu.so) +set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9") find_package(Python REQUIRED COMPONENTS Development NumPy) + +add_compile_definitions(NPY_NO_DEPRECATED_API=0) add_subdirectory("pandas/_libs") add_subdirectory("pandas/io/sas") diff --git a/Makefile b/Makefile deleted file mode 100644 index c0aa685ed47ac..0000000000000 --- a/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -.PHONY : develop build clean clean_pyc doc lint-diff black test-scripts - -all: develop - -clean: - -python setup.py clean - -clean_pyc: - -find . -name '*.py[co]' -exec rm {} \; - -build: clean_pyc - python setup.py build_ext - -lint-diff: - git diff upstream/main --name-only -- "*.py" | xargs flake8 - -black: - black . - -develop: build - python -m pip install --no-build-isolation -e . - -doc: - -rm -rf doc/build doc/source/generated - cd doc; \ - python make.py clean; \ - python make.py html - -test-scripts: - pytest scripts diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 86fac6b0a045e..2132e09f5eefd 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -1,5 +1,6 @@ set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_dispatch properties reshape sparse testing writers) foreach(LIB ${BASIC_LIBRARIES}) + add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) @@ -7,6 +8,7 @@ endforeach() set(KLIB_REQUIRING algos hashtable join) foreach(LIB ${KLIB_REQUIRING}) + add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) @@ -14,15 +16,18 @@ endforeach() set(KLIB_AND_TSLIBS_REQUIRING index interval) foreach(LIB ${KLIB_AND_TSLIBS_REQUIRING}) + add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() +add_custom_command(OUTPUT tslib.c COMMAND python -m cython -3 tslib.pyx) add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(tslib PUBLIC ${Python_LIBRARIES}) +add_custom_command(OUTPUT missing.c COMMAND python -m cython -3 missing.pyx) add_library(missing SHARED missing.c) target_include_directories(missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(missing PUBLIC ${Python_LIBRARIES}) @@ -30,10 +35,12 @@ target_link_libraries(missing PUBLIC ${Python_LIBRARIES}) add_subdirectory("tslibs") add_subdirectory("window") +add_custom_command(OUTPUT lib.c COMMAND python -m cython -3 lib.pyx) add_library(lib SHARED lib.c src/parser/tokenizer.c) target_include_directories(lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) +add_custom_command(OUTPUT parsers.c COMMAND python -m cython -3 parsers.pyx) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index 6add980bb6caf..b87df0b0fcb7f 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -1,5 +1,6 @@ set(BASIC_LIBRARIES base ccalendar dtypes nattype strptime timezones) foreach(LIB ${BASIC_LIBRARIES}) + add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) @@ -7,15 +8,18 @@ endforeach() set(NP_DATETIME_REQUIRING conversion fields offsets period timedeltas timestamps tzconversion vectorized) foreach(LIB ${NP_DATETIME_REQUIRING}) + add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c src/datetime/np_datetime.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() +add_custom_command(OUTPUT np_datetime.c COMMAND python -m cython -3 np_datetime.pyx) add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c src/datetime/np_datetime_strings.c) target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(np_datetime PUBLIC ${Python_LIBRARIES}) +add_custom_command(OUTPUT parsing.c COMMAND python -m cython -3 parsing.pyx) add_library(parsing SHARED parsing.c "../src/parser/tokenizer.c") target_include_directories(parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") target_link_libraries(parsing PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index 21a5b32c7a358..08eed8717ee8c 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -1,7 +1,9 @@ +add_custom_command(OUTPUT indexers.c COMMAND python -m cython -3 indexers.pyx) add_library(indexers SHARED indexers.c) target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(indexers PUBLIC ${Python_LIBRARIES}) +add_custom_command(OUTPUT aggregations.cpp COMMAND python -m cython -3 --cplus aggregations.pyx) add_library(aggregations SHARED aggregations.cpp) target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(aggregations PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index e4ddd51d7f422..1a4a23993f0bf 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,3 +1,4 @@ +add_custom_command(OUTPUT _sas.c COMMAND python -m cython -3 _sas.pyx) add_library(_sas SHARED sas.c) target_include_directories(_sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(_sas PUBLIC ${Python_LIBRARIES}) diff --git a/setup.py b/setup.py deleted file mode 100755 index 3988efca75f50..0000000000000 --- a/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python3 - -""" -Parts of this file were taken from the pyzmq project -(https://github.com/zeromq/pyzmq) which have been permitted for use under the -BSD license. Parts are from lxml (https://github.com/lxml/lxml) -""" - -import pathlib -from Cython.Build import cythonize - - -if __name__ == "__main__": - # TODO: this creates files but never cleans them up. Also runs serially - # should integrate into CMake process so it is included as part of make / - # make clean - for x in pathlib.Path(".").glob("**/*.pyx"): - cythonize(str(x), language_level=3) From a2dfd3e64cf378ac8f065725cf83a813ee75af4c Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 16 Jun 2022 16:51:27 -0700 Subject: [PATCH 14/68] python suffix --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b9148539ee3eb..de2f958106786 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,11 +7,10 @@ set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_SHARED_LIBRARY_PREFIX "") -set(CMAKE_SHARED_LIBRARY_SUFFIX .cpython-38-x86_64-linux-gnu.so) set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9") find_package(Python REQUIRED COMPONENTS Development NumPy) - +set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}.so") add_compile_definitions(NPY_NO_DEPRECATED_API=0) add_subdirectory("pandas/_libs") add_subdirectory("pandas/io/sas") From 40d18f998e1754fb9c8003de7c77cb775f15a345 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 16 Jun 2022 17:03:19 -0700 Subject: [PATCH 15/68] correct min version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de2f958106786..c4acbd88482ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.21) +cmake_minimum_required(VERSION 3.17) project(pandas VERSION 1.4) set(CMAKE_C_STANDARD 99) From d1c2aefd6be59957e0473165177d14c5768524ac Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 16 Jun 2022 21:54:50 -0700 Subject: [PATCH 16/68] some progress --- pandas/_libs/CMakeLists.txt | 44 +++++++++++++++++++++++------ pandas/_libs/cythonize_templates.py | 29 +++++++++++++++++++ pandas/io/sas/CMakeLists.txt | 8 +++--- setup.py | 19 +++++++++++++ 4 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 pandas/_libs/cythonize_templates.py create mode 100644 setup.py diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 2132e09f5eefd..d05a6a49b084b 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -1,4 +1,11 @@ -set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_dispatch properties reshape sparse testing writers) +add_custom_target(khash COMMAND python -m cythonize_templates khash header) + +set(HEADERS arrays dtypes hashtable lib missing util) +foreach(HEADER ${HEADERS}) + add_custom_command(OUTPUT ${HEADER}.h COMMAND python -m cython -3 ${HEADER}.pxd) +endforeach() + +set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_dispatch properties reshape testing writers) foreach(LIB ${BASIC_LIBRARIES}) add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c) @@ -6,18 +13,37 @@ foreach(LIB ${BASIC_LIBRARIES}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() -set(KLIB_REQUIRING algos hashtable join) +set(TEMPLATED_BASIC sparse) +foreach(LIB ${TEMPLATED_BASIC}) + add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cythonize_templates ${LIB} lib) + add_library(${LIB} SHARED ${LIB}.c) + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) +endforeach() + +set(KLIB_REQUIRING join) foreach(LIB ${KLIB_REQUIRING}) add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c) + add_dependencies(${LIB} khash) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() -set(KLIB_AND_TSLIBS_REQUIRING index interval) -foreach(LIB ${KLIB_AND_TSLIBS_REQUIRING}) - add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) +set(TEMPLATED_KLIB algos hashtable) +foreach(LIB ${TEMPLATED_KLIB_REQUIRING}) + add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cythonize_templates ${LIB} lib) + add_library(${LIB} SHARED ${LIB}.c) + add_dependencies(${LIB} khash) + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) +endforeach() + +set(TEMPLATED_KLIB_AND_TSLIBS_REQUIRING index interval) +foreach(LIB ${TEMPLATED_KLIB_AND_TSLIBS_REQUIRING}) + add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cythonize_templates ${LIB} lib) add_library(${LIB} SHARED ${LIB}.c) + add_dependencies(${LIB} khash) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() @@ -32,16 +58,15 @@ add_library(missing SHARED missing.c) target_include_directories(missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(missing PUBLIC ${Python_LIBRARIES}) -add_subdirectory("tslibs") -add_subdirectory("window") - add_custom_command(OUTPUT lib.c COMMAND python -m cython -3 lib.pyx) add_library(lib SHARED lib.c src/parser/tokenizer.c) +add_dependencies(lib khash) target_include_directories(lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) add_custom_command(OUTPUT parsers.c COMMAND python -m cython -3 parsers.pyx) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) +add_dependencies(parsers khash) target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) @@ -51,3 +76,6 @@ add_library(json SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c tslibs/src/datetime/np_datetime.c tslibs/src/datetime/np_datetime_strings.c) target_include_directories(json PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) target_link_libraries(json PUBLIC ${Python_LIBRARIES}) + +add_subdirectory("tslibs") +add_subdirectory("window") diff --git a/pandas/_libs/cythonize_templates.py b/pandas/_libs/cythonize_templates.py new file mode 100644 index 0000000000000..1f64a8ca75078 --- /dev/null +++ b/pandas/_libs/cythonize_templates.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import argparse +import pathlib +from Cython import Tempita +from Cython.Build import cythonize + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("target") + parser.add_argument("file_type") + parsed = parser.parse_args() + file_type = parsed.file_type + if file_type == "header": + suffix = "pxd" + elif file_type == "lib": + suffix = "pyx" + else: + raise ValueError() + + target = parsed.target + templates = pathlib.Path(".").glob(f"**/*{target}*.in") + for template in templates: + pyxcontent = Tempita.sub(open(template).read()) + with open(template.with_suffix(""), "w") as outfile: + outfile.write(pyxcontent) + + cythonize(f"{target}.{suffix}") diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index 1a4a23993f0bf..96522f3237e49 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,4 +1,4 @@ -add_custom_command(OUTPUT _sas.c COMMAND python -m cython -3 _sas.pyx) -add_library(_sas SHARED sas.c) -target_include_directories(_sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(_sas PUBLIC ${Python_LIBRARIES}) +add_custom_command(OUTPUT sas.c COMMAND python -m cython -3 sas.pyx) +add_library(sas SHARED sas.c) +target_include_directories(sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_link_libraries(sas PUBLIC ${Python_LIBRARIES}) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000..d82214acd5465 --- /dev/null +++ b/setup.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +import argparse +import pathlib +from Cython import Tempita + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("target_pyx") + target = parser.parse_args().target_pyx + + files = pathlib.Path("pandas/_libs").glob(f"**/*{target}*.in") + for file in files: + pyxcontent = Tempita.sub(open(file).read()) + with open(file.with_suffix(""), "w") as outfile: + outfile.write(pyxcontent) + breakpoint() + print('done') From 97530090a41602e64dfb4ab0f017e050af4e10cb Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 16 Jun 2022 22:18:01 -0700 Subject: [PATCH 17/68] better dependencies --- pandas/_libs/CMakeLists.txt | 56 +++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index d05a6a49b084b..9b5726b8c2ed4 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -1,4 +1,4 @@ -add_custom_target(khash COMMAND python -m cythonize_templates khash header) +add_custom_command(OUTPUT khash.h khash_for_primitive_helper.pxi COMMAND python -m cythonize_templates khash header) set(HEADERS arrays dtypes hashtable lib missing util) foreach(HEADER ${HEADERS}) @@ -13,40 +13,38 @@ foreach(LIB ${BASIC_LIBRARIES}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() -set(TEMPLATED_BASIC sparse) -foreach(LIB ${TEMPLATED_BASIC}) - add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cythonize_templates ${LIB} lib) - add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) -endforeach() +add_custom_command(OUTPUT sparse.c sparse_op_helper.pxi COMMAND python -m cythonize_templates sparse lib) +add_library(sparse SHARED sparse.c) +target_include_directories(sparse PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") +target_link_libraries(sparse PUBLIC ${Python_LIBRARIES}) set(KLIB_REQUIRING join) foreach(LIB ${KLIB_REQUIRING}) - add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) + add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx DEPENDS khash.h) add_library(${LIB} SHARED ${LIB}.c) - add_dependencies(${LIB} khash) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() -set(TEMPLATED_KLIB algos hashtable) -foreach(LIB ${TEMPLATED_KLIB_REQUIRING}) - add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cythonize_templates ${LIB} lib) - add_library(${LIB} SHARED ${LIB}.c) - add_dependencies(${LIB} khash) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) -endforeach() +add_custom_command(OUTPUT algos.c algos_common_helper.pxi algos_take_helper.pxi COMMAND python -m cythonize_templates algos lib DEPENDS khash.h) +add_library(algos SHARED algos.c) +target_include_directories(algos PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") +target_link_libraries(algos PUBLIC ${Python_LIBRARIES}) -set(TEMPLATED_KLIB_AND_TSLIBS_REQUIRING index interval) -foreach(LIB ${TEMPLATED_KLIB_AND_TSLIBS_REQUIRING}) - add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cythonize_templates ${LIB} lib) - add_library(${LIB} SHARED ${LIB}.c) - add_dependencies(${LIB} khash) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) -endforeach() +add_custom_command(OUTPUT hashtable.c hashtable_class_helper.pxi hashtable_func_helper.pxi COMMAND python -m cythonize_templates hashtable lib DEPENDS khash.h) +add_library(hashtable SHARED hashtable.c) +target_include_directories(hashtable PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") +target_link_libraries(hashtable PUBLIC ${Python_LIBRARIES}) + +add_custom_command(OUTPUT index.c index_class_helper.pxi COMMAND python -m cythonize_templates index lib DEPENDS khash.h) +add_library(index SHARED index.c) +target_include_directories(index PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") +target_link_libraries(index PUBLIC ${Python_LIBRARIES}) + +add_custom_command(OUTPUT interval.c intervaltree.pxi COMMAND python -m cythonize_templates interval lib DEPENDS khash.h) +add_library(interval SHARED interval.c) +target_include_directories(interval PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") +target_link_libraries(interval PUBLIC ${Python_LIBRARIES}) add_custom_command(OUTPUT tslib.c COMMAND python -m cython -3 tslib.pyx) add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) @@ -58,15 +56,13 @@ add_library(missing SHARED missing.c) target_include_directories(missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(missing PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT lib.c COMMAND python -m cython -3 lib.pyx) +add_custom_command(OUTPUT lib.c COMMAND python -m cython -3 lib.pyx DEPENDS khash.h) add_library(lib SHARED lib.c src/parser/tokenizer.c) -add_dependencies(lib khash) target_include_directories(lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT parsers.c COMMAND python -m cython -3 parsers.pyx) +add_custom_command(OUTPUT parsers.c COMMAND python -m cython -3 parsers.pyx DEPENDS khash.h) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) -add_dependencies(parsers khash) target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) From 42e9b6d6662f4f9bbb3e09b19dbf80dbe12c4a9b Mon Sep 17 00:00:00 2001 From: William Ayd Date: Fri, 17 Jun 2022 10:23:22 -0700 Subject: [PATCH 18/68] optimize template generation --- pandas/_libs/CMakeLists.txt | 86 +++++++++++++++++++---------- pandas/_libs/cythonize_templates.py | 29 ---------- 2 files changed, 56 insertions(+), 59 deletions(-) delete mode 100644 pandas/_libs/cythonize_templates.py diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 9b5726b8c2ed4..043275ba74dda 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -1,51 +1,77 @@ -add_custom_command(OUTPUT khash.h khash_for_primitive_helper.pxi COMMAND python -m cythonize_templates khash header) +add_custom_command(OUTPUT + algos_common_helper.pxi + algos_take_helper.pxi + hashtable_class_helper.pxi + hashtable_func_helper.pxi + index_class_helper.pxi + intervaltree.pxi + khash_for_primitive_helper.pxi + sparse_op_helper.pxi + COMMAND python generate_templates.py +) -set(HEADERS arrays dtypes hashtable lib missing util) -foreach(HEADER ${HEADERS}) - add_custom_command(OUTPUT ${HEADER}.h COMMAND python -m cython -3 ${HEADER}.pxd) -endforeach() - -set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_dispatch properties reshape testing writers) -foreach(LIB ${BASIC_LIBRARIES}) - add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) - add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) -endforeach() - -add_custom_command(OUTPUT sparse.c sparse_op_helper.pxi COMMAND python -m cythonize_templates sparse lib) -add_library(sparse SHARED sparse.c) -target_include_directories(sparse PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") -target_link_libraries(sparse PUBLIC ${Python_LIBRARIES}) +add_custom_command(OUTPUT khash.h + COMMAND python -m cython -3 khash.pxd + DEPENDS khash_for_primitive_helper.pxi +) -set(KLIB_REQUIRING join) -foreach(LIB ${KLIB_REQUIRING}) - add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx DEPENDS khash.h) - add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) -endforeach() - -add_custom_command(OUTPUT algos.c algos_common_helper.pxi algos_take_helper.pxi COMMAND python -m cythonize_templates algos lib DEPENDS khash.h) +add_custom_command(OUTPUT algos.c + COMMAND python -m cython -3 algos.pyx + DEPENDS algos_common_helper.pxi algos_take_helper.pxi +) add_library(algos SHARED algos.c) target_include_directories(algos PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(algos PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT hashtable.c hashtable_class_helper.pxi hashtable_func_helper.pxi COMMAND python -m cythonize_templates hashtable lib DEPENDS khash.h) +add_custom_command( + OUTPUT hashtable.c + COMMAND python -m cython -3 hashtable.pyx + DEPENDS hashtable_class_helper.pxi hashtable_func_helper.pxi +) + add_library(hashtable SHARED hashtable.c) target_include_directories(hashtable PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(hashtable PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT index.c index_class_helper.pxi COMMAND python -m cythonize_templates index lib DEPENDS khash.h) +add_custom_command( + OUTPUT index.c + COMMAND python -m cython -3 index.pyx + DEPENDS index_class_helper.pxi +) add_library(index SHARED index.c) target_include_directories(index PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(index PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT interval.c intervaltree.pxi COMMAND python -m cythonize_templates interval lib DEPENDS khash.h) +add_custom_command( + OUTPUT interval.c + COMMAND python -m cython -3 interval.pyx + DEPENDS intervaltree.pxi +) add_library(interval SHARED interval.c) target_include_directories(interval PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(interval PUBLIC ${Python_LIBRARIES}) +add_custom_command(OUTPUT sparse.c + COMMAND python -m cython -3 sparse.pyx + DEPENDS sparse_op_helper.pxi +) +add_library(sparse SHARED sparse.c) +target_include_directories(sparse PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") +target_link_libraries(sparse PUBLIC ${Python_LIBRARIES}) + +#set(HEADERS arrays dtypes hashtable lib missing util) +#foreach(HEADER ${HEADERS}) +# add_custom_command(OUTPUT ${HEADER}.h COMMAND python -m cython -3 ${HEADER}.pxd) +#endforeach() + +set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_dispatch properties reshape testing writers) +foreach(LIB ${BASIC_LIBRARIES}) + add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) + add_library(${LIB} SHARED ${LIB}.c) + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) + target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) +endforeach() + add_custom_command(OUTPUT tslib.c COMMAND python -m cython -3 tslib.pyx) add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") diff --git a/pandas/_libs/cythonize_templates.py b/pandas/_libs/cythonize_templates.py deleted file mode 100644 index 1f64a8ca75078..0000000000000 --- a/pandas/_libs/cythonize_templates.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import pathlib -from Cython import Tempita -from Cython.Build import cythonize - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("target") - parser.add_argument("file_type") - parsed = parser.parse_args() - file_type = parsed.file_type - if file_type == "header": - suffix = "pxd" - elif file_type == "lib": - suffix = "pyx" - else: - raise ValueError() - - target = parsed.target - templates = pathlib.Path(".").glob(f"**/*{target}*.in") - for template in templates: - pyxcontent = Tempita.sub(open(template).read()) - with open(template.with_suffix(""), "w") as outfile: - outfile.write(pyxcontent) - - cythonize(f"{target}.{suffix}") From 9d2f2509d7cde8072f6f688c176351a4f2eae5bd Mon Sep 17 00:00:00 2001 From: William Ayd Date: Fri, 17 Jun 2022 10:32:41 -0700 Subject: [PATCH 19/68] generation script --- pandas/_libs/generate_templates.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 pandas/_libs/generate_templates.py diff --git a/pandas/_libs/generate_templates.py b/pandas/_libs/generate_templates.py new file mode 100644 index 0000000000000..31a760318f7b5 --- /dev/null +++ b/pandas/_libs/generate_templates.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +import argparse +import pathlib +from Cython import Tempita +from Cython.Build import cythonize + + +if __name__ == "__main__": + for template in ( + "algos_common_helper.pxi.in", + "algos_take_helper.pxi.in", + "hashtable_class_helper.pxi.in", + "hashtable_func_helper.pxi.in", + "index_class_helper.pxi.in", + "intervaltree.pxi.in", + "khash_for_primitive_helper.pxi.in", + "sparse_op_helper.pxi.in", + ): + pyxcontent = Tempita.sub(open(template).read()) + with open(template.replace(".in", ""), "w") as outfile: + outfile.write(pyxcontent) From 8f2a49a6ef118c8fc508a0ce84b54464722c431f Mon Sep 17 00:00:00 2001 From: William Ayd Date: Fri, 17 Jun 2022 10:36:01 -0700 Subject: [PATCH 20/68] checkpoint --- pandas/_libs/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 043275ba74dda..3f1484348ec9c 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -72,6 +72,8 @@ foreach(LIB ${BASIC_LIBRARIES}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() +add_subdirectory("tslibs") + add_custom_command(OUTPUT tslib.c COMMAND python -m cython -3 tslib.pyx) add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") @@ -99,5 +101,4 @@ add_library(json SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c target_include_directories(json PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) target_link_libraries(json PUBLIC ${Python_LIBRARIES}) -add_subdirectory("tslibs") add_subdirectory("window") From 933d5b0359bad096c94cf6c11ec6c5492c2fe7da Mon Sep 17 00:00:00 2001 From: William Ayd Date: Fri, 17 Jun 2022 11:50:33 -0700 Subject: [PATCH 21/68] renamed json to ujson to avoid stdlib conflicts --- pandas/_libs/CMakeLists.txt | 6 +++--- pandas/io/json/_json.py | 2 +- pandas/tests/io/json/test_ujson.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 3f1484348ec9c..00f8983c163c3 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -94,11 +94,11 @@ add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) -add_library(json SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c +add_library(ujson SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c src/ujson/python/date_conversions.c src/ujson/python/JSONtoObj.c src/ujson/lib/ultrajsonenc.c src/ujson/lib/ultrajsondec.c tslibs/src/datetime/np_datetime.c tslibs/src/datetime/np_datetime_strings.c) -target_include_directories(json PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) -target_link_libraries(json PUBLIC ${Python_LIBRARIES}) +target_include_directories(ujson PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) +target_link_libraries(ujson PUBLIC ${Python_LIBRARIES}) add_subdirectory("window") diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index fbea7a71202eb..8eb277e41151f 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -16,7 +16,7 @@ import numpy as np -import pandas._libs.json as json +import pandas._libs.ujson as json from pandas._libs.tslibs import iNaT from pandas._typing import ( CompressionOptions, diff --git a/pandas/tests/io/json/test_ujson.py b/pandas/tests/io/json/test_ujson.py index e82a888f47388..49f5f91219685 100644 --- a/pandas/tests/io/json/test_ujson.py +++ b/pandas/tests/io/json/test_ujson.py @@ -12,7 +12,7 @@ import pytest import pytz -import pandas._libs.json as ujson +import pandas._libs.ujson as ujson from pandas.compat import ( IS64, is_platform_windows, From fa25fe1b0d66c9455a134139fb311aae075557ea Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 17 Jun 2022 12:08:47 -0700 Subject: [PATCH 22/68] added missing join build --- pandas/_libs/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 00f8983c163c3..ab41722eb4271 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -89,9 +89,14 @@ add_library(lib SHARED lib.c src/parser/tokenizer.c) target_include_directories(lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) +add_custom_command(OUTPUT join.c COMMAND python -m cython -3 join.pyx DEPENDS khash.h) +add_library(join SHARED join.c) +target_include_directories(join PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") +target_link_libraries(join PUBLIC ${Python_LIBRARIES}) + add_custom_command(OUTPUT parsers.c COMMAND python -m cython -3 parsers.pyx DEPENDS khash.h) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) -target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") +target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) add_library(ujson SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c From 496c0155b816d74e55f31a85d47506d34df07c12 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 17 Jun 2022 12:18:43 -0700 Subject: [PATCH 23/68] Working build --- pandas/_libs/CMakeLists.txt | 13 ++++--------- pandas/_libs/src/ujson/python/ujson.c | 2 +- pandas/io/excel/_odswriter.py | 2 +- pandas/io/excel/_xlsxwriter.py | 2 +- pandas/io/excel/_xlwt.py | 2 +- pandas/io/json/_table_schema.py | 2 +- 6 files changed, 9 insertions(+), 14 deletions(-) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index ab41722eb4271..b285116111a91 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -10,11 +10,6 @@ add_custom_command(OUTPUT COMMAND python generate_templates.py ) -add_custom_command(OUTPUT khash.h - COMMAND python -m cython -3 khash.pxd - DEPENDS khash_for_primitive_helper.pxi -) - add_custom_command(OUTPUT algos.c COMMAND python -m cython -3 algos.pyx DEPENDS algos_common_helper.pxi algos_take_helper.pxi @@ -84,19 +79,19 @@ add_library(missing SHARED missing.c) target_include_directories(missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(missing PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT lib.c COMMAND python -m cython -3 lib.pyx DEPENDS khash.h) +add_custom_command(OUTPUT lib.c COMMAND python -m cython -3 lib.pyx) add_library(lib SHARED lib.c src/parser/tokenizer.c) target_include_directories(lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT join.c COMMAND python -m cython -3 join.pyx DEPENDS khash.h) +add_custom_command(OUTPUT join.c COMMAND python -m cython -3 join.pyx) add_library(join SHARED join.c) target_include_directories(join PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(join PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT parsers.c COMMAND python -m cython -3 parsers.pyx DEPENDS khash.h) +add_custom_command(OUTPUT parsers.c COMMAND python -m cython -3 parsers.pyx) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) -target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) add_library(ujson SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c diff --git a/pandas/_libs/src/ujson/python/ujson.c b/pandas/_libs/src/ujson/python/ujson.c index def06cdf2db84..64d0605797222 100644 --- a/pandas/_libs/src/ujson/python/ujson.c +++ b/pandas/_libs/src/ujson/python/ujson.c @@ -74,7 +74,7 @@ static PyModuleDef moduledef = { }; -PyMODINIT_FUNC PyInit_json(void) { +PyMODINIT_FUNC PyInit_ujson(void) { import_array() initObjToJSON(); // TODO(username): clean up, maybe via tp_free? return PyModuleDef_Init(&moduledef); diff --git a/pandas/io/excel/_odswriter.py b/pandas/io/excel/_odswriter.py index f5367df6f228d..fadf6ea822100 100644 --- a/pandas/io/excel/_odswriter.py +++ b/pandas/io/excel/_odswriter.py @@ -9,7 +9,7 @@ cast, ) -import pandas._libs.json as json +import pandas._libs.ujson as json from pandas._typing import ( FilePath, StorageOptions, diff --git a/pandas/io/excel/_xlsxwriter.py b/pandas/io/excel/_xlsxwriter.py index 302d0281019f5..c7388f8c2fa3d 100644 --- a/pandas/io/excel/_xlsxwriter.py +++ b/pandas/io/excel/_xlsxwriter.py @@ -2,7 +2,7 @@ from typing import Any -import pandas._libs.json as json +import pandas._libs.ujson as json from pandas._typing import ( FilePath, StorageOptions, diff --git a/pandas/io/excel/_xlwt.py b/pandas/io/excel/_xlwt.py index 234d9e72de10d..538b5f2a2c5e3 100644 --- a/pandas/io/excel/_xlwt.py +++ b/pandas/io/excel/_xlwt.py @@ -7,7 +7,7 @@ cast, ) -import pandas._libs.json as json +import pandas._libs.ujson as json from pandas._typing import ( FilePath, StorageOptions, diff --git a/pandas/io/json/_table_schema.py b/pandas/io/json/_table_schema.py index c630f0d7613e0..5675b98b37bc2 100644 --- a/pandas/io/json/_table_schema.py +++ b/pandas/io/json/_table_schema.py @@ -12,7 +12,7 @@ ) import warnings -import pandas._libs.json as json +import pandas._libs.ujson as json from pandas._typing import ( DtypeObj, JSONSerializable, From 27fbd0afc44df3ddde1eea2bb2e35550a12586ab Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 17 Jun 2022 12:22:24 -0700 Subject: [PATCH 24/68] used detected python executable --- CMakeLists.txt | 2 +- pandas/_libs/CMakeLists.txt | 26 +++++++++++++------------- pandas/_libs/tslibs/CMakeLists.txt | 8 ++++---- pandas/_libs/window/CMakeLists.txt | 4 ++-- pandas/io/sas/CMakeLists.txt | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c4acbd88482ea..cf2e486ee2390 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_SHARED_LIBRARY_PREFIX "") set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9") -find_package(Python REQUIRED COMPONENTS Development NumPy) +find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}.so") add_compile_definitions(NPY_NO_DEPRECATED_API=0) add_subdirectory("pandas/_libs") diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index b285116111a91..78aa2a459d34c 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -7,11 +7,11 @@ add_custom_command(OUTPUT intervaltree.pxi khash_for_primitive_helper.pxi sparse_op_helper.pxi - COMMAND python generate_templates.py + COMMAND ${Python_EXECUTABLE} generate_templates.py ) add_custom_command(OUTPUT algos.c - COMMAND python -m cython -3 algos.pyx + COMMAND ${Python_EXECUTABLE} -m cython -3 algos.pyx DEPENDS algos_common_helper.pxi algos_take_helper.pxi ) add_library(algos SHARED algos.c) @@ -20,7 +20,7 @@ target_link_libraries(algos PUBLIC ${Python_LIBRARIES}) add_custom_command( OUTPUT hashtable.c - COMMAND python -m cython -3 hashtable.pyx + COMMAND ${Python_EXECUTABLE} -m cython -3 hashtable.pyx DEPENDS hashtable_class_helper.pxi hashtable_func_helper.pxi ) @@ -30,7 +30,7 @@ target_link_libraries(hashtable PUBLIC ${Python_LIBRARIES}) add_custom_command( OUTPUT index.c - COMMAND python -m cython -3 index.pyx + COMMAND ${Python_EXECUTABLE} -m cython -3 index.pyx DEPENDS index_class_helper.pxi ) add_library(index SHARED index.c) @@ -39,7 +39,7 @@ target_link_libraries(index PUBLIC ${Python_LIBRARIES}) add_custom_command( OUTPUT interval.c - COMMAND python -m cython -3 interval.pyx + COMMAND ${Python_EXECUTABLE} -m cython -3 interval.pyx DEPENDS intervaltree.pxi ) add_library(interval SHARED interval.c) @@ -47,7 +47,7 @@ target_include_directories(interval PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy target_link_libraries(interval PUBLIC ${Python_LIBRARIES}) add_custom_command(OUTPUT sparse.c - COMMAND python -m cython -3 sparse.pyx + COMMAND ${Python_EXECUTABLE} -m cython -3 sparse.pyx DEPENDS sparse_op_helper.pxi ) add_library(sparse SHARED sparse.c) @@ -56,12 +56,12 @@ target_link_libraries(sparse PUBLIC ${Python_LIBRARIES}) #set(HEADERS arrays dtypes hashtable lib missing util) #foreach(HEADER ${HEADERS}) -# add_custom_command(OUTPUT ${HEADER}.h COMMAND python -m cython -3 ${HEADER}.pxd) +# add_custom_command(OUTPUT ${HEADER}.h COMMAND ${Python_EXECUTABLE} -m cython -3 ${HEADER}.pxd) #endforeach() set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_dispatch properties reshape testing writers) foreach(LIB ${BASIC_LIBRARIES}) - add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) @@ -69,27 +69,27 @@ endforeach() add_subdirectory("tslibs") -add_custom_command(OUTPUT tslib.c COMMAND python -m cython -3 tslib.pyx) +add_custom_command(OUTPUT tslib.c COMMAND ${Python_EXECUTABLE} -m cython -3 tslib.pyx) add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(tslib PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT missing.c COMMAND python -m cython -3 missing.pyx) +add_custom_command(OUTPUT missing.c COMMAND ${Python_EXECUTABLE} -m cython -3 missing.pyx) add_library(missing SHARED missing.c) target_include_directories(missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(missing PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT lib.c COMMAND python -m cython -3 lib.pyx) +add_custom_command(OUTPUT lib.c COMMAND ${Python_EXECUTABLE} -m cython -3 lib.pyx) add_library(lib SHARED lib.c src/parser/tokenizer.c) target_include_directories(lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT join.c COMMAND python -m cython -3 join.pyx) +add_custom_command(OUTPUT join.c COMMAND ${Python_EXECUTABLE} -m cython -3 join.pyx) add_library(join SHARED join.c) target_include_directories(join PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(join PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT parsers.c COMMAND python -m cython -3 parsers.pyx) +add_custom_command(OUTPUT parsers.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsers.pyx) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index b87df0b0fcb7f..bb21e200a4e95 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -1,6 +1,6 @@ set(BASIC_LIBRARIES base ccalendar dtypes nattype strptime timezones) foreach(LIB ${BASIC_LIBRARIES}) - add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) @@ -8,18 +8,18 @@ endforeach() set(NP_DATETIME_REQUIRING conversion fields offsets period timedeltas timestamps tzconversion vectorized) foreach(LIB ${NP_DATETIME_REQUIRING}) - add_custom_command(OUTPUT ${LIB}.c COMMAND python -m cython -3 ${LIB}.pyx) + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c src/datetime/np_datetime.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() -add_custom_command(OUTPUT np_datetime.c COMMAND python -m cython -3 np_datetime.pyx) +add_custom_command(OUTPUT np_datetime.c COMMAND ${Python_EXECUTABLE} -m cython -3 np_datetime.pyx) add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c src/datetime/np_datetime_strings.c) target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(np_datetime PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT parsing.c COMMAND python -m cython -3 parsing.pyx) +add_custom_command(OUTPUT parsing.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsing.pyx) add_library(parsing SHARED parsing.c "../src/parser/tokenizer.c") target_include_directories(parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") target_link_libraries(parsing PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index 08eed8717ee8c..3d55260026c9d 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -1,9 +1,9 @@ -add_custom_command(OUTPUT indexers.c COMMAND python -m cython -3 indexers.pyx) +add_custom_command(OUTPUT indexers.c COMMAND ${Python_EXECUTABLE} -m cython -3 indexers.pyx) add_library(indexers SHARED indexers.c) target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(indexers PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT aggregations.cpp COMMAND python -m cython -3 --cplus aggregations.pyx) +add_custom_command(OUTPUT aggregations.cpp COMMAND ${Python_EXECUTABLE} -m cython -3 --cplus aggregations.pyx) add_library(aggregations SHARED aggregations.cpp) target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(aggregations PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index 96522f3237e49..fbe06b672ceea 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,4 +1,4 @@ -add_custom_command(OUTPUT sas.c COMMAND python -m cython -3 sas.pyx) +add_custom_command(OUTPUT sas.c COMMAND ${Python_EXECUTABLE} -m cython -3 sas.pyx) add_library(sas SHARED sas.c) target_include_directories(sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(sas PUBLIC ${Python_LIBRARIES}) From abdd163f9f1b84cc52902b9dea99a2948d4c75d3 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 17 Jun 2022 12:24:19 -0700 Subject: [PATCH 25/68] cmake-format usage --- pandas/_libs/CMakeLists.txt | 141 ++++++++++++++++++----------- pandas/_libs/tslibs/CMakeLists.txt | 41 ++++++--- pandas/_libs/window/CMakeLists.txt | 13 ++- pandas/io/sas/CMakeLists.txt | 6 +- 4 files changed, 131 insertions(+), 70 deletions(-) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 78aa2a459d34c..0d42d8b23ea55 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -1,104 +1,141 @@ -add_custom_command(OUTPUT - algos_common_helper.pxi - algos_take_helper.pxi - hashtable_class_helper.pxi - hashtable_func_helper.pxi - index_class_helper.pxi - intervaltree.pxi - khash_for_primitive_helper.pxi - sparse_op_helper.pxi - COMMAND ${Python_EXECUTABLE} generate_templates.py -) - -add_custom_command(OUTPUT algos.c +add_custom_command( + OUTPUT algos_common_helper.pxi + algos_take_helper.pxi + hashtable_class_helper.pxi + hashtable_func_helper.pxi + index_class_helper.pxi + intervaltree.pxi + khash_for_primitive_helper.pxi + sparse_op_helper.pxi + COMMAND ${Python_EXECUTABLE} generate_templates.py) + +add_custom_command( + OUTPUT algos.c COMMAND ${Python_EXECUTABLE} -m cython -3 algos.pyx - DEPENDS algos_common_helper.pxi algos_take_helper.pxi -) + DEPENDS algos_common_helper.pxi algos_take_helper.pxi) add_library(algos SHARED algos.c) -target_include_directories(algos PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") +target_include_directories(algos PUBLIC ${Python_INCLUDE_DIRS} + ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(algos PUBLIC ${Python_LIBRARIES}) add_custom_command( OUTPUT hashtable.c COMMAND ${Python_EXECUTABLE} -m cython -3 hashtable.pyx - DEPENDS hashtable_class_helper.pxi hashtable_func_helper.pxi -) + DEPENDS hashtable_class_helper.pxi hashtable_func_helper.pxi) add_library(hashtable SHARED hashtable.c) -target_include_directories(hashtable PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") +target_include_directories( + hashtable PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} + "src/klib") target_link_libraries(hashtable PUBLIC ${Python_LIBRARIES}) add_custom_command( OUTPUT index.c COMMAND ${Python_EXECUTABLE} -m cython -3 index.pyx - DEPENDS index_class_helper.pxi -) + DEPENDS index_class_helper.pxi) add_library(index SHARED index.c) -target_include_directories(index PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") +target_include_directories( + index PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" + "./tslibs") target_link_libraries(index PUBLIC ${Python_LIBRARIES}) add_custom_command( OUTPUT interval.c COMMAND ${Python_EXECUTABLE} -m cython -3 interval.pyx - DEPENDS intervaltree.pxi -) + DEPENDS intervaltree.pxi) add_library(interval SHARED interval.c) -target_include_directories(interval PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") +target_include_directories( + interval PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} + "src/klib" "./tslibs") target_link_libraries(interval PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT sparse.c +add_custom_command( + OUTPUT sparse.c COMMAND ${Python_EXECUTABLE} -m cython -3 sparse.pyx - DEPENDS sparse_op_helper.pxi -) + DEPENDS sparse_op_helper.pxi) add_library(sparse SHARED sparse.c) -target_include_directories(sparse PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") +target_include_directories( + sparse PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(sparse PUBLIC ${Python_LIBRARIES}) -#set(HEADERS arrays dtypes hashtable lib missing util) -#foreach(HEADER ${HEADERS}) -# add_custom_command(OUTPUT ${HEADER}.h COMMAND ${Python_EXECUTABLE} -m cython -3 ${HEADER}.pxd) -#endforeach() - -set(BASIC_LIBRARIES arrays groupby hashing indexing internals reduction ops ops_dispatch properties reshape testing writers) +# set(HEADERS arrays dtypes hashtable lib missing util) foreach(HEADER +# ${HEADERS}) add_custom_command(OUTPUT ${HEADER}.h COMMAND ${Python_EXECUTABLE} +# -m cython -3 ${HEADER}.pxd) endforeach() + +set(BASIC_LIBRARIES + arrays + groupby + hashing + indexing + internals + reduction + ops + ops_dispatch + properties + reshape + testing + writers) foreach(LIB ${BASIC_LIBRARIES}) - add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 ${LIB}.pyx) + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 + ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} + ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() add_subdirectory("tslibs") -add_custom_command(OUTPUT tslib.c COMMAND ${Python_EXECUTABLE} -m cython -3 tslib.pyx) +add_custom_command(OUTPUT tslib.c COMMAND ${Python_EXECUTABLE} -m cython -3 + tslib.pyx) add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) -target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") +target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} + ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(tslib PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT missing.c COMMAND ${Python_EXECUTABLE} -m cython -3 missing.pyx) +add_custom_command(OUTPUT missing.c COMMAND ${Python_EXECUTABLE} -m cython -3 + missing.pyx) add_library(missing SHARED missing.c) -target_include_directories(missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") +target_include_directories( + missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(missing PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT lib.c COMMAND ${Python_EXECUTABLE} -m cython -3 lib.pyx) +add_custom_command(OUTPUT lib.c COMMAND ${Python_EXECUTABLE} -m cython -3 + lib.pyx) add_library(lib SHARED lib.c src/parser/tokenizer.c) -target_include_directories(lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") +target_include_directories( + lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" + "./tslibs") target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT join.c COMMAND ${Python_EXECUTABLE} -m cython -3 join.pyx) +add_custom_command(OUTPUT join.c COMMAND ${Python_EXECUTABLE} -m cython -3 + join.pyx) add_library(join SHARED join.c) -target_include_directories(join PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") +target_include_directories(join PUBLIC ${Python_INCLUDE_DIRS} + ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(join PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT parsers.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsers.pyx) +add_custom_command(OUTPUT parsers.c COMMAND ${Python_EXECUTABLE} -m cython -3 + parsers.pyx) add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) -target_include_directories(parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") +target_include_directories( + parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" + "src") target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) -add_library(ujson SHARED src/ujson/python/ujson.c src/ujson/python/objToJSON.c - src/ujson/python/date_conversions.c src/ujson/python/JSONtoObj.c - src/ujson/lib/ultrajsonenc.c src/ujson/lib/ultrajsondec.c - tslibs/src/datetime/np_datetime.c tslibs/src/datetime/np_datetime_strings.c) -target_include_directories(ujson PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) +add_library( + ujson SHARED + src/ujson/python/ujson.c + src/ujson/python/objToJSON.c + src/ujson/python/date_conversions.c + src/ujson/python/JSONtoObj.c + src/ujson/lib/ultrajsonenc.c + src/ujson/lib/ultrajsondec.c + tslibs/src/datetime/np_datetime.c + tslibs/src/datetime/np_datetime_strings.c) +target_include_directories( + ujson PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} + src/ujson/python src/ujson/lib src/datetime) target_link_libraries(ujson PUBLIC ${Python_LIBRARIES}) add_subdirectory("window") diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index bb21e200a4e95..023daf10b028b 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -1,27 +1,44 @@ set(BASIC_LIBRARIES base ccalendar dtypes nattype strptime timezones) foreach(LIB ${BASIC_LIBRARIES}) - add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 ${LIB}.pyx) + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 + ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) + target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} + ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() -set(NP_DATETIME_REQUIRING conversion fields offsets period timedeltas timestamps tzconversion vectorized) +set(NP_DATETIME_REQUIRING + conversion + fields + offsets + period + timedeltas + timestamps + tzconversion + vectorized) foreach(LIB ${NP_DATETIME_REQUIRING}) - add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 ${LIB}.pyx) + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 + ${LIB}.pyx) add_library(${LIB} SHARED ${LIB}.c src/datetime/np_datetime.c) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") + target_include_directories( + ${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} + "src/datetime") target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) endforeach() -add_custom_command(OUTPUT np_datetime.c COMMAND ${Python_EXECUTABLE} -m cython -3 np_datetime.pyx) -add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c src/datetime/np_datetime_strings.c) -target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +add_custom_command(OUTPUT np_datetime.c COMMAND ${Python_EXECUTABLE} -m cython + -3 np_datetime.pyx) +add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c + src/datetime/np_datetime_strings.c) +target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} + ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(np_datetime PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT parsing.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsing.pyx) +add_custom_command(OUTPUT parsing.c COMMAND ${Python_EXECUTABLE} -m cython -3 + parsing.pyx) add_library(parsing SHARED parsing.c "../src/parser/tokenizer.c") -target_include_directories(parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") +target_include_directories( + parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} + "../src/klib") target_link_libraries(parsing PUBLIC ${Python_LIBRARIES}) - - diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index 3d55260026c9d..b8877be98a6fd 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -1,9 +1,14 @@ -add_custom_command(OUTPUT indexers.c COMMAND ${Python_EXECUTABLE} -m cython -3 indexers.pyx) +add_custom_command(OUTPUT indexers.c COMMAND ${Python_EXECUTABLE} -m cython -3 + indexers.pyx) add_library(indexers SHARED indexers.c) -target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} + ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(indexers PUBLIC ${Python_LIBRARIES}) -add_custom_command(OUTPUT aggregations.cpp COMMAND ${Python_EXECUTABLE} -m cython -3 --cplus aggregations.pyx) +add_custom_command( + OUTPUT aggregations.cpp COMMAND ${Python_EXECUTABLE} -m cython -3 --cplus + aggregations.pyx) add_library(aggregations SHARED aggregations.cpp) -target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} + ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(aggregations PUBLIC ${Python_LIBRARIES}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index fbe06b672ceea..4ac13980bb6c2 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,4 +1,6 @@ -add_custom_command(OUTPUT sas.c COMMAND ${Python_EXECUTABLE} -m cython -3 sas.pyx) +add_custom_command(OUTPUT sas.c COMMAND ${Python_EXECUTABLE} -m cython -3 + sas.pyx) add_library(sas SHARED sas.c) -target_include_directories(sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_include_directories(sas PUBLIC ${Python_INCLUDE_DIRS} + ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(sas PUBLIC ${Python_LIBRARIES}) From 3e714e1ef32db30b97daa072c4162e7dc01c15de Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 17 Jun 2022 12:36:03 -0700 Subject: [PATCH 26/68] CI updates --- .github/actions/build_pandas/action.yml | 3 ++- .github/workflows/32-bit-linux.yml | 3 ++- CMakeLists.txt | 4 +++- Dockerfile | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 39d5998b4ee74..178a882e72891 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -12,7 +12,8 @@ runs: - name: Build Pandas run: | - python setup.py build_ext -j $N_JOBS + cmake . + cmake --build . --parallel python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index shell: bash -el {0} env: diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index be894e6a5a63e..d079eb30a7021 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -36,7 +36,8 @@ jobs: . ~/virtualenvs/pandas-dev/bin/activate && \ python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ - python setup.py build_ext -q -j2 && \ + cmake . && \ + cmake --build . --parallel python -m pip install --no-build-isolation --no-use-pep517 -e . && \ export PANDAS_CI=1 && \ pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" diff --git a/CMakeLists.txt b/CMakeLists.txt index cf2e486ee2390..04f03aff3f391 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,12 @@ cmake_minimum_required(VERSION 3.17) -project(pandas VERSION 1.4) +project(pandas) set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED True) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_SOURCE_DIR .) +set(CMAKE_BINARY_DIR .) set(CMAKE_SHARED_LIBRARY_PREFIX "") set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9") diff --git a/Dockerfile b/Dockerfile index 650ba14271092..8fd0bc0b927f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,5 +49,6 @@ RUN . /opt/conda/etc/profile.d/conda.sh \ && conda activate base \ && cd "$pandas_home" \ && export \ - && python setup.py build_ext -j 4 \ + && cmake . \ + && cmake --build . --parallel \ && python -m pip install --no-build-isolation -e . From 0dec991fea0462b01613861004536d7860fa1bb7 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 17 Jun 2022 12:39:00 -0700 Subject: [PATCH 27/68] spaces not tabs --- .github/actions/build_pandas/action.yml | 2 +- .github/workflows/32-bit-linux.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 178a882e72891..1343cc806a5c1 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -13,7 +13,7 @@ runs: - name: Build Pandas run: | cmake . - cmake --build . --parallel + cmake --build . --parallel python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index shell: bash -el {0} env: diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index d079eb30a7021..aa6c3d555ad87 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -36,8 +36,8 @@ jobs: . ~/virtualenvs/pandas-dev/bin/activate && \ python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ - cmake . && \ - cmake --build . --parallel + cmake . && \ + cmake --build . --parallel python -m pip install --no-build-isolation --no-use-pep517 -e . && \ export PANDAS_CI=1 && \ pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" From 40973fc68e2a840992859765eaa35fe6f9714648 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 17 Jun 2022 13:13:43 -0700 Subject: [PATCH 28/68] removed setup completely --- setup.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index d82214acd5465..0000000000000 --- a/setup.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import pathlib -from Cython import Tempita - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("target_pyx") - target = parser.parse_args().target_pyx - - files = pathlib.Path("pandas/_libs").glob(f"**/*{target}*.in") - for file in files: - pyxcontent = Tempita.sub(open(file).read()) - with open(file.with_suffix(""), "w") as outfile: - outfile.write(pyxcontent) - breakpoint() - print('done') From 2881bbce849f88418d8306fafd929f2fbc4cd5c3 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 17 Jun 2022 13:27:12 -0700 Subject: [PATCH 29/68] remove pep-517 exclusion --- .github/actions/build_pandas/action.yml | 2 +- .github/workflows/32-bit-linux.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 1343cc806a5c1..4a92c13c5ccdb 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -14,7 +14,7 @@ runs: run: | cmake . cmake --build . --parallel - python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index + python -m pip install -e . --no-build-isolation --no-index shell: bash -el {0} env: # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index aa6c3d555ad87..a58ba031e8877 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -38,7 +38,7 @@ jobs: pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ cmake . && \ cmake --build . --parallel - python -m pip install --no-build-isolation --no-use-pep517 -e . && \ + python -m pip install --no-build-isolation -e . && \ export PANDAS_CI=1 && \ pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" From 5181d993bdb0f3671b9d8c41906682399641082c Mon Sep 17 00:00:00 2001 From: William Ayd Date: Fri, 17 Jun 2022 17:48:30 -0700 Subject: [PATCH 30/68] added python link options --- CMakeLists.txt | 1 - pandas/_libs/CMakeLists.txt | 26 ++++++++++++++++++++++++-- pandas/_libs/tslibs/CMakeLists.txt | 4 ++++ pandas/_libs/window/CMakeLists.txt | 2 ++ pandas/io/sas/CMakeLists.txt | 1 + 5 files changed, 31 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 04f03aff3f391..cb799beabab36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,6 @@ set(CMAKE_SOURCE_DIR .) set(CMAKE_BINARY_DIR .) set(CMAKE_SHARED_LIBRARY_PREFIX "") -set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9") find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}.so") diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 0d42d8b23ea55..d7aa256f96209 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -17,6 +17,17 @@ add_library(algos SHARED algos.c) target_include_directories(algos PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(algos PUBLIC ${Python_LIBRARIES}) +target_link_options(algos PUBLIC ${Python_LINK_OPTIONS}) + +# There is a khash header file in src/klib and a cython generated +# one in _libs. Depending on the build timing one of the other could +# get picked up, though unclear why we need both? If we stick to the +# non-generated version we can remove any DEPENDS khash.h +add_custom_command( + OUTPUT khash.h + COMMAND ${Python_EXECUTABLE} -m cython -3 khash.pxd + DEPENDS hkash_for_primitive_helper.pxi +) add_custom_command( OUTPUT hashtable.c @@ -26,8 +37,9 @@ add_custom_command( add_library(hashtable SHARED hashtable.c) target_include_directories( hashtable PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} - "src/klib") + "src/klib" DEPENDS khash.h) target_link_libraries(hashtable PUBLIC ${Python_LIBRARIES}) +target_link_options(hashtable PUBLIC ${Python_LINK_OPTIONS}) add_custom_command( OUTPUT index.c @@ -38,6 +50,7 @@ target_include_directories( index PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(index PUBLIC ${Python_LIBRARIES}) +target_link_options(index PUBLIC ${Python_LINK_OPTIONS}) add_custom_command( OUTPUT interval.c @@ -48,6 +61,7 @@ target_include_directories( interval PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(interval PUBLIC ${Python_LIBRARIES}) +target_link_options(interval PUBLIC ${Python_LINK_OPTIONS}) add_custom_command( OUTPUT sparse.c @@ -57,6 +71,7 @@ add_library(sparse SHARED sparse.c) target_include_directories( sparse PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(sparse PUBLIC ${Python_LIBRARIES}) +target_link_options(sparse PUBLIC ${Python_LINK_OPTIONS}) # set(HEADERS arrays dtypes hashtable lib missing util) foreach(HEADER # ${HEADERS}) add_custom_command(OUTPUT ${HEADER}.h COMMAND ${Python_EXECUTABLE} @@ -82,6 +97,7 @@ foreach(LIB ${BASIC_LIBRARIES}) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) + target_link_options(${LIB} PUBLIC ${Python_LINK_OPTIONS}) endforeach() add_subdirectory("tslibs") @@ -92,6 +108,7 @@ add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(tslib PUBLIC ${Python_LIBRARIES}) +target_link_options(tslib PUBLIC ${Python_LINK_OPTIONS}) add_custom_command(OUTPUT missing.c COMMAND ${Python_EXECUTABLE} -m cython -3 missing.pyx) @@ -99,6 +116,7 @@ add_library(missing SHARED missing.c) target_include_directories( missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") target_link_libraries(missing PUBLIC ${Python_LIBRARIES}) +target_link_options(missing PUBLIC ${Python_LINK_OPTIONS}) add_custom_command(OUTPUT lib.c COMMAND ${Python_EXECUTABLE} -m cython -3 lib.pyx) @@ -107,13 +125,15 @@ target_include_directories( lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) +target_link_options(lib PUBLIC ${Python_LINK_OPTIONS}) add_custom_command(OUTPUT join.c COMMAND ${Python_EXECUTABLE} -m cython -3 - join.pyx) + join.pyx DEPENDS khash_for_primitive_helper.pxi) add_library(join SHARED join.c) target_include_directories(join PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") target_link_libraries(join PUBLIC ${Python_LIBRARIES}) +target_link_options(join PUBLIC ${Python_LINK_OPTIONS}) add_custom_command(OUTPUT parsers.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsers.pyx) @@ -122,6 +142,7 @@ target_include_directories( parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) +target_link_options(parsers PUBLIC ${Python_LINK_OPTIONS}) add_library( ujson SHARED @@ -137,5 +158,6 @@ target_include_directories( ujson PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) target_link_libraries(ujson PUBLIC ${Python_LIBRARIES}) +target_link_options(ujson PUBLIC ${Python_LINK_OPTIONS}) add_subdirectory("window") diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index 023daf10b028b..581f66114d32b 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -6,6 +6,7 @@ foreach(LIB ${BASIC_LIBRARIES}) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) + target_link_options(${LIB} PUBLIC ${Python_LINK_OPTIONS}) endforeach() set(NP_DATETIME_REQUIRING @@ -25,6 +26,7 @@ foreach(LIB ${NP_DATETIME_REQUIRING}) ${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) + target_link_options(${LIB} PUBLIC ${Python_LINK_OPTIONS}) endforeach() add_custom_command(OUTPUT np_datetime.c COMMAND ${Python_EXECUTABLE} -m cython @@ -34,6 +36,7 @@ add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(np_datetime PUBLIC ${Python_LIBRARIES}) +target_link_options(np_datetime PUBLIC ${Python_LINK_OPTIONS}) add_custom_command(OUTPUT parsing.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsing.pyx) @@ -42,3 +45,4 @@ target_include_directories( parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") target_link_libraries(parsing PUBLIC ${Python_LIBRARIES}) +target_link_options(parsing PUBLIC ${Python_LINK_OPTIONS}) diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index b8877be98a6fd..b8045bdb8de1e 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -4,6 +4,7 @@ add_library(indexers SHARED indexers.c) target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(indexers PUBLIC ${Python_LIBRARIES}) +target_link_options(indexers PUBLIC ${Python_LINK_OPTIONS}) add_custom_command( OUTPUT aggregations.cpp COMMAND ${Python_EXECUTABLE} -m cython -3 --cplus @@ -12,3 +13,4 @@ add_library(aggregations SHARED aggregations.cpp) target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(aggregations PUBLIC ${Python_LIBRARIES}) +target_link_options(aggregations PUBLIC ${Python_LINK_OPTIONS}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index 4ac13980bb6c2..be1b14da7fc51 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -4,3 +4,4 @@ add_library(sas SHARED sas.c) target_include_directories(sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(sas PUBLIC ${Python_LIBRARIES}) +target_link_options(sas PUBLIC ${Python_LINK_OPTIONS}) From 73162a0ae2e8b29a3bb2af7a74284137b6891795 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Fri, 17 Jun 2022 18:55:50 -0700 Subject: [PATCH 31/68] removed public linkage --- pandas/_libs/CMakeLists.txt | 36 ++++++++++-------------------- pandas/_libs/tslibs/CMakeLists.txt | 12 ++++------ pandas/_libs/window/CMakeLists.txt | 6 ++--- pandas/io/sas/CMakeLists.txt | 3 +-- 4 files changed, 19 insertions(+), 38 deletions(-) diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index d7aa256f96209..b637d21a1ba42 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -16,8 +16,7 @@ add_custom_command( add_library(algos SHARED algos.c) target_include_directories(algos PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") -target_link_libraries(algos PUBLIC ${Python_LIBRARIES}) -target_link_options(algos PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(algos ${Python_LIBRARIES}) # There is a khash header file in src/klib and a cython generated # one in _libs. Depending on the build timing one of the other could @@ -38,8 +37,7 @@ add_library(hashtable SHARED hashtable.c) target_include_directories( hashtable PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" DEPENDS khash.h) -target_link_libraries(hashtable PUBLIC ${Python_LIBRARIES}) -target_link_options(hashtable PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(hashtable ${Python_LIBRARIES}) add_custom_command( OUTPUT index.c @@ -49,8 +47,7 @@ add_library(index SHARED index.c) target_include_directories( index PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(index PUBLIC ${Python_LIBRARIES}) -target_link_options(index PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(index ${Python_LIBRARIES}) add_custom_command( OUTPUT interval.c @@ -60,8 +57,7 @@ add_library(interval SHARED interval.c) target_include_directories( interval PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(interval PUBLIC ${Python_LIBRARIES}) -target_link_options(interval PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(interval ${Python_LIBRARIES}) add_custom_command( OUTPUT sparse.c @@ -70,8 +66,7 @@ add_custom_command( add_library(sparse SHARED sparse.c) target_include_directories( sparse PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") -target_link_libraries(sparse PUBLIC ${Python_LIBRARIES}) -target_link_options(sparse PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(sparse ${Python_LIBRARIES}) # set(HEADERS arrays dtypes hashtable lib missing util) foreach(HEADER # ${HEADERS}) add_custom_command(OUTPUT ${HEADER}.h COMMAND ${Python_EXECUTABLE} @@ -96,8 +91,7 @@ foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) - target_link_options(${LIB} PUBLIC ${Python_LINK_OPTIONS}) + target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() add_subdirectory("tslibs") @@ -107,16 +101,14 @@ add_custom_command(OUTPUT tslib.c COMMAND ${Python_EXECUTABLE} -m cython -3 add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") -target_link_libraries(tslib PUBLIC ${Python_LIBRARIES}) -target_link_options(tslib PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(tslib ${Python_LIBRARIES}) add_custom_command(OUTPUT missing.c COMMAND ${Python_EXECUTABLE} -m cython -3 missing.pyx) add_library(missing SHARED missing.c) target_include_directories( missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") -target_link_libraries(missing PUBLIC ${Python_LIBRARIES}) -target_link_options(missing PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(missing ${Python_LIBRARIES}) add_custom_command(OUTPUT lib.c COMMAND ${Python_EXECUTABLE} -m cython -3 lib.pyx) @@ -124,16 +116,14 @@ add_library(lib SHARED lib.c src/parser/tokenizer.c) target_include_directories( lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(lib PUBLIC ${Python_LIBRARIES}) -target_link_options(lib PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(lib ${Python_LIBRARIES}) add_custom_command(OUTPUT join.c COMMAND ${Python_EXECUTABLE} -m cython -3 join.pyx DEPENDS khash_for_primitive_helper.pxi) add_library(join SHARED join.c) target_include_directories(join PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") -target_link_libraries(join PUBLIC ${Python_LIBRARIES}) -target_link_options(join PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(join ${Python_LIBRARIES}) add_custom_command(OUTPUT parsers.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsers.pyx) @@ -141,8 +131,7 @@ add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) target_include_directories( parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") -target_link_libraries(parsers PUBLIC ${Python_LIBRARIES}) -target_link_options(parsers PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(parsers ${Python_LIBRARIES}) add_library( ujson SHARED @@ -157,7 +146,6 @@ add_library( target_include_directories( ujson PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) -target_link_libraries(ujson PUBLIC ${Python_LIBRARIES}) -target_link_options(ujson PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(ujson ${Python_LIBRARIES}) add_subdirectory("window") diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index 581f66114d32b..3583704768141 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -5,8 +5,7 @@ foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) - target_link_options(${LIB} PUBLIC ${Python_LINK_OPTIONS}) + target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() set(NP_DATETIME_REQUIRING @@ -25,8 +24,7 @@ foreach(LIB ${NP_DATETIME_REQUIRING}) target_include_directories( ${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") - target_link_libraries(${LIB} PUBLIC ${Python_LIBRARIES}) - target_link_options(${LIB} PUBLIC ${Python_LINK_OPTIONS}) + target_link_libraries(${LIB} ${Python_LIBRARIES}) endforeach() add_custom_command(OUTPUT np_datetime.c COMMAND ${Python_EXECUTABLE} -m cython @@ -35,8 +33,7 @@ add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c src/datetime/np_datetime_strings.c) target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(np_datetime PUBLIC ${Python_LIBRARIES}) -target_link_options(np_datetime PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(np_datetime ${Python_LIBRARIES}) add_custom_command(OUTPUT parsing.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsing.pyx) @@ -44,5 +41,4 @@ add_library(parsing SHARED parsing.c "../src/parser/tokenizer.c") target_include_directories( parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") -target_link_libraries(parsing PUBLIC ${Python_LIBRARIES}) -target_link_options(parsing PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(parsing ${Python_LIBRARIES}) diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index b8045bdb8de1e..8f39db2a7e0d9 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -3,8 +3,7 @@ add_custom_command(OUTPUT indexers.c COMMAND ${Python_EXECUTABLE} -m cython -3 add_library(indexers SHARED indexers.c) target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(indexers PUBLIC ${Python_LIBRARIES}) -target_link_options(indexers PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(indexers ${Python_LIBRARIES}) add_custom_command( OUTPUT aggregations.cpp COMMAND ${Python_EXECUTABLE} -m cython -3 --cplus @@ -12,5 +11,4 @@ add_custom_command( add_library(aggregations SHARED aggregations.cpp) target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(aggregations PUBLIC ${Python_LIBRARIES}) -target_link_options(aggregations PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(aggregations ${Python_LIBRARIES}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index be1b14da7fc51..443c0a5a3c54b 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -3,5 +3,4 @@ add_custom_command(OUTPUT sas.c COMMAND ${Python_EXECUTABLE} -m cython -3 add_library(sas SHARED sas.c) target_include_directories(sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(sas PUBLIC ${Python_LIBRARIES}) -target_link_options(sas PUBLIC ${Python_LINK_OPTIONS}) +target_link_libraries(sas ${Python_LIBRARIES}) From 9f0dc3ad1595e82c32ec66ef41a7359522b9a8d3 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Fri, 17 Jun 2022 19:42:18 -0700 Subject: [PATCH 32/68] LINK_LIBRARIES for macOS --- CMakeLists.txt | 2 ++ pandas/_libs/CMakeLists.txt | 24 ++++++++++++------------ pandas/_libs/tslibs/CMakeLists.txt | 8 ++++---- pandas/_libs/window/CMakeLists.txt | 4 ++-- pandas/io/sas/CMakeLists.txt | 2 +- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cb799beabab36..21caf91a39a57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,8 @@ set(CMAKE_SOURCE_DIR .) set(CMAKE_BINARY_DIR .) set(CMAKE_SHARED_LIBRARY_PREFIX "") +# for macOS +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup") find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}.so") diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index b637d21a1ba42..1be12e4a34d9c 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -16,7 +16,7 @@ add_custom_command( add_library(algos SHARED algos.c) target_include_directories(algos PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") -target_link_libraries(algos ${Python_LIBRARIES}) +target_link_libraries(algos LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) # There is a khash header file in src/klib and a cython generated # one in _libs. Depending on the build timing one of the other could @@ -37,7 +37,7 @@ add_library(hashtable SHARED hashtable.c) target_include_directories( hashtable PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" DEPENDS khash.h) -target_link_libraries(hashtable ${Python_LIBRARIES}) +target_link_libraries(hashtable LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command( OUTPUT index.c @@ -47,7 +47,7 @@ add_library(index SHARED index.c) target_include_directories( index PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(index ${Python_LIBRARIES}) +target_link_libraries(index LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command( OUTPUT interval.c @@ -57,7 +57,7 @@ add_library(interval SHARED interval.c) target_include_directories( interval PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(interval ${Python_LIBRARIES}) +target_link_libraries(interval LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command( OUTPUT sparse.c @@ -66,7 +66,7 @@ add_custom_command( add_library(sparse SHARED sparse.c) target_include_directories( sparse PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") -target_link_libraries(sparse ${Python_LIBRARIES}) +target_link_libraries(sparse LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) # set(HEADERS arrays dtypes hashtable lib missing util) foreach(HEADER # ${HEADERS}) add_custom_command(OUTPUT ${HEADER}.h COMMAND ${Python_EXECUTABLE} @@ -91,7 +91,7 @@ foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_link_libraries(${LIB} LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) endforeach() add_subdirectory("tslibs") @@ -101,14 +101,14 @@ add_custom_command(OUTPUT tslib.c COMMAND ${Python_EXECUTABLE} -m cython -3 add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") -target_link_libraries(tslib ${Python_LIBRARIES}) +target_link_libraries(tslib LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command(OUTPUT missing.c COMMAND ${Python_EXECUTABLE} -m cython -3 missing.pyx) add_library(missing SHARED missing.c) target_include_directories( missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") -target_link_libraries(missing ${Python_LIBRARIES}) +target_link_libraries(missing LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command(OUTPUT lib.c COMMAND ${Python_EXECUTABLE} -m cython -3 lib.pyx) @@ -116,14 +116,14 @@ add_library(lib SHARED lib.c src/parser/tokenizer.c) target_include_directories( lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(lib ${Python_LIBRARIES}) +target_link_libraries(lib LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command(OUTPUT join.c COMMAND ${Python_EXECUTABLE} -m cython -3 join.pyx DEPENDS khash_for_primitive_helper.pxi) add_library(join SHARED join.c) target_include_directories(join PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") -target_link_libraries(join ${Python_LIBRARIES}) +target_link_libraries(join LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command(OUTPUT parsers.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsers.pyx) @@ -131,7 +131,7 @@ add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) target_include_directories( parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") -target_link_libraries(parsers ${Python_LIBRARIES}) +target_link_libraries(parsers LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_library( ujson SHARED @@ -146,6 +146,6 @@ add_library( target_include_directories( ujson PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) -target_link_libraries(ujson ${Python_LIBRARIES}) +target_link_libraries(ujson LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_subdirectory("window") diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index 3583704768141..a32a2ee79c84f 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -5,7 +5,7 @@ foreach(LIB ${BASIC_LIBRARIES}) add_library(${LIB} SHARED ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_link_libraries(${LIB} LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) endforeach() set(NP_DATETIME_REQUIRING @@ -24,7 +24,7 @@ foreach(LIB ${NP_DATETIME_REQUIRING}) target_include_directories( ${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") - target_link_libraries(${LIB} ${Python_LIBRARIES}) + target_link_libraries(${LIB} LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) endforeach() add_custom_command(OUTPUT np_datetime.c COMMAND ${Python_EXECUTABLE} -m cython @@ -33,7 +33,7 @@ add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c src/datetime/np_datetime_strings.c) target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(np_datetime ${Python_LIBRARIES}) +target_link_libraries(np_datetime LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command(OUTPUT parsing.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsing.pyx) @@ -41,4 +41,4 @@ add_library(parsing SHARED parsing.c "../src/parser/tokenizer.c") target_include_directories( parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") -target_link_libraries(parsing ${Python_LIBRARIES}) +target_link_libraries(parsing LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index 8f39db2a7e0d9..9cff7232ee41c 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -3,7 +3,7 @@ add_custom_command(OUTPUT indexers.c COMMAND ${Python_EXECUTABLE} -m cython -3 add_library(indexers SHARED indexers.c) target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(indexers ${Python_LIBRARIES}) +target_link_libraries(indexers LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command( OUTPUT aggregations.cpp COMMAND ${Python_EXECUTABLE} -m cython -3 --cplus @@ -11,4 +11,4 @@ add_custom_command( add_library(aggregations SHARED aggregations.cpp) target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(aggregations ${Python_LIBRARIES}) +target_link_libraries(aggregations LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index 443c0a5a3c54b..1bda7e2e63156 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -3,4 +3,4 @@ add_custom_command(OUTPUT sas.c COMMAND ${Python_EXECUTABLE} -m cython -3 add_library(sas SHARED sas.c) target_include_directories(sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(sas ${Python_LIBRARIES}) +target_link_libraries(sas LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) From 0983ef7515ebdb3130b329bd973e26b0b4effe8b Mon Sep 17 00:00:00 2001 From: William Ayd Date: Fri, 17 Jun 2022 23:53:39 -0700 Subject: [PATCH 33/68] apple compat --- CMakeLists.txt | 5 ++++- pandas/io/sas/CMakeLists.txt | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 21caf91a39a57..ebb7db4174db0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,9 +10,12 @@ set(CMAKE_BINARY_DIR .) set(CMAKE_SHARED_LIBRARY_PREFIX "") # for macOS -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup") +if (APPLE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup") +endif() find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) +message("${Python_LINK_OPTIONS}") set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}.so") add_compile_definitions(NPY_NO_DEPRECATED_API=0) add_subdirectory("pandas/_libs") diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index 1bda7e2e63156..7b0691de1e6d3 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,6 +1,6 @@ add_custom_command(OUTPUT sas.c COMMAND ${Python_EXECUTABLE} -m cython -3 sas.pyx) -add_library(sas SHARED sas.c) -target_include_directories(sas PUBLIC ${Python_INCLUDE_DIRS} +add_library(_sas SHARED sas.c) +target_include_directories(_sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(sas LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) +target_link_libraries(_sas LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) From 3328dd6c931d43879f3798c5d9dfb9779e4fb939 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 Jun 2022 09:23:41 -0700 Subject: [PATCH 34/68] fixed sas import --- pandas/io/sas/CMakeLists.txt | 6 +++--- pandas/io/sas/{sas.pyx => _sas.pyx} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename pandas/io/sas/{sas.pyx => _sas.pyx} (100%) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index 7b0691de1e6d3..30cc5dcef338e 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,6 +1,6 @@ -add_custom_command(OUTPUT sas.c COMMAND ${Python_EXECUTABLE} -m cython -3 - sas.pyx) -add_library(_sas SHARED sas.c) +add_custom_command(OUTPUT _sas.c COMMAND ${Python_EXECUTABLE} -m cython -3 + _sas.pyx) +add_library(_sas SHARED _sas.c) target_include_directories(_sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) target_link_libraries(_sas LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) diff --git a/pandas/io/sas/sas.pyx b/pandas/io/sas/_sas.pyx similarity index 100% rename from pandas/io/sas/sas.pyx rename to pandas/io/sas/_sas.pyx From 3e973bb7475b8cbd2b1653efc0961cb89a4679da Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 Jun 2022 09:35:25 -0700 Subject: [PATCH 35/68] fixed macOS / linux lib names --- CMakeLists.txt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ebb7db4174db0..d3d61a2e02d99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,22 +1,24 @@ cmake_minimum_required(VERSION 3.17) -project(pandas) +set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9") set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED True) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_SOURCE_DIR .) set(CMAKE_BINARY_DIR .) +project(pandas) + +find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) set(CMAKE_SHARED_LIBRARY_PREFIX "") -# for macOS if (APPLE) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup") + set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}.so") +else() + set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}${CMAKE_SHARED_LIBRARY_SUFFIX}") endif() -find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) -message("${Python_LINK_OPTIONS}") -set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}.so") add_compile_definitions(NPY_NO_DEPRECATED_API=0) add_subdirectory("pandas/_libs") add_subdirectory("pandas/io/sas") From a1f41fc0c988bf60ad24be040c6143c3aac67199 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 Jun 2022 09:58:47 -0700 Subject: [PATCH 36/68] implemented project version --- CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d3d61a2e02d99..aadbef0086a53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,9 +7,14 @@ set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_SOURCE_DIR .) set(CMAKE_BINARY_DIR .) -project(pandas) find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) +execute_process( + COMMAND ${Python_EXECUTABLE} -c "import versioneer as v; print(str('.'.join(v.get_version().split('.')[:3])))" + OUTPUT_VARIABLE PANDAS_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE +) +project(pandas VERSION ${PANDAS_VERSION}) set(CMAKE_SHARED_LIBRARY_PREFIX "") if (APPLE) From 06ed34585c3da3afa4fbd9d07fb081fc6d18f45e Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 Jun 2022 10:02:00 -0700 Subject: [PATCH 37/68] removed failing build element --- CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aadbef0086a53..a1cd05a16c3f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,8 +5,6 @@ set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED True) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) -set(CMAKE_SOURCE_DIR .) -set(CMAKE_BINARY_DIR .) find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) execute_process( From dd319b89232a128bdc336cada1ed27f50763721b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 Jun 2022 14:16:57 -0700 Subject: [PATCH 38/68] windows builds --- .github/actions/build_pandas/action.yml | 2 +- CMakeLists.txt | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 4a92c13c5ccdb..b7fec8e37f2da 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -13,7 +13,7 @@ runs: - name: Build Pandas run: | cmake . - cmake --build . --parallel + cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-index shell: bash -el {0} env: diff --git a/CMakeLists.txt b/CMakeLists.txt index a1cd05a16c3f7..809c7d0830380 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,7 @@ set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) + execute_process( COMMAND ${Python_EXECUTABLE} -c "import versioneer as v; print(str('.'.join(v.get_version().split('.')[:3])))" OUTPUT_VARIABLE PANDAS_VERSION @@ -18,6 +19,10 @@ set(CMAKE_SHARED_LIBRARY_PREFIX "") if (APPLE) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup") set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}.so") +elseif(WIN32) + set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}") + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE "") + link_directories(${Python_LIBRARY_DIRS}) else() set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}${CMAKE_SHARED_LIBRARY_SUFFIX}") endif() From 46848ff592eaf21f2cd1e3fed67437bdc620a0fa Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 Jun 2022 20:20:35 -0700 Subject: [PATCH 39/68] code checks --- pandas/_libs/generate_templates.py | 16 ++++++++-------- pandas/io/json/_json.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pandas/_libs/generate_templates.py b/pandas/_libs/generate_templates.py index 31a760318f7b5..cf17a98761552 100644 --- a/pandas/_libs/generate_templates.py +++ b/pandas/_libs/generate_templates.py @@ -8,14 +8,14 @@ if __name__ == "__main__": for template in ( - "algos_common_helper.pxi.in", - "algos_take_helper.pxi.in", - "hashtable_class_helper.pxi.in", - "hashtable_func_helper.pxi.in", - "index_class_helper.pxi.in", - "intervaltree.pxi.in", - "khash_for_primitive_helper.pxi.in", - "sparse_op_helper.pxi.in", + "algos_common_helper.pxi.in", + "algos_take_helper.pxi.in", + "hashtable_class_helper.pxi.in", + "hashtable_func_helper.pxi.in", + "index_class_helper.pxi.in", + "intervaltree.pxi.in", + "khash_for_primitive_helper.pxi.in", + "sparse_op_helper.pxi.in", ): pyxcontent = Tempita.sub(open(template).read()) with open(template.replace(".in", ""), "w") as outfile: diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index 8eb277e41151f..33bd0842482df 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -16,8 +16,8 @@ import numpy as np -import pandas._libs.ujson as json from pandas._libs.tslibs import iNaT +import pandas._libs.ujson as json from pandas._typing import ( CompressionOptions, DtypeArg, From 739f2cbfc362913ea91e607e945a2b180d488937 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 Jun 2022 22:04:49 -0700 Subject: [PATCH 40/68] script import fixes --- pandas/_libs/generate_templates.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pandas/_libs/generate_templates.py b/pandas/_libs/generate_templates.py index cf17a98761552..658378f012aad 100644 --- a/pandas/_libs/generate_templates.py +++ b/pandas/_libs/generate_templates.py @@ -1,9 +1,4 @@ -#!/usr/bin/env python3 - -import argparse -import pathlib from Cython import Tempita -from Cython.Build import cythonize if __name__ == "__main__": From 659016a562fa6eb941b7a4c429e21a9f4294d7ab Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 19 Jun 2022 11:55:24 -0700 Subject: [PATCH 41/68] isort fixup --- pandas/_libs/generate_templates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/_libs/generate_templates.py b/pandas/_libs/generate_templates.py index 658378f012aad..56d61acc5abee 100644 --- a/pandas/_libs/generate_templates.py +++ b/pandas/_libs/generate_templates.py @@ -1,6 +1,5 @@ from Cython import Tempita - if __name__ == "__main__": for template in ( "algos_common_helper.pxi.in", From c8ac167ab19bc372e805213e6732db9f16b446f1 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 19 Jun 2022 12:22:16 -0700 Subject: [PATCH 42/68] docker fixups --- .github/workflows/32-bit-linux.yml | 3 ++- Dockerfile | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index a58ba031e8877..c1ff50aafd323 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -31,13 +31,14 @@ jobs: docker pull quay.io/pypa/manylinux2014_i686 docker run --platform linux/386 -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ /bin/bash -xc "cd pandas && \ + yum install -y cmake && \ git config --global --add safe.directory /pandas && \ /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ . ~/virtualenvs/pandas-dev/bin/activate && \ python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ cmake . && \ - cmake --build . --parallel + cmake --build . --parallel && \ python -m pip install --no-build-isolation -e . && \ export PANDAS_CI=1 && \ pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" diff --git a/Dockerfile b/Dockerfile index 8fd0bc0b927f2..615c9cb6b94d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM quay.io/condaforge/miniforge3 # if you forked pandas, you can pass in your own GitHub username to use your fork # i.e. gh_username=myname -ARG gh_username=pandas-dev +ARG gh_username=WillAyd ARG pandas_home="/home/pandas" # Avoid warnings by switching to noninteractive @@ -18,7 +18,7 @@ RUN apt-get update \ && dpkg-reconfigure -f noninteractive tzdata \ # # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed - && apt-get -y install git iproute2 procps iproute2 lsb-release \ + && apt-get -y install cmake git iproute2 procps iproute2 lsb-release \ # # cleanup && apt-get autoremove -y \ @@ -32,6 +32,7 @@ ENV DEBIAN_FRONTEND=dialog RUN mkdir "$pandas_home" \ && git clone "https://github.com/$gh_username/pandas.git" "$pandas_home" \ && cd "$pandas_home" \ + && git checkout cmake-build \ && git remote add upstream "https://github.com/pandas-dev/pandas.git" \ && git pull upstream main From ca4d6a0d5b4e05bb6648d6fcc2972b5c47670fe1 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 19 Jun 2022 13:56:22 -0700 Subject: [PATCH 43/68] more docker --- Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 615c9cb6b94d7..a434de98bd1bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,13 +18,19 @@ RUN apt-get update \ && dpkg-reconfigure -f noninteractive tzdata \ # # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed - && apt-get -y install cmake git iproute2 procps iproute2 lsb-release \ + && apt-get -y install git iproute2 procps iproute2 lsb-release \ # # cleanup && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* +# apt only has version 3.16 of cmake but we require 3.17 +RUN wget "https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-linux-x86_64.sh" \ + && chmod 700 cmake-3.17.5-linux-x86_64.sh \ + && ./cmake-3.17.5-linux-x86_64.sh --skip-license --prefix=/opt --include-subdir \ + && ln -s /opt/cmake-3.17.5-Linux-x86_64/bin/cmake /usr/bin/cmake + # Switch back to dialog for any ad-hoc use of apt-get ENV DEBIAN_FRONTEND=dialog From dc5ff2d92d1c0d3e472b447c5d84e582884006aa Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 08:39:50 -0700 Subject: [PATCH 44/68] Use Python_add_library --- CMakeLists.txt | 12 ---------- pandas/_libs/CMakeLists.txt | 38 ++++++++++-------------------- pandas/_libs/tslibs/CMakeLists.txt | 12 ++++------ pandas/_libs/window/CMakeLists.txt | 6 ++--- pandas/io/sas/CMakeLists.txt | 3 +-- 5 files changed, 20 insertions(+), 51 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 809c7d0830380..fe10fdeeb77c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,18 +15,6 @@ execute_process( ) project(pandas VERSION ${PANDAS_VERSION}) -set(CMAKE_SHARED_LIBRARY_PREFIX "") -if (APPLE) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup") - set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}.so") -elseif(WIN32) - set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}") - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE "") - link_directories(${Python_LIBRARY_DIRS}) -else() - set(CMAKE_SHARED_LIBRARY_SUFFIX ".${Python_SOABI}${CMAKE_SHARED_LIBRARY_SUFFIX}") -endif() - add_compile_definitions(NPY_NO_DEPRECATED_API=0) add_subdirectory("pandas/_libs") add_subdirectory("pandas/io/sas") diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 1be12e4a34d9c..c151b2d9641c4 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -13,10 +13,9 @@ add_custom_command( OUTPUT algos.c COMMAND ${Python_EXECUTABLE} -m cython -3 algos.pyx DEPENDS algos_common_helper.pxi algos_take_helper.pxi) -add_library(algos SHARED algos.c) +Python_add_library(algos MODULE WITH_SOABI algos.c) target_include_directories(algos PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") -target_link_libraries(algos LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) # There is a khash header file in src/klib and a cython generated # one in _libs. Depending on the build timing one of the other could @@ -33,40 +32,36 @@ add_custom_command( COMMAND ${Python_EXECUTABLE} -m cython -3 hashtable.pyx DEPENDS hashtable_class_helper.pxi hashtable_func_helper.pxi) -add_library(hashtable SHARED hashtable.c) +Python_add_library(hashtable MODULE WITH_SOABI hashtable.c) target_include_directories( hashtable PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" DEPENDS khash.h) -target_link_libraries(hashtable LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command( OUTPUT index.c COMMAND ${Python_EXECUTABLE} -m cython -3 index.pyx DEPENDS index_class_helper.pxi) -add_library(index SHARED index.c) +Python_add_library(index MODULE WITH_SOABI index.c) target_include_directories( index PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(index LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command( OUTPUT interval.c COMMAND ${Python_EXECUTABLE} -m cython -3 interval.pyx DEPENDS intervaltree.pxi) -add_library(interval SHARED interval.c) +Python_add_library(interval MODULE WITH_SOABI interval.c) target_include_directories( interval PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(interval LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command( OUTPUT sparse.c COMMAND ${Python_EXECUTABLE} -m cython -3 sparse.pyx DEPENDS sparse_op_helper.pxi) -add_library(sparse SHARED sparse.c) +Python_add_library(sparse MODULE WITH_SOABI sparse.c) target_include_directories( sparse PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") -target_link_libraries(sparse LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) # set(HEADERS arrays dtypes hashtable lib missing util) foreach(HEADER # ${HEADERS}) add_custom_command(OUTPUT ${HEADER}.h COMMAND ${Python_EXECUTABLE} @@ -88,53 +83,47 @@ set(BASIC_LIBRARIES foreach(LIB ${BASIC_LIBRARIES}) add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 ${LIB}.pyx) - add_library(${LIB} SHARED ${LIB}.c) + Python_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) endforeach() add_subdirectory("tslibs") add_custom_command(OUTPUT tslib.c COMMAND ${Python_EXECUTABLE} -m cython -3 tslib.pyx) -add_library(tslib ${LIB} SHARED tslib.c tslibs/src/datetime/np_datetime.c) +Python_add_library(tslib ${LIB} MODULE WITH_SOABI tslib.c tslibs/src/datetime/np_datetime.c) target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") -target_link_libraries(tslib LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command(OUTPUT missing.c COMMAND ${Python_EXECUTABLE} -m cython -3 missing.pyx) -add_library(missing SHARED missing.c) +Python_add_library(missing MODULE WITH_SOABI missing.c) target_include_directories( missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") -target_link_libraries(missing LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command(OUTPUT lib.c COMMAND ${Python_EXECUTABLE} -m cython -3 lib.pyx) -add_library(lib SHARED lib.c src/parser/tokenizer.c) +Python_add_library(lib MODULE WITH_SOABI lib.c src/parser/tokenizer.c) target_include_directories( lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -target_link_libraries(lib LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command(OUTPUT join.c COMMAND ${Python_EXECUTABLE} -m cython -3 join.pyx DEPENDS khash_for_primitive_helper.pxi) -add_library(join SHARED join.c) +Python_add_library(join MODULE WITH_SOABI join.c) target_include_directories(join PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") -target_link_libraries(join LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command(OUTPUT parsers.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsers.pyx) -add_library(parsers SHARED parsers.c src/parser/tokenizer.c src/parser/io.c) +Python_add_library(parsers MODULE WITH_SOABI parsers.c src/parser/tokenizer.c src/parser/io.c) target_include_directories( parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" "src") -target_link_libraries(parsers LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) -add_library( - ujson SHARED +Python_add_library( + ujson MODULE WITH_SOABI src/ujson/python/ujson.c src/ujson/python/objToJSON.c src/ujson/python/date_conversions.c @@ -146,6 +135,5 @@ add_library( target_include_directories( ujson PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) -target_link_libraries(ujson LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_subdirectory("window") diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index a32a2ee79c84f..a6f7ca945adaa 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -2,10 +2,9 @@ set(BASIC_LIBRARIES base ccalendar dtypes nattype strptime timezones) foreach(LIB ${BASIC_LIBRARIES}) add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 ${LIB}.pyx) - add_library(${LIB} SHARED ${LIB}.c) + Python_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) - target_link_libraries(${LIB} LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) endforeach() set(NP_DATETIME_REQUIRING @@ -20,25 +19,22 @@ set(NP_DATETIME_REQUIRING foreach(LIB ${NP_DATETIME_REQUIRING}) add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 ${LIB}.pyx) - add_library(${LIB} SHARED ${LIB}.c src/datetime/np_datetime.c) + Python_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c src/datetime/np_datetime.c) target_include_directories( ${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/datetime") - target_link_libraries(${LIB} LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) endforeach() add_custom_command(OUTPUT np_datetime.c COMMAND ${Python_EXECUTABLE} -m cython -3 np_datetime.pyx) -add_library(np_datetime SHARED np_datetime.c src/datetime/np_datetime.c +Python_add_library(np_datetime MODULE WITH_SOABI np_datetime.c src/datetime/np_datetime.c src/datetime/np_datetime_strings.c) target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(np_datetime LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command(OUTPUT parsing.c COMMAND ${Python_EXECUTABLE} -m cython -3 parsing.pyx) -add_library(parsing SHARED parsing.c "../src/parser/tokenizer.c") +Python_add_library(parsing MODULE WITH_SOABI parsing.c "../src/parser/tokenizer.c") target_include_directories( parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "../src/klib") -target_link_libraries(parsing LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index 9cff7232ee41c..efdd89fc78434 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -1,14 +1,12 @@ add_custom_command(OUTPUT indexers.c COMMAND ${Python_EXECUTABLE} -m cython -3 indexers.pyx) -add_library(indexers SHARED indexers.c) +Python_add_library(indexers MODULE WITH_SOABI indexers.c) target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(indexers LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) add_custom_command( OUTPUT aggregations.cpp COMMAND ${Python_EXECUTABLE} -m cython -3 --cplus aggregations.pyx) -add_library(aggregations SHARED aggregations.cpp) +Python_add_library(aggregations MODULE WITH_SOABI aggregations.cpp) target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(aggregations LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index 30cc5dcef338e..c13675693f592 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,6 +1,5 @@ add_custom_command(OUTPUT _sas.c COMMAND ${Python_EXECUTABLE} -m cython -3 _sas.pyx) -add_library(_sas SHARED _sas.c) +Python_add_library(_sas MODULE WITH_SOABI _sas.c) target_include_directories(_sas PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) -target_link_libraries(_sas LINK_INTERFACE_LIBRARIES ${Python_LIBRARIES}) From 73b0962a71574bd1cdd02ede8f584abed32418f8 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 11:25:48 -0700 Subject: [PATCH 45/68] fix 32bit docker image --- CMakeLists.txt | 4 +- Dockerfile | 10 ++-- pandas/_libs/CMakeLists.txt | 84 +++++++++++++++--------------- pandas/_libs/tslibs/CMakeLists.txt | 28 +++++----- pandas/_libs/window/CMakeLists.txt | 16 +++--- pandas/io/sas/CMakeLists.txt | 8 +-- 6 files changed, 75 insertions(+), 75 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fe10fdeeb77c7..2b249d479299a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.17) +cmake_minimum_required(VERSION 3.18) set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9") set(CMAKE_C_STANDARD 99) @@ -6,7 +6,7 @@ set(CMAKE_C_STANDARD_REQUIRED True) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) -find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) +find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) execute_process( COMMAND ${Python_EXECUTABLE} -c "import versioneer as v; print(str('.'.join(v.get_version().split('.')[:3])))" diff --git a/Dockerfile b/Dockerfile index a434de98bd1bf..aa0bdb71b70e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,11 +25,11 @@ RUN apt-get update \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* -# apt only has version 3.16 of cmake but we require 3.17 -RUN wget "https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-linux-x86_64.sh" \ - && chmod 700 cmake-3.17.5-linux-x86_64.sh \ - && ./cmake-3.17.5-linux-x86_64.sh --skip-license --prefix=/opt --include-subdir \ - && ln -s /opt/cmake-3.17.5-Linux-x86_64/bin/cmake /usr/bin/cmake +# apt only has version 3.16 of cmake but we require 3.18 +RUN wget "https://github.com/Kitware/CMake/releases/download/v3.18.5/cmake-3.18.5-linux-x86_64.sh" \ + && chmod 700 cmake-3.18.5-linux-x86_64.sh \ + && ./cmake-3.18.5-linux-x86_64.sh --skip-license --prefix=/opt --include-subdir \ + && ln -s /opt/cmake-3.18.5-Linux-x86_64/bin/cmake /usr/bin/cmake # Switch back to dialog for any ad-hoc use of apt-get ENV DEBIAN_FRONTEND=dialog diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index c151b2d9641c4..66c3f2aaf2286 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -7,15 +7,15 @@ add_custom_command( intervaltree.pxi khash_for_primitive_helper.pxi sparse_op_helper.pxi - COMMAND ${Python_EXECUTABLE} generate_templates.py) + COMMAND ${Python3_EXECUTABLE} generate_templates.py) add_custom_command( OUTPUT algos.c - COMMAND ${Python_EXECUTABLE} -m cython -3 algos.pyx + COMMAND ${Python3_EXECUTABLE} -m cython -3 algos.pyx DEPENDS algos_common_helper.pxi algos_take_helper.pxi) -Python_add_library(algos MODULE WITH_SOABI algos.c) -target_include_directories(algos PUBLIC ${Python_INCLUDE_DIRS} - ${Python_NumPy_INCLUDE_DIRS} "src/klib") +Python3_add_library(algos MODULE WITH_SOABI algos.c) +target_include_directories(algos PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS} "src/klib") # There is a khash header file in src/klib and a cython generated # one in _libs. Depending on the build timing one of the other could @@ -23,48 +23,48 @@ target_include_directories(algos PUBLIC ${Python_INCLUDE_DIRS} # non-generated version we can remove any DEPENDS khash.h add_custom_command( OUTPUT khash.h - COMMAND ${Python_EXECUTABLE} -m cython -3 khash.pxd + COMMAND ${Python3_EXECUTABLE} -m cython -3 khash.pxd DEPENDS hkash_for_primitive_helper.pxi ) add_custom_command( OUTPUT hashtable.c - COMMAND ${Python_EXECUTABLE} -m cython -3 hashtable.pyx + COMMAND ${Python3_EXECUTABLE} -m cython -3 hashtable.pyx DEPENDS hashtable_class_helper.pxi hashtable_func_helper.pxi) -Python_add_library(hashtable MODULE WITH_SOABI hashtable.c) +Python3_add_library(hashtable MODULE WITH_SOABI hashtable.c) target_include_directories( - hashtable PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} + hashtable PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" DEPENDS khash.h) add_custom_command( OUTPUT index.c - COMMAND ${Python_EXECUTABLE} -m cython -3 index.pyx + COMMAND ${Python3_EXECUTABLE} -m cython -3 index.pyx DEPENDS index_class_helper.pxi) -Python_add_library(index MODULE WITH_SOABI index.c) +Python3_add_library(index MODULE WITH_SOABI index.c) target_include_directories( - index PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" + index PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") add_custom_command( OUTPUT interval.c - COMMAND ${Python_EXECUTABLE} -m cython -3 interval.pyx + COMMAND ${Python3_EXECUTABLE} -m cython -3 interval.pyx DEPENDS intervaltree.pxi) -Python_add_library(interval MODULE WITH_SOABI interval.c) +Python3_add_library(interval MODULE WITH_SOABI interval.c) target_include_directories( - interval PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} + interval PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") add_custom_command( OUTPUT sparse.c - COMMAND ${Python_EXECUTABLE} -m cython -3 sparse.pyx + COMMAND ${Python3_EXECUTABLE} -m cython -3 sparse.pyx DEPENDS sparse_op_helper.pxi) -Python_add_library(sparse MODULE WITH_SOABI sparse.c) +Python3_add_library(sparse MODULE WITH_SOABI sparse.c) target_include_directories( - sparse PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib") + sparse PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib") # set(HEADERS arrays dtypes hashtable lib missing util) foreach(HEADER -# ${HEADERS}) add_custom_command(OUTPUT ${HEADER}.h COMMAND ${Python_EXECUTABLE} +# ${HEADERS}) add_custom_command(OUTPUT ${HEADER}.h COMMAND ${Python3_EXECUTABLE} # -m cython -3 ${HEADER}.pxd) endforeach() set(BASIC_LIBRARIES @@ -81,48 +81,48 @@ set(BASIC_LIBRARIES testing writers) foreach(LIB ${BASIC_LIBRARIES}) - add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python3_EXECUTABLE} -m cython -3 ${LIB}.pyx) - Python_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} - ${Python_NumPy_INCLUDE_DIRS}) + Python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) + target_include_directories(${LIB} PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) endforeach() add_subdirectory("tslibs") -add_custom_command(OUTPUT tslib.c COMMAND ${Python_EXECUTABLE} -m cython -3 +add_custom_command(OUTPUT tslib.c COMMAND ${Python3_EXECUTABLE} -m cython -3 tslib.pyx) -Python_add_library(tslib ${LIB} MODULE WITH_SOABI tslib.c tslibs/src/datetime/np_datetime.c) -target_include_directories(tslib PUBLIC ${Python_INCLUDE_DIRS} - ${Python_NumPy_INCLUDE_DIRS} "./tslibs") +Python3_add_library(tslib ${LIB} MODULE WITH_SOABI tslib.c tslibs/src/datetime/np_datetime.c) +target_include_directories(tslib PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS} "./tslibs") -add_custom_command(OUTPUT missing.c COMMAND ${Python_EXECUTABLE} -m cython -3 +add_custom_command(OUTPUT missing.c COMMAND ${Python3_EXECUTABLE} -m cython -3 missing.pyx) -Python_add_library(missing MODULE WITH_SOABI missing.c) +Python3_add_library(missing MODULE WITH_SOABI missing.c) target_include_directories( - missing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "./tslibs") + missing PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "./tslibs") -add_custom_command(OUTPUT lib.c COMMAND ${Python_EXECUTABLE} -m cython -3 +add_custom_command(OUTPUT lib.c COMMAND ${Python3_EXECUTABLE} -m cython -3 lib.pyx) -Python_add_library(lib MODULE WITH_SOABI lib.c src/parser/tokenizer.c) +Python3_add_library(lib MODULE WITH_SOABI lib.c src/parser/tokenizer.c) target_include_directories( - lib PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" + lib PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -add_custom_command(OUTPUT join.c COMMAND ${Python_EXECUTABLE} -m cython -3 +add_custom_command(OUTPUT join.c COMMAND ${Python3_EXECUTABLE} -m cython -3 join.pyx DEPENDS khash_for_primitive_helper.pxi) -Python_add_library(join MODULE WITH_SOABI join.c) -target_include_directories(join PUBLIC ${Python_INCLUDE_DIRS} - ${Python_NumPy_INCLUDE_DIRS} "src/klib") +Python3_add_library(join MODULE WITH_SOABI join.c) +target_include_directories(join PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS} "src/klib") -add_custom_command(OUTPUT parsers.c COMMAND ${Python_EXECUTABLE} -m cython -3 +add_custom_command(OUTPUT parsers.c COMMAND ${Python3_EXECUTABLE} -m cython -3 parsers.pyx) -Python_add_library(parsers MODULE WITH_SOABI parsers.c src/parser/tokenizer.c src/parser/io.c) +Python3_add_library(parsers MODULE WITH_SOABI parsers.c src/parser/tokenizer.c src/parser/io.c) target_include_directories( - parsers PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} "src/klib" + parsers PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" "src") -Python_add_library( +Python3_add_library( ujson MODULE WITH_SOABI src/ujson/python/ujson.c src/ujson/python/objToJSON.c @@ -133,7 +133,7 @@ Python_add_library( tslibs/src/datetime/np_datetime.c tslibs/src/datetime/np_datetime_strings.c) target_include_directories( - ujson PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} + ujson PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} src/ujson/python src/ujson/lib src/datetime) add_subdirectory("window") diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index a6f7ca945adaa..dc4338138572e 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -1,10 +1,10 @@ set(BASIC_LIBRARIES base ccalendar dtypes nattype strptime timezones) foreach(LIB ${BASIC_LIBRARIES}) - add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python3_EXECUTABLE} -m cython -3 ${LIB}.pyx) - Python_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) - target_include_directories(${LIB} PUBLIC ${Python_INCLUDE_DIRS} - ${Python_NumPy_INCLUDE_DIRS}) + Python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) + target_include_directories(${LIB} PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) endforeach() set(NP_DATETIME_REQUIRING @@ -17,24 +17,24 @@ set(NP_DATETIME_REQUIRING tzconversion vectorized) foreach(LIB ${NP_DATETIME_REQUIRING}) - add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python_EXECUTABLE} -m cython -3 + add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python3_EXECUTABLE} -m cython -3 ${LIB}.pyx) - Python_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c src/datetime/np_datetime.c) + Python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c src/datetime/np_datetime.c) target_include_directories( - ${LIB} PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} + ${LIB} PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/datetime") endforeach() -add_custom_command(OUTPUT np_datetime.c COMMAND ${Python_EXECUTABLE} -m cython +add_custom_command(OUTPUT np_datetime.c COMMAND ${Python3_EXECUTABLE} -m cython -3 np_datetime.pyx) -Python_add_library(np_datetime MODULE WITH_SOABI np_datetime.c src/datetime/np_datetime.c +Python3_add_library(np_datetime MODULE WITH_SOABI np_datetime.c src/datetime/np_datetime.c src/datetime/np_datetime_strings.c) -target_include_directories(np_datetime PUBLIC ${Python_INCLUDE_DIRS} - ${Python_NumPy_INCLUDE_DIRS}) +target_include_directories(np_datetime PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) -add_custom_command(OUTPUT parsing.c COMMAND ${Python_EXECUTABLE} -m cython -3 +add_custom_command(OUTPUT parsing.c COMMAND ${Python3_EXECUTABLE} -m cython -3 parsing.pyx) -Python_add_library(parsing MODULE WITH_SOABI parsing.c "../src/parser/tokenizer.c") +Python3_add_library(parsing MODULE WITH_SOABI parsing.c "../src/parser/tokenizer.c") target_include_directories( - parsing PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} + parsing PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "../src/klib") diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index efdd89fc78434..2db9a08dc9c70 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -1,12 +1,12 @@ -add_custom_command(OUTPUT indexers.c COMMAND ${Python_EXECUTABLE} -m cython -3 +add_custom_command(OUTPUT indexers.c COMMAND ${Python3_EXECUTABLE} -m cython -3 indexers.pyx) -Python_add_library(indexers MODULE WITH_SOABI indexers.c) -target_include_directories(indexers PUBLIC ${Python_INCLUDE_DIRS} - ${Python_NumPy_INCLUDE_DIRS}) +Python3_add_library(indexers MODULE WITH_SOABI indexers.c) +target_include_directories(indexers PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) add_custom_command( - OUTPUT aggregations.cpp COMMAND ${Python_EXECUTABLE} -m cython -3 --cplus + OUTPUT aggregations.cpp COMMAND ${Python3_EXECUTABLE} -m cython -3 --cplus aggregations.pyx) -Python_add_library(aggregations MODULE WITH_SOABI aggregations.cpp) -target_include_directories(aggregations PUBLIC ${Python_INCLUDE_DIRS} - ${Python_NumPy_INCLUDE_DIRS}) +Python3_add_library(aggregations MODULE WITH_SOABI aggregations.cpp) +target_include_directories(aggregations PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index c13675693f592..5a85abc7ed9d9 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,5 +1,5 @@ -add_custom_command(OUTPUT _sas.c COMMAND ${Python_EXECUTABLE} -m cython -3 +add_custom_command(OUTPUT _sas.c COMMAND ${Python3_EXECUTABLE} -m cython -3 _sas.pyx) -Python_add_library(_sas MODULE WITH_SOABI _sas.c) -target_include_directories(_sas PUBLIC ${Python_INCLUDE_DIRS} - ${Python_NumPy_INCLUDE_DIRS}) +Python3_add_library(_sas MODULE WITH_SOABI _sas.c) +target_include_directories(_sas PUBLIC ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS}) From 421657f512e0ee467b07526dda6865d5688deb03 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 11:42:24 -0700 Subject: [PATCH 46/68] reimplemented windows settings --- CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b249d479299a..354db7363647d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,12 +9,17 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) execute_process( - COMMAND ${Python_EXECUTABLE} -c "import versioneer as v; print(str('.'.join(v.get_version().split('.')[:3])))" + COMMAND ${Python_EXECUTABLE} -c "import versioneer as v; print('.'.join(v.get_version().split('.')[:3]))" OUTPUT_VARIABLE PANDAS_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE ) project(pandas VERSION ${PANDAS_VERSION}) +if(WIN32) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE "") + link_directories(${Python_LIBRARY_DIRS}) +endif() + add_compile_definitions(NPY_NO_DEPRECATED_API=0) add_subdirectory("pandas/_libs") add_subdirectory("pandas/io/sas") From 0ccf5bc6cf2ecbaa2c871acf8e0ae8347cc8e914 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 11:42:43 -0700 Subject: [PATCH 47/68] try version fix for dev Docker env --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index aa0bdb71b70e8..8209de3b694c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,13 @@ RUN wget "https://github.com/Kitware/CMake/releases/download/v3.18.5/cmake-3.18. # Switch back to dialog for any ad-hoc use of apt-get ENV DEBIAN_FRONTEND=dialog +# Without this versioneer will not be able to determine the pandas version. +# This is because of a security update to git that blocks it from reading the config folder if +# it is not owned by the current user. We hit this since the "mounted" folder is not hit by the +# Docker container. +# xref https://github.com/pypa/manylinux/issues/1309 +RUN git config --global --add safe.directory "$pandas_home" + # Clone pandas repo RUN mkdir "$pandas_home" \ && git clone "https://github.com/$gh_username/pandas.git" "$pandas_home" \ From 9cf9d8d86615a0eb4e9692e09c2c910ea6777fed Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 13:44:46 -0700 Subject: [PATCH 48/68] windows / docker fixes --- CMakeLists.txt | 16 ++++++++++------ Dockerfile | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 354db7363647d..70f1ec2b3a309 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,16 @@ set(CMAKE_C_STANDARD_REQUIRED True) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) -find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) +if(WIN32) + find_package(Python3 REQUIRED COMPONENTS Interpreter Development NumPy) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE "") + link_directories(${Python_LIBRARY_DIRS}) +else() + # we only choose Development.Module to support virtual environments where + # libpython may not be available + # see https://github.com/pypa/manylinux/issues/484 + find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) +endif() execute_process( COMMAND ${Python_EXECUTABLE} -c "import versioneer as v; print('.'.join(v.get_version().split('.')[:3]))" @@ -15,11 +24,6 @@ execute_process( ) project(pandas VERSION ${PANDAS_VERSION}) -if(WIN32) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE "") - link_directories(${Python_LIBRARY_DIRS}) -endif() - add_compile_definitions(NPY_NO_DEPRECATED_API=0) add_subdirectory("pandas/_libs") add_subdirectory("pandas/io/sas") diff --git a/Dockerfile b/Dockerfile index 8209de3b694c1..86f51c1d81983 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update \ && dpkg-reconfigure -f noninteractive tzdata \ # # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed - && apt-get -y install git iproute2 procps iproute2 lsb-release \ + && apt-get -y install build-essential git iproute2 procps iproute2 lsb-release \ # # cleanup && apt-get autoremove -y \ From acdccf9fc7e52ac45da586abe4bcd9d6ce9d2c78 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 13:45:21 -0700 Subject: [PATCH 49/68] VERBOSE build --- .github/actions/build_pandas/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index b7fec8e37f2da..3543b01acba5f 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -13,7 +13,7 @@ runs: - name: Build Pandas run: | cmake . - cmake --build . --config Release --parallel + VERBOSE=1 cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-index shell: bash -el {0} env: From 7310bf07716206cf28ca1a3663ef38c088eac464 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 20 Jun 2022 15:31:59 -0700 Subject: [PATCH 50/68] windows support --- CMakeLists.txt | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 70f1ec2b3a309..f80fffa88d062 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,23 +6,29 @@ set(CMAKE_C_STANDARD_REQUIRED True) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) +project(pandas) + if(WIN32) + if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) + endif() find_package(Python3 REQUIRED COMPONENTS Interpreter Development NumPy) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE "") - link_directories(${Python_LIBRARY_DIRS}) -else() + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE "") + link_directories(${Python3_LIBRARY_DIRS}) +else () # we only choose Development.Module to support virtual environments where # libpython may not be available # see https://github.com/pypa/manylinux/issues/484 find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) endif() -execute_process( - COMMAND ${Python_EXECUTABLE} -c "import versioneer as v; print('.'.join(v.get_version().split('.')[:3]))" - OUTPUT_VARIABLE PANDAS_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE -) -project(pandas VERSION ${PANDAS_VERSION}) +#execute_process( +# COMMAND ${Python_EXECUTABLE} -c "import versioneer as v; print('.'.join(v.get_version().split('.')[:3]))" +# OUTPUT_VARIABLE PANDAS_VERSION +# OUTPUT_STRIP_TRAILING_WHITESPACE +#) + + add_compile_definitions(NPY_NO_DEPRECATED_API=0) add_subdirectory("pandas/_libs") From 64a2bf1ae11ade605a9b709ddcac3e3511876713 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 20 Jun 2022 16:15:47 -0700 Subject: [PATCH 51/68] test fixes / compat --- pandas/tests/groupby/test_groupby.py | 3 --- pandas/tests/io/parser/common/test_float.py | 9 ++++++--- pandas/tests/io/parser/test_c_parser_only.py | 10 ++-------- pandas/tests/window/test_rolling.py | 5 ++++- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 97e616ef14cef..a557e9d6a547a 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -5,7 +5,6 @@ import pytest from pandas._libs import lib -from pandas.compat import IS64 from pandas.errors import ( PerformanceWarning, SpecificationError, @@ -2539,7 +2538,6 @@ def test_groupby_series_with_tuple_name(): tm.assert_series_equal(result, expected) -@pytest.mark.xfail(not IS64, reason="GH#38778: fail on 32-bit system") @pytest.mark.parametrize( "func, values", [("sum", [97.0, 98.0]), ("mean", [24.25, 24.5])] ) @@ -2552,7 +2550,6 @@ def test_groupby_numerical_stability_sum_mean(func, values): tm.assert_frame_equal(result, expected) -@pytest.mark.xfail(not IS64, reason="GH#38778: fail on 32-bit system") def test_groupby_numerical_stability_cumsum(): # GH#38934 data = [1e16, 1e16, 97, 98, -5e15, -5e15, -5e15, -5e15] diff --git a/pandas/tests/io/parser/common/test_float.py b/pandas/tests/io/parser/common/test_float.py index 2ca98de914f9e..e245b2ca019ee 100644 --- a/pandas/tests/io/parser/common/test_float.py +++ b/pandas/tests/io/parser/common/test_float.py @@ -7,7 +7,10 @@ import numpy as np import pytest -from pandas.compat import is_platform_linux +from pandas.compat import ( + is_platform_linux, + is_platform_mac, +) from pandas import DataFrame import pandas._testing as tm @@ -53,8 +56,8 @@ def test_too_many_exponent_digits(all_parsers_all_precisions, exp, request): data = f"data\n10E{exp}" result = parser.read_csv(StringIO(data), float_precision=precision) if precision == "round_trip": - if exp == 999999999999999999 and is_platform_linux(): - mark = pytest.mark.xfail(reason="GH38794, on Linux gives object result") + if exp == 999999999999999999 and (is_platform_linux() or is_platform_mac()): + mark = pytest.mark.xfail(reason="GH38794, on Unix gives object result") request.node.add_marker(mark) value = np.inf if exp > 0 else 0.0 diff --git a/pandas/tests/io/parser/test_c_parser_only.py b/pandas/tests/io/parser/test_c_parser_only.py index 9a81790ca3bb0..6d92225f9458d 100644 --- a/pandas/tests/io/parser/test_c_parser_only.py +++ b/pandas/tests/io/parser/test_c_parser_only.py @@ -17,10 +17,7 @@ import numpy as np import pytest -from pandas.compat import ( - IS64, - is_ci_environment, -) +from pandas.compat import is_ci_environment from pandas.errors import ParserError import pandas.util._test_decorators as td @@ -678,10 +675,7 @@ def test_float_precision_options(c_parser_only): df3 = parser.read_csv(StringIO(s), float_precision="legacy") - if IS64: - assert not df.iloc[0, 0] == df3.iloc[0, 0] - else: - assert df.iloc[0, 0] == df3.iloc[0, 0] + assert not df.iloc[0, 0] == df3.iloc[0, 0] msg = "Unrecognized float_precision option: junk" diff --git a/pandas/tests/window/test_rolling.py b/pandas/tests/window/test_rolling.py index 785603f6e05f0..bfd20ed160d32 100644 --- a/pandas/tests/window/test_rolling.py +++ b/pandas/tests/window/test_rolling.py @@ -7,6 +7,7 @@ import pytest from pandas.compat import ( + IS64, is_platform_arm, is_platform_mac, ) @@ -1187,7 +1188,9 @@ def test_rolling_sem(frame_or_series): tm.assert_series_equal(result, expected) -@pytest.mark.xfail(is_platform_arm() and not is_platform_mac(), reason="GH 38921") +@pytest.mark.xfail( + (is_platform_arm() and not is_platform_mac()) or not IS64, reason="GH 38921" +) @pytest.mark.parametrize( ("func", "third_value", "values"), [ From 357fe17c6af756573055ca6c1e23aa92093d3456 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 17:55:00 -0700 Subject: [PATCH 52/68] cmake via pip --- .github/workflows/32-bit-linux.yml | 3 +-- Dockerfile | 6 ------ environment.yml | 1 + requirements-dev.txt | 1 + 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index c1ff50aafd323..d53fad0e76fe5 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -31,12 +31,11 @@ jobs: docker pull quay.io/pypa/manylinux2014_i686 docker run --platform linux/386 -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ /bin/bash -xc "cd pandas && \ - yum install -y cmake && \ git config --global --add safe.directory /pandas && \ /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ . ~/virtualenvs/pandas-dev/bin/activate && \ python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ - pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ + pip install cmake cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ cmake . && \ cmake --build . --parallel && \ python -m pip install --no-build-isolation -e . && \ diff --git a/Dockerfile b/Dockerfile index 86f51c1d81983..41375400709ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,12 +25,6 @@ RUN apt-get update \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* -# apt only has version 3.16 of cmake but we require 3.18 -RUN wget "https://github.com/Kitware/CMake/releases/download/v3.18.5/cmake-3.18.5-linux-x86_64.sh" \ - && chmod 700 cmake-3.18.5-linux-x86_64.sh \ - && ./cmake-3.18.5-linux-x86_64.sh --skip-license --prefix=/opt --include-subdir \ - && ln -s /opt/cmake-3.18.5-Linux-x86_64/bin/cmake /usr/bin/cmake - # Switch back to dialog for any ad-hoc use of apt-get ENV DEBIAN_FRONTEND=dialog diff --git a/environment.yml b/environment.yml index 1f1583354339c..ed842fce8f86f 100644 --- a/environment.yml +++ b/environment.yml @@ -13,6 +13,7 @@ dependencies: # building # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms. + - cmake>=3.18.0 - c-compiler - cxx-compiler - cython>=0.29.30 diff --git a/requirements-dev.txt b/requirements-dev.txt index 7640587b85a1c..7d9557c3270d8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,7 @@ numpy>=1.19.5 python-dateutil>=2.8.1 pytz asv +cmake>=3.18.0 cython>=0.29.30 black==22.3.0 cpplint From fdc54c46a4a1444e79ef17599c23678f66b40e26 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 18:45:46 -0700 Subject: [PATCH 53/68] added back setuptools for sdist and bdist_wheel creation --- setup.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000..cfd781aeab35b --- /dev/null +++ b/setup.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +""" +Parts of this file were taken from the pyzmq project +(https://github.com/zeromq/pyzmq) which have been permitted for use under the +BSD license. Parts are from lxml (https://github.com/lxml/lxml) +""" +from setuptools import setup + +import versioneer + + +cmdclass = versioneer.get_cmdclass() + + +if __name__ == "__main__": + setup( + version=versioneer.get_version(), + cmdclass=cmdclass, + ) From 3bd83d1978656497a6687788a9039724f0180847 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 18:46:26 -0700 Subject: [PATCH 54/68] removed verbose build --- .github/actions/build_pandas/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 3543b01acba5f..b7fec8e37f2da 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -13,7 +13,7 @@ runs: - name: Build Pandas run: | cmake . - VERBOSE=1 cmake --build . --config Release --parallel + cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-index shell: bash -el {0} env: From f095255dc0a5f6351b1f7c0d166b4074f688ab99 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 18:54:59 -0700 Subject: [PATCH 55/68] documentation cleanups --- .circleci/setup_env.sh | 5 +++-- .../development/contributing_environment.rst | 15 ++++++++++----- doc/source/development/debugging_extensions.rst | 5 +++-- pandas/__init__.py | 3 ++- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.circleci/setup_env.sh b/.circleci/setup_env.sh index c03a7ff4be8b3..c88250d6f48d2 100755 --- a/.circleci/setup_env.sh +++ b/.circleci/setup_env.sh @@ -98,9 +98,10 @@ if [ "$(conda list -f qt --json)" != [] ]; then fi echo "Build extensions" -python setup.py build_ext -q -j3 +cmake . +cmake --build . --config Release --parallel echo "Install pandas" -python -m pip install --no-build-isolation --no-use-pep517 -e . +python -m pip install --no-build-isolation -e . echo "done" diff --git a/doc/source/development/contributing_environment.rst b/doc/source/development/contributing_environment.rst index c881770aa7584..32c9ea82f8ddf 100644 --- a/doc/source/development/contributing_environment.rst +++ b/doc/source/development/contributing_environment.rst @@ -42,7 +42,8 @@ Run Container:: If you bind your local repo for the first time, you have to build the C extensions afterwards. Run the following command inside the container:: - python setup.py build_ext -j 4 + cmake . + cmake --build . --config Release --parallel You need to rebuild the C extensions anytime the Cython code in ``pandas/_libs`` changes. This most frequently occurs when changing or merging branches. @@ -169,7 +170,8 @@ We'll now kick off a three-step process: source activate pandas-dev # Build and install pandas - python setup.py build_ext -j 4 + cmake . + cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-use-pep517 At this point you should be able to import pandas from your locally built version:: @@ -216,7 +218,8 @@ You also need to have ``setuptools`` 51.0.0 or later to build pandas. python -m pip install -r requirements-dev.txt # Build and install pandas - python setup.py build_ext -j 4 + cmake . + cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-use-pep517 **Unix**/**macOS with pyenv** @@ -240,7 +243,8 @@ Consult the docs for setting up pyenv `here `__. python -m pip install -r requirements-dev.txt # Build and install pandas - python setup.py build_ext -j 4 + cmake . + cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-use-pep517 **Windows** @@ -266,5 +270,6 @@ should already exist. python -m pip install -r requirements-dev.txt # Build and install pandas - python setup.py build_ext -j 4 + cmake . + cmake --build . --config Release --parallel python -m pip install -e . --no-build-isolation --no-use-pep517 diff --git a/doc/source/development/debugging_extensions.rst b/doc/source/development/debugging_extensions.rst index 7ba2091e18853..b2865f97e2613 100644 --- a/doc/source/development/debugging_extensions.rst +++ b/doc/source/development/debugging_extensions.rst @@ -8,11 +8,12 @@ Debugging C extensions Pandas uses select C extensions for high performance IO operations. In case you need to debug segfaults or general issues with those extensions, the following steps may be helpful. -First, be sure to compile the extensions with the appropriate flags to generate debug symbols and remove optimizations. This can be achieved as follows: +First, be sure to compile the extensions with the appropriate flags to generate debug symbols and remove optimizations. This can be achieved on Unix-like systems as follows: .. code-block:: sh - python setup.py build_ext --inplace -j4 --with-debugging-symbols + cmake . -DCMAKE_BUILD_TYPE=Debug + cmake --build . --parallel Using a debugger ================ diff --git a/pandas/__init__.py b/pandas/__init__.py index 3645e8744d8af..c0fe463c2acd1 100644 --- a/pandas/__init__.py +++ b/pandas/__init__.py @@ -28,7 +28,8 @@ raise ImportError( f"C extension: {_module} not built. If you want to import " "pandas from the source directory, you may need to run " - "'python setup.py build_ext --force' to build the C extensions first." + "'cmake . && cmake --build . --config Release --parallel' " + "to build the C extensions first." ) from _err else: del _tslib, _lib, _hashtable From dee3e6ff4df8a35764c0a7ba94bf770b84db1ded Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 19:04:05 -0700 Subject: [PATCH 56/68] isort setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index cfd781aeab35b..72e6bb1e37557 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,6 @@ import versioneer - cmdclass = versioneer.get_cmdclass() From 55a0a3ff8322e4b5df2934d17a9cff1d8b3c8bb6 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 19:24:45 -0700 Subject: [PATCH 57/68] comment cleanup --- CMakeLists.txt | 8 -------- pandas/_libs/CMakeLists.txt | 4 ---- 2 files changed, 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f80fffa88d062..2614dcc79b31e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,14 +22,6 @@ else () find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) endif() -#execute_process( -# COMMAND ${Python_EXECUTABLE} -c "import versioneer as v; print('.'.join(v.get_version().split('.')[:3]))" -# OUTPUT_VARIABLE PANDAS_VERSION -# OUTPUT_STRIP_TRAILING_WHITESPACE -#) - - - add_compile_definitions(NPY_NO_DEPRECATED_API=0) add_subdirectory("pandas/_libs") add_subdirectory("pandas/io/sas") diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 66c3f2aaf2286..1fbb090fca2e7 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -63,10 +63,6 @@ Python3_add_library(sparse MODULE WITH_SOABI sparse.c) target_include_directories( sparse PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib") -# set(HEADERS arrays dtypes hashtable lib missing util) foreach(HEADER -# ${HEADERS}) add_custom_command(OUTPUT ${HEADER}.h COMMAND ${Python3_EXECUTABLE} -# -m cython -3 ${HEADER}.pxd) endforeach() - set(BASIC_LIBRARIES arrays groupby From ffb1f3d84c2d69ada2bdd19e8682712537c872ec Mon Sep 17 00:00:00 2001 From: William Ayd Date: Mon, 20 Jun 2022 19:26:55 -0700 Subject: [PATCH 58/68] cmake-format all files --- CMakeLists.txt | 10 +++-- pandas/_libs/CMakeLists.txt | 67 +++++++++++++++++------------- pandas/_libs/tslibs/CMakeLists.txt | 13 +++--- pandas/_libs/window/CMakeLists.txt | 4 +- pandas/io/sas/CMakeLists.txt | 4 +- 5 files changed, 55 insertions(+), 43 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2614dcc79b31e..fcf450e557e39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,15 +10,17 @@ project(pandas) if(WIN32) if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) + set(CMAKE_BUILD_TYPE + Release + CACHE STRING "Build type" FORCE) endif() find_package(Python3 REQUIRED COMPONENTS Interpreter Development NumPy) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE "") link_directories(${Python3_LIBRARY_DIRS}) -else () +else() # we only choose Development.Module to support virtual environments where - # libpython may not be available - # see https://github.com/pypa/manylinux/issues/484 + # libpython may not be available see + # https://github.com/pypa/manylinux/issues/484 find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) endif() diff --git a/pandas/_libs/CMakeLists.txt b/pandas/_libs/CMakeLists.txt index 1fbb090fca2e7..3aa1e7a225476 100644 --- a/pandas/_libs/CMakeLists.txt +++ b/pandas/_libs/CMakeLists.txt @@ -13,26 +13,25 @@ add_custom_command( OUTPUT algos.c COMMAND ${Python3_EXECUTABLE} -m cython -3 algos.pyx DEPENDS algos_common_helper.pxi algos_take_helper.pxi) -Python3_add_library(algos MODULE WITH_SOABI algos.c) -target_include_directories(algos PUBLIC ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS} "src/klib") - -# There is a khash header file in src/klib and a cython generated -# one in _libs. Depending on the build timing one of the other could -# get picked up, though unclear why we need both? If we stick to the -# non-generated version we can remove any DEPENDS khash.h +python3_add_library(algos MODULE WITH_SOABI algos.c) +target_include_directories( + algos PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib") + +# There is a khash header file in src/klib and a cython generated one in _libs. +# Depending on the build timing one of the other could get picked up, though +# unclear why we need both? If we stick to the non-generated version we can +# remove any DEPENDS khash.h add_custom_command( OUTPUT khash.h COMMAND ${Python3_EXECUTABLE} -m cython -3 khash.pxd - DEPENDS hkash_for_primitive_helper.pxi -) + DEPENDS hkash_for_primitive_helper.pxi) add_custom_command( OUTPUT hashtable.c COMMAND ${Python3_EXECUTABLE} -m cython -3 hashtable.pyx DEPENDS hashtable_class_helper.pxi hashtable_func_helper.pxi) -Python3_add_library(hashtable MODULE WITH_SOABI hashtable.c) +python3_add_library(hashtable MODULE WITH_SOABI hashtable.c) target_include_directories( hashtable PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" DEPENDS khash.h) @@ -41,7 +40,7 @@ add_custom_command( OUTPUT index.c COMMAND ${Python3_EXECUTABLE} -m cython -3 index.pyx DEPENDS index_class_helper.pxi) -Python3_add_library(index MODULE WITH_SOABI index.c) +python3_add_library(index MODULE WITH_SOABI index.c) target_include_directories( index PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") @@ -50,7 +49,7 @@ add_custom_command( OUTPUT interval.c COMMAND ${Python3_EXECUTABLE} -m cython -3 interval.pyx DEPENDS intervaltree.pxi) -Python3_add_library(interval MODULE WITH_SOABI interval.c) +python3_add_library(interval MODULE WITH_SOABI interval.c) target_include_directories( interval PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") @@ -59,9 +58,10 @@ add_custom_command( OUTPUT sparse.c COMMAND ${Python3_EXECUTABLE} -m cython -3 sparse.pyx DEPENDS sparse_op_helper.pxi) -Python3_add_library(sparse MODULE WITH_SOABI sparse.c) +python3_add_library(sparse MODULE WITH_SOABI sparse.c) target_include_directories( - sparse PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib") + sparse PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + "src/klib") set(BASIC_LIBRARIES arrays @@ -79,7 +79,7 @@ set(BASIC_LIBRARIES foreach(LIB ${BASIC_LIBRARIES}) add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python3_EXECUTABLE} -m cython -3 ${LIB}.pyx) - Python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) + python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}) endforeach() @@ -88,38 +88,45 @@ add_subdirectory("tslibs") add_custom_command(OUTPUT tslib.c COMMAND ${Python3_EXECUTABLE} -m cython -3 tslib.pyx) -Python3_add_library(tslib ${LIB} MODULE WITH_SOABI tslib.c tslibs/src/datetime/np_datetime.c) -target_include_directories(tslib PUBLIC ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS} "./tslibs") +python3_add_library(tslib ${LIB} MODULE WITH_SOABI tslib.c + tslibs/src/datetime/np_datetime.c) +target_include_directories( + tslib PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "./tslibs") add_custom_command(OUTPUT missing.c COMMAND ${Python3_EXECUTABLE} -m cython -3 missing.pyx) -Python3_add_library(missing MODULE WITH_SOABI missing.c) +python3_add_library(missing MODULE WITH_SOABI missing.c) target_include_directories( - missing PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "./tslibs") + missing PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + "./tslibs") add_custom_command(OUTPUT lib.c COMMAND ${Python3_EXECUTABLE} -m cython -3 lib.pyx) -Python3_add_library(lib MODULE WITH_SOABI lib.c src/parser/tokenizer.c) +python3_add_library(lib MODULE WITH_SOABI lib.c src/parser/tokenizer.c) target_include_directories( lib PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" "./tslibs") -add_custom_command(OUTPUT join.c COMMAND ${Python3_EXECUTABLE} -m cython -3 - join.pyx DEPENDS khash_for_primitive_helper.pxi) -Python3_add_library(join MODULE WITH_SOABI join.c) +add_custom_command( + OUTPUT join.c + COMMAND ${Python3_EXECUTABLE} -m cython -3 join.pyx + DEPENDS khash_for_primitive_helper.pxi) +python3_add_library(join MODULE WITH_SOABI join.c) target_include_directories(join PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib") add_custom_command(OUTPUT parsers.c COMMAND ${Python3_EXECUTABLE} -m cython -3 parsers.pyx) -Python3_add_library(parsers MODULE WITH_SOABI parsers.c src/parser/tokenizer.c src/parser/io.c) +python3_add_library(parsers MODULE WITH_SOABI parsers.c src/parser/tokenizer.c + src/parser/io.c) target_include_directories( - parsers PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib" - "src") + parsers PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} + "src/klib" "src") -Python3_add_library( - ujson MODULE WITH_SOABI +python3_add_library( + ujson + MODULE + WITH_SOABI src/ujson/python/ujson.c src/ujson/python/objToJSON.c src/ujson/python/date_conversions.c diff --git a/pandas/_libs/tslibs/CMakeLists.txt b/pandas/_libs/tslibs/CMakeLists.txt index dc4338138572e..0f8ebdeb8991c 100644 --- a/pandas/_libs/tslibs/CMakeLists.txt +++ b/pandas/_libs/tslibs/CMakeLists.txt @@ -2,7 +2,7 @@ set(BASIC_LIBRARIES base ccalendar dtypes nattype strptime timezones) foreach(LIB ${BASIC_LIBRARIES}) add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python3_EXECUTABLE} -m cython -3 ${LIB}.pyx) - Python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) + python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c) target_include_directories(${LIB} PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}) endforeach() @@ -19,7 +19,8 @@ set(NP_DATETIME_REQUIRING foreach(LIB ${NP_DATETIME_REQUIRING}) add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python3_EXECUTABLE} -m cython -3 ${LIB}.pyx) - Python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c src/datetime/np_datetime.c) + python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c + src/datetime/np_datetime.c) target_include_directories( ${LIB} PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/datetime") @@ -27,14 +28,16 @@ endforeach() add_custom_command(OUTPUT np_datetime.c COMMAND ${Python3_EXECUTABLE} -m cython -3 np_datetime.pyx) -Python3_add_library(np_datetime MODULE WITH_SOABI np_datetime.c src/datetime/np_datetime.c - src/datetime/np_datetime_strings.c) +python3_add_library( + np_datetime MODULE WITH_SOABI np_datetime.c src/datetime/np_datetime.c + src/datetime/np_datetime_strings.c) target_include_directories(np_datetime PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}) add_custom_command(OUTPUT parsing.c COMMAND ${Python3_EXECUTABLE} -m cython -3 parsing.pyx) -Python3_add_library(parsing MODULE WITH_SOABI parsing.c "../src/parser/tokenizer.c") +python3_add_library(parsing MODULE WITH_SOABI parsing.c + "../src/parser/tokenizer.c") target_include_directories( parsing PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "../src/klib") diff --git a/pandas/_libs/window/CMakeLists.txt b/pandas/_libs/window/CMakeLists.txt index 2db9a08dc9c70..862abf42fd89f 100644 --- a/pandas/_libs/window/CMakeLists.txt +++ b/pandas/_libs/window/CMakeLists.txt @@ -1,12 +1,12 @@ add_custom_command(OUTPUT indexers.c COMMAND ${Python3_EXECUTABLE} -m cython -3 indexers.pyx) -Python3_add_library(indexers MODULE WITH_SOABI indexers.c) +python3_add_library(indexers MODULE WITH_SOABI indexers.c) target_include_directories(indexers PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}) add_custom_command( OUTPUT aggregations.cpp COMMAND ${Python3_EXECUTABLE} -m cython -3 --cplus aggregations.pyx) -Python3_add_library(aggregations MODULE WITH_SOABI aggregations.cpp) +python3_add_library(aggregations MODULE WITH_SOABI aggregations.cpp) target_include_directories(aggregations PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}) diff --git a/pandas/io/sas/CMakeLists.txt b/pandas/io/sas/CMakeLists.txt index 5a85abc7ed9d9..a8742edc7a939 100644 --- a/pandas/io/sas/CMakeLists.txt +++ b/pandas/io/sas/CMakeLists.txt @@ -1,5 +1,5 @@ add_custom_command(OUTPUT _sas.c COMMAND ${Python3_EXECUTABLE} -m cython -3 _sas.pyx) -Python3_add_library(_sas MODULE WITH_SOABI _sas.c) +python3_add_library(_sas MODULE WITH_SOABI _sas.c) target_include_directories(_sas PUBLIC ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS}) + ${Python3_NumPy_INCLUDE_DIRS}) From d27fe22c9cdb1897ae95f3a19990652f3a111c6a Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 20 Jun 2022 21:28:32 -0700 Subject: [PATCH 59/68] sdist test fix --- .github/workflows/sdist.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 5ae2280c5069f..2280af63c5002 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -43,11 +43,16 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade pip cmake setuptools wheel # GH 39416 pip install numpy + - name: Build pandas + run: | + cmake . + cmake --build . --parallel + - name: Build pandas sdist run: | pip list From c1c463bb51ff00110a191d9109ce2e42fa69b974 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 20 Jun 2022 21:48:02 -0700 Subject: [PATCH 60/68] updated MANFIEST --- MANIFEST.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index d2b1b8cb887bc..c8d901952333e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -42,6 +42,9 @@ global-exclude *~ global-exclude .DS_Store global-exclude .git* global-exclude \#* +recursive-exclude **/CMakeFiles * +global-exclude *.cmake +global-exclude CMakeCache.txt global-exclude *.c global-exclude *.cpp From 73f2e9486e4901d0828872e40f5ca520cbac6007 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 20 Jun 2022 23:00:19 -0700 Subject: [PATCH 61/68] config updates --- pyproject.toml | 1 + setup.cfg | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 0e2e41fba461c..d7dd57aff6364 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ # Minimum requirements for the build system to execute. # See https://github.com/scipy/scipy/pull/12940 for the AIX issue. requires = [ + "cmake>=3.18.0", "setuptools>=51.0.0", "wheel", "Cython>=0.29.24,<3", # Note: sync with setup.py, environment.yml and asv.conf.json diff --git a/setup.cfg b/setup.cfg index 05b7975cbc0a8..9ace5193c44c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ project_urls = [options] packages = find: install_requires = + cmake>=3.18.0 numpy>=1.18.5; platform_machine!='aarch64' and platform_machine!='arm64' and python_version<'3.10' numpy>=1.19.2; platform_machine=='aarch64' and python_version<'3.10' numpy>=1.20.0; platform_machine=='arm64' and python_version<'3.10' From fde002a9cf201b46448216b4c2f73502aa4be351 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Tue, 21 Jun 2022 08:47:39 -0700 Subject: [PATCH 62/68] remove cmake from install_requires --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9ace5193c44c6..05b7975cbc0a8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,6 @@ project_urls = [options] packages = find: install_requires = - cmake>=3.18.0 numpy>=1.18.5; platform_machine!='aarch64' and platform_machine!='arm64' and python_version<'3.10' numpy>=1.19.2; platform_machine=='aarch64' and python_version<'3.10' numpy>=1.20.0; platform_machine=='arm64' and python_version<'3.10' From d43a292198aa49586dbb9c8985bd63c36dd9ca96 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Tue, 21 Jun 2022 18:42:26 -0700 Subject: [PATCH 63/68] pep517 build backend --- MANIFEST.in | 2 ++ build_support/build_backend.py | 21 +++++++++++++++++++++ pyproject.toml | 6 +++--- setup.py | 6 ++++++ 4 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 build_support/build_backend.py diff --git a/MANIFEST.in b/MANIFEST.in index c8d901952333e..873f680b9a147 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include RELEASE.md include versioneer.py +include CMakeLists.txt graft doc prune doc/build @@ -7,6 +8,7 @@ prune doc/build graft LICENSES graft pandas +graft build_support global-exclude *.bz2 global-exclude *.csv diff --git a/build_support/build_backend.py b/build_support/build_backend.py new file mode 100644 index 0000000000000..90fe28fb9b33a --- /dev/null +++ b/build_support/build_backend.py @@ -0,0 +1,21 @@ +import pathlib +import subprocess + +from setuptools import build_meta as _orig + +prepare_metadata_for_build_wheel = _orig.prepare_metadata_for_build_wheel +build_sdist = _orig.build_sdist +get_requires_for_build_wheel = _orig.get_requires_for_build_wheel +get_requires_for_build_sdist = _orig.get_requires_for_build_sdist + + +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + filedir = pathlib.Path(__file__).resolve().parent.parent + subprocess.run(["cmake", "."], cwd=filedir, check=True) + subprocess.run( + ["cmake", "--build", ".", "--config", "Release", "--parallel"], + cwd=filedir, + check=True, + ) + + return _orig.build_wheel(wheel_directory, config_settings, metadata_directory) diff --git a/pyproject.toml b/pyproject.toml index d7dd57aff6364..25d9295f5d0ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,9 +8,9 @@ requires = [ "Cython>=0.29.24,<3", # Note: sync with setup.py, environment.yml and asv.conf.json "oldest-supported-numpy>=0.10" ] -# uncomment to enable pep517 after versioneer problem is fixed. -# https://github.com/python-versioneer/python-versioneer/issues/193 -# build-backend = "setuptools.build_meta" + +build-backend = "build_backend" +backend-path = ["build_support"] [tool.black] target-version = ['py38', 'py39'] diff --git a/setup.py b/setup.py index 72e6bb1e37557..79e4fea9fdf36 100644 --- a/setup.py +++ b/setup.py @@ -5,8 +5,14 @@ (https://github.com/zeromq/pyzmq) which have been permitted for use under the BSD license. Parts are from lxml (https://github.com/lxml/lxml) """ +import os +import sys + from setuptools import setup +# uncomment to enable pep517 after versioneer problem is fixed. +# https://github.com/python-versioneer/python-versioneer/issues/193 +sys.path.insert(0, os.path.dirname(__file__)) import versioneer cmdclass = versioneer.get_cmdclass() From b1e6be88d806a42cb6187cff1482e85289045e1e Mon Sep 17 00:00:00 2001 From: William Ayd Date: Tue, 21 Jun 2022 19:21:36 -0700 Subject: [PATCH 64/68] revert sdist ci changes --- .github/workflows/sdist.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 2280af63c5002..5ae2280c5069f 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -43,16 +43,11 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip cmake setuptools wheel + python -m pip install --upgrade pip setuptools wheel # GH 39416 pip install numpy - - name: Build pandas - run: | - cmake . - cmake --build . --parallel - - name: Build pandas sdist run: | pip list From d6592dc52d7c97e489695d2436f396bc382d8e66 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 3 Aug 2022 22:54:42 -0700 Subject: [PATCH 65/68] removed errant global-exclude of sos --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 873f680b9a147..cd13a469043b2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -30,7 +30,6 @@ global-exclude *.odt global-exclude *.orc global-exclude *.sas7bdat global-exclude *.sav -global-exclude *.so global-exclude *.xls global-exclude *.xlsb global-exclude *.xlsm From 115578761fb82f642b440b3abbc3eb144a87ab46 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 3 Aug 2022 23:31:54 -0700 Subject: [PATCH 66/68] code check --- pandas/_libs/generate_templates.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/_libs/generate_templates.py b/pandas/_libs/generate_templates.py index 56d61acc5abee..013568a2812b8 100644 --- a/pandas/_libs/generate_templates.py +++ b/pandas/_libs/generate_templates.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from Cython import Tempita if __name__ == "__main__": From c57200c199f00aceb6320bf434a829f86ec6524b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 4 Aug 2022 07:46:26 -0700 Subject: [PATCH 67/68] reverted Dockerfile hacks --- Dockerfile | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 41375400709ce..8fd0bc0b927f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM quay.io/condaforge/miniforge3 # if you forked pandas, you can pass in your own GitHub username to use your fork # i.e. gh_username=myname -ARG gh_username=WillAyd +ARG gh_username=pandas-dev ARG pandas_home="/home/pandas" # Avoid warnings by switching to noninteractive @@ -18,7 +18,7 @@ RUN apt-get update \ && dpkg-reconfigure -f noninteractive tzdata \ # # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed - && apt-get -y install build-essential git iproute2 procps iproute2 lsb-release \ + && apt-get -y install git iproute2 procps iproute2 lsb-release \ # # cleanup && apt-get autoremove -y \ @@ -28,18 +28,10 @@ RUN apt-get update \ # Switch back to dialog for any ad-hoc use of apt-get ENV DEBIAN_FRONTEND=dialog -# Without this versioneer will not be able to determine the pandas version. -# This is because of a security update to git that blocks it from reading the config folder if -# it is not owned by the current user. We hit this since the "mounted" folder is not hit by the -# Docker container. -# xref https://github.com/pypa/manylinux/issues/1309 -RUN git config --global --add safe.directory "$pandas_home" - # Clone pandas repo RUN mkdir "$pandas_home" \ && git clone "https://github.com/$gh_username/pandas.git" "$pandas_home" \ && cd "$pandas_home" \ - && git checkout cmake-build \ && git remote add upstream "https://github.com/pandas-dev/pandas.git" \ && git pull upstream main From d62f894c3ec08b721de0499cabe2c58a6e322496 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 4 Aug 2022 07:47:33 -0700 Subject: [PATCH 68/68] renamed typestub --- pandas/_libs/{json.pyi => ujson.pyi} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pandas/_libs/{json.pyi => ujson.pyi} (100%) diff --git a/pandas/_libs/json.pyi b/pandas/_libs/ujson.pyi similarity index 100% rename from pandas/_libs/json.pyi rename to pandas/_libs/ujson.pyi