From 13aa2f99fec89c369f2c9ff82523f5e17126d350 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 20 May 2022 20:23:40 -0700 Subject: [PATCH 1/5] BLD: Build shared c dependencies as a library --- setup.py | 61 ++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index bca919a3aa6f8..7484b566aa439 100755 --- a/setup.py +++ b/setup.py @@ -102,6 +102,7 @@ def build_extensions(self): if _CYTHON_INSTALLED: self.render_templates(_pxifiles) + self.run_command("build_clib") super().build_extensions() @@ -438,6 +439,33 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "pandas/_libs/tslibs/src/datetime/np_datetime_strings.h", ] +import sysconfig + +ext_libraries = [ + [ + "np_datetime", + { + "sources": [ + "pandas/_libs/tslibs/src/datetime/np_datetime.c", + "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", + ], + "include_dirs": [ + "pandas/_libs/tslibs/src/datetime", + sysconfig.get_path("include"), + numpy.get_include(), + ], + }, + ], + [ + "tokenizer", + { + "sources": ["pandas/_libs/src/parser/tokenizer.c"], + "depends": ["pandas/_libs/src/parser/tokenizer.h"], + "include_dirs": [sysconfig.get_path("include")] + klib_include, + }, + ], +] + ext_data = { "_libs.algos": { "pyxfile": "_libs/algos", @@ -472,7 +500,6 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "pyxfile": "_libs/lib", "depends": lib_depends + tseries_depends, "include": klib_include, # due to tokenizer import - "sources": ["pandas/_libs/src/parser/tokenizer.c"], }, "_libs.missing": {"pyxfile": "_libs/missing", "depends": tseries_depends}, "_libs.parsers": { @@ -482,8 +509,8 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "pandas/_libs/src/parser/tokenizer.h", "pandas/_libs/src/parser/io.h", ], + "libraries": ["tokenizer"], "sources": [ - "pandas/_libs/src/parser/tokenizer.c", "pandas/_libs/src/parser/io.c", ], }, @@ -501,21 +528,18 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "_libs.tslibs.conversion": { "pyxfile": "_libs/tslibs/conversion", "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + "libraries": ["np_datetime"], }, "_libs.tslibs.fields": { "pyxfile": "_libs/tslibs/fields", "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + "libraries": ["np_datetime"], }, "_libs.tslibs.nattype": {"pyxfile": "_libs/tslibs/nattype"}, "_libs.tslibs.np_datetime": { "pyxfile": "_libs/tslibs/np_datetime", "depends": tseries_depends, - "sources": [ - "pandas/_libs/tslibs/src/datetime/np_datetime.c", - "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", - ], + "libraries": ["np_datetime"], }, "_libs.tslibs.offsets": { "pyxfile": "_libs/tslibs/offsets", @@ -525,12 +549,12 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "pyxfile": "_libs/tslibs/parsing", "include": klib_include, "depends": ["pandas/_libs/src/parser/tokenizer.h"], - "sources": ["pandas/_libs/src/parser/tokenizer.c"], + "libraries": ["tokenizer"], }, "_libs.tslibs.period": { "pyxfile": "_libs/tslibs/period", "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + "libraries": ["np_datetime"], }, "_libs.tslibs.strptime": { "pyxfile": "_libs/tslibs/strptime", @@ -539,12 +563,12 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "_libs.tslibs.timedeltas": { "pyxfile": "_libs/tslibs/timedeltas", "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + "libraries": ["np_datetime"], }, "_libs.tslibs.timestamps": { "pyxfile": "_libs/tslibs/timestamps", "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + "libraries": ["np_datetime"], }, "_libs.tslibs.timezones": {"pyxfile": "_libs/tslibs/timezones"}, "_libs.tslibs.tzconversion": { @@ -593,6 +617,7 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): depends=data.get("depends", []), include_dirs=include, language=data.get("language", "c"), + libraries=data.get("libraries", []), define_macros=data.get("macros", macros), extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, @@ -626,17 +651,18 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "pandas/_libs/src/ujson/lib/ultrajsonenc.c", "pandas/_libs/src/ujson/lib/ultrajsondec.c", ] - + [ - "pandas/_libs/tslibs/src/datetime/np_datetime.c", - "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", - ] + # + [ + # "pandas/_libs/tslibs/src/datetime/np_datetime.c", + # "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", + # ] ), include_dirs=[ "pandas/_libs/src/ujson/python", "pandas/_libs/src/ujson/lib", - "pandas/_libs/src/datetime", + "pandas/_libs/tslibs/src/datetime", numpy.get_include(), ], + libraries=["np_datetime"], extra_compile_args=(["-D_GNU_SOURCE"] + extra_compile_args), extra_link_args=extra_link_args, define_macros=macros, @@ -654,5 +680,6 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): setup( version=versioneer.get_version(), ext_modules=maybe_cythonize(extensions, compiler_directives=directives), + libraries=ext_libraries, cmdclass=cmdclass, ) From 6cd14497e89b7833b0e06e5eb22804a40f99ce04 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 20 May 2022 21:24:00 -0700 Subject: [PATCH 2/5] Update action.yml --- .github/actions/build_pandas/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 5e5a3bdf0f024..2218627d5a068 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -16,5 +16,4 @@ runs: python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index shell: bash -el {0} env: - # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 - N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }} + N_JOBS: 4 From 8325ff8d12babd0d7e19f777c46d3ead8d560c4e Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 24 May 2022 08:24:05 -0700 Subject: [PATCH 3/5] clean --- setup.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 7484b566aa439..5b22a2989c4ab 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,10 @@ import platform import shutil import sys -from sysconfig import get_config_vars +from sysconfig import ( + get_config_vars, + get_path, +) import numpy from pkg_resources import parse_version @@ -439,9 +442,7 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "pandas/_libs/tslibs/src/datetime/np_datetime_strings.h", ] -import sysconfig - -ext_libraries = [ +internal_libraries = [ [ "np_datetime", { @@ -451,7 +452,7 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): ], "include_dirs": [ "pandas/_libs/tslibs/src/datetime", - sysconfig.get_path("include"), + get_path("include"), numpy.get_include(), ], }, @@ -461,7 +462,7 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): { "sources": ["pandas/_libs/src/parser/tokenizer.c"], "depends": ["pandas/_libs/src/parser/tokenizer.h"], - "include_dirs": [sysconfig.get_path("include")] + klib_include, + "include_dirs": [get_path("include")] + klib_include, }, ], ] @@ -651,10 +652,6 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "pandas/_libs/src/ujson/lib/ultrajsonenc.c", "pandas/_libs/src/ujson/lib/ultrajsondec.c", ] - # + [ - # "pandas/_libs/tslibs/src/datetime/np_datetime.c", - # "pandas/_libs/tslibs/src/datetime/np_datetime_strings.c", - # ] ), include_dirs=[ "pandas/_libs/src/ujson/python", @@ -680,6 +677,6 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): setup( version=versioneer.get_version(), ext_modules=maybe_cythonize(extensions, compiler_directives=directives), - libraries=ext_libraries, + libraries=internal_libraries, cmdclass=cmdclass, ) From b5545a71de6b2acfbbd697f8030908b1c75144bd Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Sun, 21 Aug 2022 13:27:01 -0400 Subject: [PATCH 4/5] change np_datetime.c references to be to the library (#16) --- setup.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 8de4e6d0454ab..167beff82f23a 100755 --- a/setup.py +++ b/setup.py @@ -543,7 +543,8 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "_libs.tslibs.offsets": { "pyxfile": "_libs/tslibs/offsets", "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + "libraries": ["np_datetime"], + # "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], }, "_libs.tslibs.parsing": { "pyxfile": "_libs/tslibs/parsing", @@ -574,12 +575,14 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "_libs.tslibs.tzconversion": { "pyxfile": "_libs/tslibs/tzconversion", "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + "libraries": ["np_datetime"], + # "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], }, "_libs.tslibs.vectorized": { "pyxfile": "_libs/tslibs/vectorized", "depends": tseries_depends, - "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], + "libraries": ["np_datetime"], + # "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"], }, "_libs.testing": {"pyxfile": "_libs/testing"}, "_libs.window.aggregations": { From 681eacbc128afccd2f295450188efbe54505cfa5 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 10 Sep 2022 07:46:59 -0400 Subject: [PATCH 5/5] pre-commit --- .github/actions/build_pandas/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 7dfb09d320d07..bc9bbb82fea06 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -16,4 +16,4 @@ runs: python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index shell: bash -el {0} env: - N_JOBS: 4 \ No newline at end of file + N_JOBS: 4