Use CMake for Build System

.circleci/setup_env.sh

@@ -98,9 +98,10 @@ if [ "$(conda list -f qt --json)" != [] ]; then
 fi
 
 echo "Build extensions"
-python setup.py build_ext -q -j3
+cmake .
+cmake --build . --config Release --parallel
 
 echo "Install pandas"
-python -m pip install --no-build-isolation --no-use-pep517 -e .
+python -m pip install --no-build-isolation -e .
 
 echo "done"

.github/actions/build_pandas/action.yml

@@ -12,8 +12,9 @@ runs:
 
     - name: Build Pandas
       run: |
-        python setup.py build_ext -j $N_JOBS
-        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
+        cmake .
+        cmake --build . --config Release --parallel
+        python -m pip install -e . --no-build-isolation  --no-index
       shell: bash -el {0}
       env:
         # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873

.github/workflows/32-bit-linux.yml

@@ -38,10 +38,11 @@ jobs:
           /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
-          pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
-          python setup.py build_ext -q -j1 && \
-          python -m pip install --no-build-isolation --no-use-pep517 -e . && \
-          python -m pip list && \
+          pip install cmake cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          cmake . && \
+          cmake --build . --parallel && \
+          python -m pip install --no-build-isolation -e . && \
+	  python -m pip list && \
           export PANDAS_CI=1 && \
           pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
 

.gitignore

@@ -37,6 +37,10 @@
 *.so
 .build_cache_dir
 MANIFEST
+Makefile
+CMakeCache.txt
+CMakeFiles/
+cmake_install.cmake
 
 # Python files #
 ################

CMakeLists.txt

@@ -0,0 +1,29 @@
+cmake_minimum_required(VERSION 3.18)
+
+set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9")
+set(CMAKE_C_STANDARD 99)
+set(CMAKE_C_STANDARD_REQUIRED True)
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+
+project(pandas)
+
+if(WIN32)
+  if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE
+        Release
+        CACHE STRING "Build type" FORCE)
+  endif()
+  find_package(Python3 REQUIRED COMPONENTS Interpreter Development NumPy)
+  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE "")
+  link_directories(${Python3_LIBRARY_DIRS})
+else()
+  # we only choose Development.Module to support virtual environments where
+  # libpython may not be available see
+  # https://github.com/pypa/manylinux/issues/484
+  find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module NumPy)
+endif()
+
+add_compile_definitions(NPY_NO_DEPRECATED_API=0)
+add_subdirectory("pandas/_libs")
+add_subdirectory("pandas/io/sas")

Dockerfile

@@ -49,5 +49,6 @@ RUN . /opt/conda/etc/profile.d/conda.sh \
     && conda activate base \
     && cd "$pandas_home" \
     && export \
-    && python setup.py build_ext -j 4 \
+    && cmake . \
+    && cmake --build . --parallel \
     && python -m pip install --no-build-isolation -e .

MANIFEST.in

@@ -1,12 +1,14 @@
 include RELEASE.md
 include versioneer.py
+include CMakeLists.txt
 
 graft doc
 prune doc/build
 
 graft LICENSES
 
 graft pandas
+graft build_support
 
 global-exclude *.bz2
 global-exclude *.csv
@@ -28,7 +30,6 @@ global-exclude *.odt
 global-exclude *.orc
 global-exclude *.sas7bdat
 global-exclude *.sav
-global-exclude *.so
 global-exclude *.xls
 global-exclude *.xlsb
 global-exclude *.xlsm
@@ -42,6 +43,9 @@ global-exclude *~
 global-exclude .DS_Store
 global-exclude .git*
 global-exclude \#*
+recursive-exclude **/CMakeFiles *
+global-exclude *.cmake
+global-exclude CMakeCache.txt
 
 global-exclude *.c
 global-exclude *.cpp

build_support/build_backend.py

@@ -0,0 +1,21 @@
+import pathlib
+import subprocess
+
+from setuptools import build_meta as _orig
+
+prepare_metadata_for_build_wheel = _orig.prepare_metadata_for_build_wheel
+build_sdist = _orig.build_sdist
+get_requires_for_build_wheel = _orig.get_requires_for_build_wheel
+get_requires_for_build_sdist = _orig.get_requires_for_build_sdist
+
+
+def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
+    filedir = pathlib.Path(__file__).resolve().parent.parent
+    subprocess.run(["cmake", "."], cwd=filedir, check=True)
+    subprocess.run(
+        ["cmake", "--build", ".", "--config", "Release", "--parallel"],
+        cwd=filedir,
+        check=True,
+    )
+
+    return _orig.build_wheel(wheel_directory, config_settings, metadata_directory)

doc/source/development/contributing_environment.rst

@@ -42,7 +42,8 @@ Run Container::
     If you bind your local repo for the first time, you have to build the C extensions afterwards.
     Run the following command inside the container::
 
-        python setup.py build_ext -j 4
+        cmake .
+        cmake --build . --config Release --parallel
 
     You need to rebuild the C extensions anytime the Cython code in ``pandas/_libs`` changes.
     This most frequently occurs when changing or merging branches.
@@ -169,7 +170,8 @@ We'll now kick off a three-step process:
    source activate pandas-dev
 
    # Build and install pandas
-   python setup.py build_ext -j 4
+   cmake .
+   cmake --build . --config Release --parallel
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 At this point you should be able to import pandas from your locally built version::
@@ -216,7 +218,8 @@ You also need to have ``setuptools`` 51.0.0 or later to build pandas.
    python -m pip install -r requirements-dev.txt
 
    # Build and install pandas
-   python setup.py build_ext -j 4
+   cmake .
+   cmake --build . --config Release --parallel
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 **Unix**/**macOS with pyenv**
@@ -240,7 +243,8 @@ Consult the docs for setting up pyenv `here <https://github.com/pyenv/pyenv>`__.
    python -m pip install -r requirements-dev.txt
 
    # Build and install pandas
-   python setup.py build_ext -j 4
+   cmake .
+   cmake --build . --config Release --parallel
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 **Windows**
@@ -266,5 +270,6 @@ should already exist.
    python -m pip install -r requirements-dev.txt
 
    # Build and install pandas
-   python setup.py build_ext -j 4
+   cmake .
+   cmake --build . --config Release --parallel
    python -m pip install -e . --no-build-isolation --no-use-pep517

doc/source/development/debugging_extensions.rst

@@ -8,11 +8,12 @@ Debugging C extensions
 
 Pandas uses select C extensions for high performance IO operations. In case you need to debug segfaults or general issues with those extensions, the following steps may be helpful.
 
-First, be sure to compile the extensions with the appropriate flags to generate debug symbols and remove optimizations. This can be achieved as follows:
+First, be sure to compile the extensions with the appropriate flags to generate debug symbols and remove optimizations. This can be achieved on Unix-like systems as follows:
 
 .. code-block:: sh
 
-   python setup.py build_ext --inplace -j4 --with-debugging-symbols
+   cmake . -DCMAKE_BUILD_TYPE=Debug
+   cmake --build . --parallel
 
 Using a debugger
 ================

environment.yml

@@ -79,6 +79,7 @@ dependencies:
   - asv
 
   # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms.
+  - cmake>=3.18.0
   - c-compiler
   - cxx-compiler
 

pandas/__init__.py

@@ -28,7 +28,8 @@
     raise ImportError(
         f"C extension: {_module} not built. If you want to import "
         "pandas from the source directory, you may need to run "
-        "'python setup.py build_ext --force' to build the C extensions first."
+        "'cmake . && cmake --build . --config Release --parallel' "
+        "to build the C extensions first."
     ) from _err
 else:
     del _tslib, _lib, _hashtable

pandas/_libs/CMakeLists.txt

@@ -0,0 +1,142 @@
+add_custom_command(
+  OUTPUT algos_common_helper.pxi
+         algos_take_helper.pxi
+         hashtable_class_helper.pxi
+         hashtable_func_helper.pxi
+         index_class_helper.pxi
+         intervaltree.pxi
+         khash_for_primitive_helper.pxi
+         sparse_op_helper.pxi
+  COMMAND ${Python3_EXECUTABLE} generate_templates.py)
+
+add_custom_command(
+  OUTPUT algos.c
+  COMMAND ${Python3_EXECUTABLE} -m cython -3 algos.pyx
+  DEPENDS algos_common_helper.pxi algos_take_helper.pxi)
+python3_add_library(algos MODULE WITH_SOABI algos.c)
+target_include_directories(
+  algos PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib")
+
+# There is a khash header file in src/klib and a cython generated one in _libs.
+# Depending on the build timing one of the other could get picked up, though
+# unclear why we need both? If we stick to the non-generated version we can
+# remove any DEPENDS khash.h
+add_custom_command(
+  OUTPUT khash.h
+  COMMAND ${Python3_EXECUTABLE} -m cython -3 khash.pxd
+  DEPENDS hkash_for_primitive_helper.pxi)
+
+add_custom_command(
+  OUTPUT hashtable.c
+  COMMAND ${Python3_EXECUTABLE} -m cython -3 hashtable.pyx
+  DEPENDS hashtable_class_helper.pxi hashtable_func_helper.pxi)
+
+python3_add_library(hashtable MODULE WITH_SOABI hashtable.c)
+target_include_directories(
+  hashtable PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}
+                   "src/klib" DEPENDS khash.h)
+
+add_custom_command(
+  OUTPUT index.c
+  COMMAND ${Python3_EXECUTABLE} -m cython -3 index.pyx
+  DEPENDS index_class_helper.pxi)
+python3_add_library(index MODULE WITH_SOABI index.c)
+target_include_directories(
+  index PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib"
+               "./tslibs")
+
+add_custom_command(
+  OUTPUT interval.c
+  COMMAND ${Python3_EXECUTABLE} -m cython -3 interval.pyx
+  DEPENDS intervaltree.pxi)
+python3_add_library(interval MODULE WITH_SOABI interval.c)
+target_include_directories(
+  interval PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}
+                  "src/klib" "./tslibs")
+
+add_custom_command(
+  OUTPUT sparse.c
+  COMMAND ${Python3_EXECUTABLE} -m cython -3 sparse.pyx
+  DEPENDS sparse_op_helper.pxi)
+python3_add_library(sparse MODULE WITH_SOABI sparse.c)
+target_include_directories(
+  sparse PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}
+                "src/klib")
+
+set(BASIC_LIBRARIES
+    arrays
+    groupby
+    hashing
+    indexing
+    internals
+    reduction
+    ops
+    ops_dispatch
+    properties
+    reshape
+    testing
+    writers)
+foreach(LIB ${BASIC_LIBRARIES})
+  add_custom_command(OUTPUT ${LIB}.c COMMAND ${Python3_EXECUTABLE} -m cython -3
+                                             ${LIB}.pyx)
+  python3_add_library(${LIB} MODULE WITH_SOABI ${LIB}.c)
+  target_include_directories(${LIB} PUBLIC ${Python3_INCLUDE_DIRS}
+                                           ${Python3_NumPy_INCLUDE_DIRS})
+endforeach()
+
+add_subdirectory("tslibs")
+
+add_custom_command(OUTPUT tslib.c COMMAND ${Python3_EXECUTABLE} -m cython -3
+                                          tslib.pyx)
+python3_add_library(tslib ${LIB} MODULE WITH_SOABI tslib.c
+                    tslibs/src/datetime/np_datetime.c)
+target_include_directories(
+  tslib PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "./tslibs")
+
+add_custom_command(OUTPUT missing.c COMMAND ${Python3_EXECUTABLE} -m cython -3
+                                            missing.pyx)
+python3_add_library(missing MODULE WITH_SOABI missing.c)
+target_include_directories(
+  missing PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}
+                 "./tslibs")
+
+add_custom_command(OUTPUT lib.c COMMAND ${Python3_EXECUTABLE} -m cython -3
+                                        lib.pyx)
+python3_add_library(lib MODULE WITH_SOABI lib.c src/parser/tokenizer.c)
+target_include_directories(
+  lib PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} "src/klib"
+             "./tslibs")
+
+add_custom_command(
+  OUTPUT join.c
+  COMMAND ${Python3_EXECUTABLE} -m cython -3 join.pyx
+  DEPENDS khash_for_primitive_helper.pxi)
+python3_add_library(join MODULE WITH_SOABI join.c)
+target_include_directories(join PUBLIC ${Python3_INCLUDE_DIRS}
+                                       ${Python3_NumPy_INCLUDE_DIRS} "src/klib")
+
+add_custom_command(OUTPUT parsers.c COMMAND ${Python3_EXECUTABLE} -m cython -3
+                                            parsers.pyx)
+python3_add_library(parsers MODULE WITH_SOABI parsers.c src/parser/tokenizer.c
+                    src/parser/io.c)
+target_include_directories(
+  parsers PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}
+                 "src/klib" "src")
+
+python3_add_library(
+  ujson
+  MODULE
+  WITH_SOABI
+  src/ujson/python/ujson.c
+  src/ujson/python/objToJSON.c
+  src/ujson/python/date_conversions.c
+  src/ujson/python/JSONtoObj.c
+  src/ujson/lib/ultrajsonenc.c
+  src/ujson/lib/ultrajsondec.c
+  tslibs/src/datetime/np_datetime.c
+  tslibs/src/datetime/np_datetime_strings.c)
+target_include_directories(
+  ujson PUBLIC ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}
+               src/ujson/python src/ujson/lib src/datetime)
+
+add_subdirectory("window")