Merge pull request numpy#24153 from mattip/cpp-random

BLD, TST: refactor test to use meson not setup.py, improve spin test -m ...

Author: rgommers

.github/workflows/build_test.yml

@@ -96,7 +96,7 @@ jobs:
         # for add-apt-repository
         sudo apt install software-properties-common -y
         sudo add-apt-repository ppa:deadsnakes/ppa -y
-        sudo apt install python3.9-dev -y
+        sudo apt install python3.9-dev ninja-build -y
         sudo ln -s /usr/bin/python3.9 /usr/bin/pythonx
         pythonx -m pip install --upgrade pip setuptools wheel
         pythonx -m pip install -r test_requirements.txt

.github/workflows/emscripten.yml

@@ -71,6 +71,7 @@ jobs:
           pyodide venv .venv-pyodide
           source .venv-pyodide/bin/activate
           pip install dist/*.whl
+          python -c "import sys; print(sys.platform)"
           pip install -r test_requirements.txt
       - name: Test
         run: |

.github/workflows/wheels.yml

@@ -192,6 +192,7 @@ jobs:
           # TODO: Don't run test suite, and instead build wheels from sdist
           # Depends on pypa/cibuildwheel#1020
           python -m pip install dist/*.gz
+          pip install ninja
           pip install -r test_requirements.txt
           cd .. # Can't import numpy within numpy src directory
           python -c "import numpy, sys; print(numpy.__version__); sys.exit(numpy.test() is False)"

.spin/cmds.py

@@ -120,13 +120,17 @@ def test(ctx, pytest_args, markexpr, n_jobs, tests, verbose):
      spin test -- -k "geometric"
      spin test -- -k "geometric and not rgeometric"
 
+    By default, spin will run `-m 'not slow'`. To run the full test suite, use
+    `spin -m full`
+
     For more, see `pytest --help`.
     """  # noqa: E501
     if (not pytest_args) and (not tests):
         pytest_args = ('numpy',)
 
     if '-m' not in pytest_args:
-        pytest_args = ('-m', markexpr) + pytest_args
+        if markexpr != "full":
+            pytest_args = ('-m', markexpr) + pytest_args
 
     if (n_jobs != "1") and ('-n' not in pytest_args):
         pytest_args = ('-n', str(n_jobs)) + pytest_args

azure-pipelines.yml

@@ -32,42 +32,6 @@ stages:
       - bash: echo "##vso[task.setvariable variable=start_main;isOutput=true]$RET"
         name: result
 
-- stage: InitialTests
-  condition: and(succeeded(), eq(dependencies.Check.outputs['Skip.result.start_main'], 'true'))
-  dependsOn: Check
-  jobs:
-
-  # Native build is based on gcc flag `-march=native`
-  - job: Linux_baseline_native
-    pool:
-      vmImage: 'ubuntu-20.04'
-    steps:
-    - script: |
-        git submodule update --init
-      displayName: 'Fetch submodules'
-    - script: |
-            if ! `gcc 2>/dev/null`; then
-                sudo apt install gcc
-            fi
-            sudo add-apt-repository ppa:deadsnakes/ppa -y
-            sudo apt install python3.9
-            sudo apt install python3.9-dev
-            sudo apt install python3.9-distutils
-            # python3 has no setuptools, so install one to get us going
-            python3.9 -m pip install --user --upgrade pip 'setuptools<49.2.0'
-            python3.9 -m pip install --user -r test_requirements.txt
-      displayName: 'install python/requirements'
-    - script: |
-            python3.9 runtests.py --show-build-log --cpu-baseline=native --cpu-dispatch=none \
-            --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml
-      displayName: 'Run native baseline Build / Tests'
-    - task: PublishTestResults@2
-      condition: succeededOrFailed()
-      inputs:
-        testResultsFiles: '**/test-*.xml'
-        failTaskOnFailedTests: true
-        testRunTitle: 'Publish test results for baseline/native'
-
 - stage: ComprehensiveTests
   condition: and(succeeded(), eq(dependencies.Check.outputs['Skip.result.start_main'], 'true'))
   dependsOn: Check
@@ -101,6 +65,7 @@ stages:
         git submodule update --init
       displayName: 'Fetch submodules'
     - script: |
+            # yum does not have a ninja package, so use the PyPI one
             docker run -v $(pwd):/numpy -e CFLAGS="-msse2 -std=c99 -UNDEBUG" \
             -e F77=gfortran-5 -e F90=gfortran-5 quay.io/pypa/manylinux2014_i686 \
             /bin/bash -xc " \
@@ -111,6 +76,7 @@ stages:
             target=\$(python3 tools/openblas_support.py) && \
             cp -r \$target/lib/* /usr/lib && \
             cp \$target/include/* /usr/include && \
+            python3 -m pip install ninja && \
             python3 -m pip install -r test_requirements.txt && \
             echo CFLAGS \$CFLAGS && \
             python3 -m pip install -v . && \
@@ -175,7 +141,7 @@ stages:
     - script: |
         python -m pip install -r test_requirements.txt
         # Don't use doc_requirements.txt since that messes up tests
-        python -m pip install vulture sphinx==4.3.0 numpydoc==1.4.0
+        python -m pip install vulture sphinx==4.3.0 numpydoc==1.4.0 ninja
       displayName: 'Install dependencies; some are optional to avoid test skips'
     - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'"
       displayName: 'Check for unreachable code paths in Python modules'

azure-steps-windows.yml

@@ -19,6 +19,7 @@ steps:
     choco install --confirm --no-progress --allow-downgrade rtools --version=4.3.5550
     choco install unzip -y
     choco install -y --checksum 6004DF17818F5A6DBF19CB335CC92702 pkgconfiglite
+    choco install ninja
     echo "##vso[task.setvariable variable=RTOOLS43_HOME]c:\rtools43"
   displayName: 'Install utilities'
 

numpy/core/tests/test_array_interface.py

@@ -1,7 +1,7 @@
 import sys
 import pytest
 import numpy as np
-from numpy.testing import extbuild
+from numpy.testing import extbuild, IS_WASM
 
 
 @pytest.fixture
@@ -12,6 +12,8 @@ def get_module(tmp_path):
 
     if not sys.platform.startswith('linux'):
         pytest.skip('link fails on cygwin')
+    if IS_WASM:
+        pytest.skip("Can't build module inside Wasm")
 
     prologue = '''
         #include <Python.h>
@@ -128,9 +130,6 @@ def get_module(tmp_path):
                                                more_init=more_init)
 
 
-# FIXME: numpy.testing.extbuild uses `numpy.distutils`, so this won't work on
-# Python 3.12 and up.
-@pytest.mark.skipif(sys.version_info >= (3, 12), reason="no numpy.distutils")
 @pytest.mark.slow
 def test_cstruct(get_module):
 

numpy/random/_examples/cython/meson.build

@@ -0,0 +1,45 @@
+project('random-build-examples', 'c', 'cpp', 'cython')
+
+py_mod = import('python')
+py3 = py_mod.find_installation(pure: false)
+
+cc = meson.get_compiler('c')
+cy = meson.get_compiler('cython')
+
+if not cy.version().version_compare('>=0.29.35')
+  error('tests requires Cython >= 0.29.35')
+endif
+
+_numpy_abs = run_command(py3, ['-c',
+               'import os; os.chdir(".."); import numpy; print(os.path.abspath(numpy.get_include() + "../../.."))'],
+                         check: true).stdout().strip()
+
+npymath_path = _numpy_abs / 'core' / 'lib'
+npy_include_path = _numpy_abs / 'core' / 'include'
+npyrandom_path = _numpy_abs / 'random' / 'lib'
+npymath_lib = cc.find_library('npymath', dirs: npymath_path)
+npyrandom_lib = cc.find_library('npyrandom', dirs: npyrandom_path)
+
+py3.extension_module(
+    'extending_distributions',
+    'extending_distributions.pyx',
+    install: false,
+    include_directories: [npy_include_path],
+    dependencies: [npyrandom_lib, npymath_lib],
+)
+py3.extension_module(
+    'extending',
+    'extending.pyx',
+    install: false,
+    include_directories: [npy_include_path],
+    dependencies: [npyrandom_lib, npymath_lib],
+)
+py3.extension_module(
+    'extending_cpp',
+    'extending_distributions.pyx',
+    install: false,
+    override_options : ['cython_language=cpp'],
+    cython_args: ['--module-name', 'extending_cpp'],
+    include_directories: [npy_include_path],
+    dependencies: [npyrandom_lib, npymath_lib],
+)

numpy/random/_examples/cython/setup.py

@@ -152,7 +152,7 @@ py.install_sources(
   [
   '_examples/cython/extending.pyx',
   '_examples/cython/extending_distributions.pyx',
-  '_examples/cython/setup.py',
+  '_examples/cython/meson.build',
   ],
   subdir: 'numpy/random/_examples/cython'
 )

numpy/random/meson.build

@@ -1,8 +1,12 @@
+from importlib.util import spec_from_file_location, module_from_spec
 import os
+import pathlib
 import pytest
 import shutil
 import subprocess
 import sys
+import sysconfig
+import textwrap
 import warnings
 
 import numpy as np
@@ -39,55 +43,62 @@
     # other fixes in the 0.29 series that are needed even for earlier
     # Python versions.
     # Note: keep in sync with the one in pyproject.toml
-    required_version = '0.29.30'
+    required_version = '0.29.35'
     if _pep440.parse(cython_version) < _pep440.Version(required_version):
         # too old or wrong cython, skip the test
         cython = None
 
 
-@pytest.mark.skipif(sys.version_info >= (3, 12),
-                    reason="numpy.distutils not supported anymore")
 @pytest.mark.skipif(IS_WASM, reason="Can't start subprocess")
 @pytest.mark.skipif(cython is None, reason="requires cython")
 @pytest.mark.slow
 def test_cython(tmp_path):
-    from numpy.distutils.misc_util import exec_mod_from_location
+    import glob
+    # build the examples in a temporary directory
     srcdir = os.path.join(os.path.dirname(__file__), '..')
     shutil.copytree(srcdir, tmp_path / 'random')
-    # build the examples and "install" them into a temporary directory
     build_dir = tmp_path / 'random' / '_examples' / 'cython'
-    subprocess.check_call([sys.executable, 'setup.py', 'build', 'install',
-                           '--prefix', str(tmp_path / 'installdir'),
-                           '--single-version-externally-managed',
-                           '--record', str(tmp_path/ 'tmp_install_log.txt'),
-                          ],
-                          cwd=str(build_dir),
-                      )
+    target_dir = build_dir / "build"
+    os.makedirs(target_dir, exist_ok=True)
+    if sys.platform == "win32":
+        subprocess.check_call(["meson", "setup",
+                               "--buildtype=release", 
+                               "--vsenv", str(build_dir)],
+                              cwd=target_dir,
+                              )
+    else:
+        subprocess.check_call(["meson", "setup", str(build_dir)],
+                              cwd=target_dir
+                              )
+    subprocess.check_call(["meson", "compile", "-vv"], cwd=target_dir)
+
     # gh-16162: make sure numpy's __init__.pxd was used for cython
     # not really part of this test, but it is a convenient place to check
-    with open(build_dir / 'extending.c') as fid:
+
+    g = glob.glob(str(target_dir / "*" / "extending.pyx.c"))
+    with open(g[0]) as fid:
         txt_to_find = 'NumPy API declarations from "numpy/__init__'
         for i, line in enumerate(fid):
             if txt_to_find in line:
                 break
         else:
             assert False, ("Could not find '{}' in C file, "
                            "wrong pxd used".format(txt_to_find))
-    # get the path to the so's
-    so1 = so2 = None
-    with open(tmp_path /'tmp_install_log.txt') as fid:
-        for line in fid:
-            if 'extending.' in line:
-                so1 = line.strip()
-            if 'extending_distributions' in line:
-                so2 = line.strip()
-    assert so1 is not None
-    assert so2 is not None
-    # import the so's without adding the directory to sys.path
-    exec_mod_from_location('extending', so1)
-    extending_distributions = exec_mod_from_location(
-                    'extending_distributions', so2)
+    # import without adding the directory to sys.path
+    suffix = sysconfig.get_config_var('EXT_SUFFIX')
+
+    def load(modname):
+        so = (target_dir / modname).with_suffix(suffix)
+        spec = spec_from_file_location(modname, so)
+        mod = module_from_spec(spec)
+        spec.loader.exec_module(mod)
+        return mod
+
+    # test that the module can be imported
+    load("extending")
+    load("extending_cpp")
     # actually test the cython c-extension
+    extending_distributions = load("extending_distributions")
     from numpy.random import PCG64
     values = extending_distributions.uniforms_ex(PCG64(0), 10, 'd')
     assert values.shape == (10,)