CI: Have NumpyDev build only test nightly numpy (#55596)

Author: mroeschke

ci/deps/actions-311-numpydev.yaml

@@ -29,5 +29,4 @@ dependencies:
     - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
     - "--pre"
     - "numpy"
-    - "scipy"
     - "tzdata>=2022.1"