Merge branch 'main' into issue-50395

Author: kostyafarber

.circleci/config.yml

@@ -18,8 +18,78 @@ jobs:
           PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
           LD_PRELOAD=$HOME/miniconda3/envs/pandas-dev/lib/libgomp.so.1:$LD_PRELOAD
           ci/run_tests.sh
+  build-aarch64:
+    parameters:
+      cibw-build:
+        type: string
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.large
+    environment:
+      ENV_FILE: ci/deps/circle-38-arm64.yaml
+      TRIGGER_SOURCE: << pipeline.trigger_source >>
+    steps:
+      - checkout
+      - run:
+          name: Check if build is necessary
+          command: |
+            # Check if tag is defined or TRIGGER_SOURCE is scheduled
+            if [[ -n "$CIRCLE_TAG" ]]; then
+                echo 'export IS_PUSH="true"' >> "$BASH_ENV"
+            elif [[ $TRIGGER_SOURCE == "scheduled_pipeline" ]]; then
+                echo 'export IS_SCHEDULE_DISPATCH="true"' >> "$BASH_ENV"
+            # Look for the build label/[wheel build] in commit
+            # grep takes a regex, so need to escape brackets
+            elif (git log --format=oneline -n 1 $CIRCLE_SHA1) | grep -q '\[wheel build\]'; then
+                : # Do nothing
+            elif ! (curl https://api.github.com/repos/pandas-dev/pandas/issues/$CIRCLE_PR_NUMBER | jq '.labels' | grep -q 'Build'); then
+                circleci-agent step halt
+            fi
+      - run:
+          name: Build aarch64 wheels
+          command: |
+            pip3 install cibuildwheel==2.9.0
+            cibuildwheel --output-dir wheelhouse
+          environment:
+            CIBW_BUILD: << parameters.cibw-build >>
+
+      - run:
+          name: Install Anaconda Client & Upload Wheels
+          command: |
+            echo "Install Mambaforge"
+            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/4.14.0-0/Mambaforge-4.14.0-0-Linux-aarch64.sh"
+            echo "Downloading $MAMBA_URL"
+            wget -q $MAMBA_URL -O minimamba.sh
+            chmod +x minimamba.sh
+
+            MAMBA_DIR="$HOME/miniconda3"
+            rm -rf $MAMBA_DIR
+            ./minimamba.sh -b -p $MAMBA_DIR
+
+            export PATH=$MAMBA_DIR/bin:$PATH
+
+            mamba install -y -c conda-forge anaconda-client
+
+            source ci/upload_wheels.sh
+            set_upload_vars
+            upload_wheels
+      - store_artifacts:
+          path: wheelhouse/
 
 workflows:
   test:
+    # Don't run trigger this one when scheduled pipeline runs
+    when:
+      not:
+        equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
       - test-arm
+  build-wheels:
+    jobs:
+      - build-aarch64:
+          filters:
+            tags:
+              only: /^v.*/
+          matrix:
+            parameters:
+              cibw-build: ["cp38-manylinux_aarch64", "cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"]

.circleci/setup_env.sh

@@ -55,8 +55,7 @@ if pip list | grep -q ^pandas; then
 fi
 
 echo "Build extensions"
-# GH 47305: Parallel build can causes flaky ImportError from pandas/_libs/tslibs
-python setup.py build_ext -q -j1
+python setup.py build_ext -q -j4
 
 echo "Install pandas"
 python -m pip install --no-build-isolation --no-use-pep517 -e .

.github/actions/build_pandas/action.yml

@@ -16,7 +16,5 @@ runs:
         python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
       shell: bash -el {0}
       env:
-        # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873
-        # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct
-        N_JOBS: 1
-        #N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }}
+        # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
+        N_JOBS: ${{ runner.os == 'macOS' && 3 || 2 }}

.github/actions/setup-conda/action.yml

@@ -18,7 +18,7 @@ runs:
     - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
       run: |
         grep -q '  - pyarrow' ${{ inputs.environment-file }}
-        sed -i"" -e "s/  - pyarrow<11/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
         cat ${{ inputs.environment-file }}
       shell: bash
       if: ${{ inputs.pyarrow-version }}
@@ -30,7 +30,7 @@ runs:
         environment-name: ${{ inputs.environment-name }}
         extra-specs: ${{ inputs.extra-specs }}
         channels: conda-forge
-        channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
+        channel-priority: 'strict'
         condarc-file: ci/condarc.yml
         cache-env: true
         cache-downloads: true

.github/workflows/32-bit-linux.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
   pull_request:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
     paths-ignore:
       - "doc/**"
 
@@ -40,7 +40,7 @@ jobs:
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
           python -m pip install versioneer[toml] && \
           python -m pip install cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.34.2 && \
-          python setup.py build_ext -q -j1 && \
+          python setup.py build_ext -q -j$(nproc) && \
           python -m pip install --no-build-isolation --no-use-pep517 -e . && \
           python -m pip list && \
           export PANDAS_CI=1 && \
@@ -52,3 +52,7 @@ jobs:
           name: Test results
           path: test-data.xml
         if: failure()
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-32bit
+      cancel-in-progress: true

.github/workflows/code-checks.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
   pull_request:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
 
 env:
   ENV_FILE: environment.yml

.github/workflows/deprecation-tracking-bot.yml

@@ -0,0 +1,31 @@
+name: Deprecations Bot
+
+on:
+  pull_request:
+    branches:
+      - main
+    types:
+      [closed]
+
+
+permissions:
+  contents: read
+
+jobs:
+  deprecation_update:
+    permissions:
+      issues: write
+    if: >-
+      contains(github.event.pull_request.labels.*.name, 'Deprecate') && github.event.pull_request.merged == true
+    runs-on: ubuntu-22.04
+    env:
+      DEPRECATION_TRACKER_ISSUE: 50578
+    steps:
+    - name: Checkout
+      run: |
+        echo "Adding deprecation PR number to deprecation tracking issue"
+        export PR=${{ github.event.pull_request.number }}
+        BODY=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${{ github.repository }}/issues/${DEPRECATION_TRACKER_ISSUE} |
+          python3 -c "import sys, json, os; x = {'body': json.load(sys.stdin)['body']}; pr = os.environ['PR']; x['body'] += f'\n- [ ] #{pr}'; print(json.dumps(x))")
+        echo ${BODY}
+        curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -X PATCH -d "${BODY}" https://api.github.com/repos/${{ github.repository }}/issues/${DEPRECATION_TRACKER_ISSUE}

.github/workflows/docbuild-and-upload.yml

@@ -4,13 +4,13 @@ on:
   push:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
 
 env:
   ENV_FILE: environment.yml
@@ -87,3 +87,10 @@ jobs:
         name: website
         path: web/build
         retention-days: 14
+
+    - name: Trigger web/doc preview
+      run: curl -X POST https://pandas.pydata.org/preview/submit/$RUN_ID/$PR_ID/
+      env:
+        RUN_ID: ${{ github.run_id }}
+        PR_ID: ${{ github.event.pull_request.number }}
+      if: github.event_name == 'pull_request'

.github/workflows/macos-windows.yml

@@ -4,20 +4,19 @@ on:
   push:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
   pull_request:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
     paths-ignore:
       - "doc/**"
+      - "web/**"
 
 env:
   PANDAS_CI: 1
   PYTEST_TARGET: pandas
   PATTERN: "not slow and not db and not network and not single_cpu"
-  ERROR_ON_WARNINGS: "1"
-
 
 permissions:
   contents: read

.github/workflows/package-checks.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
   pull_request:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
     types: [ labeled, opened, synchronize, reopened ]
 
 permissions:
@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        extra: ["test", "performance", "timezone", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "all"]
+        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "all"]
       fail-fast: false
     name: Install Extras - ${{ matrix.extra }}
     concurrency:

.github/workflows/preview-docs.yml

@@ -0,0 +1,22 @@
+name: Preview docs
+on:
+  issue_comment:
+    types: created
+
+permissions:
+  contents: read
+
+jobs:
+  preview_docs:
+    permissions:
+      issues: write
+      pull-requests: write
+    runs-on: ubuntu-22.04
+    steps:
+    - if: github.event.comment.body == '/preview'
+      run: |
+        if curl --output /dev/null --silent --head --fail "https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"; then
+          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "Website preview of this PR available at: https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
+        else
+          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "No preview found for PR #${{ github.event.issue.number }}. Did the docs build complete?"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
+        fi

.github/workflows/python-dev.yml

@@ -23,16 +23,17 @@ name: Python Dev
 on:
   push:
     branches:
-#      - main
-#      - 1.5.x
+      - main
+      - 2.0.x
       - None
   pull_request:
     branches:
-#      - main
-#      - 1.5.x
+      - main
+      - 2.0.x
       - None
     paths-ignore:
       - "doc/**"
+      - "web/**"
 
 env:
   PYTEST_WORKERS: "auto"
@@ -46,7 +47,7 @@ permissions:
 
 jobs:
   build:
-    # if: false # Uncomment this to freeze the workflow, comment it to unfreeze
+    if: false # Uncomment this to freeze the workflow, comment it to unfreeze
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -81,10 +82,9 @@ jobs:
         python -m pip install python-dateutil pytz cython hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
         python -m pip list
 
-    # GH 47305: Parallel build can cause flaky ImportError from pandas/_libs/tslibs
     - name: Build Pandas
       run: |
-        python setup.py build_ext -q -j1
+        python setup.py build_ext -q -j4
         python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
 
     - name: Build Version

.github/workflows/sdist.yml

@@ -4,14 +4,15 @@ on:
   push:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
   pull_request:
     branches:
       - main
-      - 1.5.x
+      - 2.0.x
     types: [labeled, opened, synchronize, reopened]
     paths-ignore:
       - "doc/**"
+      - "web/**"
 
 permissions:
   contents: read