From 8dd8279044500c2a4b04288073c81ab78a605676 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 8 Dec 2021 09:50:51 -0600 Subject: [PATCH 01/10] test: remove redundant NIGHTLY run from CircleCI --- .circleci/config.yml | 8 ++++---- ....9-NIGHTLY.conda => requirements-3.10-1.3.4.conda} | 11 +++++++---- ci/run_conda.sh | 10 +--------- 3 files changed, 12 insertions(+), 17 deletions(-) rename ci/{requirements-3.9-NIGHTLY.conda => requirements-3.10-1.3.4.conda} (86%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c378b3f..e93ef662 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,12 +15,12 @@ jobs: - checkout - run: ci/config_auth.sh - run: ci/run_conda.sh - "conda-3.9-NIGHTLY": + "conda-3.10": docker: - image: continuumio/miniconda3 environment: - PYTHON: "3.9" - PANDAS: "NIGHTLY" + PYTHON: "3.10" + PANDAS: "1.3.4" steps: - checkout - run: ci/config_auth.sh @@ -31,4 +31,4 @@ workflows: build: jobs: - "conda-3.7" - - "conda-3.9-NIGHTLY" + - "conda-3.10" diff --git a/ci/requirements-3.9-NIGHTLY.conda b/ci/requirements-3.10-1.3.4.conda similarity index 86% rename from ci/requirements-3.9-NIGHTLY.conda rename to ci/requirements-3.10-1.3.4.conda index 5a3e9fb7..73595253 100644 --- a/ci/requirements-3.9-NIGHTLY.conda +++ b/ci/requirements-3.10-1.3.4.conda @@ -1,10 +1,13 @@ +codecov +coverage db-dtypes -pydata-google-auth +fastavro +flake8 google-cloud-bigquery google-cloud-bigquery-storage +numpy pyarrow +pydata-google-auth pytest pytest-cov -codecov -coverage -flake8 +tqdm diff --git a/ci/run_conda.sh b/ci/run_conda.sh index e29da98a..354f9b16 100755 --- a/ci/run_conda.sh +++ b/ci/run_conda.sh @@ -9,7 +9,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" # Install dependencies using Conda conda config --set always_yes yes --set changeps1 no -conda config --add channels pandas conda config --add channels conda-forge conda update -q conda conda info -a @@ -17,14 +16,7 @@ conda create -q -n test-environment python=$PYTHON source activate test-environment REQ="ci/requirements-${PYTHON}-${PANDAS}" conda install -q --file "$REQ.conda"; - -if [[ "$PANDAS" == "NIGHTLY" ]]; then - conda install -q numpy pytz python-dateutil; - PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; - pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; -else - conda install -q pandas=$PANDAS; -fi +conda install -q pandas=$PANDAS; python setup.py develop --no-deps From 3d9bbfba3ba7aa094199944fbf6cc0d1f7042379 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 10:54:02 -0600 Subject: [PATCH 02/10] use Python 3.9 for conda session --- .circleci/config.yml | 6 +++--- ...ements-3.10-1.3.4.conda => requirements-3.9-1.3.4.conda} | 0 ci/run_conda.sh | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) rename ci/{requirements-3.10-1.3.4.conda => requirements-3.9-1.3.4.conda} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index e93ef662..7d5c9ae2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,11 +15,11 @@ jobs: - checkout - run: ci/config_auth.sh - run: ci/run_conda.sh - "conda-3.10": + "conda-3.9": docker: - image: continuumio/miniconda3 environment: - PYTHON: "3.10" + PYTHON: "3.9" PANDAS: "1.3.4" steps: - checkout @@ -31,4 +31,4 @@ workflows: build: jobs: - "conda-3.7" - - "conda-3.10" + - "conda-3.9" diff --git a/ci/requirements-3.10-1.3.4.conda b/ci/requirements-3.9-1.3.4.conda similarity index 100% rename from ci/requirements-3.10-1.3.4.conda rename to ci/requirements-3.9-1.3.4.conda diff --git a/ci/run_conda.sh b/ci/run_conda.sh index 354f9b16..a1cebbc6 100755 --- a/ci/run_conda.sh +++ b/ci/run_conda.sh @@ -15,6 +15,8 @@ conda info -a conda create -q -n test-environment python=$PYTHON source activate test-environment REQ="ci/requirements-${PYTHON}-${PANDAS}" + +# TODO: Migrate the mamba with https://github.com/mamba-org/micromamba-docker conda install -q --file "$REQ.conda"; conda install -q pandas=$PANDAS; From c09463158975cbc9f0a74b2735b091641ec6ae62 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 11:14:23 -0600 Subject: [PATCH 03/10] migrate to mamba --- .circleci/config.yml | 4 ++-- ci/run_conda.sh | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7d5c9ae2..e008054c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: # Conda "conda-3.7": docker: - - image: continuumio/miniconda3 + - image: mambaorg/micromamba environment: PYTHON: "3.7" PANDAS: "0.24.2" @@ -17,7 +17,7 @@ jobs: - run: ci/run_conda.sh "conda-3.9": docker: - - image: continuumio/miniconda3 + - image: mambaorg/micromamba environment: PYTHON: "3.9" PANDAS: "1.3.4" diff --git a/ci/run_conda.sh b/ci/run_conda.sh index a1cebbc6..b73c4b7b 100755 --- a/ci/run_conda.sh +++ b/ci/run_conda.sh @@ -15,10 +15,8 @@ conda info -a conda create -q -n test-environment python=$PYTHON source activate test-environment REQ="ci/requirements-${PYTHON}-${PANDAS}" - -# TODO: Migrate the mamba with https://github.com/mamba-org/micromamba-docker -conda install -q --file "$REQ.conda"; -conda install -q pandas=$PANDAS; +mamba install -q --file "$REQ.conda"; +mamba install -q pandas=$PANDAS; python setup.py develop --no-deps From fc15566d1da1067a887b415505dcb73e69de94cf Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 11:17:20 -0600 Subject: [PATCH 04/10] use micromamba commands --- ci/run_conda.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ci/run_conda.sh b/ci/run_conda.sh index b73c4b7b..07c35e8a 100755 --- a/ci/run_conda.sh +++ b/ci/run_conda.sh @@ -8,15 +8,12 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" # Install dependencies using Conda -conda config --set always_yes yes --set changeps1 no -conda config --add channels conda-forge -conda update -q conda -conda info -a -conda create -q -n test-environment python=$PYTHON source activate test-environment REQ="ci/requirements-${PYTHON}-${PANDAS}" -mamba install -q --file "$REQ.conda"; -mamba install -q pandas=$PANDAS; +micromamba install -q --file "$REQ.conda"; +micromamba install -q pandas=$PANDAS; +micromamba list +micromamba info python setup.py develop --no-deps From f14a856704fdd63a3d2c9077493741a21ccb1349 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 11:20:37 -0600 Subject: [PATCH 05/10] micromamba --- ci/run_conda.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/run_conda.sh b/ci/run_conda.sh index 07c35e8a..0906c224 100755 --- a/ci/run_conda.sh +++ b/ci/run_conda.sh @@ -6,12 +6,11 @@ set -e -x DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -# Install dependencies using Conda - -source activate test-environment +# Install dependencies using (micro)mamba +# https://github.com/mamba-org/micromamba-docker REQ="ci/requirements-${PYTHON}-${PANDAS}" -micromamba install -q --file "$REQ.conda"; -micromamba install -q pandas=$PANDAS; +micromamba install -q pandas=$PANDAS python=${PYTHON} -n base -c conda-forge; +micromamba install -q --file "$REQ.conda" -n base -c conda-forge; micromamba list micromamba info From 5158674f15949ef98064773c5bddb817871172a2 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 11:24:46 -0600 Subject: [PATCH 06/10] activate --- ci/run_conda.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/run_conda.sh b/ci/run_conda.sh index 0906c224..4e111009 100755 --- a/ci/run_conda.sh +++ b/ci/run_conda.sh @@ -14,6 +14,7 @@ micromamba install -q --file "$REQ.conda" -n base -c conda-forge; micromamba list micromamba info +micromamba activate base python setup.py develop --no-deps # Run the tests From e4fd93cba2910747aa0a421e33ede2cfa19ff8a7 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 11:28:04 -0600 Subject: [PATCH 07/10] activate --- ci/run_conda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_conda.sh b/ci/run_conda.sh index 4e111009..e1ecf371 100755 --- a/ci/run_conda.sh +++ b/ci/run_conda.sh @@ -9,12 +9,12 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" # Install dependencies using (micro)mamba # https://github.com/mamba-org/micromamba-docker REQ="ci/requirements-${PYTHON}-${PANDAS}" +micromamba activate micromamba install -q pandas=$PANDAS python=${PYTHON} -n base -c conda-forge; micromamba install -q --file "$REQ.conda" -n base -c conda-forge; micromamba list micromamba info -micromamba activate base python setup.py develop --no-deps # Run the tests From 78dfc6737f981f68772d50d14ba855e8f2c86fe5 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 11:32:11 -0600 Subject: [PATCH 08/10] activate --- ci/run_conda.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/run_conda.sh b/ci/run_conda.sh index e1ecf371..1a69fbdd 100755 --- a/ci/run_conda.sh +++ b/ci/run_conda.sh @@ -6,10 +6,12 @@ set -e -x DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +micromamba shell init -s bash -p ~/micromamba +source ~/.bashrc + # Install dependencies using (micro)mamba # https://github.com/mamba-org/micromamba-docker REQ="ci/requirements-${PYTHON}-${PANDAS}" -micromamba activate micromamba install -q pandas=$PANDAS python=${PYTHON} -n base -c conda-forge; micromamba install -q --file "$REQ.conda" -n base -c conda-forge; micromamba list From 3226553a59d16c3de46de739ec4359ec96131247 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 11:35:44 -0600 Subject: [PATCH 09/10] activate --- ci/run_conda.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/run_conda.sh b/ci/run_conda.sh index 1a69fbdd..8db884ef 100755 --- a/ci/run_conda.sh +++ b/ci/run_conda.sh @@ -8,6 +8,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" micromamba shell init -s bash -p ~/micromamba source ~/.bashrc +micromamba activate # Install dependencies using (micro)mamba # https://github.com/mamba-org/micromamba-docker From 52d4d4d91ce2654fe16872f7c001200073814750 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 11:37:49 -0600 Subject: [PATCH 10/10] activate with shell hook --- ci/run_conda.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ci/run_conda.sh b/ci/run_conda.sh index 8db884ef..11b5b569 100755 --- a/ci/run_conda.sh +++ b/ci/run_conda.sh @@ -6,15 +6,14 @@ set -e -x DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -micromamba shell init -s bash -p ~/micromamba -source ~/.bashrc +eval "$(micromamba shell hook --shell=bash)" micromamba activate # Install dependencies using (micro)mamba # https://github.com/mamba-org/micromamba-docker REQ="ci/requirements-${PYTHON}-${PANDAS}" -micromamba install -q pandas=$PANDAS python=${PYTHON} -n base -c conda-forge; -micromamba install -q --file "$REQ.conda" -n base -c conda-forge; +micromamba install -q pandas=$PANDAS python=${PYTHON} -c conda-forge; +micromamba install -q --file "$REQ.conda" -c conda-forge; micromamba list micromamba info