diff --git a/.travis.yml b/.travis.yml index 4d3908bc35de4..c6f6d8b81ae59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -229,14 +229,8 @@ matrix: - USE_CACHE=true before_install: - # gbq secure key - - if [ -n "$encrypted_1d9d7b1f171b_iv" ]; then - openssl aes-256-cbc -K $encrypted_1d9d7b1f171b_key - -iv $encrypted_1d9d7b1f171b_iv -in ci/travis_gbq.json.enc - -out ci/travis_gbq.json -d; - export VALID_GBQ_CREDENTIALS=True; - fi - echo "before_install" + - source ci/travis_process_gbq_encryption.sh - echo $VIRTUAL_ENV - export PATH="$HOME/miniconda/bin:$PATH" - df -h diff --git a/ci/travis_encrypt_gbq.sh b/ci/travis_encrypt_gbq.sh new file mode 100755 index 0000000000000..719db67f384e0 --- /dev/null +++ b/ci/travis_encrypt_gbq.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +GBQ_JSON_FILE=$1 +GBQ_PROJECT_ID=$2 + +if [[ $# -ne 2 ]]; then + echo -e "Too few arguments.\nUsage: ./travis_encrypt_gbq.sh "\ + " " + exit 1 +fi + +if [[ $GBQ_JSON_FILE != *.json ]]; then + echo "ERROR: Expected *.json file" + exit 1 +fi + +if [[ ! -f $GBQ_JSON_FILE ]]; then + echo "ERROR: File $GBQ_JSON_FILE does not exist" + exit 1 +fi + +echo "Encrypting $GBQ_JSON_FILE..." +read -d "\n" TRAVIS_KEY TRAVIS_IV <<<$(travis encrypt-file $GBQ_JSON_FILE \ +travis_gbq.json.enc -f | grep -o "\w*_iv\|\w*_key"); + +echo "Adding your secure key and project id to travis_gbq_config.txt ..." +echo -e "TRAVIS_IV_ENV=$TRAVIS_IV\nTRAVIS_KEY_ENV=$TRAVIS_KEY\n"\ +"GBQ_PROJECT_ID='$GBQ_PROJECT_ID'" > travis_gbq_config.txt + +echo "Done. Removing file $GBQ_JSON_FILE" +rm $GBQ_JSON_FILE + +echo -e "Created encrypted credentials file travis_gbq.json.enc.\n"\ + "NOTE: Do NOT commit the *.json file containing your unencrypted" \ + "private key" diff --git a/ci/travis_gbq_config.txt b/ci/travis_gbq_config.txt new file mode 100644 index 0000000000000..3b68d62f177cc --- /dev/null +++ b/ci/travis_gbq_config.txt @@ -0,0 +1,3 @@ +TRAVIS_IV_ENV=encrypted_1d9d7b1f171b_iv +TRAVIS_KEY_ENV=encrypted_1d9d7b1f171b_key +GBQ_PROJECT_ID='pandas-travis' diff --git a/ci/travis_process_gbq_encryption.sh b/ci/travis_process_gbq_encryption.sh new file mode 100755 index 0000000000000..7ff4c08f78e37 --- /dev/null +++ b/ci/travis_process_gbq_encryption.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +source ci/travis_gbq_config.txt + +if [[ -n ${!TRAVIS_IV_ENV} ]]; then + openssl aes-256-cbc -K ${!TRAVIS_KEY_ENV} -iv ${!TRAVIS_IV_ENV} \ + -in ci/travis_gbq.json.enc -out ci/travis_gbq.json -d; + export GBQ_PROJECT_ID=$GBQ_PROJECT_ID; + echo 'Successfully decrypted gbq credentials' +fi + diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 54de4d86a48d9..7f336abcaa6d7 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -626,6 +626,44 @@ This will display stderr from the benchmarks, and use your local Information on how to write a benchmark and how to use asv can be found in the `asv documentation `_. +.. _contributing.gbq_integration_tests: + +Running Google BigQuery Integration Tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You will need to create a Google BigQuery private key in JSON format in +order to run Google BigQuery integration tests on your local machine and +on Travis-CI. The first step is to create a `service account +`__. + +Integration tests for ``pandas.io.gbq`` are skipped in pull requests because +the credentials that are required for running Google BigQuery integration +tests are `encrypted `__ +on Travis-CI and are only accessible from the pydata/pandas repository. The +credentials won't be available on forks of pandas. Here are the steps to run +gbq integration tests on a forked repository: + +#. First, complete all the steps in the `Encrypting Files Prerequisites + `__ section. +#. Sign into `Travis `__ using your GitHub account. +#. Enable your forked repository of pandas for testing in `Travis + `__. +#. Run the following command from terminal where the current working directory + is the ``ci`` folder:: + + ./travis_encrypt_gbq.sh + +#. Create a new branch from the branch used in your pull request. Commit the + encrypted file called ``travis_gbq.json.enc`` as well as the file + ``travis_gbq_config.txt``, in an otherwise empty commit. DO NOT commit the + ``*.json`` file which contains your unencrypted private key. +#. Your branch should be tested automatically once it is pushed. You can check + the status by visiting your Travis branches page which exists at the + following location: https://travis-ci.org/your-user-name/pandas/branches . + Click on a build job for your branch. Expand the following line in the + build log: ``ci/print_skipped.py /tmp/nosetests.xml`` . Search for the + term ``test_gbq`` and confirm that gbq integration tests are not skipped. + Running the vbench performance test suite (phasing out) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -814,6 +852,11 @@ updated. Pushing them to GitHub again is done by:: This will automatically update your pull request with the latest code and restart the Travis-CI tests. +If your pull request is related to the ``pandas.io.gbq`` module, please see +the section on :ref:`Running Google BigQuery Integration Tests +` to configure a Google BigQuery service +account for your pull request on Travis-CI. + Delete your merged branch (optional) ------------------------------------ diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 7757950592da5..921fd824d6ffd 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -60,12 +60,12 @@ def _skip_if_no_private_key_contents(): def _in_travis_environment(): return 'TRAVIS_BUILD_DIR' in os.environ and \ - 'VALID_GBQ_CREDENTIALS' in os.environ + 'GBQ_PROJECT_ID' in os.environ def _get_project_id(): if _in_travis_environment(): - return 'pandas-travis' + return os.environ.get('GBQ_PROJECT_ID') else: return PROJECT_ID