From 6cc3d6e2b3dc2fc3d88ebe658f89dcef5a4fe380 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 28 Jun 2016 14:15:56 +0100 Subject: [PATCH 1/6] Diff python pyx files as a test if they have changed --- ci/prep_cython_cache.sh | 45 ++++++++++++++++++++++++++++----------- ci/submit_cython_cache.sh | 12 ++++++++--- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index 162f7a1034be6..670247b73296e 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -1,31 +1,50 @@ #!/bin/bash ls "$HOME/.cache/" + +PYX_CACHE_DIR="$HOME/.cache/pyxfiles" +pyx_file_list=`find ${TRAVIS_BUILD_DIR} -name "*.pyx"` CACHE_File="$HOME/.cache/cython_files.tar" clear_cache=0 home_dir=$(pwd) -if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ]; then +if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ] && [ -d "$PYX_CACHE_DIR" ]; then + + echo "Cache available - checking pyx diff" + + for i in ${pyx_file_list} + do + diff=`diff -u $i $PYX_CACHE_DIR${i}` + if [[ $? -ne 0 ]] + then + echo "${i##*/} can't be diffed; probably not in cache" + clear_cache=1 + fi + if [[ ! -z $diff ]] + then + echo "${i##*/} has changed:" + echo $diff + clear_cache=1 + fi + done - echo "Cache available" - clear_cache=1 - # did the last commit change cython files? - # go back 2 commits if [ "$TRAVIS_PULL_REQUEST" == "false" ] then - echo "Not a PR: checking for cython files changes from last 2 commits" - git diff HEAD~2 --numstat | grep -E "pyx|pxd" - retval=$(git diff HEAD~2 --numstat | grep -E "pyx|pxd"| wc -l) + echo "Not a PR" + # Uncomment next 2 lines to turn off cython caching not in a PR + # echo "Non PR cython caching is disabled" + # clear_cache=1 else - echo "PR: checking for any cython file changes from last 5 commits" - git diff PR_HEAD~5 --numstat | grep -E "pyx|pxd" - retval=$(git diff PR_HEAD~5 --numstat | grep -E "pyx|pxd"| wc -l) + echo "In a PR" + # Uncomment next 2 lines to turn off cython caching in a PR + # echo "PR cython caching is disabled" + # clear_cache=1 fi - echo "number of cython files changed: $retval" + fi -if [ $clear_cache -eq 1 ] && [ $retval -eq 0 ] && [ "$USE_CACHE" ] +if [ $clear_cache -eq 1 ] && [ "$USE_CACHE" ] then # nope, reuse cython files echo "Will reuse cached cython file" diff --git a/ci/submit_cython_cache.sh b/ci/submit_cython_cache.sh index 3d41d652960c9..4f60df0ccb2d8 100755 --- a/ci/submit_cython_cache.sh +++ b/ci/submit_cython_cache.sh @@ -1,17 +1,23 @@ #!/bin/bash CACHE_File="$HOME/.cache/cython_files.tar" +PYX_CACHE_DIR="$HOME/.cache/pyxfiles" +pyx_file_list=`find ${TRAVIS_BUILD_DIR} -name "*.pyx"` + rm -rf $CACHE_File +rm -rf $PYX_CACHE_DIR home_dir=$(pwd) -pyx_files=`find ${TRAVIS_BUILD_DIR} -name "*.pyx"` +mkdir $PYX_CACHE_DIR +rsync -Rv $pyx_file_list $PYX_CACHE_DIR + echo "pyx files:" -echo $pyx_files +echo $pyx_file_list tar cf ${CACHE_File} --files-from /dev/null -for i in ${pyx_files} +for i in ${pyx_file_list} do f=${i%.pyx} ls $f.{c,cpp} | tar rf ${CACHE_File} -T - From 30bd9da5798d8b25b4568df5f29d91dc047bc57a Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 28 Jun 2016 20:51:00 +0100 Subject: [PATCH 2/6] Check for the case where less cython file has been deleted --- ci/prep_cython_cache.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index 670247b73296e..b3dc94bc62a54 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -4,9 +4,22 @@ ls "$HOME/.cache/" PYX_CACHE_DIR="$HOME/.cache/pyxfiles" pyx_file_list=`find ${TRAVIS_BUILD_DIR} -name "*.pyx"` +pyx_cache_file_list=`find ${PYX_CACHE_DIR} -name "*.pyx"` + CACHE_File="$HOME/.cache/cython_files.tar" +# Clear the cython cache 0 = NO, 1 = YES clear_cache=0 + +pyx_files=`echo "$pyx_file_list" | wc -l` +pyx_cache_files=`echo "$pyx_cache_file_list" | wc -l` + +if [[ pyx_files -ne pyx_cache_files ]] +then + echo "Different number of pyx files" + clear_cache=1 +fi + home_dir=$(pwd) if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ] && [ -d "$PYX_CACHE_DIR" ]; then @@ -44,17 +57,17 @@ if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ] && [ -d "$PYX_CACHE_DIR" ]; then fi -if [ $clear_cache -eq 1 ] && [ "$USE_CACHE" ] +if [ $clear_cache -eq 0 ] && [ "$USE_CACHE" ] then - # nope, reuse cython files + # No and use_cache is set echo "Will reuse cached cython file" cd / tar xvmf $CACHE_File cd $home_dir else echo "Rebuilding cythonized files" - echo "Use cache = $USE_CACHE" - echo "Clear cache = $clear_cache" + echo "Use cache (Blank if not set) = $USE_CACHE" + echo "Clear cache (1=YES) = $clear_cache" fi From dca6e6102ae767c70e031341951cd5a24ca11f01 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 28 Jun 2016 22:49:34 +0100 Subject: [PATCH 3/6] Test commit --- do_not_merge | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 do_not_merge diff --git a/do_not_merge b/do_not_merge new file mode 100644 index 0000000000000..e69de29bb2d1d From 79fe0f0ab4b088533217643471df1adecbc1c07b Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 28 Jun 2016 22:50:21 +0100 Subject: [PATCH 4/6] Remove do_not_merge Delete pyx file Add pyx file Change pyx file Undo pyx change Un test commit Test commit after ci change Change a pyx file Bug fix --- do_not_merge => DO_NOT_MERGE | 0 ci/prep_cython_cache.sh | 2 +- pandas/src/no_merge.pyx | 0 pandas/src/period.pyx | 2 ++ 4 files changed, 3 insertions(+), 1 deletion(-) rename do_not_merge => DO_NOT_MERGE (100%) create mode 100644 pandas/src/no_merge.pyx diff --git a/do_not_merge b/DO_NOT_MERGE similarity index 100% rename from do_not_merge rename to DO_NOT_MERGE diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index b3dc94bc62a54..6f16dce2fb431 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -29,7 +29,7 @@ if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ] && [ -d "$PYX_CACHE_DIR" ]; then for i in ${pyx_file_list} do diff=`diff -u $i $PYX_CACHE_DIR${i}` - if [[ $? -ne 0 ]] + if [[ $? -eq 2 ]] then echo "${i##*/} can't be diffed; probably not in cache" clear_cache=1 diff --git a/pandas/src/no_merge.pyx b/pandas/src/no_merge.pyx new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/pandas/src/period.pyx b/pandas/src/period.pyx index aca0d0dbc107b..b2e312e66aefd 100644 --- a/pandas/src/period.pyx +++ b/pandas/src/period.pyx @@ -1,4 +1,5 @@ from datetime import datetime, date, timedelta + import operator from cpython cimport ( @@ -21,6 +22,7 @@ from pandas.tseries import offsets from pandas.tseries.tools import parse_time_string cimport cython + from datetime cimport * cimport util cimport lib From 81cbb7bcc80e5b0c0921004d1a91da708eaaefb9 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Fri, 1 Jul 2016 14:10:14 +0100 Subject: [PATCH 5/6] Reset pyx changes --- pandas/src/no_merge.pyx | 0 pandas/src/period.pyx | 2 -- 2 files changed, 2 deletions(-) delete mode 100644 pandas/src/no_merge.pyx diff --git a/pandas/src/no_merge.pyx b/pandas/src/no_merge.pyx deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/pandas/src/period.pyx b/pandas/src/period.pyx index b2e312e66aefd..aca0d0dbc107b 100644 --- a/pandas/src/period.pyx +++ b/pandas/src/period.pyx @@ -1,5 +1,4 @@ from datetime import datetime, date, timedelta - import operator from cpython cimport ( @@ -22,7 +21,6 @@ from pandas.tseries import offsets from pandas.tseries.tools import parse_time_string cimport cython - from datetime cimport * cimport util cimport lib From 8b4ad0b57f4c989bf46ab0c55f26b333adbfb08f Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Fri, 1 Jul 2016 14:12:17 +0100 Subject: [PATCH 6/6] Remove test commit --- DO_NOT_MERGE | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 DO_NOT_MERGE diff --git a/DO_NOT_MERGE b/DO_NOT_MERGE deleted file mode 100644 index e69de29bb2d1d..0000000000000