Skip to content

Commit b7bd2f8

Browse files
pitroupitrou
committed
Debug parquet failure [skip appveyor]
1 parent 6ca246a commit b7bd2f8

File tree

3 files changed

+159
-155
lines changed

3 files changed

+159
-155
lines changed

‎.travis.yml

Lines changed: 143 additions & 143 deletions
Original file line numberDiff line numberDiff line change
@@ -48,165 +48,165 @@ matrix:
4848
group: deprecated
4949
env:
5050
- ARROW_TRAVIS_USE_TOOLCHAIN=1
51-
- ARROW_TRAVIS_VALGRIND=1
52-
- ARROW_TRAVIS_PLASMA=1
53-
- ARROW_TRAVIS_ORC=1
51+
#- ARROW_TRAVIS_VALGRIND=1
52+
#- ARROW_TRAVIS_PLASMA=1
53+
#- ARROW_TRAVIS_ORC=1
5454
- ARROW_TRAVIS_CLANG_FORMAT=1
55-
- ARROW_TRAVIS_COVERAGE=1
56-
- ARROW_TRAVIS_PYTHON_BENCHMARKS=1
57-
- ARROW_TRAVIS_PYTHON_DOCS=1
55+
#- ARROW_TRAVIS_COVERAGE=1
56+
#- ARROW_TRAVIS_PYTHON_BENCHMARKS=1
57+
#- ARROW_TRAVIS_PYTHON_DOCS=1
5858
- ARROW_BUILD_WARNING_LEVEL=CHECKIN
5959
- CC="clang-6.0"
6060
- CXX="clang++-6.0"
6161
before_script:
6262
# Always run RAT checks, in case another build in matrix breaks RAT
63-
- $TRAVIS_BUILD_DIR/ci/travis_release_audit.sh
63+
#- $TRAVIS_BUILD_DIR/ci/travis_release_audit.sh
6464
# (ARROW_CI_CPP_AFFECTED implies ARROW_CI_PYTHON_AFFECTED)
6565
- if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
6666
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
6767
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
68-
- $TRAVIS_BUILD_DIR/ci/travis_lint.sh
68+
#- $TRAVIS_BUILD_DIR/ci/travis_lint.sh
6969
# If either C++ or Python changed, we must install the C++ libraries
7070
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
7171
script:
7272
# All test steps are required for accurate C++ coverage info
73-
- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
73+
#- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
7474
- $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
75-
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
75+
#- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
7676
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
77-
- $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh
78-
# [OS X] C++ & Python w/ XCode 6.4
79-
- compiler: clang
80-
language: cpp
81-
osx_image: xcode6.4
82-
os: osx
83-
cache:
84-
addons:
85-
env:
86-
- ARROW_TRAVIS_USE_TOOLCHAIN=1
87-
- ARROW_TRAVIS_PLASMA=1
88-
- ARROW_TRAVIS_ORC=1
89-
- ARROW_BUILD_WARNING_LEVEL=CHECKIN
90-
before_script:
91-
- if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
92-
# If either C++ or Python changed, we must install the C++ libraries
93-
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
94-
script:
95-
- if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
96-
- $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
97-
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
98-
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
99-
# [manylinux1] Python
100-
- language: cpp
101-
before_script:
102-
- if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest; fi
103-
script:
104-
- if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi
105-
# Java w/ OpenJDK 8
106-
- language: java
107-
os: linux
108-
jdk: openjdk8
109-
before_script:
110-
- if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
111-
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
112-
script:
113-
- $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
114-
- $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh
115-
# Java w/ Oracle JDK 9
116-
- language: java
117-
os: linux
118-
jdk: oraclejdk9
119-
before_script:
120-
- if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
121-
script:
122-
- $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
123-
addons:
124-
apt:
125-
packages:
126-
- oracle-java9-installer
127-
# Integration w/ OpenJDK 8
128-
- language: java
129-
os: linux
130-
env: ARROW_TEST_GROUP=integration
131-
jdk: openjdk8
132-
env:
133-
- CC="clang-6.0"
134-
- CXX="clang++-6.0"
135-
before_script:
136-
- if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi
137-
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
138-
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
139-
- nvm install 10.1
140-
- $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
141-
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
142-
script:
143-
- $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
144-
# NodeJS
145-
- language: node_js
146-
os: linux
147-
node_js:
148-
- '10.1'
149-
before_script:
150-
- if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi
151-
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
152-
- $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
153-
script:
154-
- $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
155-
# C++ & glib w/ gcc 4.9
156-
- compiler: gcc
157-
language: cpp
158-
os: linux
159-
env:
160-
- BUILD_TORCH_EXAMPLE=no
161-
- CC="gcc-4.9"
162-
- CXX="g++-4.9"
163-
before_script:
164-
- if [ $ARROW_CI_C_GLIB_AFFECTED != "1" ]; then exit; fi
165-
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
166-
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
167-
- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
168-
script:
169-
- $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
170-
# [OS X] C++ & glib w/ XCode 8.3 & homebrew
171-
- compiler: clang
172-
osx_image: xcode8.3
173-
os: osx
174-
cache:
175-
addons:
176-
rvm: 2.2
177-
before_script:
178-
- if [ $ARROW_CI_C_GLIB_AFFECTED != "1" ]; then exit; fi
179-
- $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh
180-
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew
181-
- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
182-
script:
183-
- $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
184-
# Rust
185-
- language: rust
186-
cache: cargo
187-
rust: stable
188-
addons:
189-
apt:
190-
packages:
191-
- libcurl4-openssl-dev
192-
- libelf-dev
193-
- libdw-dev
194-
- binutils-dev
195-
- cmake
196-
before_script:
197-
- if [ $ARROW_CI_RUST_AFFECTED != "1" ]; then exit; fi
198-
- pip install 'travis-cargo<0.2' --user && export PATH=$HOME/.local/bin:$PATH
199-
- export CARGO_TARGET_DIR=$TRAVIS_BUILD_DIR/target
200-
- cargo install cargo-travis || echo "Skipping cargo-travis installation as it already exists in cache"
201-
- export PATH=$HOME/.cargo/bin:$PATH
202-
script:
203-
- $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh
204-
after_success:
205-
- pushd ${TRAVIS_BUILD_DIR}/rust
206-
# Run coverage for codecov.io
207-
- mkdir -p target/kcov
208-
- RUST_BACKTRACE=1 cargo coverage --verbose
209-
- bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"
77+
#- $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh
78+
## [OS X] C++ & Python w/ XCode 6.4
79+
#- compiler: clang
80+
#language: cpp
81+
#osx_image: xcode6.4
82+
#os: osx
83+
#cache:
84+
#addons:
85+
#env:
86+
#- ARROW_TRAVIS_USE_TOOLCHAIN=1
87+
#- ARROW_TRAVIS_PLASMA=1
88+
#- ARROW_TRAVIS_ORC=1
89+
#- ARROW_BUILD_WARNING_LEVEL=CHECKIN
90+
#before_script:
91+
#- if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
92+
## If either C++ or Python changed, we must install the C++ libraries
93+
#- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
94+
#script:
95+
#- if [ $ARROW_CI_CPP_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh; fi
96+
#- $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
97+
#- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
98+
#- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
99+
## [manylinux1] Python
100+
#- language: cpp
101+
#before_script:
102+
#- if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest; fi
103+
#script:
104+
#- if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi
105+
## Java w/ OpenJDK 8
106+
#- language: java
107+
#os: linux
108+
#jdk: openjdk8
109+
#before_script:
110+
#- if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
111+
#- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
112+
#script:
113+
#- $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
114+
#- $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh
115+
## Java w/ Oracle JDK 9
116+
#- language: java
117+
#os: linux
118+
#jdk: oraclejdk9
119+
#before_script:
120+
#- if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
121+
#script:
122+
#- $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
123+
#addons:
124+
#apt:
125+
#packages:
126+
#- oracle-java9-installer
127+
## Integration w/ OpenJDK 8
128+
#- language: java
129+
#os: linux
130+
#env: ARROW_TEST_GROUP=integration
131+
#jdk: openjdk8
132+
#env:
133+
#- CC="clang-6.0"
134+
#- CXX="clang++-6.0"
135+
#before_script:
136+
#- if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi
137+
#- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
138+
#- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
139+
#- nvm install 10.1
140+
#- $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
141+
#- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
142+
#script:
143+
#- $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
144+
## NodeJS
145+
#- language: node_js
146+
#os: linux
147+
#node_js:
148+
#- '10.1'
149+
#before_script:
150+
#- if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi
151+
#- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
152+
#- $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
153+
#script:
154+
#- $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
155+
## C++ & glib w/ gcc 4.9
156+
#- compiler: gcc
157+
#language: cpp
158+
#os: linux
159+
#env:
160+
#- BUILD_TORCH_EXAMPLE=no
161+
#- CC="gcc-4.9"
162+
#- CXX="g++-4.9"
163+
#before_script:
164+
#- if [ $ARROW_CI_C_GLIB_AFFECTED != "1" ]; then exit; fi
165+
#- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
166+
#- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
167+
#- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
168+
#script:
169+
#- $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
170+
## [OS X] C++ & glib w/ XCode 8.3 & homebrew
171+
#- compiler: clang
172+
#osx_image: xcode8.3
173+
#os: osx
174+
#cache:
175+
#addons:
176+
#rvm: 2.2
177+
#before_script:
178+
#- if [ $ARROW_CI_C_GLIB_AFFECTED != "1" ]; then exit; fi
179+
#- $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh
180+
#- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew
181+
#- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
182+
#script:
183+
#- $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
184+
## Rust
185+
#- language: rust
186+
#cache: cargo
187+
#rust: stable
188+
#addons:
189+
#apt:
190+
#packages:
191+
#- libcurl4-openssl-dev
192+
#- libelf-dev
193+
#- libdw-dev
194+
#- binutils-dev
195+
#- cmake
196+
#before_script:
197+
#- if [ $ARROW_CI_RUST_AFFECTED != "1" ]; then exit; fi
198+
#- pip install 'travis-cargo<0.2' --user && export PATH=$HOME/.local/bin:$PATH
199+
#- export CARGO_TARGET_DIR=$TRAVIS_BUILD_DIR/target
200+
#- cargo install cargo-travis || echo "Skipping cargo-travis installation as it already exists in cache"
201+
#- export PATH=$HOME/.cargo/bin:$PATH
202+
#script:
203+
#- $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh
204+
#after_success:
205+
#- pushd ${TRAVIS_BUILD_DIR}/rust
206+
## Run coverage for codecov.io
207+
#- mkdir -p target/kcov
208+
#- RUST_BACKTRACE=1 cargo coverage --verbose
209+
#- bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"
210210

211211
after_failure:
212212
- COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1)

‎ci/travis_script_python.sh

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,9 @@ cmake -GNinja \
7070
$CMAKE_COMMON_FLAGS \
7171
-DARROW_BUILD_TESTS=on \
7272
-DARROW_BUILD_UTILITIES=off \
73-
-DARROW_PLASMA=on \
73+
-DARROW_PLASMA=off \
7474
-DARROW_PYTHON=on \
75-
-DARROW_ORC=on \
75+
-DARROW_ORC=off \
7676
-DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
7777
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
7878
$ARROW_CPP_DIR
@@ -95,28 +95,28 @@ fi
9595
export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE
9696

9797
pip install -q -r requirements.txt
98-
python setup.py build_ext -q --with-parquet --with-plasma --with-orc\
98+
python setup.py build_ext -q --with-parquet \
9999
install -q --single-version-externally-managed --record=record.text
100100
popd
101101

102102
python -c "import pyarrow.parquet"
103-
python -c "import pyarrow.plasma"
104-
python -c "import pyarrow.orc"
103+
# python -c "import pyarrow.plasma"
104+
# python -c "import pyarrow.orc"
105105

106106
if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then
107107
export PLASMA_VALGRIND=1
108108
fi
109109

110110
# Set up huge pages for plasma test
111-
if [ $TRAVIS_OS_NAME == "linux" ]; then
112-
sudo mkdir -p /mnt/hugepages
113-
sudo mount -t hugetlbfs -o uid=`id -u` -o gid=`id -g` none /mnt/hugepages
114-
sudo bash -c "echo `id -g` > /proc/sys/vm/hugetlb_shm_group"
115-
sudo bash -c "echo 20000 > /proc/sys/vm/nr_hugepages"
116-
fi
111+
# if [ $TRAVIS_OS_NAME == "linux" ]; then
112+
# sudo mkdir -p /mnt/hugepages
113+
# sudo mount -t hugetlbfs -o uid=`id -u` -o gid=`id -g` none /mnt/hugepages
114+
# sudo bash -c "echo `id -g` > /proc/sys/vm/hugetlb_shm_group"
115+
# sudo bash -c "echo 20000 > /proc/sys/vm/nr_hugepages"
116+
# fi
117117

118118
PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
119-
python -m pytest -r sxX --durations=15 $PYARROW_PATH --parquet
119+
python -m pytest -r sxX --durations=15 $PYARROW_PATH/tests/test_parquet.py::test_parquet_column_statistics_api --parquet
120120

121121
# Capture C++ coverage info and combine with previous coverage file
122122
if [ $ARROW_TRAVIS_COVERAGE == "1" ]; then

‎python/pyarrow/tests/test_parquet.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,7 @@ def make_sample_file(df):
511511
import pyarrow.parquet as pq
512512

513513
a_table = pa.Table.from_pandas(df)
514+
print(a_table.column(0))
514515

515516
buf = io.BytesIO()
516517
_write_table(a_table, buf, compression='SNAPPY', version='2.0',
@@ -602,6 +603,9 @@ def test_parquet_column_statistics_api(
602603
df = pd.DataFrame({'data': data}, dtype=dtype)
603604

604605
fileh = make_sample_file(df)
606+
table = fileh.read()
607+
column = table.column('data')
608+
print(column)
605609

606610
meta = fileh.metadata
607611

0 commit comments

Comments
 (0)