Merge remote-tracking branch 'upstream/master' into ARROW-5436-parquet-read_table

Author: jorisvandenbossche

.travis.yml

@@ -47,8 +47,6 @@ before_install:
   - if [ $TRAVIS_OS_NAME == "linux" ]; then ccache -s; fi
 
 matrix:
-  allow_failures:
-  - language: r
   fast_finish: true
   include:
   - name: "Lint, Release tests"
@@ -73,6 +71,8 @@ matrix:
     - ARROW_TRAVIS_VERBOSE=1
     - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
+    - ARROW_USE_ASAN=1
+    - ARROW_USE_UBSAN=1
     - CC="clang-7"
     - CXX="clang++-7"
     before_script:
@@ -115,7 +115,7 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1
-  - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, conda-forge toolchain, coverage"
+  - name: "Python 3.6 unit tests w/ Valgrind, conda-forge toolchain, coverage"
     compiler: gcc
     language: cpp
     os: linux
@@ -140,11 +140,9 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1
-    - ARROW_TRAVIS_PYTHON_GANDIVA=1
+    - export ARROW_TRAVIS_PYTHON_GANDIVA=1
     # Only run Plasma tests with valgrind in one of the Python builds because
     # they are slow
-    - export PLASMA_VALGRIND=0
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1
     - export PLASMA_VALGRIND=1
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh
@@ -192,12 +190,11 @@ matrix:
     before_script:
     script:
     - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
   - name: "[manylinux1] Python"
     language: cpp
     env:
-    - PYTHON_VERSIONS="2.7,32 3.6,16"
+    - PYTHON_VERSIONS="3.6,16 3.7,16"
     before_script:
     - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker-compose pull python-manylinux1; fi
     script:
@@ -339,7 +336,7 @@ matrix:
     language: r
     cache: packages
     latex: false
-    dist: trusty
+    dist: xenial
     env:
     - ARROW_TRAVIS_PARQUET=1
     - ARROW_TRAVIS_USE_SYSTEM=1
@@ -354,11 +351,16 @@ matrix:
           sudo apt-get update -qq
         fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
+    # On Xenial, installing clang-7 leads these to get removed,
+    # but that breaks R, so let's reinstall them.
+    - sudo apt-get install gfortran gfortran-5 libblas-dev libgfortran-5-dev libgfortran3 liblapack-dev liblapack3
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
     - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib
     - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig
     - pushd ${TRAVIS_BUILD_DIR}/r
+    after_success:
+    - Rscript -e 'covr::codecov()'
 
 
 after_failure:

appveyor.yml

@@ -65,9 +65,11 @@ environment:
     - JOB: "Toolchain"
       GENERATOR: Visual Studio 14 2015 Win64
       CONFIGURATION: "Release"
+      ARROW_BUILD_FLIGHT: "ON"
       ARROW_BUILD_GANDIVA: "ON"
-    - JOB: "Static_Crt_Build"
-      GENERATOR: Ninja
+    # NOTE: Since ARROW-5403 we have disabled the static CRT build
+    # - JOB: "Static_Crt_Build"
+    #   GENERATOR: Ninja
     - JOB: "Build_Debug"
       GENERATOR: Ninja
       CONFIGURATION: "Debug"

c_glib/arrow-glib/input-stream.cpp

@@ -24,6 +24,7 @@
 #include <arrow/io/interfaces.h>
 #include <arrow/io/memory.h>
 #include <arrow/ipc/reader.h>
+#include <arrow/util/string_view.h>
 
 #include <arrow-glib/buffer.hpp>
 #include <arrow-glib/codec.hpp>
@@ -386,6 +387,7 @@ garrow_seekable_input_stream_read_at(GArrowSeekableInputStream *input_stream,
  * garrow_seekable_input_stream_peek:
  * @input_stream: A #GArrowSeekableInputStream.
  * @n_bytes: The number of bytes to be peeked.
+ * @error: (nullable): Return location for a #GError or %NULL.
  *
  * Returns: (transfer full): The data of the buffer, up to the
  *   indicated number. The data becomes invalid after any operation on
@@ -397,12 +399,20 @@ garrow_seekable_input_stream_read_at(GArrowSeekableInputStream *input_stream,
  */
 GBytes *
 garrow_seekable_input_stream_peek(GArrowSeekableInputStream *input_stream,
-                                  gint64 n_bytes)
+                                  gint64 n_bytes,
+                                  GError **error)
 {
   auto arrow_random_access_file =
     garrow_seekable_input_stream_get_raw(input_stream);
-  auto string_view = arrow_random_access_file->Peek(n_bytes);
-  return g_bytes_new_static(string_view.data(), string_view.size());
+
+  arrow::util::string_view view;
+  auto status = arrow_random_access_file->Peek(n_bytes, &view);
+
+  if (garrow_error_check(error, status, "[seekable-input-stream][peek]")) {
+    return g_bytes_new_static(view.data(), view.size());
+  } else {
+    return NULL;
+  }
 }
 
 

c_glib/arrow-glib/input-stream.h

@@ -68,7 +68,8 @@ GArrowBuffer *garrow_seekable_input_stream_read_at(GArrowSeekableInputStream *in
                                                    GError **error);
 GARROW_AVAILABLE_IN_0_12
 GBytes *garrow_seekable_input_stream_peek(GArrowSeekableInputStream *input_stream,
-                                          gint64 n_bytes);
+                                          gint64 n_bytes,
+                                          GError **error);
 
 
 #define GARROW_TYPE_BUFFER_INPUT_STREAM         \

c_glib/doc/plasma-glib/meson.build

@@ -53,8 +53,8 @@ source_directories = [
   join_paths(meson.build_root(), 'plasma-glib'),
 ]
 dependencies = [
-  arrow_glib,
   plasma_glib,
+  arrow_glib,
 ]
 if arrow_cuda.found()
   dependencies += [arrow_cuda_glib]

c_glib/meson.build

@@ -138,5 +138,5 @@ test('unit test',
        'ARROW_CUDA_GLIB_TYPELIB_DIR=@0@/arrow-cuda-glib'.format(meson.build_root()),
        'GANDIVA_GLIB_TYPELIB_DIR=@0@/gandiva-glib'.format(meson.build_root()),
        'PARQUET_GLIB_TYPELIB_DIR=@0@/parquet-glib'.format(meson.build_root()),
-       'PARQUET_GLIB_TYPELIB_DIR=@0@/plasma-glib'.format(meson.build_root()),
+       'PLASMA_GLIB_TYPELIB_DIR=@0@/plasma-glib'.format(meson.build_root()),
      ])

ci/appveyor-cpp-build.bat

@@ -26,6 +26,10 @@ if "%JOB%" == "Static_Crt_Build" (
   @rem the Arrow DLL and the tests end up using a different instance of
   @rem the CRT, which wreaks havoc.
 
+  @rem ARROW-5403(wesm): Since changing to using gtest DLLs we can no
+  @rem longer run the unit tests because gtest.dll and the unit test
+  @rem executables have different static copies of the CRT
+
   mkdir cpp\build-debug
   pushd cpp\build-debug
 
@@ -100,7 +104,7 @@ set CONDA_PACKAGES=--file=ci\conda_env_python.yml python=%PYTHON% numpy=1.14 boo
 
 if "%ARROW_BUILD_GANDIVA%" == "ON" (
   @rem Install llvmdev in the toolchain if building gandiva.dll
-  set CONDA_PACKAGES=%CONDA_PACKAGES% llvmdev=%ARROW_LLVM_VERSION% clangdev=%ARROW_LLVM_VERSION%
+  set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_gandiva.yml
 )
 
 if "%JOB%" == "Toolchain" (

ci/appveyor-cpp-setup-mingw.bat

@@ -24,26 +24,36 @@ if "%MSYSTEM%" == "MINGW32" (
 )
 set PATH=%MINGW_PREFIX%\bin;C:\msys64\usr\bin;%PATH%
 
-pacman --sync --refresh --noconfirm ^
-    "ccache" ^
-    "%MINGW_PACKAGE_PREFIX%-boost" ^
-    "%MINGW_PACKAGE_PREFIX%-brotli" ^
-    "%MINGW_PACKAGE_PREFIX%-cmake" ^
-    "%MINGW_PACKAGE_PREFIX%-double-conversion" ^
-    "%MINGW_PACKAGE_PREFIX%-flatbuffers" ^
-    "%MINGW_PACKAGE_PREFIX%-gflags" ^
-    "%MINGW_PACKAGE_PREFIX%-gobject-introspection" ^
-    "%MINGW_PACKAGE_PREFIX%-gtest" ^
-    "%MINGW_PACKAGE_PREFIX%-gtk-doc" ^
-    "%MINGW_PACKAGE_PREFIX%-lz4" ^
-    "%MINGW_PACKAGE_PREFIX%-meson" ^
-    "%MINGW_PACKAGE_PREFIX%-protobuf" ^
-    "%MINGW_PACKAGE_PREFIX%-python3-numpy" ^
-    "%MINGW_PACKAGE_PREFIX%-rapidjson" ^
-    "%MINGW_PACKAGE_PREFIX%-snappy" ^
-    "%MINGW_PACKAGE_PREFIX%-thrift" ^
-    "%MINGW_PACKAGE_PREFIX%-zlib" ^
-    "%MINGW_PACKAGE_PREFIX%-zstd" || exit /B
+pacman --remove --noconfirm ^
+  %MINGW_PACKAGE_PREFIX%-gcc-ada ^
+  %MINGW_PACKAGE_PREFIX%-gcc-objc || exit /B
+pacman --sync --refresh --sysupgrade --noconfirm || exit /B
+pacman --sync --noconfirm ^
+  %MINGW_PACKAGE_PREFIX%-gcc ^
+  %MINGW_PACKAGE_PREFIX%-gcc-fortran ^
+  %MINGW_PACKAGE_PREFIX%-python3 ^
+  %MINGW_PACKAGE_PREFIX%-readline ^
+  %MINGW_PACKAGE_PREFIX%-sqlite3 || exit /B
+pacman --sync --noconfirm ^
+  ccache ^
+  %MINGW_PACKAGE_PREFIX%-boost ^
+  %MINGW_PACKAGE_PREFIX%-brotli ^
+  %MINGW_PACKAGE_PREFIX%-cmake ^
+  %MINGW_PACKAGE_PREFIX%-double-conversion ^
+  %MINGW_PACKAGE_PREFIX%-flatbuffers ^
+  %MINGW_PACKAGE_PREFIX%-gflags ^
+  %MINGW_PACKAGE_PREFIX%-gobject-introspection ^
+  %MINGW_PACKAGE_PREFIX%-gtest ^
+  %MINGW_PACKAGE_PREFIX%-gtk-doc ^
+  %MINGW_PACKAGE_PREFIX%-lz4 ^
+  %MINGW_PACKAGE_PREFIX%-meson ^
+  %MINGW_PACKAGE_PREFIX%-protobuf ^
+  %MINGW_PACKAGE_PREFIX%-python3-numpy ^
+  %MINGW_PACKAGE_PREFIX%-rapidjson ^
+  %MINGW_PACKAGE_PREFIX%-snappy ^
+  %MINGW_PACKAGE_PREFIX%-thrift ^
+  %MINGW_PACKAGE_PREFIX%-zlib ^
+  %MINGW_PACKAGE_PREFIX%-zstd || exit /B
 
 pushd c_glib
 ruby -S bundle install || exit /B

ci/appveyor-cpp-setup.bat

@@ -17,6 +17,15 @@
 
 @echo on
 
+@rem Avoid picking up AppVeyor-installed OpenSSL (linker errors with gRPC)
+@rem XXX Perhaps there is a smarter way of solving this issue?
+rd /s /q C:\OpenSSL-Win32
+rd /s /q C:\OpenSSL-Win64
+rd /s /q C:\OpenSSL-v11-Win32
+rd /s /q C:\OpenSSL-v11-Win64
+rd /s /q C:\OpenSSL-v111-Win32
+rd /s /q C:\OpenSSL-v111-Win64
+
 conda update -y -q conda
 conda config --set auto_update_conda false
 conda info -a

ci/conda_env_cpp.yml

@@ -28,15 +28,12 @@ glog
 gmock>=1.8.1
 grpc-cpp
 gtest>=1.8.1
-clangdev=7
-llvmdev=7
 libprotobuf
 lz4-c
 ninja
 pkg-config
 python
 rapidjson
-re2
 snappy
 thrift-cpp>=0.11.0
 zlib

cpp/src/parquet/util/CMakeLists.txt renamed to ci/conda_env_gandiva.yml

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Headers: util
-arrow_install_all_headers("parquet/util")
-
-add_parquet_test(memory-test)
+clangdev=7
+llvmdev=7
+re2

ci/conda_env_python.yml

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-cython>=0.29.0
+cython=0.29.7
 cloudpickle
 hypothesis
 numpy>=1.14

ci/cpp-msvc-build-main.bat

@@ -22,12 +22,10 @@ set ARROW_HOME=%CONDA_PREFIX%\Library
 set CMAKE_ARGS=-DARROW_VERBOSE_THIRDPARTY_BUILD=OFF
 
 if "%JOB%" == "Toolchain" (
-  @rem Toolchain gtest does not currently work with Visual Studio 2015
   set CMAKE_ARGS=^
       %CMAKE_ARGS% ^
       -DARROW_WITH_BZ2=ON ^
-      -DARROW_DEPENDENCY_SOURCE=CONDA ^
-      -DGTest_SOURCE=BUNDLED
+      -DARROW_DEPENDENCY_SOURCE=CONDA
 ) else (
   @rem We're in a conda enviroment but don't want to use it for the dependencies
   set CMAKE_ARGS=%CMAKE_ARGS% -DARROW_DEPENDENCY_SOURCE=AUTO
@@ -63,6 +61,7 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
       -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^
       -DARROW_CXXFLAGS="%ARROW_CXXFLAGS%" ^
       -DCMAKE_CXX_FLAGS_RELEASE="/MD %CMAKE_CXX_FLAGS_RELEASE%" ^
+      -DARROW_FLIGHT=%ARROW_BUILD_FLIGHT% ^
       -DARROW_GANDIVA=%ARROW_BUILD_GANDIVA% ^
       -DARROW_PARQUET=ON ^
       -DPARQUET_BUILD_EXECUTABLES=ON ^
@@ -89,10 +88,16 @@ pip install -r requirements.txt pickle5
 
 set PYARROW_CXXFLAGS=%ARROW_CXXFLAGS%
 set PYARROW_CMAKE_GENERATOR=%GENERATOR%
-set PYARROW_BUNDLE_ARROW_CPP=ON
+if "%ARROW_BUILD_FLIGHT%" == "ON" (
+  @rem ARROW-5441: bundling Arrow Flight libraries not implemented
+  set PYARROW_BUNDLE_ARROW_CPP=OFF
+) else (
+  set PYARROW_BUNDLE_ARROW_CPP=ON
+)
 set PYARROW_BUNDLE_BOOST=OFF
 set PYARROW_WITH_STATIC_BOOST=ON
 set PYARROW_WITH_PARQUET=ON
+set PYARROW_WITH_FLIGHT=%ARROW_BUILD_FLIGHT%
 set PYARROW_WITH_GANDIVA=%ARROW_BUILD_GANDIVA%
 set PYARROW_PARALLEL=2
 

ci/travis_before_script_cpp.sh

@@ -121,6 +121,15 @@ if [ "$ARROW_TRAVIS_VALGRIND" == "1" ]; then
   CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_TEST_MEMCHECK=ON"
 fi
 
+if [ "$ARROW_USE_ASAN" == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_USE_ASAN=ON"
+fi
+
+if [ "$ARROW_USE_UBSAN" == "1" ]; then
+  CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_USE_UBSAN=ON"
+fi
+
+
 if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then
   CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GENERATE_COVERAGE=ON"
 fi

ci/travis_install_toolchain.sh

@@ -23,23 +23,17 @@ source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 
 if [ ! -e $CPP_TOOLCHAIN ]; then
     CONDA_PACKAGES=""
-    CONDA_LABEL=""
 
-    if [ "$ARROW_TRAVIS_GANDIVA" == "1" ] && [ $TRAVIS_OS_NAME == "osx" ]; then
-        CONDA_PACKAGES="$CONDA_PACKAGES llvmdev=$CONDA_LLVM_VERSION"
-    fi
-
-    if [ "$ARROW_TRAVIS_VALGRIND" == "1" ]; then
-        # Use newer Valgrind
-        CONDA_PACKAGES="$CONDA_PACKAGES valgrind"
+    if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then
+        CONDA_PACKAGES="$CONDA_PACKAGES --file=$TRAVIS_BUILD_DIR/ci/conda_env_gandiva.yml"
     fi
 
     # Set up C++ toolchain from conda-forge packages for faster builds
     time conda create -y -q -p $CPP_TOOLCHAIN \
         --file=$TRAVIS_BUILD_DIR/ci/conda_env_cpp.yml \
         --file=$TRAVIS_BUILD_DIR/ci/conda_env_unix.yml \
-        compilers \
         $CONDA_PACKAGES \
+        compilers \
         nomkl \
         python=3.6
 fi

ci/travis_lint.sh

@@ -31,7 +31,7 @@ pre-commit install
 pre-commit run hadolint -a
 
 # CMake formatting check
-pip install cmake_format
+pip install cmake_format==0.5.2
 $TRAVIS_BUILD_DIR/run-cmake-format.py --check
 
 # C++ code linting