Merge remote-tracking branch 'upstream/master'

Author: Carlos Souza

.travis.yml

@@ -27,154 +27,71 @@ matrix:
     - language: objective-c
       os: osx
       compiler: clang
-      osx_image: xcode6.4
       cache:
         ccache: true
         directories:
           - $HOME/.cache # cython cache
           - $HOME/.ccache # compiler cache
       env:
-        - PYTHON_VERSION=3.5
-        - JOB_NAME: "35_osx"
-        - TEST_ARGS="--skip-slow --skip-network"
-        - JOB_TAG=_OSX
-        - TRAVIS_PYTHON_VERSION=3.5
-        - CACHE_NAME="35_osx"
-        - USE_CACHE=true
+        - PYTHON_VERSION=3.5 JOB_NAME="35_osx" TEST_ARGS="--skip-slow --skip-network" JOB_TAG="_OSX" TRAVIS_PYTHON_VERSION=3.5 USE_CACHE=true
     - python: 2.7
       env:
-        - PYTHON_VERSION=2.7
-        - JOB_NAME: "27_slow_nnet_LOCALE"
-        - TEST_ARGS="--only-slow --skip-network"
-        - LOCALE_OVERRIDE="zh_CN.UTF-8"
-        - FULL_DEPS=true
-        - JOB_TAG=_LOCALE
-        - CACHE_NAME="27_slow_nnet_LOCALE"
-        - USE_CACHE=true
+        - PYTHON_VERSION=2.7 JOB_NAME="27_slow_nnet_LOCALE" TEST_ARGS="--only-slow --skip-network" LOCALE_OVERRIDE="zh_CN.UTF-8" JOB_TAG="_LOCALE" USE_CACHE=true
       addons:
         apt:
           packages:
           - language-pack-zh-hans
     - python: 2.7
       env:
-        - PYTHON_VERSION=2.7
-        - JOB_NAME: "27_nslow"
-        - TEST_ARGS="--skip-slow"
-        - FULL_DEPS=true
-        - CLIPBOARD_GUI=gtk2
-        - LINT=true
-        - CACHE_NAME="27_nslow"
-        - USE_CACHE=true
+        - PYTHON_VERSION=2.7 JOB_NAME="27_nslow" TEST_ARGS="--skip-slow" LINT=true USE_CACHE=true
       addons:
         apt:
           packages:
           - python-gtk2
     - python: 3.5
       env:
-        - PYTHON_VERSION=3.5
-        - JOB_NAME: "35_nslow"
-        - TEST_ARGS="--skip-slow --skip-network"
-        - FULL_DEPS=true
-        - CLIPBOARD=xsel
-        - COVERAGE=true
-        - CACHE_NAME="35_nslow"
-        - USE_CACHE=true
+        - PYTHON_VERSION=3.5 JOB_NAME="35_nslow" TEST_ARGS="--skip-slow --skip-network" COVERAGE=true USE_CACHE=true
       addons:
         apt:
           packages:
           - xsel
     - python: 3.6
       env:
-        - PYTHON_VERSION=3.6
-        - JOB_NAME: "36"
-        - TEST_ARGS="--skip-slow --skip-network"
-        - PANDAS_TESTING_MODE="deprecate"
-        - CONDA_FORGE=true
-        - USE_CACHE=true
+        - PYTHON_VERSION=3.6 JOB_NAME="36" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" CONDA_FORGE=true USE_CACHE=true
       addons:
         apt:
           packages:
           - libatlas-base-dev
           - gfortran
-#    In allow_failures
+    # In allow_failures
     - python: 2.7
       env:
-        - PYTHON_VERSION=2.7
-        - JOB_NAME: "27_slow"
-        - JOB_TAG=_SLOW
-        - TEST_ARGS="--only-slow --skip-network"
-        - FULL_DEPS=true
-        - CACHE_NAME="27_slow"
-        - USE_CACHE=true
-#    In allow_failures
+        - PYTHON_VERSION=2.7 JOB_NAME="27_slow" JOB_TAG="_SLOW" TEST_ARGS="--only-slow --skip-network" USE_CACHE=true
+    # In allow_failures
     - python: 2.7
       env:
-        - PYTHON_VERSION=2.7
-        - JOB_NAME: "27_build_test"
-        - JOB_TAG=_BUILD_TEST
-        - TEST_ARGS="--skip-slow"
-        - FULL_DEPS=true
-        - BUILD_TEST=true
-        - CACHE_NAME="27_build_test"
-        - USE_CACHE=true
-#    In allow_failures
+        - PYTHON_VERSION=2.7 JOB_NAME="27_build_test" JOB_TAG="_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true USE_CACHE=true
+    # In allow_failures
     - python: 3.5
       env:
-        - PYTHON_VERSION=3.5
-        - JOB_NAME: "35_numpy_dev"
-        - JOB_TAG=_NUMPY_DEV
-        - TEST_ARGS="--skip-slow --skip-network"
-        - PANDAS_TESTING_MODE="deprecate"
-        - CACHE_NAME="35_numpy_dev"
-        - USE_CACHE=true
-#    In allow_failures
+        - PYTHON_VERSION=3.5 JOB_NAME="35_numpy_dev" JOB_TAG="_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" USE_CACHE=true
+    # In allow_failures
     - python: 3.5
       env:
-        - PYTHON_VERSION=3.5
-        - JOB_NAME: "doc_build"
-        - FULL_DEPS=true
-        - DOC_BUILD=true
-        - JOB_TAG=_DOC_BUILD
-        - CACHE_NAME="doc_build"
-        - USE_CACHE=true
+        - PYTHON_VERSION=3.5 JOB_NAME="doc_build" DOC_BUILD=true JOB_TAG="_DOC_BUILD" USE_CACHE=true
     allow_failures:
       - python: 2.7
         env:
-        - PYTHON_VERSION=2.7
-        - JOB_NAME: "27_slow"
-        - JOB_TAG=_SLOW
-        - TEST_ARGS="--only-slow --skip-network"
-        - FULL_DEPS=true
-        - CACHE_NAME="27_slow"
-        - USE_CACHE=true
+          - PYTHON_VERSION=2.7 JOB_NAME="27_slow" JOB_TAG="_SLOW" TEST_ARGS="--only-slow --skip-network" USE_CACHE=true
       - python: 2.7
         env:
-        - PYTHON_VERSION=2.7
-        - JOB_NAME: "27_build_test"
-        - JOB_TAG=_BUILD_TEST
-        - TEST_ARGS="--skip-slow"
-        - FULL_DEPS=true
-        - BUILD_TEST=true
-        - CACHE_NAME="27_build_test"
-        - USE_CACHE=true
+          - PYTHON_VERSION=2.7 JOB_NAME="27_build_test" JOB_TAG="_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true USE_CACHE=true
       - python: 3.5
         env:
-        - PYTHON_VERSION=3.5
-        - JOB_NAME: "35_numpy_dev"
-        - JOB_TAG=_NUMPY_DEV
-        - TEST_ARGS="--skip-slow --skip-network"
-        - PANDAS_TESTING_MODE="deprecate"
-        - CACHE_NAME="35_numpy_dev"
-        - USE_CACHE=true
+          - PYTHON_VERSION=3.5 JOB_NAME="35_numpy_dev" JOB_TAG="_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" USE_CACHE=true
       - python: 3.5
         env:
-        - PYTHON_VERSION=3.5
-        - JOB_NAME: "doc_build"
-        - FULL_DEPS=true
-        - DOC_BUILD=true
-        - JOB_TAG=_DOC_BUILD
-        - CACHE_NAME="doc_build"
-        - USE_CACHE=true
+          - PYTHON_VERSION=3.5 JOB_NAME="doc_build" DOC_BUILD=true JOB_TAG="_DOC_BUILD" USE_CACHE=true
 
 before_install:
   - echo "before_install"
@@ -186,7 +103,7 @@ before_install:
   - git --version
   - git tag
   - ci/before_install_travis.sh
-  - export DISPLAY=:99.0
+  - export DISPLAY=":99.0"
 
 install:
   - echo "install start"
@@ -209,12 +126,17 @@ script:
   - echo "script done"
 
 after_success:
-  - source activate pandas && codecov
+  - if [ "$COVERAGE" ]; then
+        source activate pandas && codecov --file /tmp/cov-single.xml /tmp/cov-multiple.xml;
+    fi
 
 after_script:
   - echo "after_script start"
-  - ci/install_test.sh
   - source activate pandas && python -c "import pandas; pandas.show_versions();"
-  - ci/print_skipped.py /tmp/single.xml
-  - ci/print_skipped.py /tmp/multiple.xml
+  - if [ -e /tmp/single.xml ]; then
+    ci/print_skipped.py /tmp/single.xml;
+    fi
+  - if [ -e /tmp/multiple.xml ]; then
+    ci/print_skipped.py /tmp/multiple.xml;
+    fi
   - echo "after_script done"

asv_bench/benchmarks/groupby.py

@@ -108,16 +108,34 @@ def setup(self):
         self.N = 10000
         self.labels = np.random.randint(0, 2000, size=self.N)
         self.labels2 = np.random.randint(0, 3, size=self.N)
-        self.df = DataFrame({'key': self.labels, 'key2': self.labels2, 'value1': randn(self.N), 'value2': (['foo', 'bar', 'baz', 'qux'] * (self.N / 4)), })
-
-    def f(self, g):
+        self.df = DataFrame({
+            'key': self.labels,
+            'key2': self.labels2,
+            'value1': np.random.randn(self.N),
+            'value2': (['foo', 'bar', 'baz', 'qux'] * (self.N // 4)),
+        })
+
+    @staticmethod
+    def scalar_function(g):
         return 1
 
-    def time_groupby_frame_apply(self):
-        self.df.groupby(['key', 'key2']).apply(self.f)
+    def time_groupby_frame_apply_scalar_function(self):
+        self.df.groupby(['key', 'key2']).apply(self.scalar_function)
+
+    def time_groupby_frame_apply_scalar_function_overhead(self):
+        self.df.groupby('key').apply(self.scalar_function)
+
+    @staticmethod
+    def df_copy_function(g):
+        # ensure that the group name is available (see GH #15062)
+        g.name
+        return g.copy()
+
+    def time_groupby_frame_df_copy_function(self):
+        self.df.groupby(['key', 'key2']).apply(self.df_copy_function)
 
-    def time_groupby_frame_apply_overhead(self):
-        self.df.groupby('key').apply(self.f)
+    def time_groupby_frame_apply_df_copy_overhead(self):
+        self.df.groupby('key').apply(self.df_copy_function)
 
 
 #----------------------------------------------------------------------

asv_bench/benchmarks/pandas_vb_common.py

@@ -25,6 +25,11 @@
     except:
         pass
 
+try:
+    Panel = Panel
+except Exception:
+    Panel = WidePanel
+
 # didn't add to namespace until later
 try:
     from pandas.core.index import MultiIndex

bench/bench_join_panel.py

@@ -45,8 +45,8 @@ def reindex_on_axis(panels, axis, axis_reindex):
         return p
 
 
-# Does the job but inefficient. It is better to handle
-# this like you read a table in pytables.
+# does the job but inefficient (better to handle like you read a table in
+# pytables...e.g create a LongPanel then convert to Wide)
 def create_panels_join(cls, panels):
         """ given an array of panels's, create a single panel """
         panels = [a for a in panels if a is not None]

ci/script_multi.sh

@@ -27,8 +27,8 @@ if [ "$BUILD_TEST" ]; then
     cd /tmp
     python -c "import pandas; pandas.test(['-n 2'])"
 elif [ "$COVERAGE" ]; then
-    echo pytest -s -n 2 -m "not single" --cov=pandas --cov-append --cov-report xml:/tmp/cov.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
-    pytest -s -n 2 -m "not single" --cov=pandas --cov-append --cov-report xml:/tmp/cov.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+    echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+    pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
 else
     echo pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
     pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas # TODO: doctest

ci/script_single.sh

@@ -20,8 +20,8 @@ fi
 if [ "$BUILD_TEST" ]; then
     echo "We are not running pytest as this is simply a build test."
 elif [ "$COVERAGE" ]; then
-    echo pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
-    pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
+    echo pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
+    pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
 else
     echo pytest -m "single" --junitxml=/tmp/single.xml $TEST_ARGS pandas
     pytest -m "single" --junitxml=/tmp/single.xml $TEST_ARGS pandas # TODO: doctest

doc/source/groupby.rst

@@ -580,9 +580,21 @@ Transformation
 --------------
 
 The ``transform`` method returns an object that is indexed the same (same size)
-as the one being grouped. Thus, the passed transform function should return a
-result that is the same size as the group chunk. For example, suppose we wished
-to standardize the data within each group:
+as the one being grouped. The transform function must:
+
+* Return a result that is either the same size as the group chunk or
+  broadcastable to the size of the group chunk (e.g., a scalar,
+  ``grouped.transform(lambda x: x.iloc[-1])``).
+* Operate column-by-column on the group chunk.  The transform is applied to
+  the first group chunk using chunk.apply.
+* Not perform in-place operations on the group chunk. Group chunks should
+  be treated as immutable, and changes to a group chunk may produce unexpected
+  results. For example, when using ``fillna``, ``inplace`` must be ``False``
+  (``grouped.transform(lambda x: x.fillna(inplace=False))``).
+* (Optionally) operates on the entire group chunk. If this is supported, a
+  fast path is used starting from the *second* chunk.
+
+For example, suppose we wished to standardize the data within each group:
 
 .. ipython:: python
 
@@ -620,6 +632,21 @@ We can also visually compare the original and transformed data sets.
    @savefig groupby_transform_plot.png
    compare.plot()
 
+Transformation functions that have lower dimension outputs are broadcast to
+match the shape of the input array.
+
+.. ipython:: python
+
+   data_range = lambda x: x.max() - x.min()
+   ts.groupby(key).transform(data_range)
+
+Alternatively the built-in methods can be could be used to produce the same
+outputs
+
+.. ipython:: python
+
+   ts.groupby(key).transform('max') - ts.groupby(key).transform('min')
+
 Another common data transform is to replace missing data with the group mean.
 
 .. ipython:: python
@@ -664,8 +691,9 @@ and that the transformed data contains no NAs.
 
 .. note::
 
-   Some functions when applied to a groupby object will automatically transform the input, returning
-   an object of the same shape as the original. Passing ``as_index=False`` will not affect these transformation methods.
+   Some functions when applied to a groupby object will automatically transform
+   the input, returning an object of the same shape as the original. Passing
+   ``as_index=False`` will not affect these transformation methods.
 
    For example: ``fillna, ffill, bfill, shift``.
 

doc/source/whatsnew/v0.20.0.txt

@@ -785,6 +785,7 @@ Deprecations
 - ``Series/DataFrame/Panel.consolidate()`` been deprecated as a public method. (:issue:`15483`)
 - The ``as_indexer`` keyword of ``Series.str.match()`` has been deprecated (ignored keyword) (:issue:`15257`).
 - The following top-level pandas functions have been deprecated and will be removed in a future version (:issue:`13790`)
+
   * ``pd.pnow()``, replaced by ``Period.now()``
   * ``pd.Term``, is removed, as it is not applicable to user code. Instead use in-line string expressions in the where clause when searching in HDFStore
   * ``pd.Expr``, is removed, as it is not applicable to user code.
@@ -812,7 +813,7 @@ Removal of prior version deprecations/changes
 - The ``Categorical`` constructor has dropped the ``name`` parameter (:issue:`10632`)
 - The ``take_last`` parameter has been dropped from ``duplicated()``, ``drop_duplicates()``, ``nlargest()``, and ``nsmallest()`` methods (:issue:`10236`, :issue:`10792`, :issue:`10920`)
 - ``Series``, ``Index``, and ``DataFrame`` have dropped the ``sort`` and ``order`` methods (:issue:`10726`)
-- The ``LongPanel`` and ``WidePanel`` classes have been removed (:issue:`10892`)
+- Where clauses in ``pytables`` are only accepted as strings and expressions types and not other data-types (:issue:`12027`)
 
 .. _whatsnew_0200.performance:
 
@@ -830,6 +831,9 @@ Performance Improvements
 - Improved performance when using ``.unstack()`` (:issue:`15503`)
 - Improved performance of merge/join on ``category`` columns (:issue:`10409`)
 - Improved performance of ``drop_duplicates()`` on ``bool`` columns (:issue:`12963`)
+- Improve performance of ``pd.core.groupby.GroupBy.apply`` when the applied
+  function used the ``.name`` attribute of the group DataFrame (:issue:`15062`).
+
 
 
 .. _whatsnew_0200.bug_fixes:
@@ -864,6 +868,7 @@ Bug Fixes
 - Bug in ``pd.cut()`` with a single bin on an all 0s array (:issue:`15428`)
 - Bug in ``pd.qcut()`` with a single quantile and an array with identical values (:issue:`15431`)
 - Compat with SciPy 0.19.0 for testing on ``.interpolate()`` (:issue:`15662`)
+- Bug in ``Series.asof`` which raised if the series contained all ``np.nan`` (:issue:`15713`)
 
 - Compat for 32-bit platforms for ``.qcut/cut``; bins will now be ``int64`` dtype (:issue:`14866`)
 

pandas/_libs/src/reduce.pyx

@@ -497,7 +497,7 @@ def apply_frame_axis0(object frame, object f, object names,
     # Need to infer if our low-level mucking is going to cause a segfault
     if n > 0:
         chunk = frame.iloc[starts[0]:ends[0]]
-        shape_before = chunk.shape
+        object.__setattr__(chunk, 'name', names[0])
         try:
             result = f(chunk)
             if result is chunk: