get_indexer_non_unique for orderable indexes

.travis.yml

@@ -320,7 +320,8 @@ before_script:
 script:
   - echo "script start"
   - ci/run_build_docs.sh
-  - ci/script.sh
+  - ci/script_single.sh
+  - ci/script_multi.sh
   - ci/lint.sh
   - echo "script done"
 
@@ -331,5 +332,6 @@ after_script:
   - echo "after_script start"
   - ci/install_test.sh
   - source activate pandas && python -c "import pandas; pandas.show_versions();"
-  - ci/print_skipped.py /tmp/pytest.xml
+  - ci/print_skipped.py /tmp/single.xml
+  - ci/print_skipped.py /tmp/multiple.xml
   - echo "after_script done"

MANIFEST.in

@@ -7,7 +7,6 @@ include setup.py
 graft doc
 prune doc/build
 
-graft examples
 graft pandas
 
 global-exclude *.so

appveyor.yml

@@ -18,19 +18,19 @@ environment:
 
   matrix:
 
-    - CONDA_ROOT: "C:\\Miniconda3.5_64"
+    - CONDA_ROOT: "C:\\Miniconda3_64"
       PYTHON_VERSION: "3.6"
       PYTHON_ARCH: "64"
       CONDA_PY: "36"
-      CONDA_NPY: "111"
+      CONDA_NPY: "112"
 
-    - CONDA_ROOT: "C:\\Miniconda3.5_64"
+    - CONDA_ROOT: "C:\\Miniconda3_64"
       PYTHON_VERSION: "2.7"
       PYTHON_ARCH: "64"
       CONDA_PY: "27"
       CONDA_NPY: "110"
 
-    - CONDA_ROOT: "C:\\Miniconda3.5_64"
+    - CONDA_ROOT: "C:\\Miniconda3_64"
       PYTHON_VERSION: "3.5"
       PYTHON_ARCH: "64"
       CONDA_PY: "35"
@@ -66,8 +66,7 @@ install:
 
   # install our build environment
   - cmd: conda config --set show_channel_urls true --set always_yes true --set changeps1 false
-  #- cmd: conda update -q conda
-  - cmd: conda install conda=4.2.15
+  - cmd: conda update -q conda
   - cmd: conda config --set ssl_verify false
 
   # add the pandas channel *before* defaults to have defaults take priority
@@ -79,23 +78,19 @@ install:
   # this is now the downloaded conda...
   - cmd: conda info -a
 
-  # build em using the local source checkout in the correct windows env
-  - cmd: '%CMD_IN_ENV% conda build ci\appveyor.recipe -q'
-
   # create our env
-  - cmd: conda create -q -n pandas python=%PYTHON_VERSION% nose pytest
+  - cmd: conda create -q -n pandas python=%PYTHON_VERSION% cython pytest
   - cmd: activate pandas
   - SET REQ=ci\requirements-%PYTHON_VERSION%-%PYTHON_ARCH%.run
   - cmd: echo "installing requirements from %REQ%"
   - cmd: conda install -n pandas -q --file=%REQ%
   - cmd: conda list -n pandas
   - cmd: echo "installing requirements from %REQ% - done"
-  - ps: conda install -n pandas (conda build ci\appveyor.recipe -q --output)
+
+  # build em using the local source checkout in the correct windows env
+  - cmd: '%CMD_IN_ENV% python setup.py build_ext --inplace'
 
 test_script:
   # tests
   - cmd: activate pandas
-  - cmd: conda list
-  - cmd: cd \
-  - cmd: python -c "import pandas; pandas.test(['--skip-slow', '--skip-network'])"
-
+  - cmd: test.bat

asv_bench/benchmarks/indexing.py

@@ -88,7 +88,7 @@ def setup(self):
 
     def time_getitem_scalar(self):
         self.ts[self.dt]
-    
+
 
 class DataFrameIndexing(object):
     goal_time = 0.2
@@ -189,6 +189,15 @@ def setup(self):
         self.eps_C = 5
         self.eps_D = 5000
         self.mdt2 = self.mdt.set_index(['A', 'B', 'C', 'D']).sortlevel()
+        self.miint = MultiIndex.from_product(
+            [np.arange(1000),
+             np.arange(1000)], names=['one', 'two'])
+
+        import string
+        self.mistring = MultiIndex.from_product(
+            [np.arange(1000),
+             np.arange(20), list(string.ascii_letters)],
+            names=['one', 'two', 'three'])
 
     def time_series_xs_mi_ix(self):
         self.s.ix[999]
@@ -197,7 +206,24 @@ def time_frame_xs_mi_ix(self):
         self.df.ix[999]
 
     def time_multiindex_slicers(self):
-        self.mdt2.loc[self.idx[(self.test_A - self.eps_A):(self.test_A + self.eps_A), (self.test_B - self.eps_B):(self.test_B + self.eps_B), (self.test_C - self.eps_C):(self.test_C + self.eps_C), (self.test_D - self.eps_D):(self.test_D + self.eps_D)], :]
+        self.mdt2.loc[self.idx[
+            (self.test_A - self.eps_A):(self.test_A + self.eps_A),
+            (self.test_B - self.eps_B):(self.test_B + self.eps_B),
+            (self.test_C - self.eps_C):(self.test_C + self.eps_C),
+            (self.test_D - self.eps_D):(self.test_D + self.eps_D)], :]
+
+    def time_multiindex_get_indexer(self):
+        self.miint.get_indexer(
+            np.array([(0, 10), (0, 11), (0, 12),
+                      (0, 13), (0, 14), (0, 15),
+                      (0, 16), (0, 17), (0, 18),
+                      (0, 19)], dtype=object))
+
+    def time_multiindex_string_get_loc(self):
+        self.mistring.get_loc((999, 19, 'Z'))
+
+    def time_is_monotonic(self):
+        self.miint.is_monotonic
 
 
 class PanelIndexing(object):

asv_bench/benchmarks/reindex.py

@@ -16,8 +16,8 @@ def setup(self):
                              data=np.random.rand(10000, 30), columns=range(30))
 
         # multi-index
-        N = 1000
-        K = 20
+        N = 5000
+        K = 200
         level1 = tm.makeStringIndex(N).values.repeat(K)
         level2 = np.tile(tm.makeStringIndex(K).values, N)
         index = MultiIndex.from_arrays([level1, level2])

ci/install_travis.sh

@@ -112,6 +112,7 @@ fi
 source activate pandas
 
 pip install pytest-xdist
+
 if [ "$LINT" ]; then
    conda install flake8
    pip install cpplint

ci/print_skipped.py

@@ -30,20 +30,21 @@ def parse_results(filename):
             i += 1
             assert i - 1 == len(skipped)
     assert i - 1 == len(skipped)
-    assert len(skipped) == int(root.attrib['skip'])
+    # assert len(skipped) == int(root.attrib['skip'])
     return '\n'.join(skipped)
 
 
 def main(args):
     print('SKIPPED TESTS:')
-    print(parse_results(args.filename))
+    for fn in args.filename:
+        print(parse_results(fn))
     return 0
 
 
 def parse_args():
     import argparse
     parser = argparse.ArgumentParser()
-    parser.add_argument('filename', help='XUnit file to parse')
+    parser.add_argument('filename', nargs='+', help='XUnit file to parse')
     return parser.parse_args()
 
 

ci/requirements-3.4_SLOW.run

@@ -9,7 +9,7 @@ html5lib
 patsy
 beautiful-soup
 scipy
-numexpr=2.4.4
+numexpr=2.4.6
 pytables
 matplotlib
 lxml

ci/requirements-3.5-64.run

@@ -1,6 +1,6 @@
 python-dateutil
 pytz
-numpy
+numpy=1.11*
 openpyxl
 xlsxwriter
 xlrd

ci/requirements-3.6-64.run

@@ -1,10 +1,10 @@
 python-dateutil
 pytz
-numpy
+numpy=1.12*
 openpyxl
 xlsxwriter
 xlrd
-#xlwt
+xlwt
 scipy
 feather-format
 numexpr

ci/script_multi.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+echo "[script multi]"
+
+source activate pandas
+
+# don't run the tests for the doc build
+if [ x"$DOC_BUILD" != x"" ]; then
+    exit 0
+fi
+
+if [ -n "$LOCALE_OVERRIDE" ]; then
+    export LC_ALL="$LOCALE_OVERRIDE";
+    echo "Setting LC_ALL to $LOCALE_OVERRIDE"
+
+    pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))'
+    python -c "$pycmd"
+fi
+
+# Workaround for pytest-xdist flaky collection order
+# https://github.com/pytest-dev/pytest/issues/920
+# https://github.com/pytest-dev/pytest/issues/1075
+export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
+echo PYTHONHASHSEED=$PYTHONHASHSEED
+
+if [ "$BUILD_TEST" ]; then
+    echo "We are not running pytest as this is simply a build test."
+elif [ "$COVERAGE" ]; then
+    echo pytest -s -n 2 -m "not single" --cov=pandas --cov-append --cov-report xml:/tmp/cov.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+    pytest -s -n 2 -m "not single" --cov=pandas --cov-append --cov-report xml:/tmp/cov.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+else
+    echo pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+    pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas # TODO: doctest
+fi
+
+RET="$?"
+
+exit "$RET"

ci/script.sh → ci/script_single.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-echo "inside $0"
+echo "[script_single]"
 
 source activate pandas
 
@@ -20,11 +20,11 @@ fi
 if [ "$BUILD_TEST" ]; then
     echo "We are not running pytest as this is simply a build test."
 elif [ "$COVERAGE" ]; then
-    echo pytest -s --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
-    pytest -s --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
+    echo pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
+    pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
 else
-    echo pytest $TEST_ARGS pandas
-    pytest $TEST_ARGS pandas # TODO: doctest
+    echo pytest -m "single" --junitxml=/tmp/single.xml $TEST_ARGS pandas
+    pytest -m "single" --junitxml=/tmp/single.xml $TEST_ARGS pandas # TODO: doctest
 fi
 
 RET="$?"

doc/source/advanced.rst

@@ -59,7 +59,7 @@ Creating a MultiIndex (hierarchical index) object
 
 The ``MultiIndex`` object is the hierarchical analogue of the standard
 ``Index`` object which typically stores the axis labels in pandas objects. You
-can think of ``MultiIndex`` an array of tuples where each tuple is unique. A
+can think of ``MultiIndex`` as an array of tuples where each tuple is unique. A
 ``MultiIndex`` can be created from a list of arrays (using
 ``MultiIndex.from_arrays``), an array of tuples (using
 ``MultiIndex.from_tuples``), or a crossed set of iterables (using

doc/source/install.rst

@@ -226,7 +226,7 @@ Recommended Dependencies
 
 * `numexpr <https://github.com/pydata/numexpr>`__: for accelerating certain numerical operations.
   ``numexpr`` uses multiple cores as well as smart chunking and caching to achieve large speedups.
-  If installed, must be Version 2.1 or higher (excluding a buggy 2.4.4). Version 2.4.6 or higher is highly recommended.
+  If installed, must be Version 2.4.6 or higher.
 
 * `bottleneck <http://berkeleyanalytics.com/bottleneck>`__: for accelerating certain types of ``nan``
   evaluations. ``bottleneck`` uses specialized cython routines to achieve large speedups.

doc/source/io.rst

@@ -2777,6 +2777,7 @@ Added support for Openpyxl >= 2.2
     ``'xlsxwriter'`` will produce an Excel 2007-format workbook (xlsx). If
     omitted, an Excel 2007-formatted workbook is produced.
 
+
 .. _io.excel.writers:
 
 Excel writer engines
@@ -2823,6 +2824,18 @@ argument to ``to_excel`` and to ``ExcelWriter``. The built-in engines are:
 
    df.to_excel('path_to_file.xlsx', sheet_name='Sheet1')
 
+.. _io.excel.style:
+
+Style and Formatting
+''''''''''''''''''''
+
+The look and feel of Excel worksheets created from pandas can be modified using the following parameters on the ``DataFrame``'s ``to_excel`` method.
+
+- ``float_format`` : Format string for floating point numbers (default None)
+- ``freeze_panes`` : A tuple of two integers representing the bottommost row and rightmost column to freeze. Each of these parameters is one-based, so (1, 1) will freeze the first row and first column (default None)
+
+
+
 .. _io.clipboard:
 
 Clipboard

doc/source/r_interface.rst

@@ -41,15 +41,17 @@ In the remainder of this page, a few examples of explicit conversion is given. T
 Transferring R data sets into Python
 ------------------------------------
 
-The ``pandas2ri.ri2py`` function retrieves an R data set and converts it to the
-appropriate pandas object (most likely a DataFrame):
+Once the pandas conversion is activated (``pandas2ri.activate()``), many conversions
+of R to pandas objects will be done automatically. For example, to obtain the 'iris' dataset as a pandas DataFrame:
 
 .. ipython:: python
 
     r.data('iris')
-    df_iris = pandas2ri.ri2py(r['iris'])
-    df_iris.head()
+    r['iris'].head()
 
+If the pandas conversion was not activated, the above could also be accomplished
+by explicitly converting it with the ``pandas2ri.ri2py`` function
+(``pandas2ri.ri2py(r['iris'])``).
 
 Converting DataFrames into R objects
 ------------------------------------
@@ -65,7 +67,6 @@ DataFrames into the equivalent R object (that is, **data.frame**):
    print(type(r_dataframe))
    print(r_dataframe)
 
-
 The DataFrame's index is stored as the ``rownames`` attribute of the
 data.frame instance.