DOC: Added docs to clarify that MultiIndex.set_levels() interprets passed values as new components of the .levels attribute (#28294)

.travis.yml

@@ -30,6 +30,12 @@ matrix:
       - python: 3.5
 
     include:
+    - dist: bionic
+      # 18.04
+      python: 3.8-dev
+      env:
+        - JOB="3.8-dev" PATTERN="(not slow and not network)"
+
     - dist: trusty
       env:
         - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)"
@@ -71,24 +77,27 @@ before_install:
   # This overrides travis and tells it to look nowhere.
   - export BOTO_CONFIG=/dev/null
 
+
 install:
   - echo "install start"
   - ci/prep_cython_cache.sh
   - ci/setup_env.sh
   - ci/submit_cython_cache.sh
   - echo "install done"
 
+
 before_script:
   # display server (for clipboard functionality) needs to be started here,
   # does not work if done in install:setup_env.sh (GH-26103)
   - export DISPLAY=":99.0"
   - echo "sh -e /etc/init.d/xvfb start"
-  - sh -e /etc/init.d/xvfb start
+  - if [ "$JOB" != "3.8-dev" ]; then sh -e /etc/init.d/xvfb start; fi
   - sleep 3
 
 script:
   - echo "script start"
-  - source activate pandas-dev
+  - echo "$JOB"
+  - if [ "$JOB" != "3.8-dev" ]; then source activate pandas-dev; fi
   - ci/run_tests.sh
 
 after_script:

asv_bench/benchmarks/ctors.py

@@ -67,7 +67,7 @@ class SeriesConstructors:
     def setup(self, data_fmt, with_index, dtype):
         if data_fmt in (gen_of_str, gen_of_tuples) and with_index:
             raise NotImplementedError(
-                "Series constructors do not support " "using generators with indexes"
+                "Series constructors do not support using generators with indexes"
             )
         N = 10 ** 4
         if dtype == "float":

asv_bench/benchmarks/eval.py

@@ -27,7 +27,7 @@ def time_add(self, engine, threads):
 
     def time_and(self, engine, threads):
         pd.eval(
-            "(self.df > 0) & (self.df2 > 0) & " "(self.df3 > 0) & (self.df4 > 0)",
+            "(self.df > 0) & (self.df2 > 0) & (self.df3 > 0) & (self.df4 > 0)",
             engine=engine,
         )
 

asv_bench/benchmarks/io/hdf.py

@@ -88,11 +88,11 @@ def time_write_store_table_dc(self):
 
     def time_query_store_table_wide(self):
         self.store.select(
-            "table_wide", where="index > self.start_wide and " "index < self.stop_wide"
+            "table_wide", where="index > self.start_wide and index < self.stop_wide"
         )
 
     def time_query_store_table(self):
-        self.store.select("table", where="index > self.start and " "index < self.stop")
+        self.store.select("table", where="index > self.start and index < self.stop")
 
     def time_store_repr(self):
         repr(self.store)

ci/build38.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+# Special build for python3.8 until numpy puts its own wheels up
+
+sudo apt-get install build-essential gcc xvfb
+pip install --no-deps -U pip wheel setuptools
+pip install python-dateutil pytz cython pytest pytest-xdist hypothesis
+
+# Possible alternative for getting numpy:
+pip install --pre -f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com/ numpy
+
+python setup.py build_ext -inplace
+python -m pip install --no-build-isolation -e .
+
+python -c "import sys; print(sys.version_info)"
+python -c "import pandas as pd"
+python -c "import hypothesis"
+
+# TODO: Is there anything else in setup_env that we really want to do?
+# ci/setup_env.sh

ci/code_checks.sh

@@ -122,7 +122,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     MSG='Check for non-standard imports' ; echo $MSG
     invgrep -R --include="*.py*" -E "from pandas.core.common import " pandas
     invgrep -R --include="*.py*" -E "from collections.abc import " pandas
-    # invgrep -R --include="*.py*" -E "from numpy import nan " pandas  # GH#24822 not yet implemented since the offending imports have not all been removed
+    invgrep -R --include="*.py*" -E "from numpy import nan " pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for use of exec' ; echo $MSG

ci/setup_env.sh

@@ -1,5 +1,9 @@
 #!/bin/bash -e
 
+if [ "$JOB" == "3.8-dev" ]; then
+    /bin/bash ci/build38.sh
+    exit 0
+fi
 
 # edit the locale file if needed
 if [ -n "$LOCALE_OVERRIDE" ]; then
@@ -51,6 +55,7 @@ echo
 echo "update conda"
 conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
+conda install pip  # create conda to create a historical artifact for pip & setuptools
 conda update -n base conda
 
 echo "conda info -a"

doc/source/conf.py

@@ -120,6 +120,9 @@
 plot_pre_code = """import numpy as np
 import pandas as pd"""
 
+# nbsphinx do not use requirejs (breaks bootstrap)
+nbsphinx_requirejs_path = ""
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["../_templates"]
 
@@ -628,11 +631,11 @@ def linkcode_resolve(domain, info):
     fn = os.path.relpath(fn, start=os.path.dirname(pandas.__file__))
 
     if "+" in pandas.__version__:
-        return "http://github.com/pandas-dev/pandas/blob/master/pandas/" "{}{}".format(
+        return "http://github.com/pandas-dev/pandas/blob/master/pandas/{}{}".format(
             fn, linespec
         )
     else:
-        return "http://github.com/pandas-dev/pandas/blob/" "v{}/pandas/{}{}".format(
+        return "http://github.com/pandas-dev/pandas/blob/v{}/pandas/{}{}".format(
             pandas.__version__, fn, linespec
         )
 

doc/source/development/contributing.rst

@@ -172,7 +172,7 @@ installed (or you wish to install a newer version) you can install a compiler
     yum groupinstall "Development Tools"
 
 For other Linux distributions, consult your favourite search engine for
-commpiler installation instructions.
+compiler installation instructions.
 
 Let us know if you have any difficulties by opening an issue or reaching out on
 `Gitter`_.
@@ -949,6 +949,9 @@ the expected correct result::
 
         assert_frame_equal(pivoted, expected)
 
+Please remember to add the Github Issue Number as a comment to a new test.
+E.g. "# brief comment, see GH#28907"
+
 Transitioning to ``pytest``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/getting_started/install.rst

@@ -18,7 +18,7 @@ Instructions for installing from source,
 Python version support
 ----------------------
 
-Officially Python 3.5.3 and above, 3.6, and 3.7.
+Officially Python 3.5.3 and above, 3.6, 3.7, and 3.8.
 
 Installing pandas
 -----------------

doc/source/user_guide/advanced.rst

@@ -783,27 +783,41 @@ values **not** in the categories, similarly to how you can reindex **any** panda
 
 .. ipython:: python
 
-   df2.reindex(['a', 'e'])
-   df2.reindex(['a', 'e']).index
-   df2.reindex(pd.Categorical(['a', 'e'], categories=list('abcde')))
-   df2.reindex(pd.Categorical(['a', 'e'], categories=list('abcde'))).index
+   df3 = pd.DataFrame({'A': np.arange(3),
+                       'B': pd.Series(list('abc')).astype('category')})
+   df3 = df3.set_index('B')
+   df3
+
+.. ipython:: python
+
+   df3.reindex(['a', 'e'])
+   df3.reindex(['a', 'e']).index
+   df3.reindex(pd.Categorical(['a', 'e'], categories=list('abe')))
+   df3.reindex(pd.Categorical(['a', 'e'], categories=list('abe'))).index
 
 .. warning::
 
    Reshaping and Comparison operations on a ``CategoricalIndex`` must have the same categories
    or a ``TypeError`` will be raised.
 
-   .. code-block:: ipython
+   .. ipython:: python
 
-    In [9]: df3 = pd.DataFrame({'A': np.arange(6), 'B': pd.Series(list('aabbca')).astype('category')})
+      df4 = pd.DataFrame({'A': np.arange(2),
+                          'B': list('ba')})
+      df4['B'] = df4['B'].astype(CategoricalDtype(list('ab')))
+      df4 = df4.set_index('B')
+      df4.index
 
-    In [11]: df3 = df3.set_index('B')
+      df5 = pd.DataFrame({'A': np.arange(2),
+                          'B': list('bc')})
+      df5['B'] = df5['B'].astype(CategoricalDtype(list('bc')))
+      df5 = df5.set_index('B')
+      df5.index
 
-    In [11]: df3.index
-    Out[11]: CategoricalIndex(['a', 'a', 'b', 'b', 'c', 'a'], categories=['a', 'b', 'c'], ordered=False, name='B', dtype='category')
+   .. code-block:: ipython
 
-    In [12]: pd.concat([df2, df3])
-    TypeError: categories must match existing categories when appending
+      In [1]: pd.concat([df4, df5])
+      TypeError: categories must match existing categories when appending
 
 .. _indexing.rangeindex:
 

doc/source/user_guide/io.rst

@@ -3811,6 +3811,8 @@ storing/selecting from homogeneous index ``DataFrames``.
         # the levels are automatically included as data columns
         store.select('df_mi', 'foo=bar')
 
+.. note::
+   The ``index`` keyword is reserved and cannot be use as a level name.
 
 .. _io.hdf5-query:
 
@@ -3829,6 +3831,7 @@ A query is specified using the ``Term`` class under the hood, as a boolean expre
 
 * ``index`` and ``columns`` are supported indexers of ``DataFrames``.
 * if ``data_columns`` are specified, these can be used as additional indexers.
+* level name in a MultiIndex, with default name  ``level_0``, ``level_1``, … if not provided.
 
 Valid comparison operators are:
 
@@ -3947,7 +3950,7 @@ space. These are in terms of the total number of rows in a table.
 
 .. _io.hdf5-timedelta:
 
-Using timedelta64[ns]
+Query timedelta64[ns]
 +++++++++++++++++++++
 
 You can store and query using the ``timedelta64[ns]`` type. Terms can be
@@ -3966,6 +3969,35 @@ specified in the format: ``<float>(<unit>)``, where float may be signed (and fra
    store.append('dftd', dftd, data_columns=True)
    store.select('dftd', "C<'-3.5D'")
 
+Query MultiIndex
+++++++++++++++++
+
+Selecting from a ``MultiIndex`` can be achieved by using the name of the level.
+
+.. ipython:: python
+
+   df_mi.index.names
+   store.select('df_mi', "foo=baz and bar=two")
+
+If the ``MultiIndex`` levels names are ``None``, the levels are automatically made available via
+the ``level_n`` keyword with ``n`` the level of the ``MultiIndex`` you want to select from.
+
+.. ipython:: python
+
+   index = pd.MultiIndex(
+       levels=[["foo", "bar", "baz", "qux"], ["one", "two", "three"]],
+       codes=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3], [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
+   )
+   df_mi_2 = pd.DataFrame(np.random.randn(10, 3),
+                          index=index, columns=["A", "B", "C"])
+   df_mi_2
+
+   store.append("df_mi_2", df_mi_2)
+
+   # the levels are automatically included as data columns with keyword level_n
+   store.select("df_mi_2", "level_0=foo and level_1=two")
+
+
 Indexing
 ++++++++
 

doc/source/user_guide/reshaping.rst

@@ -728,14 +728,14 @@ Suppose we wanted to pivot ``df`` such that the ``col`` values are columns,
 ``row`` values are the index, and the mean of ``val0`` are the values? In
 particular, the resulting DataFrame should look like:
 
-.. note::
-
-   col   col0   col1   col2   col3  col4
-   row
-   row0  0.77  0.605    NaN  0.860  0.65
-   row2  0.13    NaN  0.395  0.500  0.25
-   row3   NaN  0.310    NaN  0.545   NaN
-   row4   NaN  0.100  0.395  0.760  0.24
+.. code-block:: text
+
+    col   col0   col1   col2   col3  col4
+    row
+    row0  0.77  0.605    NaN  0.860  0.65
+    row2  0.13    NaN  0.395  0.500  0.25
+    row3   NaN  0.310    NaN  0.545   NaN
+    row4   NaN  0.100  0.395  0.760  0.24
 
 This solution uses :func:`~pandas.pivot_table`. Also note that
 ``aggfunc='mean'`` is the default. It is included here to be explicit.