Merge remote-tracking branch 'upstream/master' into 26760-converter

Author: TomAugspurger

azure-pipelines.yml

@@ -24,7 +24,6 @@ jobs:
       # XXX next command should avoid redefining the path in every step, but
       # made the process crash as it couldn't find deactivate
       #echo '##vso[task.prependpath]$HOME/miniconda3/bin'
-      echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev'
       echo '##vso[task.setvariable variable=ENV_FILE]environment.yml'
       echo '##vso[task.setvariable variable=AZURE]true'
     displayName: 'Setting environment variables'
@@ -116,3 +115,65 @@ jobs:
       fi
     displayName: 'Running benchmarks'
     condition: true
+
+- job: 'Docs'
+  pool:
+    vmImage: ubuntu-16.04
+  timeoutInMinutes: 90
+  steps:
+  - script: |
+      echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/travis-36-doc.yaml'
+    displayName: 'Setting environment variables'
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      sudo apt-get install -y libc6-dev-i386
+      ci/setup_env.sh
+    displayName: 'Setup environment and build pandas'
+
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      doc/make.py
+    displayName: 'Build documentation'
+
+  - script: |
+      cd doc/build/html
+      git init
+      touch .nojekyll
+      echo "dev.pandas.io" > CNAME
+      git add --all .
+      git config user.email "[email protected]"
+      git config user.name "pandas-docs-bot"
+      git commit -m "pandas documentation in master"
+    displayName: 'Create git repo for docs build'
+    condition : |
+      and(not(eq(variables['Build.Reason'], 'PullRequest')),
+          eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+
+  # For `InstallSSHKey@0` to work, next steps are required:
+  # 1. Generate a pair of private/public keys (i.e. `ssh-keygen -t rsa -b 4096 -C "[email protected]"`)
+  # 2. Go to "Library > Secure files" in the Azure Pipelines dashboard: https://dev.azure.com/pandas-dev/pandas/_library?itemType=SecureFiles
+  # 3. Click on "+ Secure file"
+  # 4. Upload the private key (the name of the file must match with the specified in "sshKeySecureFile" input below, "pandas_docs_key")
+  # 5. Click on file name after it is created, tick the box "Authorize for use in all pipelines" and save
+  # 6. The public key specified in "sshPublicKey" is the pair of the uploaded private key, and needs to be set as a deploy key of the repo where the docs will be pushed (with write access): https://github.com/pandas-dev/pandas-dev.github.io/settings/keys
+  - task: InstallSSHKey@0
+    inputs:
+      hostName: 'github.com,192.30.252.128 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=='
+      sshPublicKey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDHmz3l/EdqrgNxEUKkwDUuUcLv91unig03pYFGO/DMIgCmPdMG96zAgfnESd837Rm0wSSqylwSzkRJt5MV/TpFlcVifDLDQmUhqCeO8Z6dLl/oe35UKmyYICVwcvQTAaHNnYRpKC5IUlTh0JEtw9fGlnp1Ta7U1ENBLbKdpywczElhZu+hOQ892zqOj3CwA+U2329/d6cd7YnqIKoFN9DWT3kS5K6JE4IoBfQEVekIOs23bKjNLvPoOmi6CroAhu/K8j+NCWQjge5eJf2x/yTnIIP1PlEcXoHIr8io517posIx3TBup+CN8bNS1PpDW3jyD3ttl1uoBudjOQrobNnJeR6Rn67DRkG6IhSwr3BWj8alwUG5mTdZzwV5Pa9KZFdIiqX7NoDGg+itsR39QCn0thK8lGRNSR8KrWC1PSjecwelKBO7uQ7rnk/rkrZdBWR4oEA8YgNH8tirUw5WfOr5a0AIaJicKxGKNdMxZt+zmC+bS7F4YCOGIm9KHa43RrKhoGRhRf9fHHHKUPwFGqtWG4ykcUgoamDOURJyepesBAO3FiRE9rLU6ILbB3yEqqoekborHmAJD5vf7PWItW3Q/YQKuk3kkqRcKnexPyzyyq5lUgTi8CxxZdaASIOu294wjBhhdyHlXEkVTNJ9JKkj/obF+XiIIp0cBDsOXY9hDQ== [email protected]'
+      sshKeySecureFile: 'pandas_docs_key'
+    displayName: 'Install GitHub ssh deployment key'
+    condition : |
+      and(not(eq(variables['Build.Reason'], 'PullRequest')),
+          eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+
+  - script: |
+      cd doc/build/html
+      git remote add origin [email protected]:pandas-dev/pandas-dev.github.io.git
+      git push -f origin master
+      exit 0  # FIXME this will leave the build green even if the step fails. To be removed when we are confident with this.
+    displayName: 'Publish docs to GitHub pages'
+    condition : |
+      and(not(eq(variables['Build.Reason'], 'PullRequest')),
+          eq(variables['Build.SourceBranch'], 'refs/heads/master'))

ci/deps/travis-36-doc.yaml

@@ -33,8 +33,8 @@ dependencies:
   - pytz
   - scipy
   - seaborn
-  # recursion error with sphinx 2.1.0. https://github.com/pandas-dev/pandas/issues/26723
-  - sphinx==2.0.1
+  # some styling is broken with sphinx >= 2 (https://github.com/pandas-dev/pandas/issues/26058)
+  - sphinx=1.8.5
   - sqlalchemy
   - statsmodels
   - xarray

doc/source/conf.py

@@ -18,7 +18,7 @@
 import jinja2
 from sphinx.ext.autosummary import _import_by_name
 from numpydoc.docscrape import NumpyDocString
-from numpydoc.docscrape_sphinx import SphinxDocString
+
 
 logger = logging.getLogger(__name__)
 
@@ -110,6 +110,9 @@
                                  else None)))
 autosummary_generate = True if pattern is None else ['index']
 
+# numpydoc
+numpydoc_attributes_as_param_list = False
+
 # matplotlib plot directive
 plot_include_source = True
 plot_formats = [("png", 90)]
@@ -422,62 +425,6 @@
 ]
 
 
-def sphinxdocstring_str(self, indent=0, func_role="obj"):
-    # Pandas displays Attributes section in style like Methods section
-
-    # Function is copy of `SphinxDocString.__str__`
-    ns = {
-        'signature': self._str_signature(),
-        'index': self._str_index(),
-        'summary': self._str_summary(),
-        'extended_summary': self._str_extended_summary(),
-        'parameters': self._str_param_list('Parameters'),
-        'returns': self._str_returns('Returns'),
-        'yields': self._str_returns('Yields'),
-        'other_parameters': self._str_param_list('Other Parameters'),
-        'raises': self._str_param_list('Raises'),
-        'warns': self._str_param_list('Warns'),
-        'warnings': self._str_warnings(),
-        'see_also': self._str_see_also(func_role),
-        'notes': self._str_section('Notes'),
-        'references': self._str_references(),
-        'examples': self._str_examples(),
-        # Replaced `self._str_param_list('Attributes', fake_autosummary=True)`
-        # with `self._str_member_list('Attributes')`
-        'attributes': self._str_member_list('Attributes'),
-        'methods': self._str_member_list('Methods'),
-    }
-    ns = {k: '\n'.join(v) for k, v in ns.items()}
-
-    rendered = self.template.render(**ns)
-    return '\n'.join(self._str_indent(rendered.split('\n'), indent))
-
-
-SphinxDocString.__str__ = sphinxdocstring_str
-
-
-# Fix "WARNING: Inline strong start-string without end-string."
-# PR #155 "Escape the * in *args and **kwargs" from numpydoc
-# Can be removed after PR merges in v0.9.0
-def decorate_process_param(func):
-    def _escape_args_and_kwargs(name):
-        if name[:2] == '**':
-            return r'\*\*' + name[2:]
-        elif name[:1] == '*':
-            return r'\*' + name[1:]
-        else:
-            return name
-
-    def func_wrapper(self, param, desc, fake_autosummary):
-        param = _escape_args_and_kwargs(param.strip())
-        return func(self, param, desc, fake_autosummary)
-
-    return func_wrapper
-
-
-func = SphinxDocString._process_param
-SphinxDocString._process_param = decorate_process_param(func)
-
 # Add custom Documenter to handle attributes/methods of an AccessorProperty
 # eg pandas.Series.str and pandas.Series.dt (see GH9322)
 

doc/source/development/contributing.rst

@@ -499,6 +499,21 @@ as possible to avoid mass breakages.
 Additional standards are outlined on the `code style wiki
 page <https://github.com/pandas-dev/pandas/wiki/Code-Style-and-Conventions>`_.
 
+Optional dependencies
+---------------------
+
+Optional dependencies (e.g. matplotlib) should be imported with the private helper
+``pandas.compat._optional.import_optional_dependency``. This ensures a
+consistent error message when the dependency is not met.
+
+All methods using an optional dependency should include a test asserting that an
+``ImportError`` is raised when the optional dependency is not found. This test
+should be skipped if the library is present.
+
+All optional dependencies should be documented in
+:ref:`install.optional_dependencies` and the minimum required version should be
+set in the ``pandas.compat._optional.VERSIONS`` dict.
+
 C (cpplint)
 ~~~~~~~~~~~
 

doc/source/install.rst

@@ -252,87 +252,69 @@ Recommended Dependencies
 Optional Dependencies
 ~~~~~~~~~~~~~~~~~~~~~
 
-* `Cython <http://www.cython.org>`__: Only necessary to build development
-  version. Version 0.28.2 or higher.
-* `SciPy <http://www.scipy.org>`__: miscellaneous statistical functions, Version 0.19.0 or higher
-* `xarray <http://xarray.pydata.org>`__: pandas like handling for > 2 dims. Version 0.8.2 or higher is recommended.
-* `PyTables <http://www.pytables.org>`__: necessary for HDF5-based storage, Version 3.4.2 or higher
-* `pyarrow <http://arrow.apache.org/docs/python/>`__ (>= 0.9.0): necessary for feather-based storage.
-* `Apache Parquet <https://parquet.apache.org/>`__, either `pyarrow <http://arrow.apache.org/docs/python/>`__ (>= 0.9.0) or `fastparquet <https://fastparquet.readthedocs.io/en/latest>`__ (>= 0.2.1) for parquet-based storage. The `snappy <https://pypi.org/project/python-snappy>`__ and `brotli <https://pypi.org/project/brotlipy>`__ are available for compression support.
-* `SQLAlchemy <http://www.sqlalchemy.org>`__: for SQL database support. Version 1.1.4 or higher recommended. Besides SQLAlchemy, you also need a database specific driver. You can find an overview of supported drivers for each SQL dialect in the `SQLAlchemy docs <http://docs.sqlalchemy.org/en/latest/dialects/index.html>`__. Some common drivers are:
-
-    * `psycopg2 <http://initd.org/psycopg/>`__: for PostgreSQL
-    * `pymysql <https://github.com/PyMySQL/PyMySQL>`__: for MySQL.
-    * `SQLite <https://docs.python.org/3/library/sqlite3.html>`__: for SQLite, this is included in Python's standard library by default.
-
-* `matplotlib <http://matplotlib.org/>`__: for plotting, Version 2.2.2 or higher.
-* For Excel I/O:
-
-    * `xlrd/xlwt <http://www.python-excel.org/>`__: Excel reading (xlrd), version 1.0.0 or higher required, and writing (xlwt)
-    * `openpyxl <https://openpyxl.readthedocs.io/en/stable/>`__: openpyxl version 2.4.0
-      for writing .xlsx files (xlrd >= 1.0.0)
-    * `XlsxWriter <https://pypi.org/project/XlsxWriter>`__: Alternative Excel writer
-
-* `Jinja2 <http://jinja.pocoo.org/>`__: Template engine for conditional HTML formatting.
-* `s3fs <http://s3fs.readthedocs.io/>`__: necessary for Amazon S3 access (s3fs >= 0.0.8).
-* `blosc <https://pypi.org/project/blosc>`__: for msgpack compression using ``blosc``
-* `gcsfs <http://gcsfs.readthedocs.io/>`__: necessary for Google Cloud Storage access (gcsfs >= 0.1.0).
-* One of
-  `qtpy  <https://github.com/spyder-ide/qtpy>`__ (requires PyQt or PySide),
-  `PyQt5 <https://www.riverbankcomputing.com/software/pyqt/download5>`__,
-  `PyQt4 <http://www.riverbankcomputing.com/software/pyqt/download>`__,
-  `xsel <http://www.vergenet.net/~conrad/software/xsel/>`__, or
-  `xclip <https://github.com/astrand/xclip/>`__: necessary to use
-  :func:`~pandas.read_clipboard`. Most package managers on Linux distributions will have ``xclip`` and/or ``xsel`` immediately available for installation.
-* `pandas-gbq
-  <https://pandas-gbq.readthedocs.io/en/latest/install.html#dependencies>`__:
-  for Google BigQuery I/O. (pandas-gbq >= 0.8.0)
-
-* One of the following combinations of libraries is needed to use the
-  top-level :func:`~pandas.read_html` function:
-
-  .. versionchanged:: 0.23.0
-
-  .. note::
-
-     If using BeautifulSoup4 a minimum version of 4.4.1 is required
-
-  * `BeautifulSoup4`_ and `html5lib`_ (Any recent version of `html5lib`_ is
-    okay.)
-  * `BeautifulSoup4`_ and `lxml`_
-  * `BeautifulSoup4`_ and `html5lib`_ and `lxml`_
-  * Only `lxml`_, although see :ref:`HTML Table Parsing <io.html.gotchas>`
-    for reasons as to why you should probably **not** take this approach.
-
-  .. warning::
-
-     * if you install `BeautifulSoup4`_ you must install either
-       `lxml`_ or `html5lib`_ or both.
-       :func:`~pandas.read_html` will **not** work with *only*
-       `BeautifulSoup4`_ installed.
-     * You are highly encouraged to read :ref:`HTML Table Parsing gotchas <io.html.gotchas>`.
-       It explains issues surrounding the installation and
-       usage of the above three libraries.
-
-  .. note::
-
-     * if you're on a system with ``apt-get`` you can do
-
-       .. code-block:: sh
-
-          sudo apt-get build-dep python-lxml
-
-       to get the necessary dependencies for installation of `lxml`_. This
-       will prevent further headaches down the line.
-
+Pandas has many optional dependencies that are only used for specific methods.
+For example, :func:`pandas.read_hdf` requires the ``pytables`` package. If the
+optional dependency is not installed, pandas will raise an ``ImportError`` when
+the method requiring that dependency is called.
+
+========================= ================== =============================================================
+Dependency                Minimum Version    Notes
+========================= ================== =============================================================
+BeautifulSoup4            4.4.1              HTML parser for read_html (see :ref:`note <optional_html>`)
+Jinja2                                       Conditional formatting with DataFrame.style
+PyQt4                                        Clipboard I/O
+PyQt5                                        Clipboard I/O
+PyTables                  3.4.2              HDF5-based reading / writing
+SQLAlchemy                1.1.4              SQL support for databases other than sqlite
+SciPy                     0.19.0             Miscellaneous statistical functions
+XLsxWriter                                   Excel writing
+blosc                                        Compression for msgpack
+fastparquet               0.2.1              Parquet reading / writing
+gcsfs                     0.1.0              Google Cloud Storage access
+html5lib                                     HTML parser for read_html (see :ref:`note <optional_html>`)
+lxml                                         HTML parser for read_html (see :ref:`note <optional_html>`)
+matplotlib                2.2.2              Visualization
+openpyxl                  2.4.0              Reading / writing for xlsx files
+pandas-gbq                0.8.0              Google Big Query access
+psycopg2                                     PostgreSQL engine for sqlalchemy
+pyarrow                   0.9.0              Parquet and feather reading / writing
+pymysql                                      MySQL engine for sqlalchemy
+qtpy                                         Clipboard I/O
+s3fs                      0.0.8              Amazon S3 access
+xarray                    0.8.2              pandas-like API for N-dimensional data
+xclip                                        Clipboard I/O on linux
+xlrd                      1.0.0              Excel reading
+xlwt                      2.4.0              Excel writing
+xsel                                         Clipboard I/O on linux
+zlib                                         Compression for msgpack
+========================= ================== =============================================================
+
+.. _optional_html:
+
+Optional Dependencies for Parsing HTML
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+One of the following combinations of libraries is needed to use the
+top-level :func:`~pandas.read_html` function:
+
+.. versionchanged:: 0.23.0
+
+* `BeautifulSoup4`_ and `html5lib`_
+* `BeautifulSoup4`_ and `lxml`_
+* `BeautifulSoup4`_ and `html5lib`_ and `lxml`_
+* Only `lxml`_, although see :ref:`HTML Table Parsing <io.html.gotchas>`
+  for reasons as to why you should probably **not** take this approach.
+
+.. warning::
+
+    * if you install `BeautifulSoup4`_ you must install either
+      `lxml`_ or `html5lib`_ or both.
+      :func:`~pandas.read_html` will **not** work with *only*
+      `BeautifulSoup4`_ installed.
+    * You are highly encouraged to read :ref:`HTML Table Parsing gotchas <io.html.gotchas>`.
+      It explains issues surrounding the installation and
+      usage of the above three libraries.
 
 .. _html5lib: https://github.com/html5lib/html5lib-python
 .. _BeautifulSoup4: http://www.crummy.com/software/BeautifulSoup
 .. _lxml: http://lxml.de
-
-.. note::
-
-   Without the optional dependencies, many useful features will not
-   work. Hence, it is highly recommended that you install these. A packaged
-   distribution like `Anaconda <http://docs.continuum.io/anaconda/>`__, `ActivePython <https://www.activestate.com/activepython/downloads>`__  (version 2.7 or 3.5), or `Enthought Canopy
-   <http://enthought.com/products/canopy>`__ may be worth considering.

doc/source/reference/indexing.rst

@@ -190,6 +190,9 @@ Numeric Index
 .. autosummary::
    :toctree: api/
 
+   RangeIndex.start
+   RangeIndex.stop
+   RangeIndex.step
    RangeIndex.from_range
 
 .. _api.categoricalindex:

doc/source/reference/series.rst

@@ -472,6 +472,8 @@ strings and apply several methods to it. These can be accessed like
        Series.str
        Series.cat
        Series.dt
+       Series.sparse
+       DataFrame.sparse
        Index.str
 
 .. _api.series.cat:

doc/source/user_guide/missing_data.rst

@@ -466,7 +466,7 @@ at the new values.
 .. _missing_data.interp_limits:
 
 Interpolation Limits
-^^^^^^^^^^^^^^^^^^^^
+--------------------
 
 Like other pandas fill methods, :meth:`~DataFrame.interpolate` accepts a ``limit`` keyword
 argument. Use this argument to limit the number of consecutive ``NaN`` values

doc/source/user_guide/reshaping.rst

@@ -705,7 +705,7 @@ handling of NaN:
     you can use  ``df["cat_col"] = pd.Categorical(df["col"])`` or
     ``df["cat_col"] = df["col"].astype("category")``. For full docs on :class:`~pandas.Categorical`,
     see the :ref:`Categorical introduction <categorical>` and the
-    :ref:`API documentation <api.categorical>`.
+    :ref:`API documentation <api.arrays.categorical>`.
 
 Examples
 --------

doc/source/user_guide/sparse.rst

@@ -77,6 +77,8 @@ A sparse array can be converted to a regular (dense) ndarray with :meth:`numpy.a
    np.asarray(sparr)
 
 
+.. _sparse.dtype:
+
 SparseDtype
 -----------