Merge branch 'master' into pandas-devGH-26206-datetime-unit-out-of-bounds

Author: jorisvandenbossche

.travis.yml

@@ -48,17 +48,10 @@ matrix:
       env:
         - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow"
 
-    # In allow_failures
-    - dist: trusty
-      env:
-        - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true
     allow_failures:
       - dist: trusty
         env:
           - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow"
-      - dist: trusty
-        env:
-          - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true
 
 before_install:
   - echo "before_install"
@@ -97,7 +90,6 @@ before_script:
 script:
   - echo "script start"
   - source activate pandas-dev
-  - ci/build_docs.sh
   - ci/run_tests.sh
 
 after_script:

README.md

@@ -224,7 +224,7 @@ Most development discussion is taking place on github in this repo. Further, the
 
 All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome.
 
-A detailed overview on how to contribute can be found in the **[contributing guide](https://pandas-docs.github.io/pandas-docs-travis/contributing.html)**. There is also an [overview](.github/CONTRIBUTING.md) on GitHub.
+A detailed overview on how to contribute can be found in the **[contributing guide](https://dev.pandas.io/contributing.html)**. There is also an [overview](.github/CONTRIBUTING.md) on GitHub.
 
 If you are simply looking to start working with the pandas codebase, navigate to the [GitHub "issues" tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?labels=Docs&sort=updated&state=open) and [good first issue](https://github.com/pandas-dev/pandas/issues?labels=good+first+issue&sort=updated&state=open) where you could start out.
 

asv_bench/benchmarks/ctors.py

@@ -55,7 +55,14 @@ class SeriesConstructors:
               [False, True],
               ['float', 'int']]
 
+    # Generators get exhausted on use, so run setup before every call
+    number = 1
+    repeat = (3, 250, 10)
+
     def setup(self, data_fmt, with_index, dtype):
+        if data_fmt in (gen_of_str, gen_of_tuples) and with_index:
+            raise NotImplementedError('Series constructors do not support '
+                                      'using generators with indexes')
         N = 10**4
         if dtype == 'float':
             arr = np.random.randn(N)

asv_bench/benchmarks/frame_ctor.py

@@ -72,6 +72,10 @@ class FromRecords:
     params = [None, 1000]
     param_names = ['nrows']
 
+    # Generators get exhausted on use, so run setup before every call
+    number = 1
+    repeat = (3, 250, 10)
+
     def setup(self, nrows):
         N = 100000
         self.gen = ((x, (x * 20), (x * 100)) for x in range(N))

azure-pipelines.yml

@@ -123,7 +123,7 @@ jobs:
   timeoutInMinutes: 90
   steps:
   - script: |
-      echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/travis-36-doc.yaml'
+      echo '##vso[task.setvariable variable=ENV_FILE]environment.yml'
     displayName: 'Setting environment variables'
 
   - script: |
@@ -146,6 +146,7 @@ jobs:
       git init
       touch .nojekyll
       echo "dev.pandas.io" > CNAME
+      printf "User-agent: *\nDisallow: /" > robots.txt
       git add --all .
       git config user.email "[email protected]"
       git config user.name "pandas-docs-bot"
@@ -176,7 +177,6 @@ jobs:
       cd doc/build/html
       git remote add origin [email protected]:pandas-dev/pandas-dev.github.io.git
       git push -f origin master
-      exit 0  # FIXME this will leave the build green even if the step fails. To be removed when we are confident with this.
     displayName: 'Publish docs to GitHub pages'
     condition : |
       and(not(eq(variables['Build.Reason'], 'PullRequest')),

ci/build_docs.sh

@@ -460,7 +460,7 @@ Building master branch documentation
 
 When pull requests are merged into the *pandas* ``master`` branch, the main parts of
 the documentation are also built by Travis-CI. These docs are then hosted `here
-<http://pandas-docs.github.io/pandas-docs-travis>`__, see also
+<https://dev.pandas.io>`__, see also
 the :ref:`Continuous Integration <contributing.ci>` section.
 
 .. _contributing.code:

ci/deps/travis-36-doc.yaml

@@ -286,6 +286,7 @@ psycopg2                                     PostgreSQL engine for sqlalchemy
 pyarrow                   0.9.0              Parquet and feather reading / writing
 pymysql                                      MySQL engine for sqlalchemy
 pyreadstat                                   SPSS files (.sav) reading
+pytables                  3.4.2              HDF5 reading / writing
 qtpy                                         Clipboard I/O
 s3fs                      0.0.8              Amazon S3 access
 xarray                    0.8.2              pandas-like API for N-dimensional data

doc/source/development/contributing.rst

@@ -182,15 +182,15 @@ on a deeper level.
 Defined Levels
 ~~~~~~~~~~~~~~
 
-The repr of a ``MultiIndex`` shows all the defined levels of an index, even
+The :class:`MultiIndex` keeps all the defined levels of an index, even
 if they are not actually used. When slicing an index, you may notice this.
 For example:
 
 .. ipython:: python
 
-   df.columns  # original MultiIndex
+   df.columns.levels  # original MultiIndex
 
-   df[['foo','qux']].columns  # sliced
+   df[['foo','qux']].columns.levels  # sliced
 
 This is done to avoid a recomputation of the levels in order to make slicing
 highly performant. If you want to see only the used levels, you can use the
@@ -210,7 +210,8 @@ To reconstruct the ``MultiIndex`` with only the used levels, the
 
 .. ipython:: python
 
-   df[['foo', 'qux']].columns.remove_unused_levels()
+   new_mi = df[['foo', 'qux']].columns.remove_unused_levels()
+   new_mi.levels
 
 Data alignment and using ``reindex``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/install.rst

@@ -74,6 +74,38 @@ a dict to a Series groupby aggregation (:ref:`whatsnew_0200.api_breaking.depreca
 
 See :ref:`groupby.aggregate.named` for more.
 
+
+.. _whatsnew_0250.enhancements.multi_index_repr:
+
+Better repr for MultiIndex
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Printing of :class:`MultiIndex` instances now shows tuples of each row and ensures
+that the tuple items are vertically aligned, so it's now easier to understand
+the structure of the ``MultiIndex``. (:issue:`13480`):
+
+The repr now looks like this:
+
+.. ipython:: python
+
+   pd.MultiIndex.from_product([['a', 'abc'], range(500)])
+
+Previously, outputting a :class:`MultiIndex` printed all the ``levels`` and
+``codes`` of the ``MultiIndex``, which was visually unappealing and made
+the output more difficult to navigate. For example (limiting the range to 5):
+
+.. code-block:: ipython
+
+   In [1]: pd.MultiIndex.from_product([['a', 'abc'], range(5)])
+   Out[1]: MultiIndex(levels=[['a', 'abc'], [0, 1, 2, 3]],
+      ...:            codes=[[0, 0, 0, 0, 1, 1, 1, 1], [0, 1, 2, 3, 0, 1, 2, 3]])
+
+In the new repr, all values will be shown, if the number of rows is smaller
+than :attr:`options.display.max_seq_items` (default: 100 items). Horizontally,
+the output will truncate, if it's wider than :attr:`options.display.width`
+(default: 80 characters).
+
+
 .. _whatsnew_0250.enhancements.other:
 
 Other Enhancements
@@ -411,7 +443,7 @@ If installed, we now require:
 | pytest (dev)    | 4.0.2           |          |
 +-----------------+-----------------+----------+
 
-For `optional libraries <https://pandas-docs.github.io/pandas-docs-travis/install.html#dependencies>`_ the general recommendation is to use the latest version.
+For `optional libraries <https://dev.pandas.io/install.html#dependencies>`_ the general recommendation is to use the latest version.
 The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
 Optional libraries below the lowest tested version may still work, but are not considered supported.
 
@@ -503,6 +535,7 @@ Other Deprecations
 - The :meth:`Series.ftype`, :meth:`Series.ftypes` and :meth:`DataFrame.ftypes` methods are deprecated and will be removed in a future version.
   Instead, use :meth:`Series.dtype` and :meth:`DataFrame.dtypes` (:issue:`26705`).
 - :meth:`Timedelta.resolution` is deprecated and replaced with :meth:`Timedelta.resolution_string`.  In a future version, :meth:`Timedelta.resolution` will be changed to behave like the standard library :attr:`timedelta.resolution` (:issue:`21344`)
+- :meth:`Series.to_sparse`, :meth:`DataFrame.to_sparse`, :meth:`Series.to_dense` and :meth:`DataFrame.to_dense` are deprecated and will be removed in a future version. (:issue:`26557`).
 
 .. _whatsnew_0250.prior_deprecations:
 
@@ -568,6 +601,7 @@ Datetimelike
 - Bug in :func:`to_datetime` which raises ``TypeError`` for ``format='%Y%m%d'`` when called for invalid integer dates with length >= 6 digits with ``errors='ignore'``
 - Bug when comparing a :class:`PeriodIndex` against a zero-dimensional numpy array (:issue:`26689`)
 - Bug in constructing a ``Series`` or ``DataFrame`` from a numpy ``datetime64`` array with a non-ns unit and out-of-bound timestamps generating rubbish data, which will now correctly raise an ``OutOfBoundsDatetime`` error (:issue:`26206`).
+- Bug in :func:`date_range` with unnecessary ``OverflowError`` being raised for very large or very small dates (:issue:`26651`)
 
 Timedelta
 ^^^^^^^^^
@@ -710,6 +744,7 @@ Groupby/Resample/Rolling
 - Bug in :meth:`pandas.core.groupby.SeriesGroupBy.transform` where transforming an empty group would raise a ``ValueError`` (:issue:`26208`)
 - Bug in :meth:`pandas.core.frame.DataFrame.groupby` where passing a :class:`pandas.core.groupby.grouper.Grouper` would return incorrect groups when using the ``.groups`` accessor (:issue:`26326`)
 - Bug in :meth:`pandas.core.groupby.GroupBy.agg` where incorrect results are returned for uint64 columns. (:issue:`26310`)
+- Bug in :meth:`pandas.core.window.Rolling.median` and :meth:`pandas.core.window.Rolling.quantile` where MemoryError is raised with empty window (:issue:`26005`)
 
 Reshaping
 ^^^^^^^^^
@@ -737,6 +772,7 @@ Sparse
 - Bug in :class:`SparseFrame` constructor where passing ``None`` as the data would cause ``default_fill_value`` to be ignored (:issue:`16807`)
 - Bug in :class:`SparseDataFrame` when adding a column in which the length of values does not match length of index, ``AssertionError`` is raised instead of raising ``ValueError`` (:issue:`25484`)
 - Introduce a better error message in :meth:`Series.sparse.from_coo` so it returns a ``TypeError`` for inputs that are not coo matrices (:issue:`26554`)
+- Bug in :func:`numpy.modf` on a :class:`SparseArray`. Now a tuple of :class:`SparseArray` is returned.
 
 Other
 ^^^^^

doc/source/user_guide/advanced.rst

@@ -275,6 +275,10 @@ cdef convert_to_tsobject(object ts, object tz, object unit,
         - iso8601 string object
         - python datetime object
         - another timestamp object
+
+    Raises
+    ------
+    OutOfBoundsDatetime : ts cannot be converted within implementation bounds
     """
     cdef:
         _TSObject obj
@@ -294,6 +298,11 @@ cdef convert_to_tsobject(object ts, object tz, object unit,
         if obj.value != NPY_NAT:
             dt64_to_dtstruct(obj.value, &obj.dts)
     elif is_integer_object(ts):
+        try:
+            ts = <int64_t>ts
+        except OverflowError:
+            # GH#26651 re-raise as OutOfBoundsDatetime
+            raise OutOfBoundsDatetime(ts)
         if ts == NPY_NAT:
             obj.value = NPY_NAT
         else:

doc/source/whatsnew/v0.25.0.rst

@@ -1099,6 +1099,10 @@ def roll_median_c(ndarray[float64_t] values, int64_t win, int64_t minp,
         use_mock=False)
     output = np.empty(N, dtype=float)
 
+    if win == 0:
+        output[:] = NaN
+        return output
+
     sl = skiplist_init(<int>win)
     if sl == NULL:
         raise MemoryError("skiplist_init failed")
@@ -1486,6 +1490,11 @@ def roll_quantile(ndarray[float64_t, cast=True] values, int64_t win,
         minp, index, closed,
         use_mock=False)
     output = np.empty(N, dtype=float)
+
+    if win == 0:
+        output[:] = NaN
+        return output
+
     skiplist = skiplist_init(<int>win)
     if skiplist == NULL:
         raise MemoryError("skiplist_init failed")

pandas/_libs/tslibs/conversion.pyx

@@ -19,6 +19,7 @@
     "s3fs": "0.0.8",
     "scipy": "0.19.0",
     "sqlalchemy": "1.1.4",
+    "tables": "3.4.2",
     "xarray": "0.8.2",
     "xlrd": "1.1.0",
     "xlwt": "1.2.0",

pandas/_libs/window.pyx

@@ -1,9 +1,9 @@
 """ support numpy compatiblitiy across versions """
 
-import re
-import numpy as np
 from distutils.version import LooseVersion
+import re
 
+import numpy as np
 
 # numpy versioning
 _np_version = np.__version__
@@ -62,6 +62,7 @@ def np_array_datetime64_compat(arr, *args, **kwargs):
 
 
 __all__ = ['np',
+           '_np_version',
            '_np_version_under1p14',
            '_np_version_under1p15',
            '_np_version_under1p16',

pandas/compat/_optional.py

@@ -196,7 +196,7 @@ def decorator(accessor):
     return decorator
 
 
-_doc = """\
+_doc = """
 Register a custom accessor on %(klass)s objects.
 
 Parameters