Merge remote-tracking branch 'upstream/master' into from_ordered_dict

Author: mazayo

.travis.yml

@@ -48,17 +48,10 @@ matrix:
       env:
         - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow"
 
-    # In allow_failures
-    - dist: trusty
-      env:
-        - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true
     allow_failures:
       - dist: trusty
         env:
           - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow"
-      - dist: trusty
-        env:
-          - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true
 
 before_install:
   - echo "before_install"
@@ -97,7 +90,6 @@ before_script:
 script:
   - echo "script start"
   - source activate pandas-dev
-  - ci/build_docs.sh
   - ci/run_tests.sh
 
 after_script:

LICENSES/HAVEN_LICENSE

@@ -0,0 +1,2 @@
+YEAR: 2013-2016
+COPYRIGHT HOLDER: Hadley Wickham; RStudio; and Evan Miller

LICENSES/HAVEN_MIT

@@ -0,0 +1,32 @@
+Based on http://opensource.org/licenses/MIT
+
+This is a template. Complete and ship as file LICENSE the following 2
+lines (only)
+
+YEAR:
+COPYRIGHT HOLDER:
+
+and specify as
+
+License: MIT + file LICENSE
+
+Copyright (c) <YEAR>, <COPYRIGHT HOLDER>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -224,7 +224,7 @@ Most development discussion is taking place on github in this repo. Further, the
 
 All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome.
 
-A detailed overview on how to contribute can be found in the **[contributing guide](https://pandas-docs.github.io/pandas-docs-travis/contributing.html)**. There is also an [overview](.github/CONTRIBUTING.md) on GitHub.
+A detailed overview on how to contribute can be found in the **[contributing guide](https://dev.pandas.io/contributing.html)**. There is also an [overview](.github/CONTRIBUTING.md) on GitHub.
 
 If you are simply looking to start working with the pandas codebase, navigate to the [GitHub "issues" tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?labels=Docs&sort=updated&state=open) and [good first issue](https://github.com/pandas-dev/pandas/issues?labels=good+first+issue&sort=updated&state=open) where you could start out.
 

asv_bench/benchmarks/ctors.py

@@ -55,7 +55,14 @@ class SeriesConstructors:
               [False, True],
               ['float', 'int']]
 
+    # Generators get exhausted on use, so run setup before every call
+    number = 1
+    repeat = (3, 250, 10)
+
     def setup(self, data_fmt, with_index, dtype):
+        if data_fmt in (gen_of_str, gen_of_tuples) and with_index:
+            raise NotImplementedError('Series constructors do not support '
+                                      'using generators with indexes')
         N = 10**4
         if dtype == 'float':
             arr = np.random.randn(N)

asv_bench/benchmarks/frame_ctor.py

@@ -72,6 +72,10 @@ class FromRecords:
     params = [None, 1000]
     param_names = ['nrows']
 
+    # Generators get exhausted on use, so run setup before every call
+    number = 1
+    repeat = (3, 250, 10)
+
     def setup(self, nrows):
         N = 100000
         self.gen = ((x, (x * 20), (x * 100)) for x in range(N))

asv_bench/benchmarks/sparse.py

@@ -1,9 +1,8 @@
-import itertools
-
 import numpy as np
 import scipy.sparse
-from pandas import (SparseSeries, SparseDataFrame, SparseArray, Series,
-                    date_range, MultiIndex)
+
+import pandas as pd
+from pandas import MultiIndex, Series, SparseArray, date_range
 
 
 def make_array(size, dense_proportion, fill_value, dtype):
@@ -25,10 +24,10 @@ def setup(self):
             data = np.random.randn(N)[:-i]
             idx = rng[:-i]
             data[100:] = np.nan
-            self.series[i] = SparseSeries(data, index=idx)
+            self.series[i] = pd.Series(pd.SparseArray(data), index=idx)
 
     def time_series_to_frame(self):
-        SparseDataFrame(self.series)
+        pd.DataFrame(self.series)
 
 
 class SparseArrayConstructor:
@@ -51,16 +50,9 @@ def setup(self):
         N = 1000
         self.arr = np.arange(N)
         self.sparse = scipy.sparse.rand(N, N, 0.005)
-        self.dict = dict(zip(range(N), itertools.repeat([0])))
-
-    def time_constructor(self):
-        SparseDataFrame(columns=self.arr, index=self.arr)
 
     def time_from_scipy(self):
-        SparseDataFrame(self.sparse)
-
-    def time_from_dict(self):
-        SparseDataFrame(self.dict)
+        pd.DataFrame.sparse.from_spmatrix(self.sparse)
 
 
 class FromCoo:
@@ -71,7 +63,7 @@ def setup(self):
                                               shape=(100, 100))
 
     def time_sparse_series_from_coo(self):
-        SparseSeries.from_coo(self.matrix)
+        pd.Series.sparse.from_coo(self.matrix)
 
 
 class ToCoo:
@@ -82,12 +74,12 @@ def setup(self):
         s[100] = -1.0
         s[999] = 12.1
         s.index = MultiIndex.from_product([range(10)] * 4)
-        self.ss = s.to_sparse()
+        self.ss = s.astype("Sparse")
 
     def time_sparse_series_to_coo(self):
-        self.ss.to_coo(row_levels=[0, 1],
-                       column_levels=[2, 3],
-                       sort_labels=True)
+        self.ss.sparse.to_coo(row_levels=[0, 1],
+                              column_levels=[2, 3],
+                              sort_labels=True)
 
 
 class Arithmetic:

azure-pipelines.yml

@@ -5,6 +5,7 @@ jobs:
   parameters:
     name: macOS
     vmImage: xcode9-macos10.13
+
 - template: ci/azure/posix.yml
   parameters:
     name: Linux
@@ -122,7 +123,7 @@ jobs:
   timeoutInMinutes: 90
   steps:
   - script: |
-      echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/travis-36-doc.yaml'
+      echo '##vso[task.setvariable variable=ENV_FILE]environment.yml'
     displayName: 'Setting environment variables'
 
   - script: |
@@ -134,14 +135,18 @@ jobs:
   - script: |
       export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev
-      doc/make.py
+      # Next we should simply have `doc/make.py --warnings-are-errors`, everything else is required because the ipython directive doesn't fail the build on errors (https://github.com/ipython/ipython/issues/11547)
+      doc/make.py --warnings-are-errors | tee sphinx.log ; SPHINX_RET=${PIPESTATUS[0]}
+      grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; IPY_RET=$(( $? != 1 ))
+      exit $(( $SPHINX_RET + $IPY_RET ))
     displayName: 'Build documentation'
 
   - script: |
       cd doc/build/html
       git init
       touch .nojekyll
       echo "dev.pandas.io" > CNAME
+      printf "User-agent: *\nDisallow: /" > robots.txt
       git add --all .
       git config user.email "[email protected]"
       git config user.name "pandas-docs-bot"
@@ -172,7 +177,6 @@ jobs:
       cd doc/build/html
       git remote add origin [email protected]:pandas-dev/pandas-dev.github.io.git
       git push -f origin master
-      exit 0  # FIXME this will leave the build green even if the step fails. To be removed when we are confident with this.
     displayName: 'Publish docs to GitHub pages'
     condition : |
       and(not(eq(variables['Build.Reason'], 'PullRequest')),

ci/build_docs.sh

@@ -23,6 +23,7 @@ dependencies:
   - xlsxwriter
   - xlwt
   - pip:
+    - pyreadstat
     # universal
     - pytest==4.5.0
     - pytest-xdist

ci/deps/azure-macos-35.yaml

@@ -30,3 +30,4 @@ dependencies:
   - pytest-mock
   - moto
   - hypothesis>=3.58.0
+  - pyreadstat

ci/deps/azure-windows-37.yaml

@@ -19,5 +19,6 @@ dependencies:
   - hypothesis>=3.58.0
   - s3fs
   - pip
+  - pyreadstat
   - pip:
     - moto

ci/deps/travis-36-doc.yaml

@@ -221,7 +221,7 @@ You'll need to have at least python3.5 installed on your system.
    # Use an ENV_DIR of your choice. We'll use ~/virtualenvs/pandas-dev
    # Any parent directories should already exist
    python3 -m venv ~/virtualenvs/pandas-dev
-   # Activate the virtulaenv
+   # Activate the virtualenv
    . ~/virtualenvs/pandas-dev/bin/activate
 
    # Install the build dependencies
@@ -460,7 +460,7 @@ Building master branch documentation
 
 When pull requests are merged into the *pandas* ``master`` branch, the main parts of
 the documentation are also built by Travis-CI. These docs are then hosted `here
-<http://pandas-docs.github.io/pandas-docs-travis>`__, see also
+<https://dev.pandas.io>`__, see also
 the :ref:`Continuous Integration <contributing.ci>` section.
 
 .. _contributing.code:

ci/deps/travis-37.yaml

@@ -712,6 +712,7 @@ See the :ref:`Plotting <visualization>` docs.
    plt.close('all')
 
 .. ipython:: python
+   :okwarning:
 
    ts = pd.Series(np.random.randn(1000),
                   index=pd.date_range('1/1/2000', periods=1000))

doc/source/development/contributing.rst

@@ -38,8 +38,7 @@ See the :ref:`overview` for more detail about what's in the library.
     :maxdepth: 3
     :hidden:
 {% endif %}
-
-    {% if not single_doc -%}
+{% if not single_doc %}
     What's New in 0.25.0 <whatsnew/v0.25.0>
     install
     getting_started/index
@@ -52,8 +51,7 @@ See the :ref:`overview` for more detail about what's in the library.
     {% if not single_doc -%}
     development/index
     whatsnew/index
-{% endif -%}
-
+{% endif %}
 
 * :doc:`whatsnew/v0.25.0`
 * :doc:`install`

doc/source/getting_started/10min.rst

@@ -285,6 +285,8 @@ pandas-gbq                0.8.0              Google Big Query access
 psycopg2                                     PostgreSQL engine for sqlalchemy
 pyarrow                   0.9.0              Parquet and feather reading / writing
 pymysql                                      MySQL engine for sqlalchemy
+pyreadstat                                   SPSS files (.sav) reading
+pytables                  3.4.2              HDF5 reading / writing
 qtpy                                         Clipboard I/O
 s3fs                      0.0.8              Amazon S3 access
 xarray                    0.8.2              pandas-like API for N-dimensional data