Merge branch 'master' of https://github.com/pandas-dev/pandas into boilerplate-4

Author: jbrockmendel

asv_bench/benchmarks/arithmetic.py

@@ -466,7 +466,7 @@ def setup(self, offset):
         self.rng = rng
 
     def time_apply_index(self, offset):
-        offset.apply_index(self.rng)
+        self.rng + offset
 
 
 class BinaryOpsMultiIndex:

asv_bench/benchmarks/io/json.py

@@ -53,12 +53,18 @@ def time_read_json_lines(self, index):
     def time_read_json_lines_concat(self, index):
         concat(read_json(self.fname, orient="records", lines=True, chunksize=25000))
 
+    def time_read_json_lines_nrows(self, index):
+        read_json(self.fname, orient="records", lines=True, nrows=25000)
+
     def peakmem_read_json_lines(self, index):
         read_json(self.fname, orient="records", lines=True)
 
     def peakmem_read_json_lines_concat(self, index):
         concat(read_json(self.fname, orient="records", lines=True, chunksize=25000))
 
+    def peakmem_read_json_lines_nrows(self, index):
+        read_json(self.fname, orient="records", lines=True, nrows=15000)
+
 
 class ToJSON(BaseIO):
 

ci/deps/travis-36-locale.yaml

@@ -27,7 +27,7 @@ dependencies:
   - numexpr
   - numpy
   - openpyxl
-  - pandas-gbq=0.8.0
+  - pandas-gbq=0.12.0
   - psycopg2=2.6.2
   - pymysql=0.7.11
   - pytables

doc/source/development/contributing.rst

@@ -270,7 +270,7 @@ Creating a Python environment (pip)
 If you aren't using conda for your development environment, follow these instructions.
 You'll need to have at least Python 3.6.1 installed on your system.
 
-**Unix**/**Mac OS**
+**Unix**/**Mac OS with virtualenv**
 
 .. code-block:: bash
 
@@ -286,7 +286,31 @@ You'll need to have at least Python 3.6.1 installed on your system.
    python -m pip install -r requirements-dev.txt
 
    # Build and install pandas
-   python setup.py build_ext --inplace -j 0
+   python setup.py build_ext --inplace -j 4
+   python -m pip install -e . --no-build-isolation --no-use-pep517
+
+**Unix**/**Mac OS with pyenv**
+
+Consult the docs for setting up pyenv `here <https://github.com/pyenv/pyenv>`__.
+
+.. code-block:: bash
+
+   # Create a virtual environment
+   # Use an ENV_DIR of your choice. We'll use ~/Users/<yourname>/.pyenv/versions/pandas-dev
+
+   pyenv virtualenv <version> <name-to-give-it>
+
+   # For instance:
+   pyenv virtualenv 3.7.6 pandas-dev
+
+   # Activate the virtualenv
+   pyenv activate pandas-dev
+
+   # Now install the build dependencies in the cloned pandas repo
+   python -m pip install -r requirements-dev.txt
+
+   # Build and install pandas
+   python setup.py build_ext --inplace -j 4
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 **Windows**
@@ -312,7 +336,7 @@ should already exist.
    python -m pip install -r requirements-dev.txt
 
    # Build and install pandas
-   python setup.py build_ext --inplace -j 0
+   python setup.py build_ext --inplace -j 4
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 Creating a branch

doc/source/getting_started/comparison/comparison_with_sas.rst

@@ -115,7 +115,7 @@ Reading external data
 
 Like SAS, pandas provides utilities for reading in data from
 many formats.  The ``tips`` dataset, found within the pandas
-tests (`csv <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/data/tips.csv>`_)
+tests (`csv <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/io/data/csv/tips.csv>`_)
 will be used in many of the following examples.
 
 SAS provides ``PROC IMPORT`` to read csv data into a data set.
@@ -131,7 +131,7 @@ The pandas method is :func:`read_csv`, which works similarly.
 .. ipython:: python
 
    url = ('https://raw.github.com/pandas-dev/'
-          'pandas/master/pandas/tests/data/tips.csv')
+          'pandas/master/pandas/tests/io/data/csv/tips.csv')
    tips = pd.read_csv(url)
    tips.head()
 

doc/source/getting_started/comparison/comparison_with_sql.rst

@@ -25,7 +25,7 @@ structure.
 .. ipython:: python
 
     url = ('https://raw.github.com/pandas-dev'
-           '/pandas/master/pandas/tests/data/tips.csv')
+           '/pandas/master/pandas/tests/io/data/csv/tips.csv')
     tips = pd.read_csv(url)
     tips.head()
 

doc/source/getting_started/comparison/comparison_with_stata.rst

@@ -112,7 +112,7 @@ Reading external data
 
 Like Stata, pandas provides utilities for reading in data from
 many formats.  The ``tips`` data set, found within the pandas
-tests (`csv <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/data/tips.csv>`_)
+tests (`csv <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/io/data/csv/tips.csv>`_)
 will be used in many of the following examples.
 
 Stata provides ``import delimited`` to read csv data into a data set in memory.
@@ -128,7 +128,7 @@ the data set if presented with a url.
 .. ipython:: python
 
    url = ('https://raw.github.com/pandas-dev'
-          '/pandas/master/pandas/tests/data/tips.csv')
+          '/pandas/master/pandas/tests/io/data/csv/tips.csv')
    tips = pd.read_csv(url)
    tips.head()
 

doc/source/getting_started/install.rst

@@ -274,7 +274,7 @@ lxml                      3.8.0              HTML parser for read_html (see :ref
 matplotlib                2.2.2              Visualization
 numba                     0.46.0             Alternative execution engine for rolling operations
 openpyxl                  2.5.7              Reading / writing for xlsx files
-pandas-gbq                0.8.0              Google Big Query access
+pandas-gbq                0.12.0             Google Big Query access
 psycopg2                                     PostgreSQL engine for sqlalchemy
 pyarrow                   0.12.0             Parquet, ORC (requires 0.13.0), and feather reading / writing
 pymysql                   0.7.11             MySQL engine for sqlalchemy

doc/source/reference/frame.rst

@@ -47,8 +47,6 @@ Conversion
    DataFrame.convert_dtypes
    DataFrame.infer_objects
    DataFrame.copy
-   DataFrame.isna
-   DataFrame.notna
    DataFrame.bool
 
 Indexing, iteration
@@ -211,10 +209,18 @@ Missing data handling
 .. autosummary::
    :toctree: api/
 
+   DataFrame.backfill
+   DataFrame.bfill
    DataFrame.dropna
+   DataFrame.ffill
    DataFrame.fillna
-   DataFrame.replace
    DataFrame.interpolate
+   DataFrame.isna
+   DataFrame.isnull
+   DataFrame.notna
+   DataFrame.notnull
+   DataFrame.pad
+   DataFrame.replace
 
 Reshaping, sorting, transposing
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/reference/groupby.rst

@@ -50,6 +50,7 @@ Computations / descriptive stats
    GroupBy.all
    GroupBy.any
    GroupBy.bfill
+   GroupBy.backfill
    GroupBy.count
    GroupBy.cumcount
    GroupBy.cummax
@@ -67,6 +68,7 @@ Computations / descriptive stats
    GroupBy.ngroup
    GroupBy.nth
    GroupBy.ohlc
+   GroupBy.pad
    GroupBy.prod
    GroupBy.rank
    GroupBy.pct_change
@@ -88,10 +90,12 @@ application to columns of a specific data type.
 
    DataFrameGroupBy.all
    DataFrameGroupBy.any
+   DataFrameGroupBy.backfill
    DataFrameGroupBy.bfill
    DataFrameGroupBy.corr
    DataFrameGroupBy.count
    DataFrameGroupBy.cov
+   DataFrameGroupBy.cumcount
    DataFrameGroupBy.cummax
    DataFrameGroupBy.cummin
    DataFrameGroupBy.cumprod
@@ -106,6 +110,7 @@ application to columns of a specific data type.
    DataFrameGroupBy.idxmin
    DataFrameGroupBy.mad
    DataFrameGroupBy.nunique
+   DataFrameGroupBy.pad
    DataFrameGroupBy.pct_change
    DataFrameGroupBy.plot
    DataFrameGroupBy.quantile

doc/source/reference/series.rst

@@ -214,11 +214,18 @@ Missing data handling
 .. autosummary::
    :toctree: api/
 
-   Series.isna
-   Series.notna
+   Series.backfill
+   Series.bfill
    Series.dropna
+   Series.ffill
    Series.fillna
    Series.interpolate
+   Series.isna
+   Series.isnull
+   Series.notna
+   Series.notnull
+   Series.pad
+   Series.replace
 
 Reshaping, sorting
 ------------------

doc/source/user_guide/gotchas.rst

@@ -321,7 +321,7 @@ Byte-ordering issues
 --------------------
 Occasionally you may have to deal with data that were created on a machine with
 a different byte order than the one on which you are running Python. A common
-symptom of this issue is an error like:::
+symptom of this issue is an error like::
 
     Traceback
         ...

doc/source/user_guide/indexing.rst

@@ -1866,29 +1866,39 @@ A chained assignment can also crop up in setting in a mixed dtype frame.
 
    These setting rules apply to all of ``.loc/.iloc``.
 
-This is the correct access method:
+The following is the recommended access method using ``.loc`` for multiple items (using ``mask``) and a single item using a fixed index:
 
 .. ipython:: python
 
-   dfc = pd.DataFrame({'A': ['aaa', 'bbb', 'ccc'], 'B': [1, 2, 3]})
-   dfc.loc[0, 'A'] = 11
-   dfc
+   dfc = pd.DataFrame({'a': ['one', 'one', 'two',
+                             'three', 'two', 'one', 'six'],
+                       'c': np.arange(7)})
+   dfd = dfc.copy()
+   # Setting multiple items using a mask
+   mask = dfd['a'].str.startswith('o')
+   dfd.loc[mask, 'c'] = 42
+   dfd
+
+   # Setting a single item
+   dfd = dfc.copy()
+   dfd.loc[2, 'a'] = 11
+   dfd
 
-This *can* work at times, but it is not guaranteed to, and therefore should be avoided:
+The following *can* work at times, but it is not guaranteed to, and therefore should be avoided:
 
 .. ipython:: python
    :okwarning:
 
-   dfc = dfc.copy()
-   dfc['A'][0] = 111
-   dfc
+   dfd = dfc.copy()
+   dfd['a'][2] = 111
+   dfd
 
-This will **not** work at all, and so should be avoided:
+Last, the subsequent example will **not** work at all, and so should be avoided:
 
 ::
 
    >>> pd.set_option('mode.chained_assignment','raise')
-   >>> dfc.loc[0]['A'] = 1111
+   >>> dfd.loc[0]['a'] = 1111
    Traceback (most recent call last)
         ...
    SettingWithCopyException:

doc/source/user_guide/visualization.rst

@@ -865,7 +865,7 @@ for more information. By coloring these curves differently for each class
 it is possible to visualize data clustering. Curves belonging to samples
 of the same class will usually be closer together and form larger structures.
 
-**Note**: The "Iris" dataset is available `here <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/data/iris.csv>`__.
+**Note**: The "Iris" dataset is available `here <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/io/data/csv/iris.csv>`__.
 
 .. ipython:: python
 
@@ -1025,7 +1025,7 @@ be colored differently.
 See the R package `Radviz <https://cran.r-project.org/package=Radviz/>`__
 for more information.
 
-**Note**: The "Iris" dataset is available `here <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/data/iris.csv>`__.
+**Note**: The "Iris" dataset is available `here <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/io/data/csv/iris.csv>`__.
 
 .. ipython:: python