Merge remote-tracking branch 'upstream/master' into KeyError

Author: Dr-Irv

.travis.yml

@@ -14,13 +14,9 @@ cache:
 
 env:
   global:
-  # scatterci API key
-  #- secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ="
-  # ironcache API key
-  #- secure: "e4eEFn9nDQc3Xa5BWYkzfX37jaWVq89XidVX+rcCNEr5OlOImvveeXnF1IzbRXznH4Sv0YsLwUd8RGUWOmyCvkONq/VJeqCHWtTMyfaCIdqSyhIP9Odz8r9ahch+Y0XFepBey92AJHmlnTh+2GjCDgIiqq4fzglojnp56Vg1ojA="
-  #- secure: "CjmYmY5qEu3KrvMtel6zWFEtMq8ORBeS1S1odJHnjQpbwT1KY2YFZRVlLphfyDQXSz6svKUdeRrCNp65baBzs3DQNA8lIuXGIBYFeJxqVGtYAZZs6+TzBPfJJK798sGOj5RshrOJkFG2rdlWNuTq/XphI0JOrN3nPUkRrdQRpAw="
-  # pandas-docs-bot GH
-  - secure: "PCzUFR8CHmw9lH84p4ygnojdF7Z8U5h7YfY0RyT+5K/aiQ1ZTU3ZkDTPI0/rR5FVMxsEEKEQKMcc5fvqW0PeD7Q2wRmluloKgT9w4EVEJ1ppKf7lITPcvZR2QgVOvjv4AfDtibLHFNiaSjzoqyJVjM4igjOu8WTlF3JfZcmOQjQ="
+
+  # pandas-docs-travis GH
+  - secure: "UJK7kUtkcnV9PFP4IBXAvgmRQKdwARlfqF4UZQ5tBwrpnD1a3n7FLBijcuXQ3jkvwpEc/FZB9RJDXmsqYXJPvq3BC++2Cv2tFDvKr/c+y8KffszAyVk47jKEHMNmGgauwaNMggsE/rH8YHe4so9LsJHTRbzmLo8lXPNTldoIu5s="
 
 git:
     # for cloning

asv_bench/benchmarks/frame_methods.py

@@ -1012,3 +1012,14 @@ def setup(self):
 
     def time_frame_quantile_axis1(self):
         self.df.quantile([0.1, 0.5], axis=1)
+
+
+class frame_nlargest(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.df = DataFrame(np.random.randn(1000, 3),
+                            columns=list('ABC'))
+
+    def time_frame_nlargest(self):
+        self.df.nlargest(100, 'A')

asv_bench/benchmarks/replace.py

@@ -32,6 +32,30 @@ def time_replace_large_dict(self):
         self.s.replace(self.to_rep, inplace=True)
 
 
+class replace_convert(object):
+    goal_time = 0.5
+
+    def setup(self):
+        self.n = (10 ** 3)
+        self.to_ts = dict(((i, pd.Timestamp(i)) for i in range(self.n)))
+        self.to_td = dict(((i, pd.Timedelta(i)) for i in range(self.n)))
+        self.s = Series(np.random.randint(self.n, size=(10 ** 3)))
+        self.df = DataFrame({'A': np.random.randint(self.n, size=(10 ** 3)),
+                             'B': np.random.randint(self.n, size=(10 ** 3))})
+
+    def time_replace_series_timestamp(self):
+        self.s.replace(self.to_ts)
+
+    def time_replace_series_timedelta(self):
+        self.s.replace(self.to_td)
+
+    def time_replace_frame_timestamp(self):
+        self.df.replace(self.to_ts)
+
+    def time_replace_frame_timedelta(self):
+        self.df.replace(self.to_td)
+
+
 class replace_replacena(object):
     goal_time = 0.2
 

ci/build_docs.sh

@@ -43,7 +43,9 @@ if [ x"$DOC_BUILD" != x"" ]; then
     cd /tmp/doc/build/html
     git config --global user.email "[email protected]"
     git config --global user.name "pandas-docs-bot"
+    git config --global credential.helper cache
 
+    # create the repo
     git init
     touch README
     git add README
@@ -53,7 +55,7 @@ if [ x"$DOC_BUILD" != x"" ]; then
     touch .nojekyll
     git add --all .
     git commit -m "Version" --allow-empty
-    git remote add origin https://$GH_TOKEN@github.com/pandas-docs/pandas-docs-travis
+    git remote add origin "https://$GH_TOKEN@github.com/pandas-docs/pandas-docs-travis"
     git push origin gh-pages -f
 fi
 

ci/lint.sh

@@ -35,6 +35,18 @@ if [ "$LINT" ]; then
     done
     echo "Linting *.pxi.in DONE"
 
+    # readability/casting: Warnings about C casting instead of C++ casting
+    # runtime/int: Warnings about using C number types instead of C++ ones
+    # build/include_subdir: Warnings about prefacing included header files with directory
+    pip install cpplint
+
+    echo "Linting *.c and *.h"
+    cpplint --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/src/parser
+    if [ $? -ne "0" ]; then
+        RET=1
+    fi
+    echo "Linting *.c and *.h DONE"
+
     echo "Check for invalid testing"
     grep -r -E --include '*.py' --exclude nosetester.py --exclude testing.py '(numpy|np)\.testing' pandas
     if [ $? = "0" ]; then

doc/source/api.rst

@@ -27,6 +27,7 @@ Flat File
    read_table
    read_csv
    read_fwf
+   read_msgpack
 
 Clipboard
 ~~~~~~~~~
@@ -691,6 +692,7 @@ Serialization / IO / Conversion
    Series.to_pickle
    Series.to_csv
    Series.to_dict
+   Series.to_excel
    Series.to_frame
    Series.to_xarray
    Series.to_hdf

doc/source/cookbook.rst

@@ -107,10 +107,8 @@ Splitting
    df = pd.DataFrame(
         {'AAA' : [4,5,6,7], 'BBB' : [10,20,30,40],'CCC' : [100,50,-30,-50]}); df
 
-   dflow = df[df.AAA <= 5]
-   dfhigh = df[df.AAA > 5]
-
-   dflow; dfhigh
+   dflow = df[df.AAA <= 5]; dflow
+   dfhigh = df[df.AAA > 5]; dfhigh
 
 Building Criteria
 *****************

doc/source/io.rst

@@ -126,13 +126,23 @@ index_col :  int or sequence or ``False``, default ``None``
   MultiIndex is used. If you have a malformed file with delimiters at the end of
   each line, you might consider ``index_col=False`` to force pandas to *not* use
   the first column as the index (row names).
-usecols : array-like, default ``None``
-  Return a subset of the columns. All elements in this array must either
+usecols : array-like or callable, default ``None``
+  Return a subset of the columns. If array-like, all elements must either
   be positional (i.e. integer indices into the document columns) or strings
   that correspond to column names provided either by the user in `names` or
-  inferred from the document header row(s). For example, a valid `usecols`
-  parameter would be [0, 1, 2] or ['foo', 'bar', 'baz']. Using this parameter
-  results in much faster parsing time and lower memory usage.
+  inferred from the document header row(s). For example, a valid array-like
+  `usecols` parameter would be [0, 1, 2] or ['foo', 'bar', 'baz'].
+
+  If callable, the callable function will be evaluated against the column names,
+  returning names where the callable function evaluates to True:
+
+  .. ipython:: python
+
+     data = 'col1,col2,col3\na,b,1\na,b,2\nc,d,3'
+     pd.read_csv(StringIO(data))
+     pd.read_csv(StringIO(data), usecols=lambda x: x.upper() in ['COL1', 'COL3'])
+
+  Using this parameter results in much faster parsing time and lower memory usage.
 as_recarray : boolean, default ``False``
   DEPRECATED: this argument will be removed in a future version. Please call
   ``pd.read_csv(...).to_records()`` instead.
@@ -617,14 +627,17 @@ Filtering columns (``usecols``)
 +++++++++++++++++++++++++++++++
 
 The ``usecols`` argument allows you to select any subset of the columns in a
-file, either using the column names or position numbers:
+file, either using the column names, position numbers or a callable:
+
+.. versionadded:: 0.20.0 support for callable `usecols` arguments
 
 .. ipython:: python
 
     data = 'a,b,c,d\n1,2,3,foo\n4,5,6,bar\n7,8,9,baz'
     pd.read_csv(StringIO(data))
     pd.read_csv(StringIO(data), usecols=['b', 'd'])
     pd.read_csv(StringIO(data), usecols=[0, 2, 3])
+    pd.read_csv(StringIO(data), usecols=lambda x: x.upper() in ['A', 'C'])
 
 Comments and Empty Lines
 ''''''''''''''''''''''''
@@ -1268,11 +1281,22 @@ is whitespace).
    df = pd.read_fwf('bar.csv', header=None, index_col=0)
    df
 
+.. versionadded:: 0.20.0
+
+``read_fwf`` supports the ``dtype`` parameter for specifying the types of
+parsed columns to be different from the inferred type.
+
+.. ipython:: python
+
+   pd.read_fwf('bar.csv', header=None, index_col=0).dtypes
+   pd.read_fwf('bar.csv', header=None, dtype={2: 'object'}).dtypes
+
 .. ipython:: python
    :suppress:
 
    os.remove('bar.csv')
 
+
 Indexes
 '''''''
 
@@ -2527,6 +2551,20 @@ missing data to recover integer dtype:
    cfun = lambda x: int(x) if x else -1
    read_excel('path_to_file.xls', 'Sheet1', converters={'MyInts': cfun})
 
+dtype Specifications
+++++++++++++++++++++
+
+.. versionadded:: 0.20
+
+As an alternative to converters, the type for an entire column can
+be specified using the `dtype` keyword, which takes a dictionary
+mapping column names to types.  To interpret data with
+no type inference, use the type ``str`` or ``object``.
+
+.. code-block:: python
+
+   read_excel('path_to_file.xls', dtype={'MyInts': 'int64', 'MyText': str})
+
 .. _io.excel_writer:
 
 Writing Excel Files

doc/source/whatsnew/v0.19.2.txt

@@ -21,6 +21,7 @@ Highlights include:
 Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
+- Improved performance of ``.replace()`` (:issue:`12745`)
 
 .. _whatsnew_0192.bug_fixes:
 
@@ -32,9 +33,10 @@ Bug Fixes
 - Bug in ``pd.read_csv`` where reading files fails, if the number of headers is equal to the number of lines in the file (:issue:`14515`)
 - Bug in ``pd.read_csv`` for the Python engine in which an unhelpful error message was being raised when multi-char delimiters were not being respected with quotes (:issue:`14582`)
 - Fix bugs (:issue:`14734`, :issue:`13654`) in ``pd.read_sas`` and ``pandas.io.sas.sas7bdat.SAS7BDATReader`` that caused problems when reading a SAS file incrementally.
+- Bug in ``pd.read_csv`` for the Python engine in which an unhelpful error message was being raised when ``skipfooter`` was not being respected by Python's CSV library (:issue:`13879`)
 
 
-
+- Bug in ``.groupby(..., sort=True)`` of a non-lexsorted MultiIndex when grouping with multiple levels (:issue:`14776`)
 
 
 
@@ -56,6 +58,10 @@ Bug Fixes
 
 
 
+- Bug in ``HDFStore`` when writing a ``MultiIndex`` when using ``data_columns=True`` (:issue:`14435`)
+- Bug in ``HDFStore.append()`` when writing a ``Series`` and passing a ``min_itemsize`` argument containing a value for the ``index`` (:issue:`11412`)
+- Bug in ``Series.groupby.nunique()`` raising an ``IndexError`` for an empty ``Series`` (:issue:`12553`)
+- Bug in ``DataFrame.nlargest`` and ``DataFrame.nsmallest`` when the index had duplicate values (:issue:`13412`)
 
 
 
@@ -65,6 +71,7 @@ Bug Fixes
 
 
 - Bug in ``pd.read_csv()`` in which the ``dtype`` parameter was not being respected for empty data (:issue:`14712`)
+- Bug in ``pd.read_csv()`` in which the ``nrows`` parameter was not being respected for large input when using the C engine for parsing (:issue:`7626`)
 
 
 

doc/source/whatsnew/v0.20.0.txt

@@ -22,8 +22,8 @@ New features
 ~~~~~~~~~~~~
 
 
-``read_csv`` supports ``dtype`` keyword for python engine
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+``dtype`` keyword for data io
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The ``dtype`` keyword argument in the :func:`read_csv` function for specifying the types of parsed columns
  is now supported with the ``'python'`` engine (:issue:`14295`). See the :ref:`io docs <io.dtypes>` for more information.
@@ -34,17 +34,30 @@ The ``dtype`` keyword argument in the :func:`read_csv` function for specifying t
    pd.read_csv(StringIO(data), engine='python').dtypes
    pd.read_csv(StringIO(data), engine='python', dtype={'a':'float64', 'b':'object'}).dtypes
 
+The ``dtype`` keyword argument is also now supported in the :func:`read_fwf` function for parsing
+fixed-width text files, and :func:`read_excel` for parsing Excel files.
+
+.. ipython:: python
+
+   data = "a  b\n1  2\n3  4"
+   pd.read_fwf(StringIO(data)).dtypes
+   pd.read_fwf(StringIO(data), dtype={'a':'float64', 'b':'object'}).dtypes
+
 .. _whatsnew_0200.enhancements.other:
 
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
+- ``Series.sort_index`` accepts parameters ``kind`` and ``na_position`` (:issue:`13589`, :issue:`14444`)
 
 - ``pd.read_excel`` now preserves sheet order when using ``sheetname=None`` (:issue:`9930`)
 
 - New ``UnsortedIndexError`` (subclass of ``KeyError``) raised when indexing/slicing into an
   unsorted MultiIndex (:issue:`11897`). This allows differentiation between errors due to lack
   of sorting or an incorrect key. See :ref:`here <advanced.unsorted>`
 
+- ``pd.cut`` and ``pd.qcut`` now support datetime64 and timedelta64 dtypes (issue:`14714`)
+- ``Series`` provides a ``to_excel`` method to output Excel files (:issue:`8825`)
+- The ``usecols`` argument in ``pd.read_csv`` now accepts a callable function as a value  (:issue:`14154`)
 
 .. _whatsnew_0200.api_breaking:
 

pandas/core/algorithms.py

@@ -684,11 +684,12 @@ def select_n_slow(dropped, n, keep, method):
 _select_methods = {'nsmallest': nsmallest, 'nlargest': nlargest}
 
 
-def select_n(series, n, keep, method):
-    """Implement n largest/smallest.
+def select_n_series(series, n, keep, method):
+    """Implement n largest/smallest for pandas Series
 
     Parameters
     ----------
+    series : pandas.Series object
     n : int
     keep : {'first', 'last'}, default 'first'
     method : str, {'nlargest', 'nsmallest'}
@@ -717,6 +718,31 @@ def select_n(series, n, keep, method):
     return dropped.iloc[inds]
 
 
+def select_n_frame(frame, columns, n, method, keep):
+    """Implement n largest/smallest for pandas DataFrame
+
+    Parameters
+    ----------
+    frame : pandas.DataFrame object
+    columns : list or str
+    n : int
+    keep : {'first', 'last'}, default 'first'
+    method : str, {'nlargest', 'nsmallest'}
+
+    Returns
+    -------
+    nordered : DataFrame
+    """
+    from pandas.core.series import Series
+    if not is_list_like(columns):
+        columns = [columns]
+    columns = list(columns)
+    ser = getattr(frame[columns[0]], method)(n, keep=keep)
+    if isinstance(ser, Series):
+        ser = ser.to_frame()
+    return ser.merge(frame, on=columns[0], left_index=True)[frame.columns]
+
+
 def _finalize_nsmallest(arr, kth_val, n, keep, narr):
     ns, = np.nonzero(arr <= kth_val)
     inds = ns[arr[ns].argsort(kind='mergesort')][:n]