Merge branch 'master' of github.com:pydata/pandas

Author: Artemy Kolchinsky

doc/make.py

@@ -31,48 +31,48 @@
 SPHINX_BUILD = 'sphinxbuild'
 
 
-def upload_dev():
+def upload_dev(user='pandas'):
     'push a copy to the pydata dev directory'
-    if os.system('cd build/html; rsync -avz . pandas@pandas.pydata.org'
-                 ':/usr/share/nginx/pandas/pandas-docs/dev/ -essh'):
+    if os.system('cd build/html; rsync -avz . {0}@pandas.pydata.org'
+                 ':/usr/share/nginx/pandas/pandas-docs/dev/ -essh'.format(user)):
         raise SystemExit('Upload to Pydata Dev failed')
 
 
-def upload_dev_pdf():
+def upload_dev_pdf(user='pandas'):
     'push a copy to the pydata dev directory'
-    if os.system('cd build/latex; scp pandas.pdf pandas@pandas.pydata.org'
-                 ':/usr/share/nginx/pandas/pandas-docs/dev/'):
+    if os.system('cd build/latex; scp pandas.pdf {0}@pandas.pydata.org'
+                 ':/usr/share/nginx/pandas/pandas-docs/dev/'.format(user)):
         raise SystemExit('PDF upload to Pydata Dev failed')
 
 
-def upload_stable():
+def upload_stable(user='pandas'):
     'push a copy to the pydata stable directory'
-    if os.system('cd build/html; rsync -avz . pandas@pandas.pydata.org'
-                 ':/usr/share/nginx/pandas/pandas-docs/stable/ -essh'):
+    if os.system('cd build/html; rsync -avz . {0}@pandas.pydata.org'
+                 ':/usr/share/nginx/pandas/pandas-docs/stable/ -essh'.format(user)):
         raise SystemExit('Upload to stable failed')
 
 
-def upload_stable_pdf():
+def upload_stable_pdf(user='pandas'):
     'push a copy to the pydata dev directory'
-    if os.system('cd build/latex; scp pandas.pdf pandas@pandas.pydata.org'
-                 ':/usr/share/nginx/pandas/pandas-docs/stable/'):
+    if os.system('cd build/latex; scp pandas.pdf {0}@pandas.pydata.org'
+                 ':/usr/share/nginx/pandas/pandas-docs/stable/'.format(user)):
         raise SystemExit('PDF upload to stable failed')
 
 
-def upload_prev(ver, doc_root='./'):
+def upload_prev(ver, doc_root='./', user='pandas'):
     'push a copy of older release to appropriate version directory'
     local_dir = doc_root + 'build/html'
     remote_dir = '/usr/share/nginx/pandas/pandas-docs/version/%s/' % ver
-    cmd = 'cd %s; rsync -avz . pandas@pandas.pydata.org:%s -essh'
-    cmd = cmd % (local_dir, remote_dir)
+    cmd = 'cd %s; rsync -avz . %s@pandas.pydata.org:%s -essh'
+    cmd = cmd % (local_dir, user, remote_dir)
     print(cmd)
     if os.system(cmd):
         raise SystemExit(
             'Upload to %s from %s failed' % (remote_dir, local_dir))
 
     local_dir = doc_root + 'build/latex'
-    pdf_cmd = 'cd %s; scp pandas.pdf pandas@pandas.pydata.org:%s'
-    pdf_cmd = pdf_cmd % (local_dir, remote_dir)
+    pdf_cmd = 'cd %s; scp pandas.pdf %s@pandas.pydata.org:%s'
+    pdf_cmd = pdf_cmd % (local_dir, user, remote_dir)
     if os.system(pdf_cmd):
         raise SystemExit('Upload PDF to %s from %s failed' % (ver, doc_root))
 
@@ -337,6 +337,10 @@ def generate_index(api=True, single=False, **kwds):
                    type=str,
                    default=False,
                    help='filename of section to compile, e.g. "indexing"')
+argparser.add_argument('--user',
+                   type=str,
+                   default=False,
+                   help='Username to connect to the pydata server')
 
 def main():
     args, unknown = argparser.parse_known_args()
@@ -354,16 +358,19 @@ def main():
         ver = sys.argv[2]
 
         if ftype == 'build_previous':
-            build_prev(ver)
+            build_prev(ver, user=args.user)
         if ftype == 'upload_previous':
-            upload_prev(ver)
+            upload_prev(ver, user=args.user)
     elif len(sys.argv) == 2:
         for arg in sys.argv[1:]:
             func = funcd.get(arg)
             if func is None:
                 raise SystemExit('Do not know how to handle %s; valid args are %s' % (
                     arg, list(funcd.keys())))
-            func()
+            if args.user:
+                func(user=args.user)
+            else:
+                func()
     else:
         small_docs = False
         all()

doc/source/faq.rst

@@ -50,6 +50,10 @@ when calling ``df.info()``:
 
     df.info()
 
+The ``+`` symbol indicates that the true memory usage could be higher, because
+pandas does not count the memory used by values in columns with
+``dtype=object``.
+
 By default the display option is set to ``True`` but can be explicitly
 overridden by passing the ``memory_usage`` argument when invoking ``df.info()``.
 

doc/source/io.rst

@@ -3411,7 +3411,7 @@ Of course, you can specify a more "complex" query.
 
    pd.read_sql_query("SELECT id, Col_1, Col_2 FROM data WHERE id = 42;", engine)
 
-The func:`~pandas.read_sql_query` function supports a ``chunksize`` argument.
+The :func:`~pandas.read_sql_query` function supports a ``chunksize`` argument.
 Specifying this will return an iterator through chunks of the query result:
 
 .. ipython:: python

doc/source/visualization.rst

@@ -1186,7 +1186,7 @@ with "(right)" in the legend. To turn off the automatic marking, use the
 Suppressing Tick Resolution Adjustment
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-pandas includes automatically tick resolution adjustment for regular frequency
+pandas includes automatic tick resolution adjustment for regular frequency
 time-series data. For limited cases where pandas cannot infer the frequency
 information (e.g., in an externally created ``twinx``), you can choose to
 suppress this behavior for alignment purposes.

doc/source/whatsnew.rst

@@ -18,46 +18,46 @@ What's New
 
 These are new features and improvements of note in each release.
 
-.. include:: v0.15.1.txt
+.. include:: whatsnew/v0.15.1.txt
 
-.. include:: v0.15.0.txt
+.. include:: whatsnew/v0.15.0.txt
 
-.. include:: v0.14.1.txt
+.. include:: whatsnew/v0.14.1.txt
 
-.. include:: v0.14.0.txt
+.. include:: whatsnew/v0.14.0.txt
 
-.. include:: v0.13.1.txt
+.. include:: whatsnew/v0.13.1.txt
 
-.. include:: v0.13.0.txt
+.. include:: whatsnew/v0.13.0.txt
 
-.. include:: v0.12.0.txt
+.. include:: whatsnew/v0.12.0.txt
 
-.. include:: v0.11.0.txt
+.. include:: whatsnew/v0.11.0.txt
 
-.. include:: v0.10.1.txt
+.. include:: whatsnew/v0.10.1.txt
 
-.. include:: v0.10.0.txt
+.. include:: whatsnew/v0.10.0.txt
 
-.. include:: v0.9.1.txt
+.. include:: whatsnew/v0.9.1.txt
 
-.. include:: v0.9.0.txt
+.. include:: whatsnew/v0.9.0.txt
 
-.. include:: v0.8.1.txt
+.. include:: whatsnew/v0.8.1.txt
 
-.. include:: v0.8.0.txt
+.. include:: whatsnew/v0.8.0.txt
 
-.. include:: v0.7.3.txt
+.. include:: whatsnew/v0.7.3.txt
 
-.. include:: v0.7.2.txt
+.. include:: whatsnew/v0.7.2.txt
 
-.. include:: v0.7.1.txt
+.. include:: whatsnew/v0.7.1.txt
 
-.. include:: v0.7.0.txt
+.. include:: whatsnew/v0.7.0.txt
 
-.. include:: v0.6.1.txt
+.. include:: whatsnew/v0.6.1.txt
 
-.. include:: v0.6.0.txt
+.. include:: whatsnew/v0.6.0.txt
 
-.. include:: v0.5.0.txt
+.. include:: whatsnew/v0.5.0.txt
 
-.. include:: v0.4.x.txt
+.. include:: whatsnew/v0.4.x.txt

doc/source/v0.10.0.txt renamed to doc/source/whatsnew/v0.10.0.txt

@@ -28,6 +28,8 @@ Enhancements
 
 - Added option to select columns when importing Stata files (:issue:`7935`)
 
+- Qualify memory usage in ``DataFrame.info()`` by adding ``+`` if it is a lower bound (:issue:`8578`)
+
 
 .. _whatsnew_0151.performance:
 
@@ -45,3 +47,7 @@ Experimental
 
 Bug Fixes
 ~~~~~~~~~
+
+- Bug in ``cut``/``qcut`` when using ``Series`` and ``retbins=True`` (:issue:`8589`)
+
+- Fix ``shape`` attribute for ``MultiIndex`` (:issue:`8609`)

doc/source/v0.10.1.txt renamed to doc/source/whatsnew/v0.10.1.txt

@@ -297,7 +297,7 @@ def transpose(self):
     @property
     def shape(self):
         """ return a tuple of the shape of the underlying data """
-        return self._data.shape
+        return self.values.shape
 
     @property
     def ndim(self):

doc/source/v0.11.0.txt renamed to doc/source/whatsnew/v0.11.0.txt

@@ -1478,13 +1478,13 @@ def _verbose_repr():
         def _non_verbose_repr():
             lines.append(self.columns.summary(name='Columns'))
 
-        def _sizeof_fmt(num):
+        def _sizeof_fmt(num, size_qualifier):
             # returns size in human readable format
             for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:
                 if num < 1024.0:
-                    return "%3.1f %s" % (num, x)
+                    return "%3.1f%s %s" % (num, size_qualifier, x)
                 num /= 1024.0
-            return "%3.1f %s" % (num, 'PB')
+            return "%3.1f%s %s" % (num, size_qualifier, 'PB')
 
         if verbose:
             _verbose_repr()
@@ -1502,8 +1502,14 @@ def _sizeof_fmt(num):
         if memory_usage is None:
             memory_usage = get_option('display.memory_usage')
         if memory_usage:  # append memory usage of df to display
+            # size_qualifier is just a best effort; not guaranteed to catch all
+            # cases (e.g., it misses categorical data even with object
+            # categories)
+            size_qualifier = ('+' if 'object' in counts
+                              or self.index.dtype.kind == 'O' else '')
+            mem_usage = self.memory_usage(index=True).sum()
             lines.append("memory usage: %s\n" %
-                            _sizeof_fmt(self.memory_usage(index=True).sum()))
+                            _sizeof_fmt(mem_usage, size_qualifier))
         _put_lines(buf, lines)
 
     def memory_usage(self, index=False):

doc/source/v0.12.0.txt renamed to doc/source/whatsnew/v0.12.0.txt

@@ -33,8 +33,9 @@ class ParserWarning(Warning):
 
 Parameters
 ----------
-filepath_or_buffer : string or file handle / StringIO. The string could be
-    a URL. Valid URL schemes include http, ftp, s3, and file. For file URLs, a
+filepath_or_buffer : string or file handle / StringIO
+    The string could be a URL. Valid URL schemes include
+    http, ftp, s3, and file. For file URLs, a
     host is expected. For instance, a local file could be
     file ://localhost/path/to/table.csv
 %s
@@ -59,7 +60,8 @@ class ParserWarning(Warning):
 dialect : string or csv.Dialect instance, default None
     If None defaults to Excel dialect. Ignored if sep longer than 1 char
     See csv.Dialect documentation for more details
-header : int row number(s) to use as the column names, and the start of the
+header : int, list of ints
+    Row number(s) to use as the column names, and the start of the
     data.  Defaults to 0 if no ``names`` passed, otherwise ``None``. Explicitly
     pass ``header=0`` to be able to replace existing names. The header can be
     a list of integers that specify row locations for a multi-index on the
@@ -78,7 +80,7 @@ class ParserWarning(Warning):
 names : array-like
     List of column names to use. If file contains no header row, then you
     should explicitly pass header=None
-prefix : string or None (default)
+prefix : string, default None
     Prefix to add to column numbers when no header, e.g 'X' for X0, X1, ...
 na_values : list-like or dict, default None
     Additional strings to recognize as NA/NaN. If dict passed, specific
@@ -113,7 +115,7 @@ class ParserWarning(Warning):
     must be a single character. Like empty lines (as long as ``skip_blank_lines=True``),
     fully commented lines are ignored by the parameter `header`
     but not by `skiprows`. For example, if comment='#', parsing
-    '#empty\n1,2,3\na,b,c' with `header=0` will result in '1,2,3' being
+    '#empty\\na,b,c\\n1,2,3' with `header=0` will result in 'a,b,c' being
     treated as the header.
 decimal : str, default '.'
     Character to recognize as decimal point. E.g. use ',' for European data

doc/source/v0.13.0.txt renamed to doc/source/whatsnew/v0.13.0.txt

@@ -6732,6 +6732,21 @@ def test_info_memory_usage(self):
         res = buf.getvalue().splitlines()
         self.assertTrue("memory usage: " not in res[-1])
 
+        df.info(buf=buf, memory_usage=True)
+        res = buf.getvalue().splitlines()
+        # memory usage is a lower bound, so print it as XYZ+ MB
+        self.assertTrue(re.match(r"memory usage: [^+]+\+", res[-1]))
+
+        df.iloc[:, :5].info(buf=buf, memory_usage=True)
+        res = buf.getvalue().splitlines()
+        # excluded column with object dtype, so estimate is accurate
+        self.assertFalse(re.match(r"memory usage: [^+]+\+", res[-1]))
+
+        df_with_object_index = pd.DataFrame({'a': [1]}, index=['foo'])
+        df_with_object_index.info(buf=buf, memory_usage=True)
+        res = buf.getvalue().splitlines()
+        self.assertTrue(re.match(r"memory usage: [^+]+\+", res[-1]))
+
         # Test a DataFrame with duplicate columns
         dtypes = ['int64', 'int64', 'int64', 'float64']
         data = {}

doc/source/v0.13.1.txt renamed to doc/source/whatsnew/v0.13.1.txt

@@ -83,6 +83,17 @@ def f():
                 pass
         tm.assertRaisesRegexp(ValueError,'The truth value of a',f)
 
+    def test_ndarray_compat_properties(self):
+
+        idx = self.create_index()
+        self.assertTrue(idx.T.equals(idx))
+        self.assertTrue(idx.transpose().equals(idx))
+
+        values = idx.values
+        for prop in ['shape', 'ndim', 'size', 'itemsize', 'nbytes']:
+            self.assertEqual(getattr(idx, prop), getattr(values, prop))
+
+
 class TestIndex(Base, tm.TestCase):
     _holder = Index
     _multiprocess_can_split_ = True

doc/source/v0.14.0.txt renamed to doc/source/whatsnew/v0.14.0.txt

@@ -248,6 +248,16 @@ def test_qcut_return_categorical(self):
                                             ordered=True))
         tm.assert_series_equal(res, exp)
 
+    def test_series_retbins(self):
+        # GH 8589
+        s = Series(np.arange(4))
+        result, bins = cut(s, 2, retbins=True)
+        assert_equal(result.cat.codes.values, [0, 0, 1, 1])
+        assert_almost_equal(bins, [-0.003, 1.5, 3])
+
+        result, bins = qcut(s, 2, retbins=True)
+        assert_equal(result.cat.codes.values, [0, 0, 1, 1])
+        assert_almost_equal(bins, [0, 1.5, 3])
 
 
 def curpath():