Merge branch 'master' into json_improve

Author: another-green

doc/source/user_guide/io.rst

@@ -28,6 +28,7 @@ The pandas I/O API is a set of top level ``reader`` functions accessed like
     :delim: ;
 
     text;`CSV <https://en.wikipedia.org/wiki/Comma-separated_values>`__;:ref:`read_csv<io.read_csv_table>`;:ref:`to_csv<io.store_in_csv>`
+    text;`TXT <https://www.oracle.com/webfolder/technetwork/data-quality/edqhelp/Content/introduction/getting_started/configuring_fixed_width_text_file_formats.htm>`__;:ref:`read_fwf<io.fwf_reader>`
     text;`JSON <https://www.json.org/>`__;:ref:`read_json<io.json_reader>`;:ref:`to_json<io.json_writer>`
     text;`HTML <https://en.wikipedia.org/wiki/HTML>`__;:ref:`read_html<io.read_html>`;:ref:`to_html<io.html>`
     text; Local clipboard;:ref:`read_clipboard<io.clipboard>`;:ref:`to_clipboard<io.clipboard>`
@@ -1372,6 +1373,7 @@ should pass the ``escapechar`` option:
    print(data)
    pd.read_csv(StringIO(data), escapechar='\\')
 
+.. _io.fwf_reader:
 .. _io.fwf:
 
 Files with fixed width columns

doc/source/whatsnew/v0.25.1.rst

@@ -53,7 +53,7 @@ Numeric
 ^^^^^^^
 - Bug in :meth:`Series.interpolate` when using a timezone aware :class:`DatetimeIndex` (:issue:`27548`)
 - Bug when printing negative floating point complex numbers would raise an ``IndexError`` (:issue:`27484`)
--
+- Bug where :class:`DataFrame` arithmetic operators such as :meth:`DataFrame.mul` with a :class:`Series` with axis=1 would raise an ``AttributeError`` on :class:`DataFrame` larger than the minimum threshold to invoke numexpr (:issue:`27636`)
 -
 
 Conversion
@@ -103,10 +103,9 @@ MultiIndex
 I/O
 ^^^
 
-- Fix bug in :meth:`io.json.json_normalize` when nested meta paths with a nested record path. (:issue:`27220`)
 - Avoid calling ``S3File.s3`` when reading parquet, as this was removed in s3fs version 0.3.0 (:issue:`27756`)
 - Better error message when a negative header is passed in :func:`pandas.read_csv` (:issue:`27779`)
--
+- Fix bug in :meth:`io.json.json_normalize` when nested meta paths with a nested record path. (:issue:`27220`)
 
 Plotting
 ^^^^^^^^
@@ -160,6 +159,14 @@ Other
 -
 -
 
+I/O and LZMA
+~~~~~~~~~~~~
+
+Some users may unknowingly have an incomplete Python installation, which lacks the `lzma` module from the standard library. In this case, `import pandas` failed due to an `ImportError` (:issue: `27575`).
+Pandas will now warn, rather than raising an `ImportError` if the `lzma` module is not present. Any subsequent attempt to use `lzma` methods will raise a `RuntimeError`.
+A possible fix for the lack of the `lzma` module is to ensure you have the necessary libraries and then re-install Python.
+For example, on MacOS installing Python with `pyenv` may lead to an incomplete Python installation due to unmet system dependencies at compilation time (like `xz`). Compilation will succeed, but Python might fail at run time. The issue can be solved by installing the necessary dependencies and then re-installing Python.
+
 .. _whatsnew_0.251.contributors:
 
 Contributors

doc/source/whatsnew/v0.7.3.rst

@@ -25,8 +25,6 @@ New features
    from pandas.tools.plotting import scatter_matrix
    scatter_matrix(df, alpha=0.2)        # noqa F821
 
-.. image:: ../savefig/scatter_matrix_kde.png
-   :width: 5in
 
 - Add ``stacked`` argument to Series and DataFrame's ``plot`` method for
   :ref:`stacked bar plots <visualization.barplot>`.
@@ -35,15 +33,11 @@ New features
 
    df.plot(kind='bar', stacked=True)    # noqa F821
 
-.. image:: ../savefig/bar_plot_stacked_ex.png
-   :width: 4in
 
 .. code-block:: python
 
    df.plot(kind='barh', stacked=True)   # noqa F821
 
-.. image:: ../savefig/barh_plot_stacked_ex.png
-   :width: 4in
 
 - Add log x and y :ref:`scaling options <visualization.basic>` to
   ``DataFrame.plot`` and ``Series.plot``

doc/source/whatsnew/v1.0.0.rst

@@ -158,7 +158,7 @@ MultiIndex
 I/O
 ^^^
 
--
+- :meth:`read_csv` now accepts binary mode file buffers when using the Python csv engine (:issue:`23779`)
 -
 
 Plotting

pandas/_libs/parsers.pyx

@@ -2,7 +2,6 @@
 # See LICENSE for the license
 import bz2
 import gzip
-import lzma
 import os
 import sys
 import time
@@ -59,9 +58,12 @@ from pandas.core.arrays import Categorical
 from pandas.core.dtypes.concat import union_categoricals
 import pandas.io.common as icom
 
+from pandas.compat import _import_lzma, _get_lzma_file
 from pandas.errors import (ParserError, DtypeWarning,
                            EmptyDataError, ParserWarning)
 
+lzma = _import_lzma()
+
 # Import CParserError as alias of ParserError for backwards compatibility.
 # Ultimately, we want to remove this import. See gh-12665 and gh-14479.
 CParserError = ParserError
@@ -645,9 +647,9 @@ cdef class TextReader:
                                      'zip file %s', str(zip_names))
             elif self.compression == 'xz':
                 if isinstance(source, str):
-                    source = lzma.LZMAFile(source, 'rb')
+                    source = _get_lzma_file(lzma)(source, 'rb')
                 else:
-                    source = lzma.LZMAFile(filename=source)
+                    source = _get_lzma_file(lzma)(filename=source)
             else:
                 raise ValueError('Unrecognized compression type: %s' %
                                  self.compression)

pandas/compat/__init__.py

@@ -10,6 +10,7 @@
 import platform
 import struct
 import sys
+import warnings
 
 PY35 = sys.version_info[:2] == (3, 5)
 PY36 = sys.version_info >= (3, 6)
@@ -65,3 +66,32 @@ def is_platform_mac():
 
 def is_platform_32bit():
     return struct.calcsize("P") * 8 < 64
+
+
+def _import_lzma():
+    """Attempts to import lzma, warning the user when lzma is not available.
+    """
+    try:
+        import lzma
+
+        return lzma
+    except ImportError:
+        msg = (
+            "Could not import the lzma module. "
+            "Your installed Python is incomplete. "
+            "Attempting to use lzma compression will result in a RuntimeError."
+        )
+        warnings.warn(msg)
+
+
+def _get_lzma_file(lzma):
+    """Returns the lzma method LZMAFile when the module was correctly imported.
+    Otherwise, raises a RuntimeError.
+    """
+    if lzma is None:
+        raise RuntimeError(
+            "lzma module not available. "
+            "A Python re-install with the proper "
+            "dependencies might be required to solve this issue."
+        )
+    return lzma.LZMAFile

pandas/core/arrays/sparse.py

@@ -39,6 +39,7 @@
 )
 from pandas.core.dtypes.dtypes import register_extension_dtype
 from pandas.core.dtypes.generic import (
+    ABCDataFrame,
     ABCIndexClass,
     ABCSeries,
     ABCSparseArray,
@@ -1735,13 +1736,15 @@ def sparse_unary_method(self):
 
     @classmethod
     def _create_arithmetic_method(cls, op):
-        def sparse_arithmetic_method(self, other):
-            op_name = op.__name__
+        op_name = op.__name__
 
-            if isinstance(other, (ABCSeries, ABCIndexClass)):
+        def sparse_arithmetic_method(self, other):
+            if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)):
                 # Rely on pandas to dispatch to us.
                 return NotImplemented
 
+            other = lib.item_from_zerodim(other)
+
             if isinstance(other, SparseArray):
                 return _sparse_array_op(self, other, op, op_name)
 

pandas/core/computation/expressions.py

@@ -76,16 +76,17 @@ def _can_use_numexpr(op, op_str, a, b, dtype_check):
 
         # required min elements (otherwise we are adding overhead)
         if np.prod(a.shape) > _MIN_ELEMENTS:
-
             # check for dtype compatibility
             dtypes = set()
             for o in [a, b]:
-                if hasattr(o, "dtypes"):
+                # Series implements dtypes, check for dimension count as well
+                if hasattr(o, "dtypes") and o.ndim > 1:
                     s = o.dtypes.value_counts()
                     if len(s) > 1:
                         return False
                     dtypes |= set(s.index.astype(str))
-                elif isinstance(o, np.ndarray):
+                # ndarray and Series Case
+                elif hasattr(o, "dtype"):
                     dtypes |= {o.dtype.name}
 
             # allowed are a superset

pandas/core/frame.py

@@ -1190,7 +1190,7 @@ def to_numpy(self, dtype=None, copy=False):
         Parameters
         ----------
         dtype : str or numpy.dtype, optional
-            The dtype to pass to :meth:`numpy.asarray`
+            The dtype to pass to :meth:`numpy.asarray`.
         copy : bool, default False
             Whether to ensure that the returned value is a not a view on
             another array. Note that ``copy=False`` does not *ensure* that