Merge remote-tracking branch 'upstream/master' into map-na-action

Author: dsaxton

asv_bench/benchmarks/frame_ctor.py

@@ -1,5 +1,6 @@
 import numpy as np
 
+import pandas as pd
 from pandas import DataFrame, MultiIndex, Series, Timestamp, date_range
 
 from .pandas_vb_common import tm
@@ -118,4 +119,48 @@ def time_frame_from_range(self):
         self.df = DataFrame(self.data)
 
 
+class FromArrays:
+
+    goal_time = 0.2
+
+    def setup(self):
+        N_rows = 1000
+        N_cols = 1000
+        self.float_arrays = [np.random.randn(N_rows) for _ in range(N_cols)]
+        self.sparse_arrays = [
+            pd.arrays.SparseArray(np.random.randint(0, 2, N_rows), dtype="float64")
+            for _ in range(N_cols)
+        ]
+        self.int_arrays = [
+            pd.array(np.random.randint(1000, size=N_rows), dtype="Int64")
+            for _ in range(N_cols)
+        ]
+        self.index = pd.Index(range(N_rows))
+        self.columns = pd.Index(range(N_cols))
+
+    def time_frame_from_arrays_float(self):
+        self.df = DataFrame._from_arrays(
+            self.float_arrays,
+            index=self.index,
+            columns=self.columns,
+            verify_integrity=False,
+        )
+
+    def time_frame_from_arrays_int(self):
+        self.df = DataFrame._from_arrays(
+            self.int_arrays,
+            index=self.index,
+            columns=self.columns,
+            verify_integrity=False,
+        )
+
+    def time_frame_from_arrays_sparse(self):
+        self.df = DataFrame._from_arrays(
+            self.sparse_arrays,
+            index=self.index,
+            columns=self.columns,
+            verify_integrity=False,
+        )
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

doc/source/conf.py

@@ -195,7 +195,7 @@
 
 # The theme to use for HTML and HTML Help pages.  Major themes that come with
 # Sphinx are currently 'default' and 'sphinxdoc'.
-html_theme = "pandas_sphinx_theme"
+html_theme = "pydata_sphinx_theme"
 
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths

doc/source/getting_started/intro_tutorials/03_subset_data.rst

@@ -88,7 +88,7 @@ name of the column of interest.
     </ul>
 
 Each column in a :class:`DataFrame` is a :class:`Series`. As a single column is
-selected, the returned object is a pandas :class:`DataFrame`. We can verify this
+selected, the returned object is a pandas :class:`Series`. We can verify this
 by checking the type of the output:
 
 .. ipython:: python

doc/source/index.rst.template

@@ -119,7 +119,6 @@ programming language.
     :titlesonly:
 {% endif %}
 {% if not single_doc %}
-    What's New in 1.1.0 <whatsnew/v1.1.0>
     getting_started/index
     user_guide/index
     {% endif -%}

doc/source/user_guide/scale.rst

@@ -246,6 +246,7 @@ We'll import ``dask.dataframe`` and notice that the API feels similar to pandas.
 We can use Dask's ``read_parquet`` function, but provide a globstring of files to read in.
 
 .. ipython:: python
+   :okwarning:
 
    import dask.dataframe as dd
 

doc/source/whatsnew/v1.1.0.rst

@@ -343,10 +343,12 @@ I/O
   timestamps with ``version="2.0"`` (:issue:`31652`).
 - Bug in :meth:`read_csv` was raising `TypeError` when `sep=None` was used in combination with `comment` keyword (:issue:`31396`)
 - Bug in :class:`HDFStore` that caused it to set to ``int64`` the dtype of a ``datetime64`` column when reading a DataFrame in Python 3 from fixed format written in Python 2 (:issue:`31750`)
+- Bug in :meth:`DataFrame.to_json` where ``Timedelta`` objects would not be serialized correctly with ``date_format="iso"`` (:issue:`28256`)
 - :func:`read_csv` will raise a ``ValueError`` when the column names passed in `parse_dates` are missing in the Dataframe (:issue:`31251`)
 - Bug in :meth:`read_excel` where a UTF-8 string with a high surrogate would cause a segmentation violation (:issue:`23809`)
 - Bug in :meth:`read_csv` was causing a file descriptor leak on an empty file (:issue:`31488`)
 - Bug in :meth:`read_csv` was causing a segfault when there were blank lines between the header and data rows (:issue:`28071`)
+- Bug in :meth:`read_csv` was raising a misleading exception on a permissions issue (:issue:`23784`)
 
 
 Plotting
@@ -404,6 +406,7 @@ Other
 - :meth:`IntegerArray.astype` now supports ``datetime64`` dtype (:issue:32538`)
 - Fixed bug in :func:`pandas.testing.assert_series_equal` where dtypes were checked for ``Interval`` and ``ExtensionArray`` operands when ``check_dtype`` was ``False`` (:issue:`32747`)
 - Bug in :meth:`Series.map` not raising on invalid ``na_action`` (:issue:`32815`)
+- Bug in :meth:`DataFrame.__dir__` caused a segfault when using unicode surrogates in a column name (:issue:`25509`)
 
 .. ---------------------------------------------------------------------------
 

environment.yml

@@ -104,5 +104,5 @@ dependencies:
   - pyreadstat  # pandas.read_spss
   - tabulate>=0.8.3  # DataFrame.to_markdown
   - pip:
-    - git+https://github.com/pandas-dev/pandas-sphinx-theme.git@master
+    - git+https://github.com/pandas-dev/pydata-sphinx-theme.git@master
     - git+https://github.com/numpy/numpydoc

pandas/_libs/hashtable_class_helper.pxi.in

@@ -12,6 +12,9 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 from pandas._libs.tslibs.util cimport get_c_string
 from pandas._libs.missing cimport C_NA
 
+cdef extern from "Python.h":
+    void PyErr_Clear()
+
 {{py:
 
 # name, dtype, c_type
@@ -193,7 +196,7 @@ cdef class StringVector:
 
         append_data_string(self.data, x)
 
-    cdef extend(self, ndarray[:] x):
+    cdef extend(self, ndarray[object] x):
         for i in range(len(x)):
             self.append(x[i])
 
@@ -238,7 +241,7 @@ cdef class ObjectVector:
         self.external_view_exists = True
         return self.ao
 
-    cdef extend(self, ndarray[:] x):
+    cdef extend(self, ndarray[object] x):
         for i in range(len(x)):
             self.append(x[i])
 
@@ -790,6 +793,9 @@ cdef class StringHashTable(HashTable):
             else:
                 # if ignore_na is False, we also stringify NaN/None/etc.
                 v = get_c_string(<str>val)
+                if v == NULL:
+                    PyErr_Clear()
+                    v = get_c_string(<str>repr(val))
                 vecs[i] = v
 
         # compute

pandas/_libs/internals.pyx

@@ -32,7 +32,11 @@ cdef class BlockPlacement:
         self._has_slice = False
         self._has_array = False
 
-        if isinstance(val, slice):
+        if isinstance(val, int):
+            slc = slice(val, val + 1, 1)
+            self._as_slice = slc
+            self._has_slice = True
+        elif isinstance(val, slice):
             slc = slice_canonize(val)
 
             if slc.start != slc.stop:
@@ -378,25 +382,23 @@ def get_blkno_indexers(int64_t[:] blknos, bint group=True):
 
         object blkno
         object group_dict = defaultdict(list)
-        int64_t[:] res_view
 
     n = blknos.shape[0]
-
-    if n == 0:
-        return
-
+    result = list()
     start = 0
     cur_blkno = blknos[start]
 
-    if group is False:
+    if n == 0:
+        pass
+    elif group is False:
         for i in range(1, n):
             if blknos[i] != cur_blkno:
-                yield cur_blkno, slice(start, i)
+                result.append((cur_blkno, slice(start, i)))
 
                 start = i
                 cur_blkno = blknos[i]
 
-        yield cur_blkno, slice(start, n)
+        result.append((cur_blkno, slice(start, n)))
     else:
         for i in range(1, n):
             if blknos[i] != cur_blkno:
@@ -409,19 +411,20 @@ def get_blkno_indexers(int64_t[:] blknos, bint group=True):
 
         for blkno, slices in group_dict.items():
             if len(slices) == 1:
-                yield blkno, slice(slices[0][0], slices[0][1])
+                result.append((blkno, slice(slices[0][0], slices[0][1])))
             else:
                 tot_len = sum(stop - start for start, stop in slices)
-                result = np.empty(tot_len, dtype=np.int64)
-                res_view = result
+                arr = np.empty(tot_len, dtype=np.int64)
 
                 i = 0
                 for start, stop in slices:
                     for diff in range(start, stop):
-                        res_view[i] = diff
+                        arr[i] = diff
                         i += 1
 
-                yield blkno, result
+                result.append((blkno, arr))
+
+    return result
 
 
 def get_blkno_placements(blknos, group: bool = True):

pandas/_libs/parsers.pyx

@@ -241,9 +241,9 @@ cdef extern from "parser/io.h":
     void* buffer_mmap_bytes(void *source, size_t nbytes,
                             size_t *bytes_read, int *status)
 
-    void *new_file_source(char *fname, size_t buffer_size)
+    void *new_file_source(char *fname, size_t buffer_size) except NULL
 
-    void *new_rd_source(object obj)
+    void *new_rd_source(object obj) except NULL
 
     int del_file_source(void *src)
     int del_rd_source(void *src)
@@ -667,26 +667,12 @@ cdef class TextReader:
                 ptr = new_file_source(source, self.parser.chunksize)
                 self.parser.cb_io = &buffer_file_bytes
                 self.parser.cb_cleanup = &del_file_source
-
-            if ptr == NULL:
-                if not os.path.exists(source):
-
-                    raise FileNotFoundError(
-                        ENOENT,
-                        f'File {usource} does not exist',
-                        usource)
-                raise IOError('Initializing from file failed')
-
             self.parser.source = ptr
 
         elif hasattr(source, 'read'):
             # e.g., StringIO
 
             ptr = new_rd_source(source)
-            if ptr == NULL:
-                raise IOError('Initializing parser from file-like '
-                              'object failed')
-
             self.parser.source = ptr
             self.parser.cb_io = &buffer_rd_bytes
             self.parser.cb_cleanup = &del_rd_source
@@ -806,7 +792,6 @@ cdef class TextReader:
                 self._tokenize_rows(1)
 
             header = [ self.names ]
-            data_line = 0
 
             if self.parser.lines < 1:
                 field_count = len(header[0])

pandas/_libs/src/parser/io.c

@@ -28,6 +28,7 @@ The full license is in the LICENSE file, distributed with this software.
 void *new_file_source(char *fname, size_t buffer_size) {
     file_source *fs = (file_source *)malloc(sizeof(file_source));
     if (fs == NULL) {
+        PyErr_NoMemory();
         return NULL;
     }
 
@@ -41,17 +42,20 @@ void *new_file_source(char *fname, size_t buffer_size) {
         int required = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
         if (required == 0) {
             free(fs);
+            PyErr_SetFromWindowsErr(0);
             return NULL;
         }
         wname = (wchar_t*)malloc(required * sizeof(wchar_t));
         if (wname == NULL) {
             free(fs);
+            PyErr_NoMemory();
             return NULL;
         }
         if (MultiByteToWideChar(CP_UTF8, 0, fname, -1, wname, required) <
                                                                 required) {
             free(wname);
             free(fs);
+            PyErr_SetFromWindowsErr(0);
             return NULL;
         }
         fs->fd = _wopen(wname, O_RDONLY | O_BINARY);
@@ -62,6 +66,7 @@ void *new_file_source(char *fname, size_t buffer_size) {
 #endif
     if (fs->fd == -1) {
         free(fs);
+        PyErr_SetFromErrnoWithFilename(PyExc_OSError, fname);
         return NULL;
     }
 
@@ -71,6 +76,7 @@ void *new_file_source(char *fname, size_t buffer_size) {
     if (fs->buffer == NULL) {
         close(fs->fd);
         free(fs);
+        PyErr_NoMemory();
         return NULL;
     }
 
@@ -83,6 +89,10 @@ void *new_file_source(char *fname, size_t buffer_size) {
 void *new_rd_source(PyObject *obj) {
     rd_source *rds = (rd_source *)malloc(sizeof(rd_source));
 
+    if (rds == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
     /* hold on to this object */
     Py_INCREF(obj);
     rds->obj = obj;
@@ -220,20 +230,15 @@ void *new_mmap(char *fname) {
 
     mm = (memory_map *)malloc(sizeof(memory_map));
     if (mm == NULL) {
-        fprintf(stderr, "new_file_buffer: malloc() failed.\n");
-        return (NULL);
+        return NULL;
     }
     mm->fd = open(fname, O_RDONLY | O_BINARY);
     if (mm->fd == -1) {
-        fprintf(stderr, "new_file_buffer: open(%s) failed. errno =%d\n",
-          fname, errno);
         free(mm);
         return NULL;
     }
 
     if (fstat(mm->fd, &stat) == -1) {
-        fprintf(stderr, "new_file_buffer: fstat() failed. errno =%d\n",
-          errno);
         close(mm->fd);
         free(mm);
         return NULL;
@@ -242,8 +247,6 @@ void *new_mmap(char *fname) {
 
     mm->memmap = mmap(NULL, filesize, PROT_READ, MAP_SHARED, mm->fd, 0);
     if (mm->memmap == MAP_FAILED) {
-        /* XXX Eventually remove this print statement. */
-        fprintf(stderr, "new_file_buffer: mmap() failed.\n");
         close(mm->fd);
         free(mm);
         return NULL;