ENH: Provide dict object for to_dict() pandas-dev#16122

Author: dwkenefick

.travis.yml

@@ -123,7 +123,7 @@ after_success:
 
 after_script:
   - echo "after_script start"
-  - source activate pandas && python -c "import pandas; pandas.show_versions();"
+  - source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
   - if [ -e /tmp/single.xml ]; then
     ci/print_skipped.py /tmp/single.xml;
     fi

asv_bench/benchmarks/indexing.py

@@ -19,6 +19,9 @@ def time_getitem_list_like(self):
     def time_getitem_array(self):
         self.s[np.arange(10000)]
 
+    def time_getitem_lists(self):
+        self.s[np.arange(10000).tolist()]
+
     def time_iloc_array(self):
         self.s.iloc[np.arange(10000)]
 
@@ -190,9 +193,15 @@ def setup(self):
              np.arange(1000)], names=['one', 'two'])
 
         import string
-        self.mistring = MultiIndex.from_product(
-            [np.arange(1000),
-             np.arange(20), list(string.ascii_letters)],
+
+        self.mi_large = MultiIndex.from_product(
+            [np.arange(1000), np.arange(20), list(string.ascii_letters)],
+            names=['one', 'two', 'three'])
+        self.mi_med = MultiIndex.from_product(
+            [np.arange(1000), np.arange(10), list('A')],
+            names=['one', 'two', 'three'])
+        self.mi_small = MultiIndex.from_product(
+            [np.arange(100), list('A'), list('A')],
             names=['one', 'two', 'three'])
 
     def time_series_xs_mi_ix(self):
@@ -215,8 +224,14 @@ def time_multiindex_get_indexer(self):
                       (0, 16), (0, 17), (0, 18),
                       (0, 19)], dtype=object))
 
+    def time_multiindex_large_get_loc(self):
+        self.mi_large.get_loc((999, 19, 'Z'))
+
+    def time_multiindex_med_get_loc(self):
+        self.mi_med.get_loc((999, 9, 'A'))
+
     def time_multiindex_string_get_loc(self):
-        self.mistring.get_loc((999, 19, 'Z'))
+        self.mi_small.get_loc((99, 'A', 'A'))
 
     def time_is_monotonic(self):
         self.miint.is_monotonic

ci/install_travis.sh

@@ -119,15 +119,7 @@ if [ "$COVERAGE" ]; then
 fi
 
 echo
-if [ "$BUILD_TEST" ]; then
-
-    # build & install testing
-    echo ["Starting installation test."]
-    bash ci/install_release_build.sh
-    conda uninstall -y cython
-    time pip install dist/*tar.gz || exit 1
-
-else
+if [ -z "$BUILD_TEST" ]; then
 
     # build but don't install
     echo "[build em]"
@@ -163,9 +155,22 @@ fi
 # w/o removing anything else
 echo
 echo "[removing installed pandas]"
-conda remove pandas --force
+conda remove pandas -y --force
 
-if [ -z "$BUILD_TEST" ]; then
+if [ "$BUILD_TEST" ]; then
+
+    # remove any installation
+    pip uninstall -y pandas
+    conda list pandas
+    pip list --format columns |grep pandas
+
+    # build & install testing
+    echo ["building release"]
+    bash scripts/build_dist_for_release.sh
+    conda uninstall -y cython
+    time pip install dist/*tar.gz || exit 1
+
+else
 
     # install our pandas
     echo

ci/script_multi.sh

@@ -19,20 +19,26 @@ export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 429496
 echo PYTHONHASHSEED=$PYTHONHASHSEED
 
 if [ "$BUILD_TEST" ]; then
-    echo "build-test"
+    echo "[build-test]"
+
+    echo "[env]"
+    pip list --format columns |grep pandas
+
+    echo "[running]"
     cd /tmp
-    pwd
-    conda list pandas
-    echo "running"
-    python -c "import pandas; pandas.test(['-n 2'])"
+    unset PYTHONPATH
+    python -c 'import pandas; pandas.test(["-n 2", "--skip-slow", "--skip-network", "-r xX", "-m not single"])'
+
 elif [ "$DOC" ]; then
     echo "We are not running pytest as this is a doc-build"
+
 elif [ "$COVERAGE" ]; then
     echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
     pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+
 else
-    echo pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
-    pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas # TODO: doctest
+    echo pytest -n 2 -r xX -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+    pytest -n 2 -r xX -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas # TODO: doctest
 fi
 
 RET="$?"

ci/script_single.sh

@@ -20,8 +20,8 @@ elif [ "$COVERAGE" ]; then
     echo pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
     pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
 else
-    echo pytest -m "single" --junitxml=/tmp/single.xml $TEST_ARGS pandas
-    pytest -m "single" --junitxml=/tmp/single.xml $TEST_ARGS pandas # TODO: doctest
+    echo pytest -m "single" -r xX --junitxml=/tmp/single.xml $TEST_ARGS pandas
+    pytest -m "single" -r xX  --junitxml=/tmp/single.xml $TEST_ARGS pandas # TODO: doctest
 fi
 
 RET="$?"

doc/make.py

@@ -115,15 +115,28 @@ def maybe_exclude_notebooks():
     notebooks = [os.path.join(base, 'source', nb)
                  for nb in ['style.ipynb']]
     contents = {}
-    try:
-        import nbconvert
-        nbconvert.utils.pandoc.get_pandoc_version()
-    except (ImportError, nbconvert.utils.pandoc.PandocMissing):
-        print("Warning: Pandoc is not installed. Skipping Notebooks.")
+
+    def _remove_notebooks():
         for nb in notebooks:
             with open(nb, 'rt') as f:
                 contents[nb] = f.read()
             os.remove(nb)
+
+    # Skip notebook conversion if
+    # 1. nbconvert isn't installed, or
+    # 2. nbconvert is installed, but pandoc isn't
+    try:
+        import nbconvert
+    except ImportError:
+        print("Warning: nbconvert not installed. Skipping notebooks.")
+        _remove_notebooks()
+    else:
+        try:
+            nbconvert.utils.pandoc.get_pandoc_version()
+        except nbconvert.utils.pandoc.PandocMissing:
+            print("Warning: Pandoc is not installed. Skipping notebooks.")
+            _remove_notebooks()
+
     yield
     for nb, content in contents.items():
         with open(nb, 'wt') as f:

doc/source/install.rst

@@ -202,7 +202,7 @@ installed), make sure you have `pytest
 Dependencies
 ------------
 
-* `setuptools <http://pythonhosted.org/setuptools>`__
+* `setuptools <https://setuptools.readthedocs.io/en/latest/>`__
 * `NumPy <http://www.numpy.org>`__: 1.7.1 or higher
 * `python-dateutil <http://labix.org/python-dateutil>`__: 1.5 or higher
 * `pytz <http://pytz.sourceforge.net/>`__: Needed for time zone support

doc/source/style.ipynb

@@ -12,7 +12,7 @@
     "\n",
     "<span style=\"color: red\">*Provisional: This is a new feature and still under development. We'll be adding features and possibly making breaking changes in future releases. We'd love to hear your feedback.*</span>\n",
     "\n",
-    "This document is written as a Jupyter Notebook, and can be viewed or downloaded [here](http://nbviewer.ipython.org/github/pandas-dev/pandas/blob/master/doc/source/html-styling.ipynb).\n",
+    "This document is written as a Jupyter Notebook, and can be viewed or downloaded [here](http://nbviewer.ipython.org/github/pandas-dev/pandas/blob/master/doc/source/style.ipynb).\n",
     "\n",
     "You can apply **conditional formatting**, the visual styling of a DataFrame\n",
     "depending on the data within, by using the ``DataFrame.style`` property.\n",

doc/source/whatsnew.rst

@@ -18,6 +18,10 @@ What's New
 
 These are new features and improvements of note in each release.
 
+.. include:: whatsnew/v0.21.0.txt
+
+.. include:: whatsnew/v0.20.2.txt
+
 .. include:: whatsnew/v0.20.0.txt
 
 .. include:: whatsnew/v0.19.2.txt

doc/source/whatsnew/v0.20.2.txt

@@ -1,6 +1,6 @@
-.. _whatsnew_0201:
+.. _whatsnew_0202:
 
-v0.20.1 (???)
+v0.20.2 (???)
 -------------
 
 This is a minor bug-fix release in the 0.20.x series and includes some small regression fixes,
@@ -9,51 +9,56 @@ We recommend that all users upgrade to this version.
 
 Highlights include:
 
-.. contents:: What's new in v0.20.1
+.. contents:: What's new in v0.20.2
     :local:
     :backlinks: none
 
 
-.. _whatsnew_0201.enhancements:
+.. _whatsnew_0202.enhancements:
 
 Enhancements
 ~~~~~~~~~~~~
 
+- Unblocked access to additional compression types supported in pytables: 'blosc:blosclz, 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy', 'blosc:zlib', 'blosc:zstd' (:issue:`14478`) 
 
-
-.. _whatsnew_0201.performance:
+.. _whatsnew_0202.performance:
 
 Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
+- Performance regression fix when indexing with a list-like (:issue:`16285`)
+- Performance regression fix for small MultiIndexes (:issuse:`16319`)
 
-
-.. _whatsnew_0201.bug_fixes:
+.. _whatsnew_0202.bug_fixes:
 
 Bug Fixes
 ~~~~~~~~~
 
+- Bug in using ``pathlib.Path`` or ``py.path.local`` objects with io functions (:issue:`16291`)
+
 Conversion
 ^^^^^^^^^^
 
-
+- Bug in ``pd.to_numeric()`` in which empty data inputs were causing Python to crash (:issue:`16302`)
 
 
 Indexing
 ^^^^^^^^
 
-
+- Bug in ``DataFrame.reset_index(level=)`` with single level index (:issue:`16263`)
 
 
 I/O
 ^^^
 
-
+- Bug that would force importing of the clipboard routines unnecessarily, potentially causing an import error on startup (:issue:`16288`)
 
 
 Plotting
 ^^^^^^^^
 
+- Bug in ``DataFrame.plot`` with a single column and a list-like ``color`` (:issue:`3486`)
+
 
 
 
@@ -66,13 +71,12 @@ Groupby/Resample/Rolling
 Sparse
 ^^^^^^
 
-
-
+- Bug in construction of SparseDataFrame from ``scipy.sparse.dok_matrix`` (:issue:`16179`)
 
 Reshaping
 ^^^^^^^^^
 
-
+- Bug in ``DataFrame.stack`` with unsorted levels in MultiIndex columns (:issue:`16323`)
 
 
 Numeric

doc/source/whatsnew/v0.21.0.txt

@@ -27,6 +27,7 @@ New features
 Other Enhancements
 ^^^^^^^^^^^^^^^^^^
 - ``Series.to_dict()`` and ``DataFrame.to_dict()`` now support an ``into`` keyword which allows you to specify the ``collections.Mapping`` subclass that you would like returned.  The default is ``dict``, which is backwards compatible. (:issue:`16122`)
+- ``RangeIndex.append`` now returns a ``RangeIndex`` object when possible (:issue:`16212`)
 
 
 

pandas/_libs/hashtable.pxd

@@ -52,6 +52,7 @@ cdef struct Int64VectorData:
 cdef class Int64Vector:
     cdef Int64VectorData *data
     cdef ndarray ao
+    cdef bint external_view_exists
 
     cdef resize(self)
     cpdef to_array(self)

pandas/_libs/hashtable.pyx

@@ -64,6 +64,10 @@ cdef class Factorizer:
         >>> factorize(np.array([1,2,np.nan], dtype='O'), na_sentinel=20)
         array([ 0,  1, 20])
         """
+        if self.uniques.external_view_exists:
+            uniques = ObjectVector()
+            uniques.extend(self.uniques.to_array())
+            self.uniques = uniques
         labels = self.table.get_labels(values, self.uniques,
                                        self.count, na_sentinel, check_null)
         mask = (labels == na_sentinel)
@@ -99,6 +103,15 @@ cdef class Int64Factorizer:
 
     def factorize(self, int64_t[:] values, sort=False,
                   na_sentinel=-1, check_null=True):
+        """
+        Factorize values with nans replaced by na_sentinel
+        >>> factorize(np.array([1,2,np.nan], dtype='O'), na_sentinel=20)
+        array([ 0,  1, 20])
+        """
+        if self.uniques.external_view_exists:
+            uniques = Int64Vector()
+            uniques.extend(self.uniques.to_array())
+            self.uniques = uniques
         labels = self.table.get_labels(values, self.uniques,
                                        self.count, na_sentinel,
                                        check_null)