Merge remote-tracking branch 'upstream/master' into categorical-bool-fixed

Author: TomAugspurger

.travis.yml

@@ -35,6 +35,11 @@ matrix:
       language: generic
       env:
         - JOB="3.5, OSX" ENV_FILE="ci/travis-35-osx.yaml" TEST_ARGS="--skip-slow --skip-network"
+
+    - dist: trusty
+      env:
+        - JOB="3.7" ENV_FILE="ci/travis-37.yaml" TEST_ARGS="--skip-slow --skip-network"
+
     - dist: trusty
       env:
         - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true

MANIFEST.in

@@ -3,27 +3,39 @@ include LICENSE
 include RELEASE.md
 include README.md
 include setup.py
-include pyproject.toml
 
 graft doc
 prune doc/build
 
+graft LICENSES
+
 graft pandas
 
-global-exclude *.so
-global-exclude *.pyd
+global-exclude *.bz2
+global-exclude *.csv
+global-exclude *.dta
+global-exclude *.gz
+global-exclude *.h5
+global-exclude *.html
+global-exclude *.json
+global-exclude *.msgpack
+global-exclude *.pickle
+global-exclude *.png
 global-exclude *.pyc
+global-exclude *.pyd
+global-exclude *.sas7bdat
+global-exclude *.so
+global-exclude *.xls
+global-exclude *.xlsm
+global-exclude *.xlsx
+global-exclude *.xpt
+global-exclude *.xz
+global-exclude *.zip
 global-exclude *~
-global-exclude \#*
-global-exclude .git*
 global-exclude .DS_Store
-global-exclude *.png
+global-exclude .git*
+global-exclude \#*
 
-# include examples/data/*
-# recursive-include examples *.py
-# recursive-include doc/source *
-# recursive-include doc/sphinxext *
-# recursive-include LICENSES *
 include versioneer.py
 include pandas/_version.py
 include pandas/io/formats/templates/*.tpl

Makefile

@@ -23,3 +23,4 @@ doc:
 	cd doc; \
 	python make.py clean; \
 	python make.py html
+	python make.py spellcheck

README.md

@@ -10,7 +10,7 @@
 <tr>
   <td>Latest Release</td>
   <td>
-    <a href="https://pypi.python.org/pypi/pandas/">
+    <a href="https://pypi.org/project/pandas/">
     <img src="https://img.shields.io/pypi/v/pandas.svg" alt="latest release" />
     </a>
   </td>
@@ -25,7 +25,7 @@
 <tr>
   <td>Package Status</td>
   <td>
-		<a href="https://pypi.python.org/pypi/pandas/">
+		<a href="https://pypi.org/project/pandas/">
 		<img src="https://img.shields.io/pypi/status/pandas.svg" alt="status" /></td>
 		</a>
 </tr>
@@ -158,7 +158,7 @@ The source code is currently hosted on GitHub at:
 https://github.com/pandas-dev/pandas
 
 Binary installers for the latest released version are available at the [Python
-package index](https://pypi.python.org/pypi/pandas) and on conda.
+package index](https://pypi.org/project/pandas) and on conda.
 
 ```sh
 # conda
@@ -171,7 +171,7 @@ pip install pandas
 ```
 
 ## Dependencies
-- [NumPy](http://www.numpy.org): 1.9.0 or higher
+- [NumPy](https://www.numpy.org): 1.9.0 or higher
 - [python-dateutil](https://labix.org/python-dateutil): 2.5.0 or higher
 - [pytz](https://pythonhosted.org/pytz): 2011k or higher
 
@@ -233,7 +233,7 @@ All contributions, bug reports, bug fixes, documentation improvements, enhanceme
 
 A detailed overview on how to contribute can be found in the **[contributing guide.](https://pandas.pydata.org/pandas-docs/stable/contributing.html)**
 
-If you are simply looking to start working with the pandas codebase, navigate to the [GitHub “issues” tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?labels=Docs&sort=updated&state=open) and [Difficulty Novice](https://github.com/pandas-dev/pandas/issues?q=is%3Aopen+is%3Aissue+label%3A%22Difficulty+Novice%22) where you could start out.
+If you are simply looking to start working with the pandas codebase, navigate to the [GitHub “issues” tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?labels=Docs&sort=updated&state=open) and [good first issue](https://github.com/pandas-dev/pandas/issues?labels=good+first+issue&sort=updated&state=open) where you could start out.
 
 You can also triage issues which may include reproducing bug reports, or asking for vital information such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to pandas on CodeTriage](https://www.codetriage.com/pandas-dev/pandas).
 

asv_bench/benchmarks/categoricals.py

@@ -51,6 +51,7 @@ def setup(self):
 
         self.values_some_nan = list(np.tile(self.categories + [np.nan], N))
         self.values_all_nan = [np.nan] * len(self.values)
+        self.values_all_int8 = np.ones(N, 'int8')
 
     def time_regular(self):
         pd.Categorical(self.values, self.categories)
@@ -70,6 +71,9 @@ def time_with_nan(self):
     def time_all_nan(self):
         pd.Categorical(self.values_all_nan)
 
+    def time_from_codes_all_int8(self):
+        pd.Categorical.from_codes(self.values_all_int8, self.categories)
+
 
 class ValueCounts(object):
 
@@ -169,3 +173,75 @@ def setup(self, dtype):
 
     def time_isin_categorical(self, dtype):
         self.series.isin(self.sample)
+
+
+class IsMonotonic(object):
+
+    def setup(self):
+        N = 1000
+        self.c = pd.CategoricalIndex(list('a' * N + 'b' * N + 'c' * N))
+        self.s = pd.Series(self.c)
+
+    def time_categorical_index_is_monotonic_increasing(self):
+        self.c.is_monotonic_increasing
+
+    def time_categorical_index_is_monotonic_decreasing(self):
+        self.c.is_monotonic_decreasing
+
+    def time_categorical_series_is_monotonic_increasing(self):
+        self.s.is_monotonic_increasing
+
+    def time_categorical_series_is_monotonic_decreasing(self):
+        self.s.is_monotonic_decreasing
+
+
+class Contains(object):
+
+    goal_time = 0.2
+
+    def setup(self):
+        N = 10**5
+        self.ci = tm.makeCategoricalIndex(N)
+        self.c = self.ci.values
+        self.key = self.ci.categories[0]
+
+    def time_categorical_index_contains(self):
+        self.key in self.ci
+
+    def time_categorical_contains(self):
+        self.key in self.c
+
+
+class CategoricalSlicing(object):
+
+    goal_time = 0.2
+    params = ['monotonic_incr', 'monotonic_decr', 'non_monotonic']
+    param_names = ['index']
+
+    def setup(self, index):
+        N = 10**6
+        values = list('a' * N + 'b' * N + 'c' * N)
+        indices = {
+            'monotonic_incr': pd.Categorical(values),
+            'monotonic_decr': pd.Categorical(reversed(values)),
+            'non_monotonic': pd.Categorical(list('abc' * N))}
+        self.data = indices[index]
+
+        self.scalar = 10000
+        self.list = list(range(10000))
+        self.cat_scalar = 'b'
+
+    def time_getitem_scalar(self, index):
+        self.data[self.scalar]
+
+    def time_getitem_slice(self, index):
+        self.data[:self.scalar]
+
+    def time_getitem_list_like(self, index):
+        self.data[[self.scalar]]
+
+    def time_getitem_list(self, index):
+        self.data[self.list]
+
+    def time_getitem_bool_array(self, index):
+        self.data[self.data == self.cat_scalar]

asv_bench/benchmarks/frame_methods.py

@@ -501,7 +501,7 @@ def time_info(self):
 class NSort(object):
 
     goal_time = 0.2
-    params = ['first', 'last']
+    params = ['first', 'last', 'all']
     param_names = ['keep']
 
     def setup(self, keep):
@@ -512,3 +512,21 @@ def time_nlargest(self, keep):
 
     def time_nsmallest(self, keep):
         self.df.nsmallest(100, 'A', keep=keep)
+
+
+class Describe(object):
+
+    goal_time = 0.2
+
+    def setup(self):
+        self.df = DataFrame({
+            'a': np.random.randint(0, 100, int(1e6)),
+            'b': np.random.randint(0, 100, int(1e6)),
+            'c': np.random.randint(0, 100, int(1e6))
+        })
+
+    def time_series_describe(self):
+        self.df['a'].describe()
+
+    def time_dataframe_describe(self):
+        self.df.describe()

asv_bench/benchmarks/groupby.py

@@ -5,7 +5,7 @@
 
 import numpy as np
 from pandas import (DataFrame, Series, MultiIndex, date_range, period_range,
-                    TimeGrouper, Categorical)
+                    TimeGrouper, Categorical, Timestamp)
 import pandas.util.testing as tm
 
 from .pandas_vb_common import setup  # noqa
@@ -385,6 +385,25 @@ def time_dtype_as_field(self, dtype, method, application):
         self.as_field_method()
 
 
+class RankWithTies(object):
+    # GH 21237
+    goal_time = 0.2
+    param_names = ['dtype', 'tie_method']
+    params = [['float64', 'float32', 'int64', 'datetime64'],
+              ['first', 'average', 'dense', 'min', 'max']]
+
+    def setup(self, dtype, tie_method):
+        N = 10**4
+        if dtype == 'datetime64':
+            data = np.array([Timestamp("2011/01/01")] * N, dtype=dtype)
+        else:
+            data = np.array([1] * N, dtype=dtype)
+        self.df = DataFrame({'values': data, 'key': ['foo'] * N})
+
+    def time_rank_ties(self, dtype, tie_method):
+        self.df.groupby('key').rank(method=tie_method)
+
+
 class Float32(object):
     # GH 13335
     goal_time = 0.2

asv_bench/benchmarks/indexing.py

@@ -3,7 +3,8 @@
 import numpy as np
 import pandas.util.testing as tm
 from pandas import (Series, DataFrame, MultiIndex, Int64Index, Float64Index,
-                    IntervalIndex, IndexSlice, concat, date_range)
+                    IntervalIndex, CategoricalIndex,
+                    IndexSlice, concat, date_range)
 from .pandas_vb_common import setup, Panel  # noqa
 
 
@@ -230,6 +231,49 @@ def time_loc_list(self, monotonic):
         monotonic.loc[80000:]
 
 
+class CategoricalIndexIndexing(object):
+
+    goal_time = 0.2
+    params = ['monotonic_incr', 'monotonic_decr', 'non_monotonic']
+    param_names = ['index']
+
+    def setup(self, index):
+        N = 10**5
+        values = list('a' * N + 'b' * N + 'c' * N)
+        indices = {
+            'monotonic_incr': CategoricalIndex(values),
+            'monotonic_decr': CategoricalIndex(reversed(values)),
+            'non_monotonic': CategoricalIndex(list('abc' * N))}
+        self.data = indices[index]
+
+        self.int_scalar = 10000
+        self.int_list = list(range(10000))
+
+        self.cat_scalar = 'b'
+        self.cat_list = ['a', 'c']
+
+    def time_getitem_scalar(self, index):
+        self.data[self.int_scalar]
+
+    def time_getitem_slice(self, index):
+        self.data[:self.int_scalar]
+
+    def time_getitem_list_like(self, index):
+        self.data[[self.int_scalar]]
+
+    def time_getitem_list(self, index):
+        self.data[self.int_list]
+
+    def time_getitem_bool_array(self, index):
+        self.data[self.data == self.cat_scalar]
+
+    def time_get_loc_scalar(self, index):
+        self.data.get_loc(self.cat_scalar)
+
+    def time_get_indexer_list(self, index):
+        self.data.get_indexer(self.cat_list)
+
+
 class PanelIndexing(object):
 
     goal_time = 0.2

asv_bench/benchmarks/pandas_vb_common.py

@@ -2,10 +2,7 @@
 from importlib import import_module
 
 import numpy as np
-try:
-    from pandas import Panel
-except ImportError:
-    from pandas import WidePanel as Panel  # noqa
+from pandas import Panel
 
 # Compatibility import for lib
 for imp in ['pandas._libs.lib', 'pandas.lib']:

asv_bench/benchmarks/period.py

@@ -64,6 +64,11 @@ def setup(self):
     def time_setitem_period_column(self):
         self.df['col'] = self.rng
 
+    def time_set_index(self):
+        # GH#21582 limited by comparisons of Period objects
+        self.df['col2'] = self.rng
+        self.df.set_index('col2', append=True)
+
 
 class Algorithms(object):
 

asv_bench/benchmarks/series_methods.py

@@ -41,7 +41,7 @@ def time_isin(self, dtypes):
 class NSort(object):
 
     goal_time = 0.2
-    params = ['last', 'first']
+    params = ['first', 'last', 'all']
     param_names = ['keep']
 
     def setup(self, keep):
@@ -121,3 +121,16 @@ def setup(self):
 
     def time_dir_strings(self):
         dir(self.s)
+
+
+class SeriesGetattr(object):
+    # https://github.com/pandas-dev/pandas/issues/19764
+    goal_time = 0.2
+
+    def setup(self):
+        self.s = Series(1,
+                        index=date_range("2012-01-01", freq='s',
+                                         periods=int(1e6)))
+
+    def time_series_datetimeindex_repr(self):
+        getattr(self.s, 'a', None)