Merge remote-tracking branch 'upstream/master' into windows_crlf

Author: deflatSOCO

.coveragerc

@@ -2,6 +2,7 @@
 [run]
 branch = False
 omit = */tests/*
+plugins = Cython.Coverage
 
 [report]
 # Regexes for lines to exclude from consideration
@@ -22,6 +23,7 @@ exclude_lines =
     if __name__ == .__main__.:
 
 ignore_errors = False
+show_missing = True
 
 [html]
 directory = coverage_html_report

README.md

@@ -89,7 +89,7 @@
 
 
 
-## What is it
+## What is it?
 
 **pandas** is a Python package providing fast, flexible, and expressive data
 structures designed to make working with "relational" or "labeled" data both

appveyor.yml

@@ -20,12 +20,14 @@ environment:
   matrix:
 
     - CONDA_ROOT: "C:\\Miniconda3_64"
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       PYTHON_VERSION: "3.6"
       PYTHON_ARCH: "64"
       CONDA_PY: "36"
       CONDA_NPY: "113"
 
     - CONDA_ROOT: "C:\\Miniconda3_64"
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
       PYTHON_VERSION: "2.7"
       PYTHON_ARCH: "64"
       CONDA_PY: "27"

asv_bench/benchmarks/groupby.py

@@ -142,7 +142,7 @@ def time_frame_nth(self, dtype):
     def time_series_nth_any(self, dtype):
         self.df['values'].groupby(self.df['key']).nth(0, dropna='any')
 
-    def time_groupby_nth_all(self, dtype):
+    def time_series_nth_all(self, dtype):
         self.df['values'].groupby(self.df['key']).nth(0, dropna='all')
 
     def time_series_nth(self, dtype):

asv_bench/benchmarks/io/csv.py

@@ -54,7 +54,14 @@ def time_frame_date_formatting(self):
         self.data.to_csv(self.fname, date_format='%Y%m%d')
 
 
-class ReadCSVDInferDatetimeFormat(object):
+class StringIORewind(object):
+
+    def data(self, stringio_object):
+        stringio_object.seek(0)
+        return stringio_object
+
+
+class ReadCSVDInferDatetimeFormat(StringIORewind):
 
     goal_time = 0.2
     params = ([True, False], ['custom', 'iso8601', 'ymd'])
@@ -66,10 +73,12 @@ def setup(self, infer_datetime_format, format):
                    'iso8601': '%Y-%m-%d %H:%M:%S',
                    'ymd': '%Y%m%d'}
         dt_format = formats[format]
-        self.data = StringIO('\n'.join(rng.strftime(dt_format).tolist()))
+        self.StringIO_input = StringIO('\n'.join(
+                                       rng.strftime(dt_format).tolist()))
 
     def time_read_csv(self, infer_datetime_format, format):
-        read_csv(self.data, header=None, names=['foo'], parse_dates=['foo'],
+        read_csv(self.data(self.StringIO_input),
+                 header=None, names=['foo'], parse_dates=['foo'],
                  infer_datetime_format=infer_datetime_format)
 
 
@@ -95,7 +104,7 @@ def time_skipprows(self, skiprows):
         read_csv(self.fname, skiprows=skiprows)
 
 
-class ReadUint64Integers(object):
+class ReadUint64Integers(StringIORewind):
 
     goal_time = 0.2
 
@@ -108,13 +117,13 @@ def setup(self):
         self.data2 = StringIO('\n'.join(arr.astype(str).tolist()))
 
     def time_read_uint64(self):
-        read_csv(self.data1, header=None, names=['foo'])
+        read_csv(self.data(self.data1), header=None, names=['foo'])
 
     def time_read_uint64_neg_values(self):
-        read_csv(self.data2, header=None, names=['foo'])
+        read_csv(self.data(self.data2), header=None, names=['foo'])
 
     def time_read_uint64_na_values(self):
-        read_csv(self.data1, header=None, names=['foo'],
+        read_csv(self.data(self.data1), header=None, names=['foo'],
                  na_values=self.na_values)
 
 
@@ -140,19 +149,20 @@ def time_thousands(self, sep, thousands):
         read_csv(self.fname, sep=sep, thousands=thousands)
 
 
-class ReadCSVComment(object):
+class ReadCSVComment(StringIORewind):
 
     goal_time = 0.2
 
     def setup(self):
         data = ['A,B,C'] + (['1,2,3 # comment'] * 100000)
-        self.s_data = StringIO('\n'.join(data))
+        self.StringIO_input = StringIO('\n'.join(data))
 
     def time_comment(self):
-        read_csv(self.s_data, comment='#', header=None, names=list('abc'))
+        read_csv(self.data(self.StringIO_input), comment='#',
+                 header=None, names=list('abc'))
 
 
-class ReadCSVFloatPrecision(object):
+class ReadCSVFloatPrecision(StringIORewind):
 
     goal_time = 0.2
     params = ([',', ';'], ['.', '_'], [None, 'high', 'round_trip'])
@@ -164,14 +174,14 @@ def setup(self, sep, decimal, float_precision):
         rows = sep.join(['0{}'.format(decimal) + '{}'] * 3) + '\n'
         data = rows * 5
         data = data.format(*floats) * 200  # 1000 x 3 strings csv
-        self.s_data = StringIO(data)
+        self.StringIO_input = StringIO(data)
 
     def time_read_csv(self, sep, decimal, float_precision):
-        read_csv(self.s_data, sep=sep, header=None, names=list('abc'),
-                 float_precision=float_precision)
+        read_csv(self.data(self.StringIO_input), sep=sep, header=None,
+                 names=list('abc'), float_precision=float_precision)
 
     def time_read_csv_python_engine(self, sep, decimal, float_precision):
-        read_csv(self.s_data, sep=sep, header=None, engine='python',
+        read_csv(self.data(self.StringIO_input), sep=sep, header=None, engine='python',
                  float_precision=None, names=list('abc'))
 
 
@@ -193,7 +203,7 @@ def time_convert_direct(self):
         read_csv(self.fname, dtype='category')
 
 
-class ReadCSVParseDates(object):
+class ReadCSVParseDates(StringIORewind):
 
     goal_time = 0.2
 
@@ -206,12 +216,14 @@ def setup(self):
                """
         two_cols = ['KORD,19990127'] * 5
         data = data.format(*two_cols)
-        self.s_data = StringIO(data)
+        self.StringIO_input = StringIO(data)
 
     def time_multiple_date(self):
-        read_csv(self.s_data, sep=',', header=None,
-                 names=list(string.digits[:9]), parse_dates=[[1, 2], [1, 3]])
+        read_csv(self.data(self.StringIO_input), sep=',', header=None,
+                 names=list(string.digits[:9]),
+                 parse_dates=[[1, 2], [1, 3]])
 
     def time_baseline(self):
-        read_csv(self.s_data, sep=',', header=None, parse_dates=[1],
+        read_csv(self.data(self.StringIO_input), sep=',', header=None,
+                 parse_dates=[1],
                  names=list(string.digits[:9]))

asv_bench/benchmarks/reshape.py

@@ -1,7 +1,9 @@
+import string
 from itertools import product
 
 import numpy as np
 from pandas import DataFrame, MultiIndex, date_range, melt, wide_to_long
+import pandas as pd
 
 from .pandas_vb_common import setup  # noqa
 
@@ -132,3 +134,19 @@ def setup(self):
 
     def time_pivot_table(self):
         self.df.pivot_table(index='key1', columns=['key2', 'key3'])
+
+
+class GetDummies(object):
+    goal_time = 0.2
+
+    def setup(self):
+        categories = list(string.ascii_letters[:12])
+        s = pd.Series(np.random.choice(categories, size=1000000),
+                      dtype=pd.api.types.CategoricalDtype(categories))
+        self.s = s
+
+    def time_get_dummies_1d(self):
+        pd.get_dummies(self.s, sparse=False)
+
+    def time_get_dummies_1d_sparse(self):
+        pd.get_dummies(self.s, sparse=True)

asv_bench/benchmarks/series_methods.py

@@ -38,6 +38,64 @@ def time_isin(self, dtypes):
         self.s.isin(self.values)
 
 
+class IsInFloat64(object):
+
+    def setup(self):
+        self.small = Series([1, 2], dtype=np.float64)
+        self.many_different_values = np.arange(10**6, dtype=np.float64)
+        self.few_different_values = np.zeros(10**7, dtype=np.float64)
+        self.only_nans_values = np.full(10**7, np.nan, dtype=np.float64)
+
+    def time_isin_many_different(self):
+        # runtime is dominated by creation of the lookup-table
+        self.small.isin(self.many_different_values)
+
+    def time_isin_few_different(self):
+        # runtime is dominated by creation of the lookup-table
+        self.small.isin(self.few_different_values)
+
+    def time_isin_nan_values(self):
+        # runtime is dominated by creation of the lookup-table
+        self.small.isin(self.few_different_values)
+
+
+class IsInForObjects(object):
+
+    def setup(self):
+        self.s_nans = Series(np.full(10**4, np.nan)).astype(np.object)
+        self.vals_nans = np.full(10**4, np.nan).astype(np.object)
+        self.s_short = Series(np.arange(2)).astype(np.object)
+        self.s_long = Series(np.arange(10**5)).astype(np.object)
+        self.vals_short = np.arange(2).astype(np.object)
+        self.vals_long = np.arange(10**5).astype(np.object)
+        # because of nans floats are special:
+        self.s_long_floats = Series(np.arange(10**5,
+                                    dtype=np.float)).astype(np.object)
+        self.vals_long_floats = np.arange(10**5,
+                                          dtype=np.float).astype(np.object)
+
+    def time_isin_nans(self):
+        # if nan-objects are different objects,
+        # this has the potential to trigger O(n^2) running time
+        self.s_nans.isin(self.vals_nans)
+
+    def time_isin_short_series_long_values(self):
+        # running time dominated by the preprocessing
+        self.s_short.isin(self.vals_long)
+
+    def time_isin_long_series_short_values(self):
+        # running time dominated by look-up
+        self.s_long.isin(self.vals_short)
+
+    def time_isin_long_series_long_values(self):
+        # no dominating part
+        self.s_long.isin(self.vals_long)
+
+    def time_isin_long_series_long_values_floats(self):
+        # no dominating part
+        self.s_long_floats.isin(self.vals_long_floats)
+
+
 class NSort(object):
 
     goal_time = 0.2

asv_bench/benchmarks/timeseries.py

@@ -343,6 +343,25 @@ def time_iso8601_tz_spaceformat(self):
         to_datetime(self.strings_tz_space)
 
 
+class ToDatetimeNONISO8601(object):
+
+    goal_time = 0.2
+
+    def setup(self):
+        N = 10000
+        half = int(N / 2)
+        ts_string_1 = 'March 1, 2018 12:00:00+0400'
+        ts_string_2 = 'March 1, 2018 12:00:00+0500'
+        self.same_offset = [ts_string_1] * N
+        self.diff_offset = [ts_string_1] * half + [ts_string_2] * half
+
+    def time_same_offset(self):
+        to_datetime(self.same_offset)
+
+    def time_different_offset(self):
+        to_datetime(self.diff_offset)
+
+
 class ToDatetimeFormat(object):
 
     goal_time = 0.2