Merge remote-tracking branch 'upstream/master' into mcmali-s3-pub-test

Author: alimcmaster1

.travis.yml

@@ -58,12 +58,6 @@ matrix:
       services:
         - mysql
         - postgresql
-
-    - env:
-        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
-      services:
-        - mysql
-        - postgresql
   allow_failures:
     - arch: arm64
       env:

asv_bench/benchmarks/categoricals.py

@@ -34,6 +34,7 @@ def setup(self):
         self.values_all_int8 = np.ones(N, "int8")
         self.categorical = pd.Categorical(self.values, self.categories)
         self.series = pd.Series(self.categorical)
+        self.intervals = pd.interval_range(0, 1, periods=N // 10)
 
     def time_regular(self):
         pd.Categorical(self.values, self.categories)
@@ -44,6 +45,9 @@ def time_fastpath(self):
     def time_datetimes(self):
         pd.Categorical(self.datetimes)
 
+    def time_interval(self):
+        pd.Categorical(self.datetimes, categories=self.datetimes)
+
     def time_datetimes_with_nat(self):
         pd.Categorical(self.datetimes_with_nat)
 

asv_bench/benchmarks/groupby.py

@@ -16,7 +16,7 @@
 
 from .pandas_vb_common import tm
 
-method_blacklist = {
+method_blocklist = {
     "object": {
         "median",
         "prod",
@@ -403,7 +403,7 @@ class GroupByMethods:
     ]
 
     def setup(self, dtype, method, application):
-        if method in method_blacklist.get(dtype, {}):
+        if method in method_blocklist.get(dtype, {}):
             raise NotImplementedError  # skip benchmark
         ngroups = 1000
         size = ngroups * 2

asv_bench/benchmarks/io/json.py

@@ -1,3 +1,5 @@
+import sys
+
 import numpy as np
 
 from pandas import DataFrame, concat, date_range, read_json, timedelta_range
@@ -82,6 +84,7 @@ def setup(self, orient, frame):
         timedeltas = timedelta_range(start=1, periods=N, freq="s")
         datetimes = date_range(start=1, periods=N, freq="s")
         ints = np.random.randint(100000000, size=N)
+        longints = sys.maxsize * np.random.randint(100000000, size=N)
         floats = np.random.randn(N)
         strings = tm.makeStringIndex(N)
         self.df = DataFrame(np.random.randn(N, ncols), index=np.arange(N))
@@ -120,6 +123,18 @@ def setup(self, orient, frame):
             index=index,
         )
 
+        self.df_longint_float_str = DataFrame(
+            {
+                "longint_1": longints,
+                "longint_2": longints,
+                "float_1": floats,
+                "float_2": floats,
+                "str_1": strings,
+                "str_2": strings,
+            },
+            index=index,
+        )
+
     def time_to_json(self, orient, frame):
         getattr(self, frame).to_json(self.fname, orient=orient)
 
@@ -172,6 +187,7 @@ def setup(self):
         timedeltas = timedelta_range(start=1, periods=N, freq="s")
         datetimes = date_range(start=1, periods=N, freq="s")
         ints = np.random.randint(100000000, size=N)
+        longints = sys.maxsize * np.random.randint(100000000, size=N)
         floats = np.random.randn(N)
         strings = tm.makeStringIndex(N)
         self.df = DataFrame(np.random.randn(N, ncols), index=np.arange(N))
@@ -209,6 +225,17 @@ def setup(self):
             },
             index=index,
         )
+        self.df_longint_float_str = DataFrame(
+            {
+                "longint_1": longints,
+                "longint_2": longints,
+                "float_1": floats,
+                "float_2": floats,
+                "str_1": strings,
+                "str_2": strings,
+            },
+            index=index,
+        )
 
     def time_floats_with_int_idex_lines(self):
         self.df.to_json(self.fname, orient="records", lines=True)
@@ -225,6 +252,9 @@ def time_float_int_lines(self):
     def time_float_int_str_lines(self):
         self.df_int_float_str.to_json(self.fname, orient="records", lines=True)
 
+    def time_float_longint_str_lines(self):
+        self.df_longint_float_str.to_json(self.fname, orient="records", lines=True)
+
 
 class ToJSONMem:
     def setup_cache(self):

asv_bench/benchmarks/rolling.py

@@ -91,11 +91,18 @@ class EWMMethods:
     def setup(self, constructor, window, dtype, method):
         N = 10 ** 5
         arr = (100 * np.random.random(N)).astype(dtype)
+        times = pd.date_range("1900", periods=N, freq="23s")
         self.ewm = getattr(pd, constructor)(arr).ewm(halflife=window)
+        self.ewm_times = getattr(pd, constructor)(arr).ewm(
+            halflife="1 Day", times=times
+        )
 
     def time_ewm(self, constructor, window, dtype, method):
         getattr(self.ewm, method)()
 
+    def time_ewm_times(self, constructor, window, dtype, method):
+        self.ewm.mean()
+
 
 class VariableWindowMethods(Methods):
     params = (

asv_bench/benchmarks/tslibs/fields.py

@@ -0,0 +1,74 @@
+import numpy as np
+
+from pandas._libs.tslibs.fields import (
+    get_date_field,
+    get_start_end_field,
+    get_timedelta_field,
+)
+
+from .tslib import _sizes
+
+
+class TimeGetTimedeltaField:
+    params = [
+        _sizes,
+        ["days", "h", "s", "seconds", "ms", "microseconds", "us", "ns", "nanoseconds"],
+    ]
+    param_names = ["size", "field"]
+
+    def setup(self, size, field):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+    def time_get_timedelta_field(self, size, field):
+        get_timedelta_field(self.i8data, field)
+
+
+class TimeGetDateField:
+    params = [
+        _sizes,
+        [
+            "Y",
+            "M",
+            "D",
+            "h",
+            "m",
+            "s",
+            "us",
+            "ns",
+            "doy",
+            "dow",
+            "woy",
+            "q",
+            "dim",
+            "is_leap_year",
+        ],
+    ]
+    param_names = ["size", "field"]
+
+    def setup(self, size, field):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+    def time_get_date_field(self, size, field):
+        get_date_field(self.i8data, field)
+
+
+class TimeGetStartEndField:
+    params = [
+        _sizes,
+        ["start", "end"],
+        ["month", "quarter", "year"],
+        ["B", None, "QS"],
+        [12, 3, 5],
+    ]
+    param_names = ["size", "side", "period", "freqstr", "month_kw"]
+
+    def setup(self, size, side, period, freqstr, month_kw):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+        self.attrname = f"is_{period}_{side}"
+
+    def time_get_start_end_field(self, size, side, period, freqstr, month_kw):
+        get_start_end_field(self.i8data, self.attrname, freqstr, month_kw=month_kw)

asv_bench/benchmarks/tslibs/resolution.py

@@ -0,0 +1,53 @@
+"""
+ipython analogue:
+
+tr = TimeResolution()
+mi = pd.MultiIndex.from_product(tr.params[:-1] + ([str(x) for x in tr.params[-1]],))
+df = pd.DataFrame(np.nan, index=mi, columns=["mean", "stdev"])
+
+for unit in tr.params[0]:
+    for size in tr.params[1]:
+        for tz in tr.params[2]:
+            tr.setup(unit, size, tz)
+            key = (unit, size, str(tz))
+            print(key)
+
+            val = %timeit -o tr.time_get_resolution(unit, size, tz)
+
+            df.loc[key] = (val.average, val.stdev)
+
+"""
+from datetime import timedelta, timezone
+
+from dateutil.tz import gettz, tzlocal
+import numpy as np
+import pytz
+
+try:
+    from pandas._libs.tslibs import get_resolution
+except ImportError:
+    from pandas._libs.tslibs.resolution import get_resolution
+
+
+class TimeResolution:
+    params = (
+        ["D", "h", "m", "s", "us", "ns"],
+        [1, 100, 10 ** 4, 10 ** 6],
+        [
+            None,
+            timezone.utc,
+            timezone(timedelta(minutes=60)),
+            pytz.timezone("US/Pacific"),
+            gettz("Asia/Tokyo"),
+            tzlocal(),
+        ],
+    )
+    param_names = ["unit", "size", "tz"]
+
+    def setup(self, unit, size, tz):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        arr = arr.view(f"M8[{unit}]").astype("M8[ns]").view("i8")
+        self.i8data = arr
+
+    def time_get_resolution(self, unit, size, tz):
+        get_resolution(self.i8data, tz)

asv_bench/benchmarks/tslibs/timestamp.py

@@ -1,17 +1,29 @@
-import datetime
+from datetime import datetime, timedelta, timezone
 
-import dateutil
+from dateutil.tz import gettz, tzlocal, tzutc
 import numpy as np
 import pytz
 
 from pandas import Timestamp
 
+# One case for each type of tzinfo object that has its own code path
+#  in tzconversion code.
+_tzs = [
+    None,
+    pytz.timezone("Europe/Amsterdam"),
+    gettz("US/Central"),
+    pytz.UTC,
+    tzutc(),
+    timezone(timedelta(minutes=60)),
+    tzlocal(),
+]
+
 
 class TimestampConstruction:
     def setup(self):
         self.npdatetime64 = np.datetime64("2020-01-01 00:00:00")
-        self.dttime_unaware = datetime.datetime(2020, 1, 1, 0, 0, 0)
-        self.dttime_aware = datetime.datetime(2020, 1, 1, 0, 0, 0, 0, pytz.UTC)
+        self.dttime_unaware = datetime(2020, 1, 1, 0, 0, 0)
+        self.dttime_aware = datetime(2020, 1, 1, 0, 0, 0, 0, pytz.UTC)
         self.ts = Timestamp("2020-01-01 00:00:00")
 
     def time_parse_iso8601_no_tz(self):
@@ -49,7 +61,6 @@ def time_from_pd_timestamp(self):
 
 
 class TimestampProperties:
-    _tzs = [None, pytz.timezone("Europe/Amsterdam"), pytz.UTC, dateutil.tz.tzutc()]
     _freqs = [None, "B"]
     params = [_tzs, _freqs]
     param_names = ["tz", "freq"]
@@ -63,9 +74,6 @@ def time_tz(self, tz, freq):
     def time_dayofweek(self, tz, freq):
         self.ts.dayofweek
 
-    def time_weekday_name(self, tz, freq):
-        self.ts.day_name
-
     def time_dayofyear(self, tz, freq):
         self.ts.dayofyear
 
@@ -108,9 +116,12 @@ def time_microsecond(self, tz, freq):
     def time_month_name(self, tz, freq):
         self.ts.month_name()
 
+    def time_weekday_name(self, tz, freq):
+        self.ts.day_name()
+
 
 class TimestampOps:
-    params = [None, "US/Eastern", pytz.UTC, dateutil.tz.tzutc()]
+    params = _tzs
     param_names = ["tz"]
 
     def setup(self, tz):
@@ -148,7 +159,7 @@ def time_ceil(self, tz):
 
 class TimestampAcrossDst:
     def setup(self):
-        dt = datetime.datetime(2016, 3, 27, 1)
+        dt = datetime(2016, 3, 27, 1)
         self.tzinfo = pytz.timezone("CET").localize(dt, is_dst=False).tzinfo
         self.ts2 = Timestamp(dt)
 

asv_bench/benchmarks/tslibs/tslib.py

@@ -0,0 +1,58 @@
+"""
+ipython analogue:
+
+tr = TimeIntsToPydatetime()
+mi = pd.MultiIndex.from_product(
+    tr.params[:-1] + ([str(x) for x in tr.params[-1]],)
+)
+df = pd.DataFrame(np.nan, index=mi, columns=["mean", "stdev"])
+for box in tr.params[0]:
+    for size in tr.params[1]:
+        for tz in tr.params[2]:
+            tr.setup(box, size, tz)
+            key = (box, size, str(tz))
+            print(key)
+            val = %timeit -o tr.time_ints_to_pydatetime(box, size, tz)
+            df.loc[key] = (val.average, val.stdev)
+"""
+from datetime import timedelta, timezone
+
+from dateutil.tz import gettz, tzlocal
+import numpy as np
+import pytz
+
+try:
+    from pandas._libs.tslibs import ints_to_pydatetime
+except ImportError:
+    from pandas._libs.tslib import ints_to_pydatetime
+
+_tzs = [
+    None,
+    timezone.utc,
+    timezone(timedelta(minutes=60)),
+    pytz.timezone("US/Pacific"),
+    gettz("Asia/Tokyo"),
+    tzlocal(),
+]
+_sizes = [0, 1, 100, 10 ** 4, 10 ** 6]
+
+
+class TimeIntsToPydatetime:
+    params = (
+        ["time", "date", "datetime", "timestamp"],
+        _sizes,
+        _tzs,
+    )
+    param_names = ["box", "size", "tz"]
+    # TODO: fold? freq?
+
+    def setup(self, box, size, tz):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+    def time_ints_to_pydatetime(self, box, size, tz):
+        if box == "date":
+            # ints_to_pydatetime does not allow non-None tz with date;
+            #  this will mean doing some duplicate benchmarks
+            tz = None
+        ints_to_pydatetime(self.i8data, tz, box=box)