|
| 1 | +from timeit import repeat as timeit |
| 2 | + |
| 3 | +import numpy as np |
| 4 | +import seaborn as sns |
| 5 | + |
| 6 | +from pandas import DataFrame |
| 7 | + |
| 8 | +setup_common = """from pandas import DataFrame |
| 9 | +from numpy.random import randn |
| 10 | +df = DataFrame(randn(%d, 3), columns=list('abc')) |
| 11 | +%s""" |
| 12 | + |
| 13 | +setup_with = "s = 'a + b * (c ** 2 + b ** 2 - a) / (a * c) ** 3'" |
| 14 | + |
| 15 | + |
| 16 | +def bench_with(n, times=10, repeat=3, engine="numexpr"): |
| 17 | + return ( |
| 18 | + np.array( |
| 19 | + timeit( |
| 20 | + "df.eval(s, engine=%r)" % engine, |
| 21 | + setup=setup_common % (n, setup_with), |
| 22 | + repeat=repeat, |
| 23 | + number=times, |
| 24 | + ) |
| 25 | + ) |
| 26 | + / times |
| 27 | + ) |
| 28 | + |
| 29 | + |
| 30 | +setup_subset = "s = 'a <= b <= c ** 2 + b ** 2 - a and b > c'" |
| 31 | + |
| 32 | + |
| 33 | +def bench_subset(n, times=20, repeat=3, engine="numexpr"): |
| 34 | + return ( |
| 35 | + np.array( |
| 36 | + timeit( |
| 37 | + "df.query(s, engine=%r)" % engine, |
| 38 | + setup=setup_common % (n, setup_subset), |
| 39 | + repeat=repeat, |
| 40 | + number=times, |
| 41 | + ) |
| 42 | + ) |
| 43 | + / times |
| 44 | + ) |
| 45 | + |
| 46 | + |
| 47 | +def bench(mn=3, mx=7, num=100, engines=("python", "numexpr"), verbose=False): |
| 48 | + r = np.logspace(mn, mx, num=num).round().astype(int) |
| 49 | + |
| 50 | + ev = DataFrame(np.empty((num, len(engines))), columns=engines) |
| 51 | + qu = ev.copy(deep=True) |
| 52 | + |
| 53 | + ev["size"] = qu["size"] = r |
| 54 | + |
| 55 | + for engine in engines: |
| 56 | + for i, n in enumerate(r): |
| 57 | + if verbose & (i % 10 == 0): |
| 58 | + print("engine: %r, i == %d" % (engine, i)) |
| 59 | + ev_times = bench_with(n, times=1, repeat=1, engine=engine) |
| 60 | + ev.loc[i, engine] = np.mean(ev_times) |
| 61 | + qu_times = bench_subset(n, times=1, repeat=1, engine=engine) |
| 62 | + qu.loc[i, engine] = np.mean(qu_times) |
| 63 | + |
| 64 | + return ev, qu |
| 65 | + |
| 66 | + |
| 67 | +def plot_perf(df, engines, title, filename=None): |
| 68 | + from matplotlib.pyplot import figure |
| 69 | + |
| 70 | + sns.set() |
| 71 | + sns.set_palette("Set2") |
| 72 | + |
| 73 | + fig = figure(figsize=(4, 3), dpi=120) |
| 74 | + ax = fig.add_subplot(111) |
| 75 | + |
| 76 | + for engine in engines: |
| 77 | + ax.loglog(df["size"], df[engine], label=engine, lw=2) |
| 78 | + |
| 79 | + ax.set_xlabel("Number of Rows") |
| 80 | + ax.set_ylabel("Time (s)") |
| 81 | + ax.set_title(title) |
| 82 | + ax.legend(loc="best") |
| 83 | + ax.tick_params(top=False, right=False) |
| 84 | + |
| 85 | + fig.tight_layout() |
| 86 | + |
| 87 | + if filename is not None: |
| 88 | + fig.savefig(filename) |
| 89 | + |
| 90 | + |
| 91 | +if __name__ == "__main__": |
| 92 | + import os |
| 93 | + |
| 94 | + pandas_dir = os.path.dirname( |
| 95 | + os.path.dirname(os.path.abspath(os.path.dirname(__file__))) |
| 96 | + ) |
| 97 | + static_path = os.path.join(pandas_dir, "doc", "source", "_static") |
| 98 | + |
| 99 | + join = lambda p: os.path.join(static_path, p) |
| 100 | + |
| 101 | + fn = join("eval-query-perf-data.h5") |
| 102 | + |
| 103 | + engines = "python", "numexpr" |
| 104 | + |
| 105 | + ev, qu = bench(verbose=True) # only this one |
| 106 | + |
| 107 | + plot_perf(ev, engines, "DataFrame.eval()", filename=join("eval-perf.png")) |
| 108 | + plot_perf(qu, engines, "DataFrame.query()", filename=join("query-perf.png")) |
0 commit comments