test_perf.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
What
----
vbench is a library which can be used to benchmark the performance
of a codebase over time.
Although vbench can collect data over many commites, generate plots
and other niceties, for Pull-Requests the important thing is the
performance of the HEAD commit against a known-good baseline.

This script tries to automate the process of comparing these
two commits, and is meant to run out of the box on a fresh
clone.

How
---
These are the steps taken:
1) create a temp directory into which vbench will clone the temporary repo.
2) instantiate a vbench runner, using the local repo as the source repo.
3) perform a vbench run for the baseline commit, then the target commit.
4) pull the results for both commits from the db. use pandas to align
everything and calculate a ration for the timing information.
5) print the results to the log file and to stdout.

"""

# IMPORTANT NOTE
#
# This script should run on pandas versions at least as far back as 0.9.1.
# devs should be able to use the latest version of this script with
# any dusty old commit and expect it to "just work".
# One way in which this is useful is when collecting historical data,
# where writing some logic around this script may prove easier
# in some cases then running vbench directly (think perf bisection).
#
# *please*, when you modify this script for whatever reason,
# make sure you do not break it's functionality when running under older
# pandas versions.
# Note that depreaction warnings are turned off in main(), so there's
# no need to change the actual code to supress such warnings.

import shutil
import os
import sys
import argparse
import tempfile
import time
import re

import random
import numpy as np

import pandas as pd
from pandas import DataFrame, Series

from suite import REPO_PATH
VB_DIR = os.path.dirname(os.path.abspath(__file__))
DEFAULT_MIN_DURATION = 0.01
HEAD_COL="head[ms]"
BASE_COL="base[ms]"

try:
    import git # gitpython
except Exception:
    print("Error: Please install the `gitpython` package\n")
    sys.exit(1)

class RevParseAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        import subprocess
        cmd = 'git rev-parse --short -verify {0}^{{commit}}'.format(values)
        rev_parse = subprocess.check_output(cmd, shell=True)
        setattr(namespace, self.dest, rev_parse.strip())


parser = argparse.ArgumentParser(description='Use vbench to measure and compare the performance of commits.')
parser.add_argument('-H', '--head',
                    help='Execute vbenches using the currently checked out copy.',
                    dest='head',
                    action='store_true',
                    default=False)
parser.add_argument('-b', '--base-commit',
                    help='The commit serving as performance baseline ',
                    type=str, action=RevParseAction)
parser.add_argument('-t', '--target-commit',
                    help='The commit to compare against the baseline (default: HEAD).',
                    type=str, action=RevParseAction)
parser.add_argument('--base-pickle',
                    help='name of pickle file with timings data generated by a former `-H -d FILE` run. '\
                    'filename must be of the form <hash>-*.* or specify --base-commit seperately',
                    type=str)
parser.add_argument('--target-pickle',
                    help='name of pickle file with timings data generated by a former `-H -d FILE` run '\
                    'filename must be of the form <hash>-*.* or specify --target-commit seperately',
                    type=str)
parser.add_argument('-m', '--min-duration',
                    help='Minimum duration (in ms) of baseline test for inclusion in report (default: %.3f).' % DEFAULT_MIN_DURATION,
                    type=float,
                    default=0.01)
parser.add_argument('-o', '--output',
                    metavar="<file>",
                    dest='log_file',
                    help='Path of file in which to save the textual report (default: vb_suite.log).')
parser.add_argument('-d', '--outdf',
                    metavar="FNAME",
                    dest='outdf',
                    default=None,
                    help='Name of file to df.save() the result table into. Will overwrite')
parser.add_argument('-r', '--regex',
                    metavar="REGEX",
                    dest='regex',
                    default="",
                    help='Regex pat, only tests whose name matches the regext will be run.')
parser.add_argument('-e', '--extra-benchmarks', metavar='EXTRA',
                    dest='extras', action='append',
                    help='Extra modules to collect benchmarks from')
parser.add_argument('-s', '--seed',
                    metavar="SEED",
                    dest='seed',
                    default=1234,
                    type=int,
                    help='Integer value to seed PRNG with')
parser.add_argument('-n', '--repeats',
                    metavar="N",
                    dest='repeats',
                    default=3,
                    type=int,
                    help='Number of times to run each vbench, result value is the best of')
parser.add_argument('-c', '--ncalls',
                    metavar="N",
                    dest='ncalls',
                    default=3,
                    type=int,
                    help='Number of calls to in each repetition of a vbench')
parser.add_argument('-N', '--hrepeats',
                    metavar="N",
                    dest='hrepeats',
                    default=1,
                    type=int,
                    help='implies -H, number of times to run the vbench suite on the head commit.\n'
                    'Each iteration will yield another column in the output' )
parser.add_argument('-a', '--affinity',
                    metavar="a",
                    dest='affinity',
                    default=1,
                    type=int,
                    help='set processor affinity of process by default bind to cpu/core #1 only. '
                         'Requires the "affinity" or "psutil" python module, will raise Warning otherwise')
parser.add_argument('-u', '--burnin',
                    metavar="u",
                    dest='burnin',
                    default=1,
                    type=int,
                    help='Number of extra iteration per benchmark to perform first, then throw away. '  )

parser.add_argument('-S', '--stats',
                    default=False,
                    action='store_true',
                    help='when specified with -N, prints the output of describe() per vbench results. '  )

parser.add_argument('--temp-dir',
                    metavar="PATH",
                    default=None,
                    help='Specify temp work dir to use. ccache depends on builds being invoked from consistent directory.'  )

parser.add_argument('-q', '--quiet',
                    default=False,
                    action='store_true',
                    help='Suppress report output to stdout. '  )

def get_results_df(db, rev):
    """Takes a git commit hash and returns a Dataframe of benchmark results
    """
    bench = DataFrame(db.get_benchmarks())
    results = DataFrame(map(list,db.get_rev_results(rev).values()))

    # Sinch vbench.db._reg_rev_results returns an unlabeled dict,
    # we have to break encapsulation a bit.
    results.columns = db._results.c.keys()
    results = results.join(bench['name'], on='checksum').set_index("checksum")
    return results


def prprint(s):
    print("*** %s" % s)

def pre_hook():
    import gc
    gc.disable()

def post_hook():
    import gc
    gc.enable()

def profile_comparative(benchmarks):

    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import BUILD, DB_PATH, PREPARE, dependencies

    TMP_DIR = args.temp_dir or tempfile.mkdtemp()

    try:

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)
        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint("Slaughtering kittens...")
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("Removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)

        report_comparative(head_res,baseline_res)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)

def prep_pickle_for_total(df, agg_name='median'):
    """
    accepts a datafram resulting from invocation with -H -d o.pickle
    If multiple data columns are present (-N was used), the
    `agg_name` attr of the datafram will be used to reduce
    them to a single value per vbench, df.median is used by defa
    ult.

    Returns a datadrame of the form expected by prep_totals
    """
    def prep(df):
        agg = getattr(df,agg_name)
        df = DataFrame(agg(1))
        cols = list(df.columns)
        cols[0]='timing'
        df.columns=cols
        df['name'] = list(df.index)
        return df

    return prep(df)

def prep_totals(head_res, baseline_res):
    """
    Each argument should be a dataframe with  'timing' and 'name' columns
    where name is the name of the vbench.

    returns a 'totals' dataframe, suitable as input for print_report.
    """
    head_res, baseline_res = head_res.align(baseline_res)
    ratio = head_res['timing'] / baseline_res['timing']
    totals = DataFrame({HEAD_COL:head_res['timing'],
                        BASE_COL:baseline_res['timing'],
                        'ratio':ratio,
                        'name':baseline_res.name},
                        columns=[HEAD_COL, BASE_COL, "ratio", "name"])
    totals = totals.ix[totals[HEAD_COL] > args.min_duration]
    # ignore below threshold
    totals = totals.dropna(
    ).sort("ratio").set_index('name')  # sort in ascending order
    return totals

def report_comparative(head_res,baseline_res):
    try:
        r=git.Repo(VB_DIR)
    except:
        import pdb
        pdb.set_trace()

    totals = prep_totals(head_res,baseline_res)

    h_head = args.target_commit
    h_baseline = args.base_commit
    h_msg = b_msg = "Unknown"
    try:
        h_msg =  r.commit(h_head).message.strip()
    except git.exc.BadObject:
        pass
    try:
        b_msg =  r.commit(h_baseline).message.strip()
    except git.exc.BadObject:
        pass


    print_report(totals,h_head=h_head,h_msg=h_msg,
             h_baseline=h_baseline,b_msg=b_msg)

    if args.outdf:
        prprint("The results DataFrame was written to '%s'\n" %  args.outdf)
        totals.save(args.outdf)

def profile_head_single(benchmark):
    import gc
    results = []

    # just in case
    gc.collect()

    try:
        from ctypes import cdll, CDLL
        cdll.LoadLibrary("libc.so.6")
        libc = CDLL("libc.so.6")
        libc.malloc_trim(0)
    except:
        pass


    N =  args.hrepeats + args.burnin

    results = []
    try:
        for i in range(N):
            gc.disable()
            d=dict()

            try:
                d = benchmark.run()

            except KeyboardInterrupt:
                raise
            except Exception as e: # if a single vbench bursts into flames, don't die.
                err=""
                try:
                    err =  d.get("traceback")
                    if err is None:
                        err = str(e)
                except:
                    pass
                print("%s died with:\n%s\nSkipping...\n" % (benchmark.name, err))

            results.append(d.get('timing',np.nan))
            gc.enable()
            gc.collect()

    finally:
        gc.enable()

    if results:
        # throw away the burn_in
        results = results[args.burnin:]
    sys.stdout.write('.')
    sys.stdout.flush()
    return Series(results, name=benchmark.name)

    # df = DataFrame(results)
    # df.columns = ["name",HEAD_COL]
    # return df.set_index("name")[HEAD_COL]

def profile_head(benchmarks):
    print( "Performing %d benchmarks (%d runs each)" % ( len(benchmarks), args.hrepeats))

    ss= [profile_head_single(b) for b in benchmarks]
    print("\n")

    results = DataFrame(ss)
    results.columns=[ "#%d" %i for i in range(args.hrepeats)]
    # results.index = ["#%d" % i for i in range(len(ss))]
    # results = results.T

    shas, messages, _,_  = _parse_commit_log(None,REPO_PATH,base_commit="HEAD^")
    print_report(results,h_head=shas[-1],h_msg=messages[-1])


    if args.outdf:
        prprint("The results DataFrame was written to '%s'\n" %  args.outdf)
        DataFrame(results).save(args.outdf)

def print_report(df,h_head=None,h_msg="",h_baseline=None,b_msg=""):

    name_width=45
    col_width = 10

    hdr = ("{:%s}" % name_width).format("Test name")
    hdr += ("|{:^%d}"  % col_width)* len(df.columns)
    hdr += "|"
    hdr = hdr.format(*df.columns)
    hdr = "-"*len(hdr) + "\n" + hdr + "\n" + "-"*len(hdr) + "\n"
    ftr=hdr
    s = "\n"
    s+= "Invoked with :\n"
    s+= "--ncalls: %s\n" % (args.ncalls or 'Auto')
    s+= "--repeats: %s\n" % (args.repeats)
    s+= "\n\n"

    s += hdr
    # import ipdb
    # ipdb.set_trace()
    for i in range(len(df)):
        lfmt = ("{:%s}" % name_width)
        lfmt += ("| {:%d.4f} " % (col_width-2))* len(df.columns)
        lfmt += "|\n"
        s += lfmt.format(df.index[i],*list(df.irow(i).values))

    s+= ftr + "\n"

    s += "Ratio < 1.0 means the target commit is faster then the baseline.\n"
    s += "Seed used: %d\n\n" % args.seed

    if  h_head:
        s += 'Target [%s] : %s\n' % (h_head, h_msg)
    if  h_baseline:
        s += 'Base   [%s] : %s\n\n' % (
            h_baseline, b_msg)

    stats_footer = "\n"
    if args.stats :
        try:
            pd.options.display.expand_frame_repr=False
            pd.set_option('display.max_rows', None)
        except:
            pass
        stats_footer += str(df.T.describe().T) + "\n\n"

    s+= stats_footer
    logfile = open(args.log_file, 'w')
    logfile.write(s)
    logfile.close()

    if not args.quiet:
        prprint(s)

    if args.stats and args.quiet:
        prprint(stats_footer)

    prprint("Results were also written to the logfile at '%s'" %
            args.log_file)


def main():
    if not args.log_file:
        args.log_file = os.path.abspath(
            os.path.join(REPO_PATH, 'vb_suite.log'))

    saved_dir = os.path.curdir
    if args.outdf:
        # not bullet-proof but enough for us
        args.outdf = os.path.realpath(args.outdf)

    if args.log_file:
        # not bullet-proof but enough for us
        args.log_file = os.path.realpath(args.log_file)

    random.seed(args.seed)
    np.random.seed(args.seed)

    if args.base_pickle and args.target_pickle:
        baseline_res = prep_pickle_for_total(pd.load(args.base_pickle))
        target_res = prep_pickle_for_total(pd.load(args.target_pickle))

        report_comparative(target_res, baseline_res)
        sys.exit(0)

    if args.affinity is not None:
        try: # use psutil rather then stale affinity module. Thanks @yarikoptic
            import psutil
            if hasattr(psutil.Process, 'set_cpu_affinity'):
                psutil.Process(os.getpid()).set_cpu_affinity([args.affinity])
                print("CPU affinity set to %d" % args.affinity)
        except ImportError:
            print("-a/--affinity specified, but the 'psutil' module is not available, aborting.\n")
            sys.exit(1)

    print("\n")
    prprint("LOG_FILE = %s" % args.log_file)
    if args.outdf:
        prprint("PICKE_FILE = %s" % args.outdf)

    print("\n")

    # move away from the pandas root dit, to avoid possible import
    # surprises
    os.chdir(os.path.dirname(os.path.abspath(__file__)))

    from suite import discover_benchmarks, benchmarks

    benchmarks = [b for b in benchmarks]
    if args.extras:
        benchmarks.extend(discover_benchmarks(args.extras, return_as='list'))

    benchmarks = [bm for bm in benchmarks
                  if re.search(args.regex, bm.name)]

    for b in benchmarks:
        b.repeat = args.repeats
        if args.ncalls:
            b.ncalls = args.ncalls

    if benchmarks:
        if args.head:
            profile_head(benchmarks)
        else:
            profile_comparative(benchmarks)
    else:
        print( "No matching benchmarks")

    os.chdir(saved_dir)

# hack , vbench.git ignores some commits, but we
# need to be able to reference any commit.
# modified from vbench.git
def _parse_commit_log(this,repo_path,base_commit=None):
    from vbench.git import _convert_timezones
    from pandas import Series
    from dateutil import parser as dparser

    git_cmd = 'git --git-dir=%s/.git --work-tree=%s ' % (repo_path, repo_path)
    githist = git_cmd + ('log --graph --pretty=format:'+
                         '\"::%h::%cd::%s::%an\"'+
                         ('%s..' % base_commit)+
                         '> githist.txt')
    os.system(githist)
    githist = open('githist.txt').read()
    os.remove('githist.txt')

    shas = []
    timestamps = []
    messages = []
    authors = []
    for line in githist.split('\n'):
        if '*' not in line.split("::")[0]:  # skip non-commit lines
            continue

        _, sha, stamp, message, author = line.split('::', 4)

        # parse timestamp into datetime object
        stamp = dparser.parse(stamp)

        shas.append(sha)
        timestamps.append(stamp)
        messages.append(message)
        authors.append(author)

    # to UTC for now
    timestamps = _convert_timezones(timestamps)

    shas = Series(shas, timestamps)
    messages = Series(messages, shas)
    timestamps = Series(timestamps, shas)
    authors = Series(authors, shas)
    return shas[::-1], messages[::-1], timestamps[::-1], authors[::-1]

# even worse, monkey patch vbench
def _parse_wrapper(base_commit):
    def inner(repo_path):
        return _parse_commit_log(repo_path,base_commit)
    return inner

if __name__ == '__main__':
    args = parser.parse_args()
    if (not args.head
        and not (args.base_commit and args.target_commit)
        and not (args.base_pickle and args.target_pickle)):
        parser.print_help()
        sys.exit(1)
    elif ((args.base_pickle or args.target_pickle) and not
        (args.base_pickle and args.target_pickle)):
        print("Must specify Both --base-pickle and --target-pickle.")
        sys.exit(1)

    if ((args.base_pickle or args.target_pickle) and not
        (args.base_commit and args.target_commit)):
        if not args.base_commit:
            print("base_commit not specified, Assuming base_pickle is named <commit>-foo.*")
            args.base_commit = args.base_pickle.split('-')[0]
        if not args.target_commit:
            print("target_commit not specified, Assuming target_pickle is named <commit>-foo.*")
            args.target_commit = args.target_pickle.split('-')[0]

    import warnings
    warnings.filterwarnings('ignore',category=FutureWarning)
    warnings.filterwarnings('ignore',category=DeprecationWarning)

    if args.base_commit and args.target_commit:
        print("Verifying specified commits exist in repo...")
        r=git.Repo(VB_DIR)
        for c in [ args.base_commit, args.target_commit ]:
            try:
                msg =  r.commit(c).message.strip()
            except git.BadObject:
                print("The commit '%s' was not found, aborting..." % c)
                sys.exit(1)
            else:
                print("%s: %s" % (c,msg))

    main()