perf-test: New option --compare-to to include past results in tables

This simplifies performance comparison.

Author: tautschnig

scripts/perf-test/ec2.yaml

@@ -41,6 +41,9 @@ Parameters:
     WitnessCheck:
         Type: String
 
+    CompareTo:
+        Type: String
+
 Conditions:
     UseSpot: !Not [!Equals [!Ref MaxPrice, ""]]
 
@@ -68,6 +71,10 @@ Resources:
                             - "s3:GetObject"
                           Effect: Allow
                           Resource: !Join ["/", [!Sub "arn:aws:s3:::${S3Bucket}", "*"]]
+                        - Action:
+                            - "s3:ListBucket"
+                          Effect: Allow
+                          Resource: !Sub "arn:aws:s3:::${S3Bucket}"
                         - Action:
                             - "sns:Publish"
                           Effect: Allow
@@ -169,6 +176,7 @@ Resources:
                     sed -i 's#:S3BUCKET:#${S3Bucket}#g' /root/run-on-ec2.sh
                     sed -i 's#:PERFTESTID:#${PerfTestId}#g' /root/run-on-ec2.sh
                     sed -i 's#:INSTANCETYPE:#${InstanceType}#g' /root/run-on-ec2.sh
+                    sed -i 's#:COMPARETO:#${CompareTo}#g' /root/run-on-ec2.sh
                     . /root/run-on-ec2.sh
 
     AutoScalingGroup:

scripts/perf-test/perf_test.py

@@ -72,6 +72,8 @@ def parse_args():
                         help='Non-default CodeBuild template to use')
     parser.add_argument('-W', '--witness-check', action='store_true',
                         help='Run witness checks after benchmarking')
+    parser.add_argument('-C', '--compare-to', default=[], action='append',
+                        help='Include past results in tables [may repeat]')
 
     args = parser.parse_args()
 
@@ -517,7 +519,7 @@ def seed_queue(session, region, queue, task_set):
 def run_perf_test(
         session, mode, region, az, ami, instance_type, sqs_arn, sqs_url,
         parallel, snapshot_id, instance_terminated_arn, bucket_name,
-        perf_test_id, price, ssh_key_name, witness_check):
+        perf_test_id, price, ssh_key_name, witness_check, compare_to):
     # create an EC2 instance and trigger benchmarking
     logger = logging.getLogger('perf_test')
 
@@ -589,6 +591,10 @@ def run_perf_test(
                 {
                     'ParameterKey': 'WitnessCheck',
                     'ParameterValue': str(witness_check)
+                },
+                {
+                    'ParameterKey': 'CompareTo',
+                    'ParameterValue': ':'.join(compare_to)
                 }
             ],
             Capabilities=['CAPABILITY_NAMED_IAM'])
@@ -685,7 +691,7 @@ def main():
             session, args.mode, region, az, ami, args.instance_type,
             sqs_arn, sqs_url, args.parallel, snapshot_id,
             instance_terminated_arn, bucket_name, perf_test_id, price,
-            args.ssh_key_name, args.witness_check)
+            args.ssh_key_name, args.witness_check, args.compare_to)
 
     return 0
 

scripts/perf-test/run-on-ec2.sh

@@ -201,12 +201,21 @@ do
       cd ..
     done
 
+    compare_to=""
     if [ x:WITNESSCHECK: = xTrue ]
     then
-      ../benchexec/bin/table-generator logs-$t/*xml.bz2 *-logs-$t/*.xml.bz2 -o logs-$t/
-    else
-      ../benchexec/bin/table-generator logs-$t/*xml.bz2 -o logs-$t/
+      compare_to="*-logs-$t/*.xml.bz2"
     fi
+    for c in $(echo :COMPARETO: | sed 's/:/ /g')
+    do
+      if aws s3 ls s3://:S3BUCKET:/$c/$cfg/logs-$t/
+      then
+        aws s3 sync s3://:S3BUCKET:/$c/$cfg/logs-$t logs-$t-$c
+        compare_to="$compare_to logs-$t-$c/*.xml.bz2"
+      fi
+    done
+
+    ../benchexec/bin/table-generator logs-$t/*xml.bz2 $compare_to -o logs-$t/
     aws s3 cp logs-$t s3://:S3BUCKET:/:PERFTESTID:/$cfg/logs-$t/ --recursive
     for wc in *-witnesses.xml
     do