Bench unit test compile time (rust-lang#943)

* bench unit test compile time

Author: tgymnich

.github/workflows/enzyme-ci.yml

@@ -1,6 +1,10 @@
 name: Enzyme CI
 
-on: [push]
+on: 
+  push:
+    branches:
+      - main
+  pull_request:
 
 jobs:
   build-linux:
@@ -51,7 +55,10 @@ jobs:
     - name: make check-activityanalysis
       run: cd enzyme/build && make check-activityanalysis -j`nproc`
     - name: make check-enzyme
-      run: cd enzyme/build && make check-enzyme -j`nproc`
+      run: cd enzyme/build && make check-enzyme-bench -j`nproc`
+    - name: graph results
+      run: python3 enzyme/test/upload-results.py enzyme/build/test/Enzyme/results.json --url https://enzyme.mit.edu/cibench/api --token ${{ secrets.GRAPH_TOKEN }}
+      if: github.event_name != 'pull_request' && matrix.build == 'Release'
 
   build-macos:
     name: Enzyme CI LLVM ${{ matrix.llvm }} ${{ matrix.build }} macOS
@@ -86,7 +93,10 @@ jobs:
     - name: make check-activityanalysis
       run: cd enzyme/build && make check-activityanalysis -j 3
     - name: make check-enzyme
-      run: cd enzyme/build && make check-enzyme -j 3
+      run: cd enzyme/build && make check-enzyme-bench -j 3
+    - name: graph results
+      run: python3 enzyme/test/upload-results.py enzyme/build/test/Enzyme/results.json --url https://enzyme.mit.edu/cibench/api --token ${{ secrets.GRAPH_TOKEN }}
+      if: github.event_name != 'pull_request' && matrix.build == 'Release'
 
   build-xcode:
     name: Enzyme CI LLVM ${{ matrix.llvm }} ${{ matrix.build }} macOS XCode

enzyme/test/Enzyme/CMakeLists.txt

@@ -12,4 +12,12 @@ add_lit_testsuite(check-enzyme "Running enzyme regression tests"
     ARGS -v
 )
 
+add_lit_testsuite(check-enzyme-bench "Benchmarking enzyme regression tests"
+    ${CMAKE_CURRENT_BINARY_DIR}
+    DEPENDS ${ENZYME_TEST_DEPS}
+    ARGS -v --time-tests -o results.json
+)
+
+
 set_target_properties(check-enzyme PROPERTIES FOLDER "Tests")
+set_target_properties(check-enzyme-bench PROPERTIES FOLDER "Tests")

enzyme/test/upload-results.py

@@ -0,0 +1,77 @@
+import json
+import datetime
+import subprocess
+import requests
+import argparse
+import platform
+
+
+def get_git_revision_hash():
+    try:
+        return subprocess.check_output(['git', 'rev-parse', 'HEAD'], stderr=subprocess.STDOUT).decode('ascii').strip()
+    except:
+        return "N/A"
+
+
+def get_git_revision_date():
+    try:
+        time = subprocess.check_output(
+            ['git', 'show', '--quiet', '--format=%ct', 'HEAD'], stderr=subprocess.STDOUT).decode('ascii').strip()
+        dt = datetime.datetime.fromtimestamp(
+            int(time), tz=datetime.timezone.utc)
+        return dt
+    except:
+        return datetime.datetime.now(datetime.timezone.utc)
+
+
+def extract_results(json):
+    result = []
+    githash = get_git_revision_hash()
+    time = get_git_revision_date().isoformat()
+    arch = platform.platform()
+    llvm = ".".join(str(json["__version__"]))
+
+    for test in json["tests"]:
+        if test["code"] == "PASS":
+            compiletime = test["elapsed"]
+            name = test["name"]
+            pathstr = name.split(" :: ")[1]
+            comps = pathstr.split("/")
+            mode = comps[1]
+            testname = ".".join(comps[2:])
+
+            res = {
+                "mode": mode,
+                "llvm-version": llvm,
+                "test-suite": "Enzyme Unit Tests",
+                "commit": githash,
+                "test-name": testname,
+                "compile-time": compiletime,
+                "timestamp": time,
+                "platform": arch
+            }
+            result.append(res)
+
+    return result
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('file', type=argparse.FileType(
+    'r'), help="JSON file containing the benchmark results")
+parser.add_argument('-t', '--token', type=str, required=False,
+                    help="Token to authorize at graphing backend")
+parser.add_argument('-u', '--url', type=str, required=True,
+                    help="URL of the graphing backend")
+
+args = parser.parse_args()
+
+json = json.load(args.file)
+results = extract_results(json)
+
+if args.token:
+    response = requests.post(args.url, json=results,
+                             headers={"X-TOKEN": args.token})
+    response.raise_for_status()
+else:
+    response = requests.post(args.url, json=results)
+    response.raise_for_status()