Skip to content

Commit 9f6ebc0

Browse files
gendxgendx
committed
Add benchmarking scripts.
1 parent dd33815 commit 9f6ebc0

10 files changed

+292
-1
lines changed

‎Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ repository = "https://github.com/gendx/stv-rs"
88
readme = "README.md"
99
categories = ["voting"]
1010
keywords = ["voting", "election", "stv", "meek"]
11-
exclude = ["man/*", "testdata/*", ".github/*"]
11+
exclude = ["man/*", "testdata/*", "tools/*", ".github/*"]
1212
edition = "2021"
1313

1414
[dependencies]

‎tools/benchmark-all-commits.sh

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
if [ -e "${HOME}/.cargo/bin/hyperfine" ]; then
6+
HYPERFINE_PATH=${HOME}/.cargo/bin/hyperfine
7+
elif [ -e "/usr/bin/hyperfine" ]; then
8+
HYPERFINE_PATH=/usr/bin/hyperfine
9+
else
10+
echo "Hyperfine is not installed. Please install it with 'apt install hyperfine' or 'cargo +nightly install hyperfine'."
11+
exit 42
12+
fi
13+
14+
BEGIN=main
15+
END=HEAD
16+
17+
echo "[*] Listing commits"
18+
git log ${BEGIN}..${END} --reverse --format=oneline
19+
COMMITS=$(git log ${BEGIN}..${END} --reverse --format=oneline | cut -d' ' -f1)
20+
21+
echo "[*] Building at all commits"
22+
./tools/build-all-commits.sh ${COMMITS}
23+
sleep 15
24+
25+
echo "[*] Benchmarking each commit"
26+
ITER=0
27+
for COMMIT in ${COMMITS}
28+
do
29+
ITER=$(expr ${ITER} + 1)
30+
printf -v INDEX "%02d" ${ITER}
31+
echo "[${INDEX}] Benchmarking ${COMMIT}"
32+
33+
./tools/benchmark-at.sh ${INDEX} ${COMMIT} 2>&1 | tee ${INDEX}-${COMMIT}.benchmark.log
34+
done

‎tools/benchmark-at.sh

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
if [ -e "${HOME}/.cargo/bin/hyperfine" ]; then
6+
HYPERFINE_PATH=${HOME}/.cargo/bin/hyperfine
7+
elif [ -e "/usr/bin/hyperfine" ]; then
8+
HYPERFINE_PATH=/usr/bin/hyperfine
9+
else
10+
echo "Hyperfine is not installed. Please install it with 'apt install hyperfine' or 'cargo +nightly install hyperfine'."
11+
exit 42
12+
fi
13+
14+
INDEX=$1
15+
COMMIT=$2
16+
BINARY=./bin/stv-rs-${INDEX}-${COMMIT}
17+
18+
if [ ! -e "${BINARY}" ]; then
19+
echo "Binary not found at ${BINARY}. Please build it via the script at 'tools/benchmark-all-commits.sh'."
20+
exit 42
21+
fi
22+
23+
SLEEP_SECONDS=10
24+
25+
function benchmark() {
26+
ARITHMETIC=$1
27+
EQUALIZE=$2
28+
INPUT=$3
29+
30+
sleep ${SLEEP_SECONDS}
31+
${HYPERFINE_PATH} --warmup 1 \
32+
"${BINARY} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=false > /dev/null"
33+
for NUM_THREADS in 2 4 8
34+
do
35+
sleep ${SLEEP_SECONDS}
36+
RAYON_NUM_THREADS=${NUM_THREADS} ${HYPERFINE_PATH} --warmup 1 \
37+
"${BINARY} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=true > /dev/null"
38+
done
39+
}
40+
41+
benchmark fixed9 "" "testdata/ballots/random/rand_hypergeometric.blt"
42+
benchmark fixed9 "" "testdata/shuffle_ballots/rand_sorted_lexicographically.blt"
43+
benchmark fixed9 "" "testdata/shuffle_ballots/rand_10k_sorted_lexicographically.blt"
44+
benchmark fixed9 --equalize "testdata/ballots/random/rand_hypergeometric.blt"
45+
#benchmark fixed9 "" "testdata/shuffle_ballots/rand_sorted_by_product.blt"
46+
#benchmark fixed9 "" "testdata/shuffle_ballots/rand_sorted_by_lexico_product.blt"
47+
benchmark bigfixed9 "" "testdata/ballots/random/rand_hypergeometric.blt"

‎tools/benchmark-compare-commits.sh

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
if [ -e "${HOME}/.cargo/bin/hyperfine" ]; then
6+
HYPERFINE_PATH=${HOME}/.cargo/bin/hyperfine
7+
elif [ -e "/usr/bin/hyperfine" ]; then
8+
HYPERFINE_PATH=/usr/bin/hyperfine
9+
else
10+
echo "Hyperfine is not installed. Please install it with 'apt install hyperfine' or 'cargo +nightly install hyperfine'."
11+
exit 42
12+
fi
13+
14+
BEGIN=main
15+
END=HEAD
16+
17+
echo "[*] Listing commits"
18+
git log ${BEGIN}..${END} --reverse --format=oneline
19+
COMMITS=$(git log ${BEGIN}..${END} --reverse --format=oneline | cut -d' ' -f1)
20+
21+
INDEX_COMMITS=
22+
ITER=0
23+
for COMMIT in ${COMMITS}
24+
do
25+
ITER=$(expr ${ITER} + 1)
26+
printf -v INDEX "%02d" ${ITER}
27+
if [ "${INDEX_COMMITS}" != "" ]; then
28+
INDEX_COMMITS="${INDEX_COMMITS},"
29+
fi
30+
INDEX_COMMITS=${INDEX_COMMITS}${INDEX}-${COMMIT}
31+
done
32+
echo "Commits joined as a hyperfine parameter list: ${INDEX_COMMITS}"
33+
34+
echo "[*] Building at all commits"
35+
./tools/build-all-commits.sh ${COMMITS}
36+
sleep 15
37+
38+
SLEEP_SECONDS=10
39+
40+
function benchmark() {
41+
ARITHMETIC=$1
42+
EQUALIZE=$2
43+
INPUT=$3
44+
45+
${HYPERFINE_PATH} \
46+
--setup "sleep ${SLEEP_SECONDS}" \
47+
--warmup 1 \
48+
--parameter-list INDEX_COMMIT ${INDEX_COMMITS} \
49+
"./bin/stv-rs-{INDEX_COMMIT} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=false > /dev/null"
50+
51+
for NUM_THREADS in 2 4 8
52+
do
53+
RAYON_NUM_THREADS=${NUM_THREADS} ${HYPERFINE_PATH} \
54+
--setup "sleep ${SLEEP_SECONDS}" \
55+
--warmup 1 \
56+
--parameter-list INDEX_COMMIT ${INDEX_COMMITS} \
57+
"./bin/stv-rs-{INDEX_COMMIT} --arithmetic ${ARITHMETIC} --input ${INPUT} meek ${EQUALIZE} --parallel=true > /dev/null"
58+
done
59+
}
60+
61+
benchmark fixed9 "" "testdata/shuffle_ballots/rand_sorted_lexicographically.blt"
62+
benchmark fixed9 "" "testdata/shuffle_ballots/rand_10k_sorted_lexicographically.blt"
63+
benchmark fixed9 --equalize "testdata/ballots/random/rand_hypergeometric.blt"
64+
benchmark bigfixed9 "" "testdata/ballots/random/rand_hypergeometric.blt"

‎tools/build-all-commits.sh

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
COMMITS=$@
6+
echo "Building at commits: '${COMMITS}'"
7+
8+
mkdir -p bin
9+
10+
ITER=0
11+
for COMMIT in ${COMMITS}
12+
do
13+
ITER=$(expr ${ITER} + 1)
14+
printf -v INDEX "%02d" ${ITER}
15+
echo "[${INDEX}] Processing ${COMMIT}"
16+
17+
if [ -e "bin/stv-rs-${INDEX}-${COMMIT}" ]; then
18+
echo "Binary already exists at 'bin/stv-rs-${INDEX}-${COMMIT}', skipping..."
19+
else
20+
git checkout ${COMMIT}
21+
git status
22+
cargo +nightly build --release
23+
cp target/release/stv-rs bin/stv-rs-${INDEX}-${COMMIT}
24+
fi
25+
done

‎tools/disable-perf.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/sh
2+
3+
echo 3 > /proc/sys/kernel/perf_event_paranoid
4+
echo 1 > /proc/sys/kernel/kptr_restrict
5+
echo 1 > /proc/sys/kernel/nmi_watchdog

‎tools/enable-perf.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/sh
2+
3+
echo -1 > /proc/sys/kernel/perf_event_paranoid
4+
echo 0 > /proc/sys/kernel/kptr_restrict
5+
echo 0 > /proc/sys/kernel/nmi_watchdog

‎tools/perf-all-commits.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
BEGIN=main
6+
END=HEAD
7+
8+
echo "[*] Listing commits"
9+
git log ${BEGIN}..${END} --reverse --format=oneline
10+
COMMITS=$(git log ${BEGIN}..${END} --reverse --format=oneline | cut -d' ' -f1)
11+
12+
echo "[*] Building at all commits"
13+
./tools/build-all-commits.sh ${COMMITS}
14+
sleep 15
15+
16+
echo "[*] Perfing each commit"
17+
ITER=0
18+
for COMMIT in ${COMMITS}
19+
do
20+
ITER=$(expr ${ITER} + 1)
21+
printf -v INDEX "%02d" ${ITER}
22+
echo "[${INDEX}] Benchmarking ${COMMIT}"
23+
24+
./tools/perf-stat-at.sh ${INDEX} ${COMMIT} 2>&1 | tee ${INDEX}-${COMMIT}.perf.log
25+
done
26+

‎tools/perf-record.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
BALLOT_FILE=testdata/ballots/random/rand_hypergeometric.blt
6+
ARITHMETIC=fixed9
7+
8+
RUSTFLAGS='-C force-frame-pointers=y' cargo +nightly build --release
9+
BINARY=./target/release/stv-rs
10+
sleep 5
11+
12+
function record() {
13+
EVENT=$1
14+
INTERVAL=$2
15+
NUM_THREADS=4
16+
TRACE_FILE=perf-record.${NUM_THREADS}-threads.${EVENT}
17+
18+
sleep 1
19+
RAYON_NUM_THREADS=${NUM_THREADS} perf record -e ${EVENT} -c ${INTERVAL} -g --output=${TRACE_FILE}.perf \
20+
${BINARY} --arithmetic ${ARITHMETIC} --input ${BALLOT_FILE} meek --parallel=true > /dev/null
21+
perf script --input=${TRACE_FILE}.perf -F +pid > ${TRACE_FILE}.processed.perf
22+
}
23+
24+
record branch-misses 100
25+
record L1-dcache-load-misses 100
26+
record L1-icache-load-misses 100
27+
record LLC-loads 10
28+
record LLC-load-misses 1

‎tools/perf-stat-at.sh

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
BALLOT_FILE=testdata/ballots/random/rand_hypergeometric.blt
6+
ARITHMETIC=fixed9
7+
REPETITIONS=100
8+
#ARITHMETIC=bigfixed9
9+
#REPETITIONS=10
10+
11+
INDEX=$1
12+
COMMIT=$2
13+
BINARY=./bin/stv-rs-${INDEX}-${COMMIT}
14+
15+
if [ ! -e "${BINARY}" ]; then
16+
echo "Binary not found at ${BINARY}. Please build it via the script at 'tools/perf-all-commits.sh'."
17+
exit 42
18+
fi
19+
20+
SLEEP_SECONDS=5
21+
22+
function sample() {
23+
ARITHMETIC=$1
24+
TITLE=$2
25+
PARAMS=$3
26+
27+
set +x
28+
echo "****************************************"
29+
echo "* ${TITLE} *"
30+
echo "****************************************"
31+
set -x
32+
33+
sleep ${SLEEP_SECONDS}
34+
perf stat -r ${REPETITIONS} ${PARAMS} \
35+
${BINARY} --arithmetic ${ARITHMETIC} --input ${BALLOT_FILE} meek --parallel=false > /dev/null
36+
37+
for NUM_THREADS in 2 4 8
38+
do
39+
sleep ${SLEEP_SECONDS}
40+
RAYON_NUM_THREADS=${NUM_THREADS} perf stat -r ${REPETITIONS} ${PARAMS} \
41+
${BINARY} --arithmetic ${ARITHMETIC} --input ${BALLOT_FILE} meek --parallel=true > /dev/null
42+
done
43+
}
44+
45+
function sample_default() {
46+
sample $1 "STATS" "-d -d"
47+
}
48+
49+
function sample_events() {
50+
sample $1 $2 "-e $3"
51+
}
52+
53+
sample_default fixed9
54+
sample_events fixed9 "INSTRUCTIONS" "task-clock,user_time,system_time,duration_time,cycles:u,instructions:u,branch-instructions:u,branch-misses:u"
55+
sample_events fixed9 "L1-CACHE" "L1-dcache-loads,L1-dcache-load-misses,L1-icache-load-misses,LLC-loads"
56+
sample_events fixed9 "LL-CACHE" "L1-dcache-loads,LLC-loads,LLC-load-misses"
57+
#sample_events fixed9 "STORES" "L1-dcache-stores,LLC-stores,LLC-store-misses"

0 commit comments

Comments
 (0)