Skip to content

Commit 19a08e8

Browse files
per1234per1234
authored
Merge pull request #78 from per1234/benchmark
Add workflow to compare engine sync performance
2 parents 38cf7d8 + 87b2ae7 commit 19a08e8

File tree

1 file changed

+283
-0
lines changed

1 file changed

+283
-0
lines changed

‎.github/workflows/compare-performance.yml

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
name: Compare Performance
2+
3+
env:
4+
# See: https://github.com/actions/setup-go/tree/v2#readme
5+
GO_VERSION: "1.16"
6+
REPORTS_ARTIFACT_NAME: reports
7+
8+
# See: https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows
9+
on:
10+
push:
11+
paths:
12+
- ".github/workflows/compare-performance.ya?ml"
13+
- "**/go.mod"
14+
- "**/go.sum"
15+
- "Taskfile.ya?ml"
16+
- "**.go"
17+
pull_request:
18+
paths:
19+
- ".github/workflows/compare-performance.ya?ml"
20+
- "**/go.mod"
21+
- "**/go.sum"
22+
- "Taskfile.ya?ml"
23+
- "**.go"
24+
workflow_dispatch:
25+
inputs:
26+
comparison-ref:
27+
description: Comparison ref
28+
29+
jobs:
30+
init:
31+
runs-on: ubuntu-latest
32+
33+
outputs:
34+
base-ref: ${{ steps.base-ref.outputs.ref }}
35+
36+
steps:
37+
- name: Determine comparison ref
38+
id: base-ref
39+
run: |
40+
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
41+
echo "::set-output name=ref::${{ github.event.inputs.comparison-ref }}"
42+
elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
43+
echo "::set-output name=ref::${{ github.base_ref }}"
44+
else
45+
echo "::set-output name=ref::${{ github.event.before }}"
46+
fi
47+
48+
run:
49+
name: Run at ${{ matrix.data.ref }} (${{ matrix.data.description }})
50+
needs: init
51+
runs-on: ubuntu-latest
52+
53+
strategy:
54+
matrix:
55+
data:
56+
# Use two copies of each job to catch job-specific anomalous durations.
57+
- ref: ${{ github.ref }} # The tip of the branch selected in the workflow dispatch dialog's "Use workflow from" menu
58+
description: tip run 1
59+
position: after
60+
- ref: ${{ github.ref }}
61+
description: tip run 2
62+
position: after
63+
- ref: ${{ needs.init.outputs.base-ref }}
64+
description: comparison run 1
65+
position: before
66+
- ref: ${{ needs.init.outputs.base-ref }}
67+
description: comparison run 2
68+
position: before
69+
70+
steps:
71+
- name: Set environment variables
72+
run: |
73+
# See: https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
74+
ENGINE_DATA_PATH="${{ runner.temp }}/engine"
75+
mkdir --parents "$ENGINE_DATA_PATH"
76+
echo "ENGINE_DATA_PATH=${ENGINE_DATA_PATH}" >> "$GITHUB_ENV"
77+
echo "GIT_CLONES_PATH=${ENGINE_DATA_PATH}/gitclones" >> "$GITHUB_ENV"
78+
echo "LIBRARY_ARCHIVES_PATH=${ENGINE_DATA_PATH}/libraries" >> "$GITHUB_ENV"
79+
echo "LOGS_PATH=${ENGINE_DATA_PATH}/logs" >> "$GITHUB_ENV"
80+
echo "CONFIG_PATH=${ENGINE_DATA_PATH}/config.json" >> "$GITHUB_ENV"
81+
echo "REGISTRY_PATH=${ENGINE_DATA_PATH}/registry.txt" >> "$GITHUB_ENV"
82+
echo "REPORTS_PATH=${ENGINE_DATA_PATH}/reports" >> "$GITHUB_ENV"
83+
84+
- name: Checkout repository
85+
uses: actions/checkout@v2
86+
with:
87+
ref: ${{ matrix.data.ref }}
88+
89+
- name: Determine appropriate Go version
90+
id: go-version
91+
run: |
92+
if [[ -f "go.mod" ]]; then
93+
USE_GO_VERSION="${{ env.GO_VERSION }}"
94+
else
95+
# Dependency installation for old engine versions fails when not in GOPATH mode. Go <1.16 uses
96+
# GO111MODULE=auto by default, meaning it will use GOPATH mode. Old Go versions were used by the old engine
97+
# anyway.
98+
USE_GO_VERSION="1.14"
99+
fi
100+
echo "::set-output name=version::$USE_GO_VERSION"
101+
102+
- name: Install Go
103+
uses: actions/setup-go@v2
104+
with:
105+
go-version: ${{ steps.go-version.outputs.version }}
106+
107+
- name: Install Task
108+
uses: arduino/setup-task@v1
109+
with:
110+
repo-token: ${{ secrets.GITHUB_TOKEN }}
111+
version: 3.x
112+
113+
- name: Install latest release of Arduino Lint
114+
run: |
115+
ARDUINO_LINT_INSTALLATION_PATH="${{ runner.temp }}/arduino-lint"
116+
mkdir --parents "$ARDUINO_LINT_INSTALLATION_PATH"
117+
curl \
118+
-fsSL \
119+
https://raw.githubusercontent.com/arduino/arduino-lint/main/etc/install.sh \
120+
| \
121+
BINDIR="$ARDUINO_LINT_INSTALLATION_PATH" \
122+
sh
123+
124+
# Add installation folder to path
125+
echo "$ARDUINO_LINT_INSTALLATION_PATH" >> "$GITHUB_PATH"
126+
127+
- name: Configure Git for `go get` access to private repo
128+
run: |
129+
if ! [[ -f "go.mod" ]]; then
130+
# engine versions prior to 7dd8f69282232919955c82c143fefb14e50d0889 had a dependency that is hosted in a
131+
# private repo. The `go.mod` file was added at the same time the dependency was removed, so its presence can
132+
# be used as the indicator.
133+
git config \
134+
--global url."https://${{ secrets.REPO_SCOPE_TOKEN }}:[email protected]/".insteadOf "https://github.com/"
135+
fi
136+
137+
- name: Build engine
138+
run: |
139+
task go:build
140+
141+
- name: Generate configuration file
142+
run: |
143+
cat > "${{ env.CONFIG_PATH }}" << EOF
144+
{
145+
"BaseDownloadUrl": "https://downloads.arduino.cc/libraries/",
146+
"LibrariesFolder": "${{ env.LIBRARY_ARCHIVES_PATH }}",
147+
"LibrariesIndex": "${{ env.ENGINE_DATA_PATH }}/library_index.json",
148+
"LogsFolder": "${{ env.ENGINE_DATA_PATH }}/logs",
149+
"LibrariesDB": "${{ env.ENGINE_DATA_PATH }}/db.json",
150+
"GitClonesFolder": "${{ env.GIT_CLONES_PATH }}",
151+
"DoNotRunClamav": true
152+
}
153+
EOF
154+
155+
- name: Generate registry file
156+
run: |
157+
FULL_REGISTRY_PATH="${{ runner.temp }}/registry.txt"
158+
curl \
159+
--output "$FULL_REGISTRY_PATH" \
160+
https://raw.githubusercontent.com/arduino/library-registry/1c3f73b279d2845ff139883c78e733e2954437b8/registry.txt
161+
162+
# Only use the first part of the file for the test
163+
head \
164+
-300 \
165+
"$FULL_REGISTRY_PATH" > \
166+
"${{ env.REGISTRY_PATH }}"
167+
168+
- name: Run sync on empty environment
169+
id: fresh
170+
run: |
171+
SECONDS=0
172+
./libraries-repository-engine "${{ env.CONFIG_PATH }}" "${{ env.REGISTRY_PATH }}"
173+
174+
# Define step outputs with the performance data
175+
echo "::set-output name=Type::fresh"
176+
echo "::set-output name=Duration::$SECONDS"
177+
echo "::set-output name=GitClonesSize::$(du --apparent-size --bytes --summarize "${{ env.GIT_CLONES_PATH }}" | cut --fields=1)"
178+
echo "::set-output name=LibraryArchivesSize::$(du --apparent-size --bytes --summarize "${{ env.LIBRARY_ARCHIVES_PATH }}" | cut --fields=1)"
179+
echo "::set-output name=LogsSize::$(du --apparent-size --bytes --summarize "${{ env.LOGS_PATH }}" | cut --fields=1)"
180+
181+
- name: Run sync on populated database
182+
id: populated
183+
run: |
184+
SECONDS=0
185+
./libraries-repository-engine "${{ env.CONFIG_PATH }}" "${{ env.REGISTRY_PATH }}"
186+
187+
# Define step outputs with the performance data
188+
echo "::set-output name=Type::populated"
189+
echo "::set-output name=Duration::$SECONDS"
190+
echo "::set-output name=GitClonesSize::$(du --apparent-size --bytes --summarize "${{ env.GIT_CLONES_PATH }}" | cut --fields=1)"
191+
echo "::set-output name=LibraryArchivesSize::$(du --apparent-size --bytes --summarize "${{ env.LIBRARY_ARCHIVES_PATH }}" | cut --fields=1)"
192+
echo "::set-output name=LogsSize::$(du --apparent-size --bytes --summarize "${{ env.LOGS_PATH }}" | cut --fields=1)"
193+
194+
- name: Create report
195+
run: |
196+
mkdir --parents "${{ env.REPORTS_PATH }}"
197+
cat > "${{ env.REPORTS_PATH }}/$RANDOM.json" << EOF
198+
{
199+
"Ref": "${{ matrix.data.ref }}",
200+
"Description": "${{ matrix.data.description }}",
201+
"Position": "${{ matrix.data.position }}",
202+
"Results": [
203+
${{ toJSON(steps.fresh.outputs) }},
204+
${{ toJSON(steps.populated.outputs) }}
205+
]
206+
}
207+
EOF
208+
209+
- name: Upload report to a workflow artifact
210+
uses: actions/upload-artifact@v2
211+
with:
212+
if-no-files-found: error
213+
path: ${{ env.REPORTS_PATH }}
214+
name: ${{ env.REPORTS_ARTIFACT_NAME }}
215+
216+
results:
217+
needs: run
218+
runs-on: ubuntu-latest
219+
220+
env:
221+
REPORTS_PATH: reports
222+
223+
steps:
224+
- name: Download reports
225+
uses: actions/download-artifact@v2
226+
with:
227+
name: ${{ env.REPORTS_ARTIFACT_NAME }}
228+
path: ${{ env.REPORTS_PATH }}
229+
230+
- name: Print results
231+
shell: python
232+
run: |
233+
import json
234+
import pathlib
235+
236+
reports_path = pathlib.Path("${{ env.REPORTS_PATH }}")
237+
reports = []
238+
for report_path in reports_path.iterdir():
239+
with report_path.open() as report_file:
240+
reports.append(json.load(fp=report_file))
241+
242+
sample_size = 0
243+
summary_data = {
244+
"Duration": [],
245+
"GitClonesSize": [],
246+
"LibraryArchivesSize": [],
247+
"LogsSize": [],
248+
}
249+
for report in reports:
250+
if report["Position"] == "before":
251+
sample_size += 1
252+
for result in report["Results"]:
253+
for key in list(summary_data):
254+
type_index = None
255+
for index, summary_item in enumerate(summary_data[key]):
256+
if summary_item["type"] == result["Type"]:
257+
type_index = index
258+
break
259+
if type_index is None:
260+
summary_data[key].append(
261+
{"type": result["Type"], "before": 0, "after": 0}
262+
)
263+
type_index = len(summary_data[key]) - 1
264+
summary_data[key][type_index][report["Position"]] += int(result[key])
265+
266+
print("% change:")
267+
for key in list(summary_data):
268+
for type_data in summary_data[key]:
269+
print(
270+
"{key} ({type}): {value}".format(
271+
key=key,
272+
type=type_data["type"],
273+
value=round(
274+
100
275+
* (type_data["after"] - type_data["before"])
276+
/ type_data["before"]
277+
),
278+
)
279+
)
280+
281+
print("::group::Full results")
282+
print(json.dumps(obj=reports, indent=2))
283+
print("::endgroup::")

0 commit comments

Comments
 (0)