Skip to content

Commit 20449fc

Browse files
cmaglieper1234
cmaglie
and
per1234
authored
Improved JSON decoding (improves arduino-cli startup time) (#1402)
* Updated library_index.json for testing * Use easyjson for parsing library_index * Added benchmarks for json deconding. Results: $ go test -v -benchmem -bench BenchmarkIndexParsing github.com/arduino/arduino-cli/arduino/libraries/librariesindex === RUN TestIndexer --- PASS: TestIndexer (0.16s) goos: linux goarch: amd64 pkg: github.com/arduino/arduino-cli/arduino/libraries/librariesindex cpu: Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz BenchmarkIndexParsingStdJSON BenchmarkIndexParsingStdJSON-8 5 214872730 ns/op 94.52 MB/s 58956539 B/op 418973 allocs/op BenchmarkIndexParsingEasyJSON BenchmarkIndexParsingEasyJSON-8 16 69215472 ns/op 293.42 MB/s 56162664 B/op 418966 allocs/op PASS ok github.com/arduino/arduino-cli/arduino/libraries/librariesindex 4.442s easyjson is 3x faster. * Updated license check cache * Added easyjson to package_index.json parser * Allow easyjson to match tags in case-insensitive See mailru/easyjson#372 * Added easyjson generation task in Taskfile * Added github action check * Update .github/workflows/check-easyjson.yml Co-authored-by: per1234 <[email protected]> Co-authored-by: per1234 <[email protected]>
1 parent 01eab14 commit 20449fc

File tree

21 files changed

+236760
-23008
lines changed

21 files changed

+236760
-23008
lines changed

Diff for: ‎.github/workflows/check-easyjson.yml

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: Check easyjson generated files
2+
3+
env:
4+
# See: https://github.com/actions/setup-go/tree/v2#readme
5+
GO_VERSION: "1.17"
6+
7+
# See: https://docs.github.com/en/actions/reference/events-that-trigger-workflows
8+
on:
9+
push:
10+
paths:
11+
- ".github/workflows/check-easyjson.ya?ml"
12+
- "go.mod"
13+
- "go.sum"
14+
- "Taskfile.ya?ml"
15+
- "**.go"
16+
pull_request:
17+
paths:
18+
- ".github/workflows/check-easyjson.ya?ml"
19+
- "go.mod"
20+
- "go.sum"
21+
- "Taskfile.ya?ml"
22+
- "**.go"
23+
workflow_dispatch:
24+
repository_dispatch:
25+
26+
jobs:
27+
check-easyjson-generated-files:
28+
runs-on: ubuntu-latest
29+
30+
steps:
31+
- name: Checkout repository
32+
uses: actions/checkout@v3
33+
34+
- name: Install Go
35+
uses: actions/setup-go@v3
36+
with:
37+
go-version: ${{ env.GO_VERSION }}
38+
39+
- name: Install Task
40+
uses: arduino/setup-task@v1
41+
with:
42+
repo-token: ${{ secrets.GITHUB_TOKEN }}
43+
version: 3.x
44+
45+
- name: Regenerate easyjson source code
46+
run: task go:easyjson-generate
47+
48+
- name: Check for outdated cache
49+
id: diff
50+
run: |
51+
git add .
52+
if ! git diff --cached --color --exit-code; then
53+
echo
54+
echo "::error::easyjson generated files out of sync."
55+
exit 1
56+
fi

Diff for: ‎.licenses/go/github.com/josharian/intern.dep.yml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
---
2+
name: github.com/josharian/intern
3+
version: v1.0.0
4+
type: go
5+
summary: Package intern interns strings.
6+
homepage: https://pkg.go.dev/github.com/josharian/intern
7+
license: mit
8+
licenses:
9+
- sources: license.md
10+
text: |
11+
MIT License
12+
13+
Copyright (c) 2019 Josh Bleecher Snyder
14+
15+
Permission is hereby granted, free of charge, to any person obtaining a copy
16+
of this software and associated documentation files (the "Software"), to deal
17+
in the Software without restriction, including without limitation the rights
18+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
19+
copies of the Software, and to permit persons to whom the Software is
20+
furnished to do so, subject to the following conditions:
21+
22+
The above copyright notice and this permission notice shall be included in all
23+
copies or substantial portions of the Software.
24+
25+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
30+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31+
SOFTWARE.
32+
notices: []

Diff for: ‎.licenses/go/github.com/mailru/easyjson.dep.yml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
name: github.com/mailru/easyjson
3+
version: v0.7.7
4+
type: go
5+
summary: Package easyjson contains marshaler/unmarshaler interfaces and helper functions.
6+
homepage: https://pkg.go.dev/github.com/mailru/easyjson
7+
license: mit
8+
licenses:
9+
- sources: LICENSE
10+
text: |
11+
Copyright (c) 2016 Mail.Ru Group
12+
13+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
14+
15+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
16+
17+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
18+
notices: []

Diff for: ‎.licenses/go/github.com/mailru/easyjson/buffer.dep.yml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
---
2+
name: github.com/mailru/easyjson/buffer
3+
version: v0.7.7
4+
type: go
5+
summary: Package buffer implements a buffer for serialization, consisting of a chain
6+
of []byte-s to reduce copying and to allow reuse of individual chunks.
7+
homepage: https://pkg.go.dev/github.com/mailru/easyjson/buffer
8+
license: mit
9+
licenses:
10+
- sources: [email protected]/LICENSE
11+
text: |
12+
Copyright (c) 2016 Mail.Ru Group
13+
14+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
15+
16+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
17+
18+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19+
notices: []

Diff for: ‎.licenses/go/github.com/mailru/easyjson/jlexer.dep.yml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
name: github.com/mailru/easyjson/jlexer
3+
version: v0.7.7
4+
type: go
5+
summary: Package jlexer contains a JSON lexer implementation.
6+
homepage: https://pkg.go.dev/github.com/mailru/easyjson/jlexer
7+
license: mit
8+
licenses:
9+
- sources: [email protected]/LICENSE
10+
text: |
11+
Copyright (c) 2016 Mail.Ru Group
12+
13+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
14+
15+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
16+
17+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
18+
notices: []

Diff for: ‎.licenses/go/github.com/mailru/easyjson/jwriter.dep.yml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
name: github.com/mailru/easyjson/jwriter
3+
version: v0.7.7
4+
type: go
5+
summary: Package jwriter contains a JSON writer.
6+
homepage: https://pkg.go.dev/github.com/mailru/easyjson/jwriter
7+
license: mit
8+
licenses:
9+
- sources: [email protected]/LICENSE
10+
text: |
11+
Copyright (c) 2016 Mail.Ru Group
12+
13+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
14+
15+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
16+
17+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
18+
notices: []

Diff for: ‎Taskfile.yml

+7
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,13 @@ tasks:
119119
cmds:
120120
- go vet {{default .DEFAULT_GO_PACKAGES .GO_PACKAGES}}
121121

122+
go:easyjson-generate:
123+
desc: Run easyjson code generation
124+
cmds:
125+
- go install github.com/mailru/easyjson/[email protected]
126+
- easyjson arduino/cores/packageindex/index.go
127+
- easyjson arduino/libraries/librariesindex/json.go
128+
122129
# Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-markdown-task/Taskfile.yml
123130
markdown:check-links:
124131
desc: Check for broken links

Diff for: ‎arduino/cores/packageindex/index.go

+14-2
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,20 @@ import (
2424
"github.com/arduino/arduino-cli/arduino/security"
2525
"github.com/arduino/arduino-cli/i18n"
2626
"github.com/arduino/go-paths-helper"
27+
easyjson "github.com/mailru/easyjson"
2728
"github.com/sirupsen/logrus"
2829
semver "go.bug.st/relaxed-semver"
2930
)
3031

3132
// Index represents Cores and Tools struct as seen from package_index.json file.
33+
//easyjson:json
3234
type Index struct {
3335
Packages []*indexPackage `json:"packages"`
3436
IsTrusted bool
3537
}
3638

3739
// indexPackage represents a single entry from package_index.json file.
40+
//easyjson:json
3841
type indexPackage struct {
3942
Name string `json:"name"`
4043
Maintainer string `json:"maintainer"`
@@ -47,6 +50,7 @@ type indexPackage struct {
4750
}
4851

4952
// indexPlatformRelease represents a single Core Platform from package_index.json file.
53+
//easyjson:json
5054
type indexPlatformRelease struct {
5155
Name string `json:"name"`
5256
Architecture string `json:"architecture"`
@@ -65,32 +69,37 @@ type indexPlatformRelease struct {
6569
}
6670

6771
// indexToolDependency represents a single dependency of a core from a tool.
72+
//easyjson:json
6873
type indexToolDependency struct {
6974
Packager string `json:"packager"`
7075
Name string `json:"name"`
7176
Version *semver.RelaxedVersion `json:"version"`
7277
}
7378

7479
// indexDiscoveryDependency represents a single dependency of a core from a pluggable discovery tool.
80+
//easyjson:json
7581
type indexDiscoveryDependency struct {
7682
Packager string `json:"packager"`
7783
Name string `json:"name"`
7884
}
7985

8086
// indexMonitorDependency represents a single dependency of a core from a pluggable monitor tool.
87+
//easyjson:json
8188
type indexMonitorDependency struct {
8289
Packager string `json:"packager"`
8390
Name string `json:"name"`
8491
}
8592

8693
// indexToolRelease represents a single Tool from package_index.json file.
94+
//easyjson:json
8795
type indexToolRelease struct {
8896
Name string `json:"name"`
8997
Version *semver.RelaxedVersion `json:"version"`
9098
Systems []indexToolReleaseFlavour `json:"systems"`
9199
}
92100

93101
// indexToolReleaseFlavour represents a single tool flavor in the package_index.json file.
102+
//easyjson:json
94103
type indexToolReleaseFlavour struct {
95104
OS string `json:"host"`
96105
URL string `json:"url"`
@@ -100,17 +109,20 @@ type indexToolReleaseFlavour struct {
100109
}
101110

102111
// indexBoard represents a single Board as written in package_index.json file.
112+
//easyjson:json
103113
type indexBoard struct {
104114
Name string `json:"name"`
105115
ID []indexBoardID `json:"id,omitempty"`
106116
}
107117

108118
// indexBoardID represents the ID of a single board. i.e. uno, yun, diecimila, micro and the likes
119+
//easyjson:json
109120
type indexBoardID struct {
110121
USB string `json:"usb"`
111122
}
112123

113124
// indexHelp represents the help URL
125+
//easyjson:json
114126
type indexHelp struct {
115127
Online string `json:"online,omitempty"`
116128
}
@@ -352,7 +364,7 @@ func LoadIndex(jsonIndexFile *paths.Path) (*Index, error) {
352364
return nil, err
353365
}
354366
var index Index
355-
err = json.Unmarshal(buff, &index)
367+
err = easyjson.Unmarshal(buff, &index)
356368
if err != nil {
357369
return nil, err
358370
}
@@ -381,7 +393,7 @@ func LoadIndexNoSign(jsonIndexFile *paths.Path) (*Index, error) {
381393
return nil, err
382394
}
383395
var index Index
384-
err = json.Unmarshal(buff, &index)
396+
err = easyjson.Unmarshal(buff, &index)
385397
if err != nil {
386398
return nil, err
387399
}

0 commit comments

Comments
 (0)