diff --git a/tools/download.go b/tools/download.go index 78bc7d927..4fc2fc36d 100644 --- a/tools/download.go +++ b/tools/download.go @@ -40,40 +40,10 @@ import ( "golang.org/x/crypto/openpgp" "github.com/arduino/arduino-create-agent/utilities" + "github.com/arduino/arduino-create-agent/v2/pkgs" "github.com/blang/semver" - "github.com/xrash/smetrics" ) -type system struct { - Host string `json:"host"` - URL string `json:"url"` - Name string `json:"archiveFileName"` - CheckSum string `json:"checksum"` -} - -type tool struct { - Name string `json:"name"` - Version string `json:"version"` - Systems []system `json:"systems"` -} - -type index struct { - Packages []struct { - Name string `json:"name"` - Tools []tool `json:"tools"` - } `json:"packages"` -} - -var systems = map[string]string{ - "linuxamd64": "x86_64-linux-gnu", - "linux386": "i686-linux-gnu", - "darwinamd64": "i686-apple-darwin", - "darwinarm64": "arm64-apple-darwin", - "windows386": "i686-mingw32", - "windowsamd64": "i686-mingw32", - "linuxarm": "arm-linux-gnueabihf", -} - // public vars to allow override in the tests var ( OS = runtime.GOOS @@ -199,7 +169,7 @@ func (t *Tools) Download(pack, name, version, behaviour string) error { return err } - var data index + var data pkgs.Index json.Unmarshal(body, &data) // Find the tool by name @@ -245,7 +215,7 @@ func (t *Tools) Download(pack, name, version, behaviour string) error { checksum := sha256.Sum256(body) checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size]) - if checkSumString != correctSystem.CheckSum { + if checkSumString != correctSystem.Checksum { return errors.New("checksum doesn't match") } @@ -299,8 +269,8 @@ func (t *Tools) Download(pack, name, version, behaviour string) error { return t.writeMap() } -func findTool(pack, name, version string, data index) (tool, system) { - var correctTool tool +func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.System) { + var correctTool pkgs.Tool correctTool.Version = "0.0" for _, p := range data.Packages { @@ -324,16 +294,7 @@ func findTool(pack, name, version string, data index) (tool, system) { } // Find the url based on system - var correctSystem system - maxSimilarity := 0.7 - - for _, s := range correctTool.Systems { - similarity := smetrics.Jaro(s.Host, systems[OS+Arch]) - if similarity > maxSimilarity { - correctSystem = s - maxSimilarity = similarity - } - } + correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch) return correctTool, correctSystem } diff --git a/tools/download_test.go b/tools/download_test.go index af33df6c8..3d2c63bac 100644 --- a/tools/download_test.go +++ b/tools/download_test.go @@ -24,6 +24,7 @@ import ( "path" "testing" + "github.com/arduino/arduino-create-agent/v2/pkgs" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -40,14 +41,14 @@ func TestDownloadCorrectPlatform(t *testing.T) { {"darwin", "amd64", "x86_64-apple-darwin"}, {"darwin", "arm64", "arm64-apple-darwin"}, {"windows", "386", "i686-mingw32"}, - {"windows", "amd64", "i686-mingw32"}, + {"windows", "amd64", "x86_64-mingw32"}, {"linux", "arm", "arm-linux-gnueabihf"}, } testIndex := paths.New("testdata", "test_tool_index.json") buf, err := testIndex.ReadFile() require.NoError(t, err) - var data index + var data pkgs.Index err = json.Unmarshal(buf, &data) require.NoError(t, err) for _, tc := range testCases { @@ -65,6 +66,38 @@ func TestDownloadCorrectPlatform(t *testing.T) { } } +func TestDownloadFallbackPlatform(t *testing.T) { + testCases := []struct { + hostOS string + hostArch string + correctOSArch string + }{ + {"darwin", "amd64", "i386-apple-darwin11"}, + {"darwin", "arm64", "i386-apple-darwin11"}, + {"windows", "amd64", "i686-mingw32"}, + } + testIndex := paths.New("testdata", "test_tool_index.json") + buf, err := testIndex.ReadFile() + require.NoError(t, err) + + var data pkgs.Index + err = json.Unmarshal(buf, &data) + require.NoError(t, err) + for _, tc := range testCases { + t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) { + OS = tc.hostOS // override `runtime.OS` for testing purposes + Arch = tc.hostArch // override `runtime.ARCH` for testing purposes + // Find the tool by name + correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.0", data) + require.NotNil(t, correctTool) + require.NotNil(t, correctSystem) + require.Equal(t, correctTool.Name, "arduino-fwuploader") + require.Equal(t, correctTool.Version, "2.2.0") + require.Equal(t, correctSystem.Host, tc.correctOSArch) + }) + } +} + func Test_findBaseDir(t *testing.T) { cases := []struct { dirList []string diff --git a/tools/testdata/test_tool_index.json b/tools/testdata/test_tool_index.json index 9b4e315fa..8d5dbc652 100644 --- a/tools/testdata/test_tool_index.json +++ b/tools/testdata/test_tool_index.json @@ -1,117 +1,137 @@ { - "packages": [ + "packages": [ + { + "name": "arduino-test", + "maintainer": "Arduino", + "websiteURL": "http://www.arduino.cc/", + "email": "packages@arduino.cc", + "help": { + "online": "http://www.arduino.cc/en/Reference/HomePage" + }, + "platforms": [ { - "name": "arduino-test", - "maintainer": "Arduino", - "websiteURL": "http://www.arduino.cc/", - "email": "packages@arduino.cc", - "help": { - "online": "http://www.arduino.cc/en/Reference/HomePage" + "name": "Arduino megaAVR Boards - Pre-release", + "architecture": "megaavr", + "version": "1.8.102", + "category": "Arduino", + "url": "http://downloads.arduino.cc/cores/staging/core-megaavr-1.8.102.tar.bz2", + "archiveFileName": "core-megaavr-1.8.102.tar.bz2", + "checksum": "SHA-256:ad5e60b828678d9ccff957032524a4c4d68b218737e7df24b905769a04dc2a6a", + "size": "858620", + "help": { + "online": "https://github.com/arduino/ArduinoCore-megaavr/issues" + }, + "boards": [ + { + "name": "Arduino Uno WiFi Rev2" }, - "platforms": [ - { - "name": "Arduino megaAVR Boards - Pre-release", - "architecture": "megaavr", - "version": "1.8.102", - "category": "Arduino", - "url": "http://downloads.arduino.cc/cores/staging/core-megaavr-1.8.102.tar.bz2", - "archiveFileName": "core-megaavr-1.8.102.tar.bz2", - "checksum": "SHA-256:ad5e60b828678d9ccff957032524a4c4d68b218737e7df24b905769a04dc2a6a", - "size": "858620", - "help": { - "online": "https://github.com/arduino/ArduinoCore-megaavr/issues" - }, - "boards": [ - { - "name": "Arduino Uno WiFi Rev2" - }, - { - "name": "Arduino Nano Every" - } - ], - "toolsDependencies": [ - { - "packager": "arduino", - "name": "avr-gcc", - "version": "7.3.0-atmel3.6.1-arduino5" - }, - { - "packager": "arduino", - "name": "avrdude", - "version": "7.0-arduino.3" - }, - { - "packager": "arduino", - "name": "arduinoOTA", - "version": "1.3.0" - } - ] - } - ], - "tools": [ - { - "name": "arduino-fwuploader", - "version": "2.2.2", - "systems": [ - { - "host": "i686-linux-gnu", - "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Linux_32bit.tar.gz", - "archiveFileName": "arduino-fwuploader_2.2.2_Linux_32bit.tar.gz", - "checksum": "SHA-256:503b9f8b24c6e396d09eb64f0e1f625c6f9aa5a90b01a50d7dec6477f4a866f0", - "size": "7262873" - }, - { - "host": "x86_64-linux-gnu", - "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Linux_64bit.tar.gz", - "archiveFileName": "arduino-fwuploader_2.2.2_Linux_64bit.tar.gz", - "checksum": "SHA-256:8d77d0b33c8b0787fe3b80191709b69d638ef2a447d9853536cda35bfafd274b", - "size": "7306763" - }, - { - "host": "i686-mingw32", - "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Windows_32bit.zip", - "archiveFileName": "arduino-fwuploader_2.2.2_Windows_32bit.zip", - "checksum": "SHA-256:74ad9a5d369204b51be288c98d74f949ceb7a0c227ee64eb65ae179ec884c84c", - "size": "7450717" - }, - { - "host": "x86_64-mingw32", - "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Windows_64bit.zip", - "archiveFileName": "arduino-fwuploader_2.2.2_Windows_64bit.zip", - "checksum": "SHA-256:b25ac549cb0645166613c96cf899aebc541e482fe196aada6408bd7cff2c7d02", - "size": "7390999" - }, - { - "host": "x86_64-apple-darwin", - "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_macOS_64bit.tar.gz", - "archiveFileName": "arduino-fwuploader_2.2.2_macOS_64bit.tar.gz", - "checksum": "SHA-256:2cd6168ff470457b5124ba0faf118f315be2d1b9fb4fef43eb74370cd83620a2", - "size": "7306576" - }, - { - "host": "arm64-apple-darwin", - "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_macOS_ARM64.tar.gz", - "archiveFileName": "arduino-fwuploader_2.2.2_macOS_ARM64.tar.gz", - "checksum": "SHA-256:10ae5614af4d82096b6ba0e1e07aab667fa140d2bf1d5e3407dd8ad4c6748195", - "size": "6878214" - }, - { - "host": "arm-linux-gnueabihf", - "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Linux_ARMv6.tar.gz", - "archiveFileName": "arduino-fwuploader_2.2.2_Linux_ARMv6.tar.gz", - "checksum": "SHA-256:5aadf6e50ffe620635faf941fdf82c0765c8cba4830951bb53267ad125fc5af8", - "size": "6940393" - }, - { - "host": "aarch64-linux-gnu", - "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Linux_ARM64.tar.gz", - "archiveFileName": "arduino-fwuploader_2.2.2_Linux_ARM64.tar.gz", - "checksum": "SHA-256:6d11a4f4aa5a81de865f3d18ca395a2780fdbb1e1597a2b11b2b5329e09f30fd", - "size": "6829396" - } - ] - } - ] + { + "name": "Arduino Nano Every" + } + ], + "toolsDependencies": [ + { + "packager": "arduino", + "name": "avr-gcc", + "version": "7.3.0-atmel3.6.1-arduino5" + }, + { + "packager": "arduino", + "name": "avrdude", + "version": "7.0-arduino.3" + }, + { + "packager": "arduino", + "name": "arduinoOTA", + "version": "1.3.0" + } + ] + } + ], + "tools": [ + { + "name": "arduino-fwuploader", + "version": "2.2.0", + "systems": [ + { + "host": "i686-mingw32", + "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.0_Windows_32bit.zip", + "archiveFileName": "arduino-fwuploader_2.2.0_Windows_32bit.zip", + "checksum": "SHA-256:c0ff772702460fb5cbd8593f8ce731145d21fbf550342da556e45ef946c7d2f5", + "size": "7442444" + }, + { + "host": "i386-apple-darwin11", + "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.0_macOS_64bit.tar.gz", + "archiveFileName": "arduino-fwuploader_2.2.0_macOS_64bit.tar.gz", + "checksum": "SHA-256:ce4444e92ba88c6b24736adb4b0b2b6c4241e4fb916945acbc7de6391d5bfe8c", + "size": "7301372" + } + ] + }, + { + "name": "arduino-fwuploader", + "version": "2.2.2", + "systems": [ + { + "host": "i686-linux-gnu", + "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Linux_32bit.tar.gz", + "archiveFileName": "arduino-fwuploader_2.2.2_Linux_32bit.tar.gz", + "checksum": "SHA-256:503b9f8b24c6e396d09eb64f0e1f625c6f9aa5a90b01a50d7dec6477f4a866f0", + "size": "7262873" + }, + { + "host": "x86_64-linux-gnu", + "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Linux_64bit.tar.gz", + "archiveFileName": "arduino-fwuploader_2.2.2_Linux_64bit.tar.gz", + "checksum": "SHA-256:8d77d0b33c8b0787fe3b80191709b69d638ef2a447d9853536cda35bfafd274b", + "size": "7306763" + }, + { + "host": "i686-mingw32", + "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Windows_32bit.zip", + "archiveFileName": "arduino-fwuploader_2.2.2_Windows_32bit.zip", + "checksum": "SHA-256:74ad9a5d369204b51be288c98d74f949ceb7a0c227ee64eb65ae179ec884c84c", + "size": "7450717" + }, + { + "host": "x86_64-mingw32", + "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Windows_64bit.zip", + "archiveFileName": "arduino-fwuploader_2.2.2_Windows_64bit.zip", + "checksum": "SHA-256:b25ac549cb0645166613c96cf899aebc541e482fe196aada6408bd7cff2c7d02", + "size": "7390999" + }, + { + "host": "x86_64-apple-darwin", + "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_macOS_64bit.tar.gz", + "archiveFileName": "arduino-fwuploader_2.2.2_macOS_64bit.tar.gz", + "checksum": "SHA-256:2cd6168ff470457b5124ba0faf118f315be2d1b9fb4fef43eb74370cd83620a2", + "size": "7306576" + }, + { + "host": "arm64-apple-darwin", + "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_macOS_ARM64.tar.gz", + "archiveFileName": "arduino-fwuploader_2.2.2_macOS_ARM64.tar.gz", + "checksum": "SHA-256:10ae5614af4d82096b6ba0e1e07aab667fa140d2bf1d5e3407dd8ad4c6748195", + "size": "6878214" + }, + { + "host": "arm-linux-gnueabihf", + "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Linux_ARMv6.tar.gz", + "archiveFileName": "arduino-fwuploader_2.2.2_Linux_ARMv6.tar.gz", + "checksum": "SHA-256:5aadf6e50ffe620635faf941fdf82c0765c8cba4830951bb53267ad125fc5af8", + "size": "6940393" + }, + { + "host": "aarch64-linux-gnu", + "url": "http://downloads.arduino.cc/arduino-fwuploader/arduino-fwuploader_2.2.2_Linux_ARM64.tar.gz", + "archiveFileName": "arduino-fwuploader_2.2.2_Linux_ARM64.tar.gz", + "checksum": "SHA-256:6d11a4f4aa5a81de865f3d18ca395a2780fdbb1e1597a2b11b2b5329e09f30fd", + "size": "6829396" + } + ] } - ] -} \ No newline at end of file + ] + } + ] +} diff --git a/v2/pkgs/pkgs.go b/v2/pkgs/pkgs.go index 9896fdd77..f4965117c 100644 --- a/v2/pkgs/pkgs.go +++ b/v2/pkgs/pkgs.go @@ -21,6 +21,8 @@ // cores, and to download tools used for upload. package pkgs +import "regexp" + // Index is the go representation of a typical // package-index file, stripped from every non-used field. type Index struct { @@ -34,11 +36,106 @@ type Index struct { // tool contained in a package-index file, stripped from // every non-used field. type Tool struct { - Name string `json:"name"` - Version string `json:"version"` - Systems []struct { - Host string `json:"host"` - URL string `json:"url"` - Checksum string `json:"checksum"` - } `json:"systems"` + Name string `json:"name"` + Version string `json:"version"` + Systems []System `json:"systems"` +} + +// System is the go representation of the info needed to +// download a tool for a specific OS/Arch +type System struct { + Host string `json:"host"` + URL string `json:"url"` + Name string `json:"archiveFileName"` + Checksum string `json:"checksum"` +} + +// Source: https://github.com/arduino/arduino-cli/blob/master/arduino/cores/tools.go#L129-L142 +var ( + regexpLinuxArm = regexp.MustCompile("arm.*-linux-gnueabihf") + regexpLinuxArm64 = regexp.MustCompile("(aarch64|arm64)-linux-gnu") + regexpLinux64 = regexp.MustCompile("x86_64-.*linux-gnu") + regexpLinux32 = regexp.MustCompile("i[3456]86-.*linux-gnu") + regexpWindows32 = regexp.MustCompile("i[3456]86-.*(mingw32|cygwin)") + regexpWindows64 = regexp.MustCompile("(amd64|x86_64)-.*(mingw32|cygwin)") + regexpMac64 = regexp.MustCompile("x86_64-apple-darwin.*") + regexpMac32 = regexp.MustCompile("i[3456]86-apple-darwin.*") + regexpMacArm64 = regexp.MustCompile("arm64-apple-darwin.*") + regexpFreeBSDArm = regexp.MustCompile("arm.*-freebsd[0-9]*") + regexpFreeBSD32 = regexp.MustCompile("i?[3456]86-freebsd[0-9]*") + regexpFreeBSD64 = regexp.MustCompile("amd64-freebsd[0-9]*") +) + +// Source: https://github.com/arduino/arduino-cli/blob/master/arduino/cores/tools.go#L144-L176 +func (s *System) isExactMatchWith(osName, osArch string) bool { + if s.Host == "all" { + return true + } + + switch osName + "," + osArch { + case "linux,arm", "linux,armbe": + return regexpLinuxArm.MatchString(s.Host) + case "linux,arm64": + return regexpLinuxArm64.MatchString(s.Host) + case "linux,amd64": + return regexpLinux64.MatchString(s.Host) + case "linux,386": + return regexpLinux32.MatchString(s.Host) + case "windows,386": + return regexpWindows32.MatchString(s.Host) + case "windows,amd64": + return regexpWindows64.MatchString(s.Host) + case "darwin,arm64": + return regexpMacArm64.MatchString(s.Host) + case "darwin,amd64": + return regexpMac64.MatchString(s.Host) + case "darwin,386": + return regexpMac32.MatchString(s.Host) + case "freebsd,arm": + return regexpFreeBSDArm.MatchString(s.Host) + case "freebsd,386": + return regexpFreeBSD32.MatchString(s.Host) + case "freebsd,amd64": + return regexpFreeBSD64.MatchString(s.Host) + } + return false +} + +// Source: https://github.com/arduino/arduino-cli/blob/master/arduino/cores/tools.go#L178-L198 +func (s *System) isCompatibleWith(osName, osArch string) (bool, int) { + if s.isExactMatchWith(osName, osArch) { + return true, 1000 + } + + switch osName + "," + osArch { + case "windows,amd64": + return regexpWindows32.MatchString(s.Host), 10 + case "darwin,amd64": + return regexpMac32.MatchString(s.Host), 10 + case "darwin,arm64": + // Compatibility guaranteed through Rosetta emulation + if regexpMac64.MatchString(s.Host) { + // Prefer amd64 version if available + return true, 20 + } + return regexpMac32.MatchString(s.Host), 10 + } + + return false, 0 +} + +// GetFlavourCompatibleWith returns the downloadable resource (System) compatible with the specified OS/Arch +// Source: https://github.com/arduino/arduino-cli/blob/master/arduino/cores/tools.go#L206-L216 +func (t *Tool) GetFlavourCompatibleWith(osName, osArch string) System { + var correctSystem System + maxSimilarity := -1 + + for _, s := range t.Systems { + if comp, similarity := s.isCompatibleWith(osName, osArch); comp && similarity > maxSimilarity { + correctSystem = s + maxSimilarity = similarity + } + } + + return correctSystem } diff --git a/v2/pkgs/tools.go b/v2/pkgs/tools.go index a320b82be..25ab85b18 100644 --- a/v2/pkgs/tools.go +++ b/v2/pkgs/tools.go @@ -33,7 +33,6 @@ import ( "github.com/arduino/arduino-create-agent/gen/tools" "github.com/codeclysm/extract/v3" - "github.com/xrash/smetrics" ) // Tools is a client that implements github.com/arduino/arduino-create-agent/gen/tools.Service interface. @@ -161,9 +160,9 @@ func (c *Tools) Install(ctx context.Context, payload *tools.ToolPayload) (*tools if tool.Name == payload.Name && tool.Version == payload.Version { - i := findSystem(tool) + sys := tool.GetFlavourCompatibleWith(runtime.GOOS, runtime.GOARCH) - return c.install(ctx, path, tool.Systems[i].URL, tool.Systems[i].Checksum) + return c.install(ctx, path, sys.URL, sys.Checksum) } } } @@ -236,30 +235,6 @@ func rename(base string) extract.Renamer { } } -func findSystem(tool Tool) int { - var systems = map[string]string{ - "linuxamd64": "x86_64-linux-gnu", - "linux386": "i686-linux-gnu", - "darwinamd64": "apple-darwin", - "windows386": "i686-mingw32", - "windowsamd64": "i686-mingw32", - "linuxarm": "arm-linux-gnueabihf", - } - - var correctSystem int - maxSimilarity := 0.7 - - for i, system := range tool.Systems { - similarity := smetrics.Jaro(system.Host, systems[runtime.GOOS+runtime.GOARCH]) - if similarity > maxSimilarity { - correctSystem = i - maxSimilarity = similarity - } - } - - return correctSystem -} - func writeInstalled(folder, path string) error { // read installed.json installed := map[string]string{}