Use codeclysm/extract and remove custom implementation (#867)

* replace test with a meaningful one
The tools are the one coming with the cores installed in the builder

* use `codeclysm/extract` and remove extract code

* fix cleanup in tests

* enlarge the scope of the test: verify dir tree is handled correctly

* the archives are first extracted in tmp and then moved to correct place
we remove the root dir if we find one.
The logic is similar to: https://github.com/arduino/arduino-cli/blob/7a146635aaa740e748b84bf8fbfdccf1cc420c61/arduino/resources/install.go#L34

* fix test failing

Author: umbynos

tools/download.go

@@ -16,11 +16,8 @@
 package tools
 
 import (
-	"archive/tar"
-	"archive/zip"
 	"bytes"
-	"compress/bzip2"
-	"compress/gzip"
+	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
@@ -30,14 +27,13 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"runtime"
-	"strings"
 
-	"github.com/arduino/arduino-create-agent/utilities"
 	"github.com/arduino/arduino-create-agent/v2/pkgs"
+	"github.com/arduino/go-paths-helper"
 	"github.com/blang/semver"
+	"github.com/codeclysm/extract/v3"
 )
 
 // public vars to allow override in the tests
@@ -46,10 +42,6 @@ var (
 	Arch = runtime.GOARCH
 )
 
-func mimeType(data []byte) (string, error) {
-	return http.DetectContentType(data[0:512]), nil
-}
-
 func pathExists(path string) bool {
 	_, err := os.Stat(path)
 	if err == nil {
@@ -129,39 +121,46 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
 		return errors.New("checksum doesn't match")
 	}
 
-	// Decompress
-	t.logger("Unpacking tool " + name)
-
-	location := t.directory.Join(pack, correctTool.Name, correctTool.Version).String()
-	err = os.RemoveAll(location)
-
+	tempPath := paths.TempDir()
+	// Create a temporary dir to extract package
+	if err := tempPath.MkdirAll(); err != nil {
+		return fmt.Errorf("creating temp dir for extraction: %s", err)
+	}
+	tempDir, err := tempPath.MkTempDir("package-")
 	if err != nil {
-		return err
+		return fmt.Errorf("creating temp dir for extraction: %s", err)
 	}
+	defer tempDir.RemoveAll()
 
-	srcType, err := mimeType(body)
+	t.logger("Unpacking tool " + name)
+	ctx := context.Background()
+	reader := bytes.NewReader(body)
+	// Extract into temp directory
+	if err := extract.Archive(ctx, reader, tempDir.String(), nil); err != nil {
+		return fmt.Errorf("extracting archive: %s", err)
+	}
+
+	location := t.directory.Join(pack, correctTool.Name, correctTool.Version)
+	err = location.RemoveAll()
 	if err != nil {
 		return err
 	}
 
-	switch srcType {
-	case "application/zip":
-		location, err = extractZip(t.logger, body, location)
-	case "application/x-bz2":
-	case "application/octet-stream":
-		location, err = extractBz2(t.logger, body, location)
-	case "application/x-gzip":
-		location, err = extractTarGz(t.logger, body, location)
-	default:
-		return errors.New("Unknown extension for file " + correctSystem.URL)
+	// Check package content and find package root dir
+	root, err := findPackageRoot(tempDir)
+	if err != nil {
+		return fmt.Errorf("searching package root dir: %s", err)
 	}
 
-	if err != nil {
-		t.logger("Error extracting the archive: " + err.Error())
-		return err
+	if err := root.Rename(location); err != nil {
+		if err := root.CopyDirTo(location); err != nil {
+			return fmt.Errorf("moving extracted archive to destination dir: %s", err)
+		}
 	}
 
-	err = t.installDrivers(location)
+	// if the tool contains a post_install script, run it: it means it is a tool that needs to install drivers
+	// AFAIK this is only the case for the windows-driver tool
+	err = t.installDrivers(location.String())
 	if err != nil {
 		return err
 	}
@@ -170,13 +169,27 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
 	t.logger("Ensure that the files are executable")
 
 	// Update the tool map
-	t.logger("Updating map with location " + location)
+	t.logger("Updating map with location " + location.String())
 
-	t.setMapValue(name, location)
-	t.setMapValue(name+"-"+correctTool.Version, location)
+	t.setMapValue(name, location.String())
+	t.setMapValue(name+"-"+correctTool.Version, location.String())
 	return t.writeMap()
 }
 
+func findPackageRoot(parent *paths.Path) (*paths.Path, error) {
+	files, err := parent.ReadDir()
+	if err != nil {
+		return nil, fmt.Errorf("reading package root dir: %s", err)
+	}
+	files.FilterOutPrefix("__MACOSX")
+
+	// if there is only one dir, it is the root dir
+	if len(files) == 1 && files[0].IsDir() {
+		return files[0], nil
+	}
+	return parent, nil
+}
+
 func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.System) {
 	var correctTool pkgs.Tool
 	correctTool.Version = "0.0"
@@ -207,258 +220,6 @@ func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.Syst
 	return correctTool, correctSystem
 }
 
-func commonPrefix(sep byte, paths []string) string {
-	// Handle special cases.
-	switch len(paths) {
-	case 0:
-		return ""
-	case 1:
-		return path.Clean(paths[0])
-	}
-
-	c := []byte(path.Clean(paths[0]))
-
-	// We add a trailing sep to handle: common prefix directory is included in the path list
-	// (e.g. /home/user1, /home/user1/foo, /home/user1/bar).
-	// path.Clean will have cleaned off trailing / separators with
-	// the exception of the root directory, "/" making it "//"
-	// but this will get fixed up to "/" below).
-	c = append(c, sep)
-
-	// Ignore the first path since it's already in c
-	for _, v := range paths[1:] {
-		// Clean up each path before testing it
-		v = path.Clean(v) + string(sep)
-
-		// Find the first non-common byte and truncate c
-		if len(v) < len(c) {
-			c = c[:len(v)]
-		}
-		for i := 0; i < len(c); i++ {
-			if v[i] != c[i] {
-				c = c[:i]
-				break
-			}
-		}
-	}
-
-	// Remove trailing non-separator characters and the final separator
-	for i := len(c) - 1; i >= 0; i-- {
-		if c[i] == sep {
-			c = c[:i]
-			break
-		}
-	}
-
-	return string(c)
-}
-
-func removeStringFromSlice(s []string, r string) []string {
-	for i, v := range s {
-		if v == r {
-			return append(s[:i], s[i+1:]...)
-		}
-	}
-	return s
-}
-
-func findBaseDir(dirList []string) string {
-	if len(dirList) == 1 {
-		return path.Dir(dirList[0]) + "/"
-	}
-
-	// https://github.com/backdrop-ops/contrib/issues/55#issuecomment-73814500
-	dontdiff := []string{"pax_global_header"}
-	for _, v := range dontdiff {
-		dirList = removeStringFromSlice(dirList, v)
-	}
-
-	commonBaseDir := commonPrefix('/', dirList)
-	if commonBaseDir != "" {
-		commonBaseDir = commonBaseDir + "/"
-	}
-	return commonBaseDir
-}
-
-func extractZip(log func(msg string), body []byte, location string) (string, error) {
-	path, _ := utilities.SaveFileonTempDir("tooldownloaded.zip", bytes.NewReader(body))
-	r, err := zip.OpenReader(path)
-	if err != nil {
-		return location, err
-	}
-
-	var dirList []string
-
-	for _, f := range r.File {
-		dirList = append(dirList, f.Name)
-	}
-
-	basedir := findBaseDir(dirList)
-	log(fmt.Sprintf("selected baseDir %s from Zip Archive Content: %v", basedir, dirList))
-
-	for _, f := range r.File {
-		fullname := filepath.Join(location, strings.Replace(f.Name, basedir, "", -1))
-		log(fmt.Sprintf("generated fullname %s removing %s from %s", fullname, basedir, f.Name))
-		if f.FileInfo().IsDir() {
-			os.MkdirAll(fullname, f.FileInfo().Mode().Perm())
-		} else {
-			os.MkdirAll(filepath.Dir(fullname), 0755)
-			perms := f.FileInfo().Mode().Perm()
-			out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms)
-			if err != nil {
-				return location, err
-			}
-			rc, err := f.Open()
-			if err != nil {
-				return location, err
-			}
-			_, err = io.CopyN(out, rc, f.FileInfo().Size())
-			if err != nil {
-				return location, err
-			}
-			rc.Close()
-			out.Close()
-
-			mtime := f.FileInfo().ModTime()
-			err = os.Chtimes(fullname, mtime, mtime)
-			if err != nil {
-				return location, err
-			}
-		}
-	}
-	return location, nil
-}
-
-func extractTarGz(log func(msg string), body []byte, location string) (string, error) {
-	bodyCopy := make([]byte, len(body))
-	copy(bodyCopy, body)
-	tarFile, _ := gzip.NewReader(bytes.NewReader(body))
-	tarReader := tar.NewReader(tarFile)
-
-	var dirList []string
-
-	for {
-		header, err := tarReader.Next()
-		if err == io.EOF {
-			break
-		}
-		dirList = append(dirList, header.Name)
-	}
-
-	basedir := findBaseDir(dirList)
-	log(fmt.Sprintf("selected baseDir %s from TarGz Archive Content: %v", basedir, dirList))
-
-	tarFile, _ = gzip.NewReader(bytes.NewReader(bodyCopy))
-	tarReader = tar.NewReader(tarFile)
-
-	for {
-		header, err := tarReader.Next()
-		if err == io.EOF {
-			break
-		} else if err != nil {
-			return location, err
-		}
-
-		path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
-		info := header.FileInfo()
-
-		// Create parent folder
-		dirmode := info.Mode() | os.ModeDir | 0700
-		if err = os.MkdirAll(filepath.Dir(path), dirmode); err != nil {
-			return location, err
-		}
-
-		if info.IsDir() {
-			if err = os.MkdirAll(path, info.Mode()); err != nil {
-				return location, err
-			}
-			continue
-		}
-
-		if header.Typeflag == tar.TypeSymlink {
-			_ = os.Symlink(header.Linkname, path)
-			continue
-		}
-
-		file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
-		if err != nil {
-			continue
-		}
-		_, err = io.Copy(file, tarReader)
-		if err != nil {
-			return location, err
-		}
-		file.Close()
-	}
-	return location, nil
-}
-
-func extractBz2(log func(msg string), body []byte, location string) (string, error) {
-	bodyCopy := make([]byte, len(body))
-	copy(bodyCopy, body)
-	tarFile := bzip2.NewReader(bytes.NewReader(body))
-	tarReader := tar.NewReader(tarFile)
-
-	var dirList []string
-
-	for {
-		header, err := tarReader.Next()
-		if err == io.EOF {
-			break
-		}
-		dirList = append(dirList, header.Name)
-	}
-
-	basedir := findBaseDir(dirList)
-	log(fmt.Sprintf("selected baseDir %s from Bz2 Archive Content: %v", basedir, dirList))
-
-	tarFile = bzip2.NewReader(bytes.NewReader(bodyCopy))
-	tarReader = tar.NewReader(tarFile)
-
-	for {
-		header, err := tarReader.Next()
-		if err == io.EOF {
-			break
-		} else if err != nil {
-			continue
-			//return location, err
-		}
-
-		path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
-		info := header.FileInfo()
-
-		// Create parent folder
-		dirmode := info.Mode() | os.ModeDir | 0700
-		if err = os.MkdirAll(filepath.Dir(path), dirmode); err != nil {
-			return location, err
-		}
-
-		if info.IsDir() {
-			if err = os.MkdirAll(path, info.Mode()); err != nil {
-				return location, err
-			}
-			continue
-		}
-
-		if header.Typeflag == tar.TypeSymlink {
-			_ = os.Symlink(header.Linkname, path)
-			continue
-		}
-
-		file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
-		if err != nil {
-			continue
-			//return location, err
-		}
-		_, err = io.Copy(file, tarReader)
-		if err != nil {
-			return location, err
-		}
-		file.Close()
-	}
-	return location, nil
-}
-
 func (t *Tools) installDrivers(location string) error {
 	OkPressed := 6
 	extension := ".bat"

tools/download_test.go

@@ -17,16 +17,13 @@ package tools
 
 import (
 	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"path"
+	"runtime"
 	"testing"
+	"time"
 
+	"github.com/arduino/arduino-create-agent/index"
 	"github.com/arduino/arduino-create-agent/v2/pkgs"
 	"github.com/arduino/go-paths-helper"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -44,6 +41,10 @@ func TestDownloadCorrectPlatform(t *testing.T) {
 		{"windows", "amd64", "x86_64-mingw32"},
 		{"linux", "arm", "arm-linux-gnueabihf"},
 	}
+	defer func() {
+		OS = runtime.GOOS     // restore `runtime.OS`
+		Arch = runtime.GOARCH // restore `runtime.ARCH`
+	}()
 	testIndex := paths.New("testdata", "test_tool_index.json")
 	buf, err := testIndex.ReadFile()
 	require.NoError(t, err)
@@ -76,6 +77,10 @@ func TestDownloadFallbackPlatform(t *testing.T) {
 		{"darwin", "arm64", "i386-apple-darwin11"},
 		{"windows", "amd64", "i686-mingw32"},
 	}
+	defer func() {
+		OS = runtime.GOOS     // restore `runtime.OS`
+		Arch = runtime.GOARCH // restore `runtime.ARCH`
+	}()
 	testIndex := paths.New("testdata", "test_tool_index.json")
 	buf, err := testIndex.ReadFile()
 	require.NoError(t, err)
@@ -98,99 +103,59 @@ func TestDownloadFallbackPlatform(t *testing.T) {
 	}
 }
 
-func Test_findBaseDir(t *testing.T) {
-	cases := []struct {
-		dirList []string
-		want    string
+func TestDownload(t *testing.T) {
+	testCases := []struct {
+		name         string
+		version      string
+		filesCreated []string
 	}{
-		{[]string{"bin/bossac"}, "bin/"},
-		{[]string{"bin/", "bin/bossac"}, "bin/"},
-		{[]string{"bin/", "bin/bossac", "example"}, ""},
-		{[]string{"avrdude/bin/",
-			"avrdude/bin/avrdude.exe",
-			"avrdude/bin/remove_giveio.bat",
-			"avrdude/bin/status_giveio.bat",
-			"avrdude/bin/giveio.sys",
-			"avrdude/bin/loaddrv.exe",
-			"avrdude/bin/libusb0.dll",
-			"avrdude/bin/install_giveio.bat",
-			"avrdude/etc/avrdude.conf"}, "avrdude/"},
-		{[]string{"pax_global_header", "bin/", "bin/bossac"}, "bin/"},
+		{"avrdude", "6.3.0-arduino17", []string{"bin", "etc"}},
+		{"bossac", "1.6.1-arduino", []string{"bossac"}},
+		{"bossac", "1.7.0-arduino3", []string{"bossac"}},
+		{"bossac", "1.9.1-arduino2", []string{"bossac"}},
+		{"openocd", "0.11.0-arduino2", []string{"bin", "share"}},
+		{"dfu-util", "0.10.0-arduino1", []string{"dfu-prefix", "dfu-suffix", "dfu-util"}},
+		{"rp2040tools", "1.0.6", []string{"elf2uf2", "picotool", "pioasm", "rp2040load"}},
+		{"esptool_py", "4.5.1", []string{"esptool"}},
+		{"arduino-fwuploader", "2.2.2", []string{"arduino-fwuploader"}},
+		{"fwupdater", "0.1.12", []string{"firmwares", "FirmwareUploader"}}, // old legacy tool
 	}
-	for _, tt := range cases {
-		t.Run(fmt.Sprintln(tt.dirList), func(t *testing.T) {
-			if got := findBaseDir(tt.dirList); got != tt.want {
-				t.Errorf("findBaseDir() = got %v, want %v", got, tt.want)
-			}
-		})
+	// prepare the test environment
+	tempDir := t.TempDir()
+	tempDirPath := paths.New(tempDir)
+	testIndex := index.Resource{
+		IndexFile:   *paths.New("testdata", "test_tool_index.json"),
+		LastRefresh: time.Now(),
 	}
-}
-
-func TestTools_DownloadAndUnpackBehaviour(t *testing.T) {
-	urls := []string{
-		"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-armhf-pc-linux-gnu.tar.bz2",
-		"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-aarch64-pc-linux-gnu.tar.bz2",
-		"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i386-apple-darwin11.tar.bz2",
-		"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-x86_64-pc-linux-gnu.tar.bz2",
-		"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i686-pc-linux-gnu.tar.bz2",
-		"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i686-w64-mingw32.zip",
-	}
-	expectedDirList := []string{"bin", "etc"}
-
-	tmpDir, err := os.MkdirTemp("", "download_test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	for _, url := range urls {
-		t.Log("Downloading tool from " + url)
-		resp, err := http.Get(url)
-		if err != nil {
-			t.Errorf("%v", err)
-		}
-		defer resp.Body.Close()
+	testTools := New(tempDirPath, &testIndex, func(msg string) { t.Log(msg) })
 
-		// Read the body
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			t.Errorf("%v", err)
-		}
-
-		location := path.Join(tmpDir, "username", "arduino", "avrdude", "6.3.0-arduino14")
-		os.MkdirAll(location, os.ModePerm)
-		err = os.RemoveAll(location)
-
-		if err != nil {
-			t.Errorf("%v", err)
-		}
-
-		srcType, err := mimeType(body)
-		if err != nil {
-			t.Errorf("%v", err)
-		}
-
-		switch srcType {
-		case "application/zip":
-			location, err = extractZip(func(msg string) { t.Log(msg) }, body, location)
-		case "application/x-bz2":
-		case "application/octet-stream":
-			location, err = extractBz2(func(msg string) { t.Log(msg) }, body, location)
-		case "application/x-gzip":
-			location, err = extractTarGz(func(msg string) { t.Log(msg) }, body, location)
-		default:
-			t.Errorf("no suitable type found")
-		}
-		files, err := os.ReadDir(location)
-		if err != nil {
-			t.Errorf("%v", err)
-		}
-		dirList := []string{}
-		for _, f := range files {
-			dirList = append(dirList, f.Name())
-		}
+	for _, tc := range testCases {
+		t.Run(tc.name+"-"+tc.version, func(t *testing.T) {
+			// Download the tool
+			err := testTools.Download("arduino-test", tc.name, tc.version, "replace")
+			require.NoError(t, err)
+
+			// Check that the tool has been downloaded
+			toolDir := tempDirPath.Join("arduino-test", tc.name, tc.version)
+			require.DirExists(t, toolDir.String())
+
+			// Check that the files have been created
+			for _, file := range tc.filesCreated {
+				filePath := toolDir.Join(file)
+				if filePath.IsDir() {
+					require.DirExists(t, filePath.String())
+				} else {
+					if OS == "windows" {
+						require.FileExists(t, filePath.String()+".exe")
+					} else {
+						require.FileExists(t, filePath.String())
+					}
+				}
+			}
 
-		assert.ElementsMatchf(t, dirList, expectedDirList, "error message %s", "formatted")
+			// Check that the tool has been installed
+			_, ok := testTools.getMapValue(tc.name + "-" + tc.version)
+			require.True(t, ok)
+		})
 	}
-
 }

tools/testdata/test_tool_index.json

@@ -49,6 +49,390 @@
         }
       ],
       "tools": [
+        {
+          "name": "avrdude",
+          "version": "6.3.0-arduino17",
+          "systems": [
+            {
+              "size": "219631",
+              "checksum": "SHA-256:2a8e68c5d803aa6f902ef219f177ec3a4c28275d85cbe272962ad2cd374f50d1",
+              "host": "arm-linux-gnueabihf",
+              "archiveFileName": "avrdude-6.3.0-arduino17-armhf-pc-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-armhf-pc-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "229852",
+              "checksum": "SHA-256:6cf948f751acfe7b96684537f2291c766ec8b54b4f7dc95539864821456fa9fc",
+              "host": "aarch64-linux-gnu",
+              "archiveFileName": "avrdude-6.3.0-arduino17-aarch64-pc-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-aarch64-pc-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "279045",
+              "checksum": "SHA-256:120cc9edaae699e7e9ac50b1b8eb0e7d51fdfa555bac54233c2511e6ee5418c9",
+              "host": "x86_64-apple-darwin12",
+              "archiveFileName": "avrdude-6.3.0-arduino17-x86_64-apple-darwin12.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-x86_64-apple-darwin12.tar.bz2"
+            },
+            {
+              "size": "254271",
+              "checksum": "SHA-256:accdfb920af2aabf4f7461d2ac73c0751760f525216dc4e7657427a78c60d13d",
+              "host": "x86_64-linux-gnu",
+              "archiveFileName": "avrdude-6.3.0-arduino17-x86_64-pc-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-x86_64-pc-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "244550",
+              "checksum": "SHA-256:5c8cc6c17db9300e1451fe41cd7178b0442b4490ee6fdbc0aed9811aef96c05f",
+              "host": "i686-linux-gnu",
+              "archiveFileName": "avrdude-6.3.0-arduino17-i686-pc-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-i686-pc-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "328460",
+              "checksum": "SHA-256:e99188873c7c5ad8f8f906f068c33600e758b2e36cce3adbd518a21bd266749d",
+              "host": "i686-mingw32",
+              "archiveFileName": "avrdude-6.3.0-arduino17-i686-w64-mingw32.zip",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-i686-w64-mingw32.zip"
+            }
+          ]
+        },
+        {
+          "name": "bossac",
+          "version": "1.6.1-arduino",
+          "systems": [
+            {
+              "host": "arm-linux-gnueabihf",
+              "url": "http://downloads.arduino.cc/bossac-1.6.1-arduino-arm-linux-gnueabihf.tar.bz2",
+              "archiveFileName": "bossac-1.6.1-arduino-arm-linux-gnueabihf.tar.bz2",
+              "checksum": "SHA-256:8c4e63db982178919c824e7a35580dffc95c3426afa7285de3eb583982d4d391",
+              "size": "201341"
+            },
+            {
+              "host": "i686-mingw32",
+              "url": "http://downloads.arduino.cc/bossac-1.6.1-arduino-mingw32.tar.gz",
+              "archiveFileName": "bossac-1.6.1-arduino-mingw32.tar.gz",
+              "checksum": "SHA-256:d59f43e2e83a337d04c4ae88b195a4ee175b8d87fff4c43144d23412a4a9513b",
+              "size": "222918"
+            },
+            {
+              "host": "x86_64-apple-darwin",
+              "url": "http://downloads.arduino.cc/bossac-1.6.1-arduino-i386-apple-darwin14.5.0.tar.gz",
+              "archiveFileName": "bossac-1.6.1-arduino-i386-apple-darwin14.5.0.tar.gz",
+              "checksum": "SHA-256:2f80ef569a3fb19da60ab3489e49d8fe7d4699876acf30ff4938c632230a09aa",
+              "size": "64587"
+            },
+            {
+              "host": "x86_64-pc-linux-gnu",
+              "url": "http://downloads.arduino.cc/bossac-1.6.1-arduino-x86_64-linux-gnu.tar.gz",
+              "archiveFileName": "bossac-1.6.1-arduino-x86_64-linux-gnu.tar.gz",
+              "checksum": "SHA-256:b78afc66c00ccfdd69a08bd3959c260a0c64ccce78a71d5a1135ae4437ff40db",
+              "size": "30869"
+            },
+            {
+              "host": "i686-pc-linux-gnu",
+              "url": "http://downloads.arduino.cc/bossac-1.6.1-arduino-i486-linux-gnu.tar.gz",
+              "archiveFileName": "bossac-1.6.1-arduino-i486-linux-gnu.tar.gz",
+              "checksum": "SHA-256:1e211347569d75193b337296a10dd25b0ce04419e3d7dc644355178b6b514f92",
+              "size": "30320"
+            }
+          ]
+        },
+        {
+          "name": "bossac",
+          "version": "1.7.0-arduino3",
+          "systems": [
+            {
+              "host": "i686-mingw32",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-windows.tar.gz",
+              "archiveFileName": "bossac-1.7.0-arduino3-windows.tar.gz",
+              "checksum": "SHA-256:62745cc5a98c26949ec9041ef20420643c561ec43e99dae659debf44e6836526",
+              "size": "3607421"
+            },
+            {
+              "host": "x86_64-apple-darwin",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-osx.tar.gz",
+              "archiveFileName": "bossac-1.7.0-arduino3-osx.tar.gz",
+              "checksum": "SHA-256:adb3c14debd397d8135e9e970215c6972f0e592c7af7532fa15f9ce5e64b991f",
+              "size": "75510"
+            },
+            {
+              "host": "x86_64-pc-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-linux64.tar.gz",
+              "archiveFileName": "bossac-1.7.0-arduino3-linux64.tar.gz",
+              "checksum": "SHA-256:1ae54999c1f97234a5c603eb99ad39313b11746a4ca517269a9285afa05f9100",
+              "size": "207271"
+            },
+            {
+              "host": "i686-pc-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-linux32.tar.gz",
+              "archiveFileName": "bossac-1.7.0-arduino3-linux32.tar.gz",
+              "checksum": "SHA-256:4ac4354746d1a09258f49a43ef4d1baf030d81c022f8434774268b00f55d3ec3",
+              "size": "193577"
+            },
+            {
+              "host": "arm-linux-gnueabihf",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-linuxarm.tar.gz",
+              "archiveFileName": "bossac-1.7.0-arduino3-linuxarm.tar.gz",
+              "checksum": "SHA-256:626c6cc548046901143037b782bf019af1663bae0d78cf19181a876fb9abbb90",
+              "size": "193941"
+            },
+            {
+              "host": "aarch64-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-linuxaarch64.tar.gz",
+              "archiveFileName": "bossac-1.7.0-arduino3-linuxaarch64.tar.gz",
+              "checksum": "SHA-256:a098b2cc23e29f0dc468416210d097c4a808752cd5da1a7b9b8b7b931a04180b",
+              "size": "268365"
+            }
+          ]
+        },
+        {
+          "name": "bossac",
+          "version": "1.9.1-arduino2",
+          "systems": [
+            {
+              "host": "i686-mingw32",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-windows.tar.gz",
+              "archiveFileName": "bossac-1.9.1-arduino2-windows.tar.gz",
+              "checksum": "SHA-256:5c994d04354f0db8e4bea136f49866d2ba537f0af74b2e78026f2d4fc75e3e39",
+              "size": "1260628"
+            },
+            {
+              "host": "x86_64-apple-darwin",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-osx.tar.gz",
+              "archiveFileName": "bossac-1.9.1-arduino2-osx.tar.gz",
+              "checksum": "SHA-256:b7732129364a378676604db6579c9b8dab50dd965fb50d7a3afff1839c97ff80",
+              "size": "47870"
+            },
+            {
+              "host": "x86_64-pc-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-linux64.tar.gz",
+              "archiveFileName": "bossac-1.9.1-arduino2-linux64.tar.gz",
+              "checksum": "SHA-256:9eb549874391521999cee13dc823a2cfc8866b8246945339a281808d99c72d2c",
+              "size": "399532"
+            },
+            {
+              "host": "i686-pc-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-linux32.tar.gz",
+              "archiveFileName": "bossac-1.9.1-arduino2-linux32.tar.gz",
+              "checksum": "SHA-256:10d69f53f169f25afee2dd583dfd9dc803c10543e6c5260d106725cb0d174900",
+              "size": "384951"
+            },
+            {
+              "host": "arm-linux-gnueabihf",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-linuxarm.tar.gz",
+              "archiveFileName": "bossac-1.9.1-arduino2-linuxarm.tar.gz",
+              "checksum": "SHA-256:c9539d161d23231b5beb1d09a71829744216c7f5bc2857a491999c3e567f5b19",
+              "size": "361915"
+            },
+            {
+              "host": "aarch64-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-linuxaarch64.tar.gz",
+              "archiveFileName": "bossac-1.9.1-arduino2-linuxaarch64.tar.gz",
+              "checksum": "SHA-256:c167fa0ea223966f4d21f5592da3888bcbfbae385be6c5c4e41f8abff35f5cb1",
+              "size": "442853"
+            }
+          ]
+        },
+        {
+          "name": "openocd",
+          "version": "0.11.0-arduino2",
+          "systems": [
+            {
+              "size": "1902818",
+              "checksum": "SHA-256:a1aa7f1435a61eafb72ee90722f2496d6a34a7a0f085d0315c2613e4a548b824",
+              "host": "aarch64-linux-gnu",
+              "archiveFileName": "openocd-0.11.0-arduino2-static-aarch64-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-aarch64-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "1986716",
+              "checksum": "SHA-256:57041130160be086e69387cceb4616eefc9819a0ef75de1f7c11aea57fb92699",
+              "host": "arm-linux-gnueabihf",
+              "archiveFileName": "openocd-0.11.0-arduino2-static-arm-linux-gnueabihf.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-arm-linux-gnueabihf.tar.bz2"
+            },
+            {
+              "size": "1971364",
+              "checksum": "SHA-256:6f4a8b77c8076aa18afb8438472526dff8c0d161a3ca68d0326163b59fcab663",
+              "host": "i686-linux-gnu",
+              "archiveFileName": "openocd-0.11.0-arduino2-static-i686-ubuntu12.04-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-i686-ubuntu12.04-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "2460087",
+              "checksum": "SHA-256:631010980f12b1e750c4c67ce012b31c5953caabf4d30607d806e3d2b717d4b8",
+              "host": "i686-mingw32",
+              "archiveFileName": "openocd-0.11.0-arduino2-static-i686-w64-mingw32.zip",
+              "url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-i686-w64-mingw32.zip"
+            },
+            {
+              "size": "1893150",
+              "checksum": "SHA-256:280e7234eba84e830e92d791ebc685286f71d2bc1d3347f93605ef170d54fef4",
+              "host": "i386-apple-darwin11",
+              "archiveFileName": "openocd-0.11.0-arduino2-static-x86_64-apple-darwin13.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-x86_64-apple-darwin13.tar.bz2"
+            },
+            {
+              "size": "2052080",
+              "checksum": "SHA-256:4d19b6e3906de1434ec86841e0e3138235714c655d45f037c0fabfa5e5c0681b",
+              "host": "x86_64-linux-gnu",
+              "archiveFileName": "openocd-0.11.0-arduino2-static-x86_64-ubuntu12.04-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-x86_64-ubuntu12.04-linux-gnu.tar.bz2"
+            }
+          ]
+        },
+        {
+          "name": "dfu-util",
+          "version": "0.10.0-arduino1",
+          "systems": [
+            {
+              "host": "i386-apple-darwin11",
+              "url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-osx.tar.bz2",
+              "archiveFileName": "dfu-util-0.10.0-arduino1-osx.tar.bz2",
+              "size": "73921",
+              "checksum": "SHA-256:7562d128036759605828d64b8d672d42445a8d95555c4b9ba339f73a1711a640"
+            },
+            {
+              "host": "arm-linux-gnueabihf",
+              "url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-arm.tar.bz2",
+              "archiveFileName": "dfu-util-0.10.0-arduino1-arm.tar.bz2",
+              "size": "272153",
+              "checksum": "SHA-256:f1e550f40c235356b7fde1c59447bfbab28f768915d3c14bd858fe0576bfc5a9"
+            },
+            {
+              "host": "aarch64-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-arm64.tar.bz2",
+              "archiveFileName": "dfu-util-0.10.0-arduino1-arm64.tar.bz2",
+              "size": "277886",
+              "checksum": "SHA-256:ebfbd21d3030c500da1f83b9aae5b8c597bee04c3bde1ce0a51b41abeafc9614"
+            },
+            {
+              "host": "x86_64-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-linux64.tar.bz2",
+              "archiveFileName": "dfu-util-0.10.0-arduino1-linux64.tar.bz2",
+              "size": "77184",
+              "checksum": "SHA-256:13ef2ec591c1e8b0b7eb0a05da972ecd6695016e7a9607e332c7553899af9b4a"
+            },
+            {
+              "host": "i686-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-linux32.tar.bz2",
+              "archiveFileName": "dfu-util-0.10.0-arduino1-linux32.tar.bz2",
+              "size": "81826",
+              "checksum": "SHA-256:43599ec60c000e9ef016970a496d6ab2cbbe5a8b7df9d06ef3114ecf83f9d123"
+            },
+            {
+              "host": "i686-mingw32",
+              "url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-windows.tar.bz2",
+              "archiveFileName": "dfu-util-0.10.0-arduino1-windows.tar.bz2",
+              "size": "464314",
+              "checksum": "SHA-256:90816b669273ae796d734a2459c46bb340d4790783fd7aa01eb40c0443f1a9b1"
+            }
+          ]
+        },
+        {
+          "name": "rp2040tools",
+          "version": "1.0.6",
+          "systems": [
+            {
+              "host": "i386-apple-darwin11",
+              "url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-darwin_amd64.tar.bz2",
+              "archiveFileName": "rp2040tools-1.0.6-darwin_amd64.tar.bz2",
+              "size": "1717967",
+              "checksum": "SHA-256:4e32aa4b8f36db40a17bfbdfd34d80da91710e30c3887732bf0c0bf0b02840a7"
+            },
+            {
+              "host": "arm-linux-gnueabihf",
+              "url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-linux_arm.tar.bz2",
+              "archiveFileName": "rp2040tools-1.0.6-linux_arm.tar.bz2",
+              "size": "8702508",
+              "checksum": "SHA-256:084a29accf0014bc79723fbb40057b95299c7ae63876f74494a077c987014cc3"
+            },
+            {
+              "host": "aarch64-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-linux_arm64.tar.bz2",
+              "archiveFileName": "rp2040tools-1.0.6-linux_arm64.tar.bz2",
+              "size": "9037783",
+              "checksum": "SHA-256:1a2a6cb1abf1f7b8198d494c8d8e838700297d748877be8232e02aaa5ca8d0df"
+            },
+            {
+              "host": "x86_64-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-linux_amd64.tar.bz2",
+              "archiveFileName": "rp2040tools-1.0.6-linux_amd64.tar.bz2",
+              "size": "6108121",
+              "checksum": "SHA-256:6e2ea818db1ff57f2d8e1e3010fbc5bdb5f28ff44f5a68900cae41d7d709f738"
+            },
+            {
+              "host": "i686-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-linux_386.tar.bz2",
+              "archiveFileName": "rp2040tools-1.0.6-linux_386.tar.bz2",
+              "size": "6604083",
+              "checksum": "SHA-256:ef339e2e0f5c7d5464b9911b612c634767daba39a6be977a1ffa41c95b9827a1"
+            },
+            {
+              "host": "i686-mingw32",
+              "url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-windows_386.tar.bz2",
+              "archiveFileName": "rp2040tools-1.0.6-windows_386.tar.bz2",
+              "size": "3145329",
+              "checksum": "SHA-256:26a5daebba68c2348dade33716a6e379ded89895ef0e49df1332964a724f6170"
+            }
+          ]
+        },
+        {
+          "name": "esptool_py",
+          "version": "4.5.1",
+          "systems": [
+            {
+              "host": "x86_64-pc-linux-gnu",
+              "url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-src.tar.gz",
+              "archiveFileName": "esptool-v4.5.1-src.tar.gz",
+              "checksum": "SHA-256:aa06831a7d88d8ccde4ea21241e983a08dbdae967290e181658b0d18bffc8f86",
+              "size": "96922"
+            },
+            {
+              "host": "i686-pc-linux-gnu",
+              "url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-src.tar.gz",
+              "archiveFileName": "esptool-v4.5.1-src.tar.gz",
+              "checksum": "SHA-256:aa06831a7d88d8ccde4ea21241e983a08dbdae967290e181658b0d18bffc8f86",
+              "size": "96922"
+            },
+            {
+              "host": "aarch64-linux-gnu",
+              "url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-src.tar.gz",
+              "archiveFileName": "esptool-v4.5.1-src.tar.gz",
+              "checksum": "SHA-256:aa06831a7d88d8ccde4ea21241e983a08dbdae967290e181658b0d18bffc8f86",
+              "size": "96922"
+            },
+            {
+              "host": "arm-linux-gnueabihf",
+              "url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-src.tar.gz",
+              "archiveFileName": "esptool-v4.5.1-src.tar.gz",
+              "checksum": "SHA-256:aa06831a7d88d8ccde4ea21241e983a08dbdae967290e181658b0d18bffc8f86",
+              "size": "96922"
+            },
+            {
+              "host": "x86_64-apple-darwin",
+              "url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-macos.tar.gz",
+              "archiveFileName": "esptool-v4.5.1-macos.tar.gz",
+              "checksum": "SHA-256:78b52acfd51541ceb97cee893b7d4d49b8ddc284602be8c73ea47e3d849e0956",
+              "size": "5850888"
+            },
+            {
+              "host": "x86_64-mingw32",
+              "url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-win64.zip",
+              "archiveFileName": "esptool-v4.5.1-win64.zip",
+              "checksum": "SHA-256:64d0c24499d46b80d6bd7a05c98bdacc3455ab6d503cc2a99e35711310216045",
+              "size": "6638448"
+            },
+            {
+              "host": "i686-mingw32",
+              "url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-win64.zip",
+              "archiveFileName": "esptool-v4.5.1-win64.zip",
+              "checksum": "SHA-256:64d0c24499d46b80d6bd7a05c98bdacc3455ab6d503cc2a99e35711310216045",
+              "size": "6638448"
+            }
+          ]
+        },
         {
           "name": "arduino-fwuploader",
           "version": "2.2.0",
@@ -130,6 +514,61 @@
               "size": "6829396"
             }
           ]
+        },
+        {
+          "name": "fwupdater",
+          "version": "0.1.12",
+          "systems": [
+            {
+              "host": "i686-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_32bit.tar.bz2",
+              "archiveFileName": "FirmwareUploader_0.1.12_Linux_32bit.tar.bz2",
+              "checksum": "SHA-256:2fec2bdfd20ad4950bc9ba37108dc2a7c152f569174279c0697efe1f5a0db781",
+              "size": "26097546"
+            },
+            {
+              "host": "x86_64-pc-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_64bit.tar.bz2",
+              "archiveFileName": "FirmwareUploader_0.1.12_Linux_64bit.tar.bz2",
+              "checksum": "SHA-256:ce57d0afef30cb7d3513f5da326346c99d6bf4923bbc2200634086811f3fb31e",
+              "size": "26073327"
+            },
+            {
+              "host": "i686-mingw32",
+              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Windows_32bit.zip",
+              "archiveFileName": "FirmwareUploader_0.1.12_Windows_32bit.zip",
+              "checksum": "SHA-256:558568b453caa1c821def8cc6d34555d0c910eb7e7e871de3ae1c39ae6f01bdd",
+              "size": "25743641"
+            },
+            {
+              "host": "x86_64-mingw32",
+              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Windows_64bit.zip",
+              "archiveFileName": "FirmwareUploader_0.1.12_Windows_64bit.zip",
+              "checksum": "SHA-256:ec16de33620985434280c92c3c322257b89bb67adf8fd4d5dd5f9467ea1e9e40",
+              "size": "25851428"
+            },
+            {
+              "host": "i386-apple-darwin11",
+              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_macOS_64bit.tar.bz2",
+              "archiveFileName": "FirmwareUploader_0.1.12_macOS_64bit.tar.bz2",
+              "checksum": "SHA-256:a470361b57f86ddfcaecd274d844af51ee1d23a71cd6c26e30fcef2152d1a03f",
+              "size": "25792860"
+            },
+            {
+              "host": "arm-linux-gnueabihf",
+              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_ARM.tar.bz2",
+              "archiveFileName": "FirmwareUploader_0.1.12_Linux_ARM.tar.bz2",
+              "checksum": "SHA-256:855fa0a9b942c3ee18906efc510bdfe30bf3334ff28ffbb476e648ff30033847",
+              "size": "25936245"
+            },
+            {
+              "host": "aarch64-linux-gnu",
+              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_ARM64.tar.bz2",
+              "archiveFileName": "FirmwareUploader_0.1.12_Linux_ARM64.tar.bz2",
+              "checksum": "SHA-256:691461e64fe075e9a79801347c2bd895fb72f8f2c45a7cd49056c6ad9efe8fc4",
+              "size": "25967430"
+            }
+          ]
         }
       ]
     }

utilities/utilities.go

@@ -16,7 +16,6 @@
 package utilities
 
 import (
-	"archive/zip"
 	"bytes"
 	"crypto"
 	"crypto/rsa"
@@ -29,7 +28,6 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"strings"
 
@@ -120,44 +118,6 @@ func call(stack []*exec.Cmd, pipes []*io.PipeWriter) (err error) {
 	return stack[0].Wait()
 }
 
-// Unzip will unzip a file to a destination
-func Unzip(zippath string, destination string) (err error) {
-	r, err := zip.OpenReader(zippath)
-	if err != nil {
-		return err
-	}
-	for _, f := range r.File {
-		fullname := path.Join(destination, f.Name)
-		if f.FileInfo().IsDir() {
-			os.MkdirAll(fullname, f.FileInfo().Mode().Perm())
-		} else {
-			os.MkdirAll(filepath.Dir(fullname), 0755)
-			perms := f.FileInfo().Mode().Perm()
-			out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms)
-			if err != nil {
-				return err
-			}
-			rc, err := f.Open()
-			if err != nil {
-				return err
-			}
-			_, err = io.CopyN(out, rc, f.FileInfo().Size())
-			if err != nil {
-				return err
-			}
-			rc.Close()
-			out.Close()
-
-			mtime := f.FileInfo().ModTime()
-			err = os.Chtimes(fullname, mtime, mtime)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return
-}
-
 // SafeJoin performs a filepath.Join of 'parent' and 'subdir' but returns an error
 // if the resulting path points outside of 'parent'.
 func SafeJoin(parent, subdir string) (string, error) {