Skip to content

Use codeclysm/extract and remove custom implementation #867

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Dec 18, 2023
+548 −423
Merged

Use codeclysm/extract and remove custom implementation #867

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4a6f967
replace test with a meaningful one
umbynos Dec 7, 2023
872d230
use `codeclysm/extract` and remove extract code
umbynos Dec 7, 2023
970d654
fix cleanup in tests
umbynos Dec 18, 2023
ac88c78
enlarge the scope of the test: verify dir tree is handled correctly
umbynos Dec 15, 2023
4117277
the archives are first extracted in tmp and then moved to correct place
umbynos Dec 15, 2023
c8bd54a
fix test failing
umbynos Dec 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
337 changes: 49 additions & 288 deletions tools/download.go
View file
Original file line number Diff line number Diff line change
@@ -16,11 +16,8 @@
package tools

import (
"archive/tar"
"archive/zip"
"bytes"
"compress/bzip2"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
@@ -30,14 +27,13 @@ import (
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"strings"

"github.com/arduino/arduino-create-agent/utilities"
"github.com/arduino/arduino-create-agent/v2/pkgs"
"github.com/arduino/go-paths-helper"
"github.com/blang/semver"
"github.com/codeclysm/extract/v3"
)

// public vars to allow override in the tests
@@ -46,10 +42,6 @@ var (
Arch = runtime.GOARCH
)

func mimeType(data []byte) (string, error) {
return http.DetectContentType(data[0:512]), nil
}

func pathExists(path string) bool {
_, err := os.Stat(path)
if err == nil {
@@ -129,39 +121,46 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
return errors.New("checksum doesn't match")
}

// Decompress
t.logger("Unpacking tool " + name)

location := t.directory.Join(pack, correctTool.Name, correctTool.Version).String()
err = os.RemoveAll(location)

tempPath := paths.TempDir()
// Create a temporary dir to extract package
if err := tempPath.MkdirAll(); err != nil {
return fmt.Errorf("creating temp dir for extraction: %s", err)
}
tempDir, err := tempPath.MkTempDir("package-")
if err != nil {
return err
return fmt.Errorf("creating temp dir for extraction: %s", err)
}
defer tempDir.RemoveAll()

srcType, err := mimeType(body)
t.logger("Unpacking tool " + name)
ctx := context.Background()
reader := bytes.NewReader(body)
// Extract into temp directory
if err := extract.Archive(ctx, reader, tempDir.String(), nil); err != nil {
return fmt.Errorf("extracting archive: %s", err)
}

location := t.directory.Join(pack, correctTool.Name, correctTool.Version)
err = location.RemoveAll()
if err != nil {
return err
}

switch srcType {
case "application/zip":
location, err = extractZip(t.logger, body, location)
case "application/x-bz2":
case "application/octet-stream":
location, err = extractBz2(t.logger, body, location)
case "application/x-gzip":
location, err = extractTarGz(t.logger, body, location)
default:
return errors.New("Unknown extension for file " + correctSystem.URL)
// Check package content and find package root dir
root, err := findPackageRoot(tempDir)
if err != nil {
return fmt.Errorf("searching package root dir: %s", err)
}

if err != nil {
t.logger("Error extracting the archive: " + err.Error())
return err
if err := root.Rename(location); err != nil {
if err := root.CopyDirTo(location); err != nil {
return fmt.Errorf("moving extracted archive to destination dir: %s", err)
}
}

err = t.installDrivers(location)
// if the tool contains a post_install script, run it: it means it is a tool that needs to install drivers
// AFAIK this is only the case for the windows-driver tool
err = t.installDrivers(location.String())
if err != nil {
return err
}
@@ -170,13 +169,27 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
t.logger("Ensure that the files are executable")

// Update the tool map
t.logger("Updating map with location " + location)
t.logger("Updating map with location " + location.String())

t.setMapValue(name, location)
t.setMapValue(name+"-"+correctTool.Version, location)
t.setMapValue(name, location.String())
t.setMapValue(name+"-"+correctTool.Version, location.String())
return t.writeMap()
}

func findPackageRoot(parent *paths.Path) (*paths.Path, error) {
files, err := parent.ReadDir()
if err != nil {
return nil, fmt.Errorf("reading package root dir: %s", err)
}
files.FilterOutPrefix("__MACOSX")

// if there is only one dir, it is the root dir
if len(files) == 1 && files[0].IsDir() {
return files[0], nil
}
return parent, nil
}

func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.System) {
var correctTool pkgs.Tool
correctTool.Version = "0.0"
@@ -207,258 +220,6 @@ func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.Syst
return correctTool, correctSystem
}

func commonPrefix(sep byte, paths []string) string {
// Handle special cases.
switch len(paths) {
case 0:
return ""
case 1:
return path.Clean(paths[0])
}

c := []byte(path.Clean(paths[0]))

// We add a trailing sep to handle: common prefix directory is included in the path list
// (e.g. /home/user1, /home/user1/foo, /home/user1/bar).
// path.Clean will have cleaned off trailing / separators with
// the exception of the root directory, "/" making it "//"
// but this will get fixed up to "/" below).
c = append(c, sep)

// Ignore the first path since it's already in c
for _, v := range paths[1:] {
// Clean up each path before testing it
v = path.Clean(v) + string(sep)

// Find the first non-common byte and truncate c
if len(v) < len(c) {
c = c[:len(v)]
}
for i := 0; i < len(c); i++ {
if v[i] != c[i] {
c = c[:i]
break
}
}
}

// Remove trailing non-separator characters and the final separator
for i := len(c) - 1; i >= 0; i-- {
if c[i] == sep {
c = c[:i]
break
}
}

return string(c)
}

func removeStringFromSlice(s []string, r string) []string {
for i, v := range s {
if v == r {
return append(s[:i], s[i+1:]...)
}
}
return s
}

func findBaseDir(dirList []string) string {
if len(dirList) == 1 {
return path.Dir(dirList[0]) + "/"
}

// https://github.com/backdrop-ops/contrib/issues/55#issuecomment-73814500
dontdiff := []string{"pax_global_header"}
for _, v := range dontdiff {
dirList = removeStringFromSlice(dirList, v)
}

commonBaseDir := commonPrefix('/', dirList)
if commonBaseDir != "" {
commonBaseDir = commonBaseDir + "/"
}
return commonBaseDir
}

func extractZip(log func(msg string), body []byte, location string) (string, error) {
path, _ := utilities.SaveFileonTempDir("tooldownloaded.zip", bytes.NewReader(body))
r, err := zip.OpenReader(path)
if err != nil {
return location, err
}

var dirList []string

for _, f := range r.File {
dirList = append(dirList, f.Name)
}

basedir := findBaseDir(dirList)
log(fmt.Sprintf("selected baseDir %s from Zip Archive Content: %v", basedir, dirList))

for _, f := range r.File {
fullname := filepath.Join(location, strings.Replace(f.Name, basedir, "", -1))
log(fmt.Sprintf("generated fullname %s removing %s from %s", fullname, basedir, f.Name))
if f.FileInfo().IsDir() {
os.MkdirAll(fullname, f.FileInfo().Mode().Perm())
} else {
os.MkdirAll(filepath.Dir(fullname), 0755)
perms := f.FileInfo().Mode().Perm()
out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms)
if err != nil {
return location, err
}
rc, err := f.Open()
if err != nil {
return location, err
}
_, err = io.CopyN(out, rc, f.FileInfo().Size())
if err != nil {
return location, err
}
rc.Close()
out.Close()

mtime := f.FileInfo().ModTime()
err = os.Chtimes(fullname, mtime, mtime)
if err != nil {
return location, err
}
}
}
return location, nil
}

func extractTarGz(log func(msg string), body []byte, location string) (string, error) {
bodyCopy := make([]byte, len(body))
copy(bodyCopy, body)
tarFile, _ := gzip.NewReader(bytes.NewReader(body))
tarReader := tar.NewReader(tarFile)

var dirList []string

for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
dirList = append(dirList, header.Name)
}

basedir := findBaseDir(dirList)
log(fmt.Sprintf("selected baseDir %s from TarGz Archive Content: %v", basedir, dirList))

tarFile, _ = gzip.NewReader(bytes.NewReader(bodyCopy))
tarReader = tar.NewReader(tarFile)

for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
return location, err
}

path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
info := header.FileInfo()

// Create parent folder
dirmode := info.Mode() | os.ModeDir | 0700
if err = os.MkdirAll(filepath.Dir(path), dirmode); err != nil {
return location, err
}

if info.IsDir() {
if err = os.MkdirAll(path, info.Mode()); err != nil {
return location, err
}
continue
}

if header.Typeflag == tar.TypeSymlink {
_ = os.Symlink(header.Linkname, path)
continue
}

file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
if err != nil {
continue
}
_, err = io.Copy(file, tarReader)
if err != nil {
return location, err
}
file.Close()
}
return location, nil
}

func extractBz2(log func(msg string), body []byte, location string) (string, error) {
bodyCopy := make([]byte, len(body))
copy(bodyCopy, body)
tarFile := bzip2.NewReader(bytes.NewReader(body))
tarReader := tar.NewReader(tarFile)

var dirList []string

for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
dirList = append(dirList, header.Name)
}

basedir := findBaseDir(dirList)
log(fmt.Sprintf("selected baseDir %s from Bz2 Archive Content: %v", basedir, dirList))

tarFile = bzip2.NewReader(bytes.NewReader(bodyCopy))
tarReader = tar.NewReader(tarFile)

for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
continue
//return location, err
}

path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
info := header.FileInfo()

// Create parent folder
dirmode := info.Mode() | os.ModeDir | 0700
if err = os.MkdirAll(filepath.Dir(path), dirmode); err != nil {
return location, err
}

if info.IsDir() {
if err = os.MkdirAll(path, info.Mode()); err != nil {
return location, err
}
continue
}

if header.Typeflag == tar.TypeSymlink {
_ = os.Symlink(header.Linkname, path)
continue
}

file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
if err != nil {
continue
//return location, err
}
_, err = io.Copy(file, tarReader)
if err != nil {
return location, err
}
file.Close()
}
return location, nil
}

func (t *Tools) installDrivers(location string) error {
OkPressed := 6
extension := ".bat"
155 changes: 60 additions & 95 deletions tools/download_test.go
View file
Original file line number Diff line number Diff line change
@@ -17,16 +17,13 @@ package tools

import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path"
"runtime"
"testing"
"time"

"github.com/arduino/arduino-create-agent/index"
"github.com/arduino/arduino-create-agent/v2/pkgs"
"github.com/arduino/go-paths-helper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

@@ -44,6 +41,10 @@ func TestDownloadCorrectPlatform(t *testing.T) {
{"windows", "amd64", "x86_64-mingw32"},
{"linux", "arm", "arm-linux-gnueabihf"},
}
defer func() {
OS = runtime.GOOS // restore `runtime.OS`
Arch = runtime.GOARCH // restore `runtime.ARCH`
}()
testIndex := paths.New("testdata", "test_tool_index.json")
buf, err := testIndex.ReadFile()
require.NoError(t, err)
@@ -76,6 +77,10 @@ func TestDownloadFallbackPlatform(t *testing.T) {
{"darwin", "arm64", "i386-apple-darwin11"},
{"windows", "amd64", "i686-mingw32"},
}
defer func() {
OS = runtime.GOOS // restore `runtime.OS`
Arch = runtime.GOARCH // restore `runtime.ARCH`
}()
testIndex := paths.New("testdata", "test_tool_index.json")
buf, err := testIndex.ReadFile()
require.NoError(t, err)
@@ -98,99 +103,59 @@ func TestDownloadFallbackPlatform(t *testing.T) {
}
}

func Test_findBaseDir(t *testing.T) {
cases := []struct {
dirList []string
want string
func TestDownload(t *testing.T) {
testCases := []struct {
name string
version string
filesCreated []string
}{
{[]string{"bin/bossac"}, "bin/"},
{[]string{"bin/", "bin/bossac"}, "bin/"},
{[]string{"bin/", "bin/bossac", "example"}, ""},
{[]string{"avrdude/bin/",
"avrdude/bin/avrdude.exe",
"avrdude/bin/remove_giveio.bat",
"avrdude/bin/status_giveio.bat",
"avrdude/bin/giveio.sys",
"avrdude/bin/loaddrv.exe",
"avrdude/bin/libusb0.dll",
"avrdude/bin/install_giveio.bat",
"avrdude/etc/avrdude.conf"}, "avrdude/"},
{[]string{"pax_global_header", "bin/", "bin/bossac"}, "bin/"},
{"avrdude", "6.3.0-arduino17", []string{"bin", "etc"}},
{"bossac", "1.6.1-arduino", []string{"bossac"}},
{"bossac", "1.7.0-arduino3", []string{"bossac"}},
{"bossac", "1.9.1-arduino2", []string{"bossac"}},
{"openocd", "0.11.0-arduino2", []string{"bin", "share"}},
{"dfu-util", "0.10.0-arduino1", []string{"dfu-prefix", "dfu-suffix", "dfu-util"}},
{"rp2040tools", "1.0.6", []string{"elf2uf2", "picotool", "pioasm", "rp2040load"}},
{"esptool_py", "4.5.1", []string{"esptool"}},
{"arduino-fwuploader", "2.2.2", []string{"arduino-fwuploader"}},
{"fwupdater", "0.1.12", []string{"firmwares", "FirmwareUploader"}}, // old legacy tool
}
for _, tt := range cases {
t.Run(fmt.Sprintln(tt.dirList), func(t *testing.T) {
if got := findBaseDir(tt.dirList); got != tt.want {
t.Errorf("findBaseDir() = got %v, want %v", got, tt.want)
}
})
// prepare the test environment
tempDir := t.TempDir()
tempDirPath := paths.New(tempDir)
testIndex := index.Resource{
IndexFile: *paths.New("testdata", "test_tool_index.json"),
LastRefresh: time.Now(),
}
}

func TestTools_DownloadAndUnpackBehaviour(t *testing.T) {
urls := []string{
"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-armhf-pc-linux-gnu.tar.bz2",
"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-aarch64-pc-linux-gnu.tar.bz2",
"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i386-apple-darwin11.tar.bz2",
"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-x86_64-pc-linux-gnu.tar.bz2",
"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i686-pc-linux-gnu.tar.bz2",
"https://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i686-w64-mingw32.zip",
}
expectedDirList := []string{"bin", "etc"}

tmpDir, err := os.MkdirTemp("", "download_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)

for _, url := range urls {
t.Log("Downloading tool from " + url)
resp, err := http.Get(url)
if err != nil {
t.Errorf("%v", err)
}
defer resp.Body.Close()
testTools := New(tempDirPath, &testIndex, func(msg string) { t.Log(msg) })

// Read the body
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Errorf("%v", err)
}

location := path.Join(tmpDir, "username", "arduino", "avrdude", "6.3.0-arduino14")
os.MkdirAll(location, os.ModePerm)
err = os.RemoveAll(location)

if err != nil {
t.Errorf("%v", err)
}

srcType, err := mimeType(body)
if err != nil {
t.Errorf("%v", err)
}

switch srcType {
case "application/zip":
location, err = extractZip(func(msg string) { t.Log(msg) }, body, location)
case "application/x-bz2":
case "application/octet-stream":
location, err = extractBz2(func(msg string) { t.Log(msg) }, body, location)
case "application/x-gzip":
location, err = extractTarGz(func(msg string) { t.Log(msg) }, body, location)
default:
t.Errorf("no suitable type found")
}
files, err := os.ReadDir(location)
if err != nil {
t.Errorf("%v", err)
}
dirList := []string{}
for _, f := range files {
dirList = append(dirList, f.Name())
}
for _, tc := range testCases {
t.Run(tc.name+"-"+tc.version, func(t *testing.T) {
// Download the tool
err := testTools.Download("arduino-test", tc.name, tc.version, "replace")
require.NoError(t, err)

// Check that the tool has been downloaded
toolDir := tempDirPath.Join("arduino-test", tc.name, tc.version)
require.DirExists(t, toolDir.String())

// Check that the files have been created
for _, file := range tc.filesCreated {
filePath := toolDir.Join(file)
if filePath.IsDir() {
require.DirExists(t, filePath.String())
} else {
if OS == "windows" {
require.FileExists(t, filePath.String()+".exe")
} else {
require.FileExists(t, filePath.String())
}
}
}

assert.ElementsMatchf(t, dirList, expectedDirList, "error message %s", "formatted")
// Check that the tool has been installed
_, ok := testTools.getMapValue(tc.name + "-" + tc.version)
require.True(t, ok)
})
}

}
439 changes: 439 additions & 0 deletions tools/testdata/test_tool_index.json
View file
Original file line number Diff line number Diff line change
@@ -49,6 +49,390 @@
}
],
"tools": [
{
"name": "avrdude",
"version": "6.3.0-arduino17",
"systems": [
{
"size": "219631",
"checksum": "SHA-256:2a8e68c5d803aa6f902ef219f177ec3a4c28275d85cbe272962ad2cd374f50d1",
"host": "arm-linux-gnueabihf",
"archiveFileName": "avrdude-6.3.0-arduino17-armhf-pc-linux-gnu.tar.bz2",
"url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-armhf-pc-linux-gnu.tar.bz2"
},
{
"size": "229852",
"checksum": "SHA-256:6cf948f751acfe7b96684537f2291c766ec8b54b4f7dc95539864821456fa9fc",
"host": "aarch64-linux-gnu",
"archiveFileName": "avrdude-6.3.0-arduino17-aarch64-pc-linux-gnu.tar.bz2",
"url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-aarch64-pc-linux-gnu.tar.bz2"
},
{
"size": "279045",
"checksum": "SHA-256:120cc9edaae699e7e9ac50b1b8eb0e7d51fdfa555bac54233c2511e6ee5418c9",
"host": "x86_64-apple-darwin12",
"archiveFileName": "avrdude-6.3.0-arduino17-x86_64-apple-darwin12.tar.bz2",
"url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-x86_64-apple-darwin12.tar.bz2"
},
{
"size": "254271",
"checksum": "SHA-256:accdfb920af2aabf4f7461d2ac73c0751760f525216dc4e7657427a78c60d13d",
"host": "x86_64-linux-gnu",
"archiveFileName": "avrdude-6.3.0-arduino17-x86_64-pc-linux-gnu.tar.bz2",
"url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-x86_64-pc-linux-gnu.tar.bz2"
},
{
"size": "244550",
"checksum": "SHA-256:5c8cc6c17db9300e1451fe41cd7178b0442b4490ee6fdbc0aed9811aef96c05f",
"host": "i686-linux-gnu",
"archiveFileName": "avrdude-6.3.0-arduino17-i686-pc-linux-gnu.tar.bz2",
"url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-i686-pc-linux-gnu.tar.bz2"
},
{
"size": "328460",
"checksum": "SHA-256:e99188873c7c5ad8f8f906f068c33600e758b2e36cce3adbd518a21bd266749d",
"host": "i686-mingw32",
"archiveFileName": "avrdude-6.3.0-arduino17-i686-w64-mingw32.zip",
"url": "http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino17-i686-w64-mingw32.zip"
}
]
},
{
"name": "bossac",
"version": "1.6.1-arduino",
"systems": [
{
"host": "arm-linux-gnueabihf",
"url": "http://downloads.arduino.cc/bossac-1.6.1-arduino-arm-linux-gnueabihf.tar.bz2",
"archiveFileName": "bossac-1.6.1-arduino-arm-linux-gnueabihf.tar.bz2",
"checksum": "SHA-256:8c4e63db982178919c824e7a35580dffc95c3426afa7285de3eb583982d4d391",
"size": "201341"
},
{
"host": "i686-mingw32",
"url": "http://downloads.arduino.cc/bossac-1.6.1-arduino-mingw32.tar.gz",
"archiveFileName": "bossac-1.6.1-arduino-mingw32.tar.gz",
"checksum": "SHA-256:d59f43e2e83a337d04c4ae88b195a4ee175b8d87fff4c43144d23412a4a9513b",
"size": "222918"
},
{
"host": "x86_64-apple-darwin",
"url": "http://downloads.arduino.cc/bossac-1.6.1-arduino-i386-apple-darwin14.5.0.tar.gz",
"archiveFileName": "bossac-1.6.1-arduino-i386-apple-darwin14.5.0.tar.gz",
"checksum": "SHA-256:2f80ef569a3fb19da60ab3489e49d8fe7d4699876acf30ff4938c632230a09aa",
"size": "64587"
},
{
"host": "x86_64-pc-linux-gnu",
"url": "http://downloads.arduino.cc/bossac-1.6.1-arduino-x86_64-linux-gnu.tar.gz",
"archiveFileName": "bossac-1.6.1-arduino-x86_64-linux-gnu.tar.gz",
"checksum": "SHA-256:b78afc66c00ccfdd69a08bd3959c260a0c64ccce78a71d5a1135ae4437ff40db",
"size": "30869"
},
{
"host": "i686-pc-linux-gnu",
"url": "http://downloads.arduino.cc/bossac-1.6.1-arduino-i486-linux-gnu.tar.gz",
"archiveFileName": "bossac-1.6.1-arduino-i486-linux-gnu.tar.gz",
"checksum": "SHA-256:1e211347569d75193b337296a10dd25b0ce04419e3d7dc644355178b6b514f92",
"size": "30320"
}
]
},
{
"name": "bossac",
"version": "1.7.0-arduino3",
"systems": [
{
"host": "i686-mingw32",
"url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-windows.tar.gz",
"archiveFileName": "bossac-1.7.0-arduino3-windows.tar.gz",
"checksum": "SHA-256:62745cc5a98c26949ec9041ef20420643c561ec43e99dae659debf44e6836526",
"size": "3607421"
},
{
"host": "x86_64-apple-darwin",
"url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-osx.tar.gz",
"archiveFileName": "bossac-1.7.0-arduino3-osx.tar.gz",
"checksum": "SHA-256:adb3c14debd397d8135e9e970215c6972f0e592c7af7532fa15f9ce5e64b991f",
"size": "75510"
},
{
"host": "x86_64-pc-linux-gnu",
"url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-linux64.tar.gz",
"archiveFileName": "bossac-1.7.0-arduino3-linux64.tar.gz",
"checksum": "SHA-256:1ae54999c1f97234a5c603eb99ad39313b11746a4ca517269a9285afa05f9100",
"size": "207271"
},
{
"host": "i686-pc-linux-gnu",
"url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-linux32.tar.gz",
"archiveFileName": "bossac-1.7.0-arduino3-linux32.tar.gz",
"checksum": "SHA-256:4ac4354746d1a09258f49a43ef4d1baf030d81c022f8434774268b00f55d3ec3",
"size": "193577"
},
{
"host": "arm-linux-gnueabihf",
"url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-linuxarm.tar.gz",
"archiveFileName": "bossac-1.7.0-arduino3-linuxarm.tar.gz",
"checksum": "SHA-256:626c6cc548046901143037b782bf019af1663bae0d78cf19181a876fb9abbb90",
"size": "193941"
},
{
"host": "aarch64-linux-gnu",
"url": "http://downloads.arduino.cc/tools/bossac-1.7.0-arduino3-linuxaarch64.tar.gz",
"archiveFileName": "bossac-1.7.0-arduino3-linuxaarch64.tar.gz",
"checksum": "SHA-256:a098b2cc23e29f0dc468416210d097c4a808752cd5da1a7b9b8b7b931a04180b",
"size": "268365"
}
]
},
{
"name": "bossac",
"version": "1.9.1-arduino2",
"systems": [
{
"host": "i686-mingw32",
"url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-windows.tar.gz",
"archiveFileName": "bossac-1.9.1-arduino2-windows.tar.gz",
"checksum": "SHA-256:5c994d04354f0db8e4bea136f49866d2ba537f0af74b2e78026f2d4fc75e3e39",
"size": "1260628"
},
{
"host": "x86_64-apple-darwin",
"url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-osx.tar.gz",
"archiveFileName": "bossac-1.9.1-arduino2-osx.tar.gz",
"checksum": "SHA-256:b7732129364a378676604db6579c9b8dab50dd965fb50d7a3afff1839c97ff80",
"size": "47870"
},
{
"host": "x86_64-pc-linux-gnu",
"url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-linux64.tar.gz",
"archiveFileName": "bossac-1.9.1-arduino2-linux64.tar.gz",
"checksum": "SHA-256:9eb549874391521999cee13dc823a2cfc8866b8246945339a281808d99c72d2c",
"size": "399532"
},
{
"host": "i686-pc-linux-gnu",
"url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-linux32.tar.gz",
"archiveFileName": "bossac-1.9.1-arduino2-linux32.tar.gz",
"checksum": "SHA-256:10d69f53f169f25afee2dd583dfd9dc803c10543e6c5260d106725cb0d174900",
"size": "384951"
},
{
"host": "arm-linux-gnueabihf",
"url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-linuxarm.tar.gz",
"archiveFileName": "bossac-1.9.1-arduino2-linuxarm.tar.gz",
"checksum": "SHA-256:c9539d161d23231b5beb1d09a71829744216c7f5bc2857a491999c3e567f5b19",
"size": "361915"
},
{
"host": "aarch64-linux-gnu",
"url": "http://downloads.arduino.cc/tools/bossac-1.9.1-arduino2-linuxaarch64.tar.gz",
"archiveFileName": "bossac-1.9.1-arduino2-linuxaarch64.tar.gz",
"checksum": "SHA-256:c167fa0ea223966f4d21f5592da3888bcbfbae385be6c5c4e41f8abff35f5cb1",
"size": "442853"
}
]
},
{
"name": "openocd",
"version": "0.11.0-arduino2",
"systems": [
{
"size": "1902818",
"checksum": "SHA-256:a1aa7f1435a61eafb72ee90722f2496d6a34a7a0f085d0315c2613e4a548b824",
"host": "aarch64-linux-gnu",
"archiveFileName": "openocd-0.11.0-arduino2-static-aarch64-linux-gnu.tar.bz2",
"url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-aarch64-linux-gnu.tar.bz2"
},
{
"size": "1986716",
"checksum": "SHA-256:57041130160be086e69387cceb4616eefc9819a0ef75de1f7c11aea57fb92699",
"host": "arm-linux-gnueabihf",
"archiveFileName": "openocd-0.11.0-arduino2-static-arm-linux-gnueabihf.tar.bz2",
"url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-arm-linux-gnueabihf.tar.bz2"
},
{
"size": "1971364",
"checksum": "SHA-256:6f4a8b77c8076aa18afb8438472526dff8c0d161a3ca68d0326163b59fcab663",
"host": "i686-linux-gnu",
"archiveFileName": "openocd-0.11.0-arduino2-static-i686-ubuntu12.04-linux-gnu.tar.bz2",
"url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-i686-ubuntu12.04-linux-gnu.tar.bz2"
},
{
"size": "2460087",
"checksum": "SHA-256:631010980f12b1e750c4c67ce012b31c5953caabf4d30607d806e3d2b717d4b8",
"host": "i686-mingw32",
"archiveFileName": "openocd-0.11.0-arduino2-static-i686-w64-mingw32.zip",
"url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-i686-w64-mingw32.zip"
},
{
"size": "1893150",
"checksum": "SHA-256:280e7234eba84e830e92d791ebc685286f71d2bc1d3347f93605ef170d54fef4",
"host": "i386-apple-darwin11",
"archiveFileName": "openocd-0.11.0-arduino2-static-x86_64-apple-darwin13.tar.bz2",
"url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-x86_64-apple-darwin13.tar.bz2"
},
{
"size": "2052080",
"checksum": "SHA-256:4d19b6e3906de1434ec86841e0e3138235714c655d45f037c0fabfa5e5c0681b",
"host": "x86_64-linux-gnu",
"archiveFileName": "openocd-0.11.0-arduino2-static-x86_64-ubuntu12.04-linux-gnu.tar.bz2",
"url": "http://downloads.arduino.cc/tools/openocd-0.11.0-arduino2-static-x86_64-ubuntu12.04-linux-gnu.tar.bz2"
}
]
},
{
"name": "dfu-util",
"version": "0.10.0-arduino1",
"systems": [
{
"host": "i386-apple-darwin11",
"url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-osx.tar.bz2",
"archiveFileName": "dfu-util-0.10.0-arduino1-osx.tar.bz2",
"size": "73921",
"checksum": "SHA-256:7562d128036759605828d64b8d672d42445a8d95555c4b9ba339f73a1711a640"
},
{
"host": "arm-linux-gnueabihf",
"url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-arm.tar.bz2",
"archiveFileName": "dfu-util-0.10.0-arduino1-arm.tar.bz2",
"size": "272153",
"checksum": "SHA-256:f1e550f40c235356b7fde1c59447bfbab28f768915d3c14bd858fe0576bfc5a9"
},
{
"host": "aarch64-linux-gnu",
"url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-arm64.tar.bz2",
"archiveFileName": "dfu-util-0.10.0-arduino1-arm64.tar.bz2",
"size": "277886",
"checksum": "SHA-256:ebfbd21d3030c500da1f83b9aae5b8c597bee04c3bde1ce0a51b41abeafc9614"
},
{
"host": "x86_64-linux-gnu",
"url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-linux64.tar.bz2",
"archiveFileName": "dfu-util-0.10.0-arduino1-linux64.tar.bz2",
"size": "77184",
"checksum": "SHA-256:13ef2ec591c1e8b0b7eb0a05da972ecd6695016e7a9607e332c7553899af9b4a"
},
{
"host": "i686-linux-gnu",
"url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-linux32.tar.bz2",
"archiveFileName": "dfu-util-0.10.0-arduino1-linux32.tar.bz2",
"size": "81826",
"checksum": "SHA-256:43599ec60c000e9ef016970a496d6ab2cbbe5a8b7df9d06ef3114ecf83f9d123"
},
{
"host": "i686-mingw32",
"url": "http://downloads.arduino.cc/tools/dfu-util-0.10.0-arduino1-windows.tar.bz2",
"archiveFileName": "dfu-util-0.10.0-arduino1-windows.tar.bz2",
"size": "464314",
"checksum": "SHA-256:90816b669273ae796d734a2459c46bb340d4790783fd7aa01eb40c0443f1a9b1"
}
]
},
{
"name": "rp2040tools",
"version": "1.0.6",
"systems": [
{
"host": "i386-apple-darwin11",
"url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-darwin_amd64.tar.bz2",
"archiveFileName": "rp2040tools-1.0.6-darwin_amd64.tar.bz2",
"size": "1717967",
"checksum": "SHA-256:4e32aa4b8f36db40a17bfbdfd34d80da91710e30c3887732bf0c0bf0b02840a7"
},
{
"host": "arm-linux-gnueabihf",
"url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-linux_arm.tar.bz2",
"archiveFileName": "rp2040tools-1.0.6-linux_arm.tar.bz2",
"size": "8702508",
"checksum": "SHA-256:084a29accf0014bc79723fbb40057b95299c7ae63876f74494a077c987014cc3"
},
{
"host": "aarch64-linux-gnu",
"url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-linux_arm64.tar.bz2",
"archiveFileName": "rp2040tools-1.0.6-linux_arm64.tar.bz2",
"size": "9037783",
"checksum": "SHA-256:1a2a6cb1abf1f7b8198d494c8d8e838700297d748877be8232e02aaa5ca8d0df"
},
{
"host": "x86_64-linux-gnu",
"url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-linux_amd64.tar.bz2",
"archiveFileName": "rp2040tools-1.0.6-linux_amd64.tar.bz2",
"size": "6108121",
"checksum": "SHA-256:6e2ea818db1ff57f2d8e1e3010fbc5bdb5f28ff44f5a68900cae41d7d709f738"
},
{
"host": "i686-linux-gnu",
"url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-linux_386.tar.bz2",
"archiveFileName": "rp2040tools-1.0.6-linux_386.tar.bz2",
"size": "6604083",
"checksum": "SHA-256:ef339e2e0f5c7d5464b9911b612c634767daba39a6be977a1ffa41c95b9827a1"
},
{
"host": "i686-mingw32",
"url": "http://downloads.arduino.cc/tools/rp2040tools-1.0.6-windows_386.tar.bz2",
"archiveFileName": "rp2040tools-1.0.6-windows_386.tar.bz2",
"size": "3145329",
"checksum": "SHA-256:26a5daebba68c2348dade33716a6e379ded89895ef0e49df1332964a724f6170"
}
]
},
{
"name": "esptool_py",
"version": "4.5.1",
"systems": [
{
"host": "x86_64-pc-linux-gnu",
"url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-src.tar.gz",
"archiveFileName": "esptool-v4.5.1-src.tar.gz",
"checksum": "SHA-256:aa06831a7d88d8ccde4ea21241e983a08dbdae967290e181658b0d18bffc8f86",
"size": "96922"
},
{
"host": "i686-pc-linux-gnu",
"url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-src.tar.gz",
"archiveFileName": "esptool-v4.5.1-src.tar.gz",
"checksum": "SHA-256:aa06831a7d88d8ccde4ea21241e983a08dbdae967290e181658b0d18bffc8f86",
"size": "96922"
},
{
"host": "aarch64-linux-gnu",
"url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-src.tar.gz",
"archiveFileName": "esptool-v4.5.1-src.tar.gz",
"checksum": "SHA-256:aa06831a7d88d8ccde4ea21241e983a08dbdae967290e181658b0d18bffc8f86",
"size": "96922"
},
{
"host": "arm-linux-gnueabihf",
"url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-src.tar.gz",
"archiveFileName": "esptool-v4.5.1-src.tar.gz",
"checksum": "SHA-256:aa06831a7d88d8ccde4ea21241e983a08dbdae967290e181658b0d18bffc8f86",
"size": "96922"
},
{
"host": "x86_64-apple-darwin",
"url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-macos.tar.gz",
"archiveFileName": "esptool-v4.5.1-macos.tar.gz",
"checksum": "SHA-256:78b52acfd51541ceb97cee893b7d4d49b8ddc284602be8c73ea47e3d849e0956",
"size": "5850888"
},
{
"host": "x86_64-mingw32",
"url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-win64.zip",
"archiveFileName": "esptool-v4.5.1-win64.zip",
"checksum": "SHA-256:64d0c24499d46b80d6bd7a05c98bdacc3455ab6d503cc2a99e35711310216045",
"size": "6638448"
},
{
"host": "i686-mingw32",
"url": "https://github.com/espressif/arduino-esp32/releases/download/2.0.7/esptool-v4.5.1-win64.zip",
"archiveFileName": "esptool-v4.5.1-win64.zip",
"checksum": "SHA-256:64d0c24499d46b80d6bd7a05c98bdacc3455ab6d503cc2a99e35711310216045",
"size": "6638448"
}
]
},
{
"name": "arduino-fwuploader",
"version": "2.2.0",
@@ -130,6 +514,61 @@
"size": "6829396"
}
]
},
{
"name": "fwupdater",
"version": "0.1.12",
"systems": [
{
"host": "i686-linux-gnu",
"url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_32bit.tar.bz2",
"archiveFileName": "FirmwareUploader_0.1.12_Linux_32bit.tar.bz2",
"checksum": "SHA-256:2fec2bdfd20ad4950bc9ba37108dc2a7c152f569174279c0697efe1f5a0db781",
"size": "26097546"
},
{
"host": "x86_64-pc-linux-gnu",
"url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_64bit.tar.bz2",
"archiveFileName": "FirmwareUploader_0.1.12_Linux_64bit.tar.bz2",
"checksum": "SHA-256:ce57d0afef30cb7d3513f5da326346c99d6bf4923bbc2200634086811f3fb31e",
"size": "26073327"
},
{
"host": "i686-mingw32",
"url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Windows_32bit.zip",
"archiveFileName": "FirmwareUploader_0.1.12_Windows_32bit.zip",
"checksum": "SHA-256:558568b453caa1c821def8cc6d34555d0c910eb7e7e871de3ae1c39ae6f01bdd",
"size": "25743641"
},
{
"host": "x86_64-mingw32",
"url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Windows_64bit.zip",
"archiveFileName": "FirmwareUploader_0.1.12_Windows_64bit.zip",
"checksum": "SHA-256:ec16de33620985434280c92c3c322257b89bb67adf8fd4d5dd5f9467ea1e9e40",
"size": "25851428"
},
{
"host": "i386-apple-darwin11",
"url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_macOS_64bit.tar.bz2",
"archiveFileName": "FirmwareUploader_0.1.12_macOS_64bit.tar.bz2",
"checksum": "SHA-256:a470361b57f86ddfcaecd274d844af51ee1d23a71cd6c26e30fcef2152d1a03f",
"size": "25792860"
},
{
"host": "arm-linux-gnueabihf",
"url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_ARM.tar.bz2",
"archiveFileName": "FirmwareUploader_0.1.12_Linux_ARM.tar.bz2",
"checksum": "SHA-256:855fa0a9b942c3ee18906efc510bdfe30bf3334ff28ffbb476e648ff30033847",
"size": "25936245"
},
{
"host": "aarch64-linux-gnu",
"url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_ARM64.tar.bz2",
"archiveFileName": "FirmwareUploader_0.1.12_Linux_ARM64.tar.bz2",
"checksum": "SHA-256:691461e64fe075e9a79801347c2bd895fb72f8f2c45a7cd49056c6ad9efe8fc4",
"size": "25967430"
}
]
}
]
}
40 changes: 0 additions & 40 deletions utilities/utilities.go
View file
Original file line number Diff line number Diff line change
@@ -16,7 +16,6 @@
package utilities

import (
"archive/zip"
"bytes"
"crypto"
"crypto/rsa"
@@ -29,7 +28,6 @@ import (
"io"
"os"
"os/exec"
"path"
"path/filepath"
"strings"

@@ -120,44 +118,6 @@ func call(stack []*exec.Cmd, pipes []*io.PipeWriter) (err error) {
return stack[0].Wait()
}

// Unzip will unzip a file to a destination
func Unzip(zippath string, destination string) (err error) {
r, err := zip.OpenReader(zippath)
if err != nil {
return err
}
for _, f := range r.File {
fullname := path.Join(destination, f.Name)
if f.FileInfo().IsDir() {
os.MkdirAll(fullname, f.FileInfo().Mode().Perm())
} else {
os.MkdirAll(filepath.Dir(fullname), 0755)
perms := f.FileInfo().Mode().Perm()
out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms)
if err != nil {
return err
}
rc, err := f.Open()
if err != nil {
return err
}
_, err = io.CopyN(out, rc, f.FileInfo().Size())
if err != nil {
return err
}
rc.Close()
out.Close()

mtime := f.FileInfo().ModTime()
err = os.Chtimes(fullname, mtime, mtime)
if err != nil {
return err
}
}
}
return
}

// SafeJoin performs a filepath.Join of 'parent' and 'subdir' but returns an error
// if the resulting path points outside of 'parent'.
func SafeJoin(parent, subdir string) (string, error) {