Allow the builder to be aware of package_index properties

.travis.yml

@@ -12,6 +12,7 @@ install:
   - go get github.com/wadey/gocovmerge
   - go get github.com/arduino/go-properties-map
   - go get github.com/arduino/go-timeutils
+  - go get github.com/blang/semver
 
 script:
   - go build -o $HOME/arduino-builder -v github.com/arduino/arduino-builder/arduino-builder

add_missing_build_properties_from_parent_platform_txt_files.go

@@ -30,6 +30,10 @@
 package builder
 
 import (
+	"strings"
+
+	"github.com/arduino/arduino-builder/constants"
+	"github.com/arduino/arduino-builder/json_package_index"
 	"github.com/arduino/arduino-builder/types"
 )
 
@@ -48,3 +52,40 @@ func (s *AddMissingBuildPropertiesFromParentPlatformTxtFiles) Run(ctx *types.Con
 
 	return nil
 }
+
+type OverridePropertiesWithJsonInfo struct{}
+
+func (s *OverridePropertiesWithJsonInfo) Run(ctx *types.Context) error {
+
+	if ctx.JsonFolders != nil {
+
+		allowedJsons := strings.Split(ctx.BuildProperties[constants.ADDITIONALE_BOARD_MANAGER_JSON], ",")
+		jsonProperties, err := json_package_index.PackageIndexFoldersToPropertiesMap(ctx.Hardware, ctx.JsonFolders, allowedJsons)
+
+		if err != nil {
+			// doesn't matter, log the broken package in verbose mode
+		}
+
+		newBuildProperties := jsonProperties[ctx.TargetPackage.PackageId+":"+ctx.TargetPlatform.PlatformId+":"+ctx.TargetPlatform.Properties["version"]]
+
+		buildProperties := ctx.BuildProperties.Clone()
+
+		buildProperties.Merge(newBuildProperties)
+
+		// HACK!!! To overcome AVR core 1.6.12 lto problems, replace avr-gcc-4.8.1-arduino5 with
+		// 4.9.2-atmel3.5.3-arduino2 if it exists
+		if ctx.TargetBoard.Properties["build.core"] == "arduino:arduino" && buildProperties[constants.HACK_PROPERTIES_AVR_GCC_NEW] != "" {
+			buildProperties[constants.HACK_PROPERTIES_AVR_GCC_OLD] =
+				"{" + constants.HACK_PROPERTIES_AVR_GCC_NEW + "}"
+			// if base runtime property is fully specified it needs to be replaced
+			if !strings.HasPrefix(buildProperties[constants.HACK_PROPERTIES_AVR_GCC_OLD_2], "{") {
+				buildProperties[constants.HACK_PROPERTIES_AVR_GCC_OLD_2] =
+					"{" + constants.HACK_PROPERTIES_AVR_GCC_NEW + "}"
+			}
+		}
+
+		ctx.BuildProperties = buildProperties
+	}
+
+	return nil
+}

arduino-builder/main.go

@@ -59,6 +59,7 @@ const FLAG_HARDWARE = "hardware"
 const FLAG_TOOLS = "tools"
 const FLAG_BUILT_IN_LIBRARIES = "built-in-libraries"
 const FLAG_LIBRARIES = "libraries"
+const FLAG_PACKAGE_JSON = "package-index"
 const FLAG_PREFS = "prefs"
 const FLAG_FQBN = "fqbn"
 const FLAG_IDE_VERSION = "ide-version"
@@ -123,6 +124,7 @@ var hardwareFoldersFlag foldersFlag
 var toolsFoldersFlag foldersFlag
 var librariesBuiltInFoldersFlag foldersFlag
 var librariesFoldersFlag foldersFlag
+var jsonFoldersFlag foldersFlag
 var customBuildPropertiesFlag propertiesFlag
 var fqbnFlag *string
 var coreAPIVersionFlag *string
@@ -146,6 +148,7 @@ func init() {
 	flag.Var(&toolsFoldersFlag, FLAG_TOOLS, "Specify a 'tools' folder. Can be added multiple times for specifying multiple 'tools' folders")
 	flag.Var(&librariesBuiltInFoldersFlag, FLAG_BUILT_IN_LIBRARIES, "Specify a built-in 'libraries' folder. These are low priority libraries. Can be added multiple times for specifying multiple built-in 'libraries' folders")
 	flag.Var(&librariesFoldersFlag, FLAG_LIBRARIES, "Specify a 'libraries' folder. Can be added multiple times for specifying multiple 'libraries' folders")
+	flag.Var(&jsonFoldersFlag, FLAG_PACKAGE_JSON, "Specify a folder containing package_index json files. Can be added multiple times for specifying multiple folders")
 	flag.Var(&customBuildPropertiesFlag, FLAG_PREFS, "Specify a custom preference. Can be added multiple times for specifying multiple custom preferences")
 	fqbnFlag = flag.String(FLAG_FQBN, "", "fully qualified board name")
 	coreAPIVersionFlag = flag.String(FLAG_CORE_API_VERSION, "10600", "version of core APIs (used to populate ARDUINO #define)")
@@ -217,6 +220,13 @@ func main() {
 		ctx.OtherLibrariesFolders = librariesFolders
 	}
 
+	// FLAG_PACKAGE_JSON
+	if jsonFolders, err := toSliceOfUnquoted(jsonFoldersFlag); err != nil {
+		printCompleteError(err)
+	} else if len(jsonFolders) > 0 {
+		ctx.JsonFolders = jsonFolders
+	}
+
 	// FLAG_BUILT_IN_LIBRARIES
 	if librariesBuiltInFolders, err := toSliceOfUnquoted(librariesBuiltInFoldersFlag); err != nil {
 		printCompleteError(err)

constants/constants.go

@@ -232,3 +232,7 @@ const SKETCH_FOLDER_SRC = "src"
 const TOOL_NAME = "name"
 const TOOL_URL = "url"
 const TOOL_VERSION = "version"
+const ADDITIONALE_BOARD_MANAGER_JSON = "boardsmanager.additional.urls"
+const HACK_PROPERTIES_AVR_GCC_NEW = BUILD_PROPERTIES_RUNTIME_TOOLS_PREFIX + "avr-gcc-arduino-4.9.2-atmel3.5.3-arduino2" + BUILD_PROPERTIES_RUNTIME_TOOLS_SUFFIX
+const HACK_PROPERTIES_AVR_GCC_OLD = BUILD_PROPERTIES_RUNTIME_TOOLS_PREFIX + "avr-gcc-arduino-4.8.1-arduino5" + BUILD_PROPERTIES_RUNTIME_TOOLS_SUFFIX
+const HACK_PROPERTIES_AVR_GCC_OLD_2 = BUILD_PROPERTIES_RUNTIME_TOOLS_PREFIX + "avr-gcc" + BUILD_PROPERTIES_RUNTIME_TOOLS_SUFFIX

container_setup.go

@@ -52,6 +52,7 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context)
 		&LoadVIDPIDSpecificProperties{},
 		&SetCustomBuildProperties{},
 		&AddMissingBuildPropertiesFromParentPlatformTxtFiles{},
+		&OverridePropertiesWithJsonInfo{},
 	}
 
 	for _, command := range commands {

json_package_index/package_index.go

@@ -0,0 +1,249 @@
+/*
+ * This file is part of Arduino Builder.
+ *
+ * Copyright 2016 Arduino LLC (http://www.arduino.cc/)
+ *
+ * Arduino Builder is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As a special exception, you may use this file as part of a free software
+ * library without restriction.  Specifically, if other files instantiate
+ * templates or use macros or inline functions from this file, or you compile
+ * this file and link it with other files to produce an executable, this
+ * file does not by itself cause the resulting executable to be covered by
+ * the GNU General Public License.  This exception does not however
+ * invalidate any other reasons why the executable file might be covered by
+ * the GNU General Public License.
+ */
+
+package json_package_index
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/blang/semver"
+
+	"github.com/arduino/arduino-builder/constants"
+	_ "github.com/arduino/arduino-builder/i18n"
+	"github.com/arduino/arduino-builder/types"
+	"github.com/arduino/arduino-builder/utils"
+	"github.com/arduino/go-properties-map"
+)
+
+type core struct {
+	Architecture string `json:"architecture"`
+	Version      string `json:"version"`
+	URL          string `json:"url"`
+	Maintainer   string `json:"maintainer"`
+	Name         string `json:"archiveFileName"`
+	Checksum     string `json:"checksum"`
+	destination  string
+	installed    bool
+	Dependencies []struct {
+		Packager string `json:"packager"`
+		Name     string `json:"name"`
+		Version  string `json:"version"`
+	} `json:"toolsDependencies"`
+	CoreDependencies []struct {
+		Packager string `json:"packager"`
+	} `json:"coreDependencies"`
+}
+
+type tool struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+	Systems []struct {
+		Host     string `json:"host"`
+		URL      string `json:"url"`
+		Name     string `json:"archiveFileName"`
+		Checksum string `json:"checksum"`
+	} `json:"systems"`
+	url         string
+	destination string
+}
+
+type index struct {
+	Packages []struct {
+		Name       string `json:"name"`
+		Maintainer string `json:"maintainer"`
+		Platforms  []core `json:"platforms"`
+		Tools      []tool `json:"tools"`
+	} `json:"packages"`
+}
+
+var systems = map[string]string{
+	"linuxamd64":  "x86_64-linux-gnu",
+	"linux386":    "i686-linux-gnu",
+	"darwinamd64": "apple-darwin",
+	"windows386":  "i686-mingw32",
+}
+
+// globalProperties is a big map of properties maps in the form
+// globalProperties["arduino:avr:1.6.12"] = usual properties Map
+// at compile time, when de board is well defined, the relevant map
+// should be merged with the "classic" map overriding its values
+
+var globalProperties map[string]properties.Map
+
+func PackageIndexFoldersToPropertiesMap(packages *types.Packages, folders []string, specifiedFilenames []string) (map[string]properties.Map, error) {
+
+	var paths []string
+
+	for _, folder := range folders {
+		folder, err := filepath.Abs(folder)
+		if err != nil {
+			break
+		}
+		files, _ := ioutil.ReadDir(folder)
+		for _, f := range files {
+			if strings.HasPrefix(f.Name(), "package") && strings.HasSuffix(f.Name(), "index.json") {
+				// if a list of required json has been provided only add them
+				if specifiedFilenames != nil && len(specifiedFilenames) > 1 &&
+					!utils.SliceContains(specifiedFilenames, f.Name()) {
+					continue
+				} else {
+					paths = append(paths, filepath.Join(folder, f.Name()))
+				}
+			}
+		}
+	}
+	return PackageIndexesToPropertiesMap(packages, paths)
+}
+
+func PackageIndexesToPropertiesMap(packages *types.Packages, urls []string) (map[string]properties.Map, error) {
+
+	globalProperties = make(map[string]properties.Map)
+	coreDependencyMap := make(map[string]string)
+
+	data, err := PackageIndexesToGlobalIndex(packages, urls)
+
+	for _, p := range data.Packages {
+		for _, a := range p.Platforms {
+			localProperties := make(properties.Map)
+			for _, dep := range a.Dependencies {
+				localProperties[constants.BUILD_PROPERTIES_RUNTIME_TOOLS_PREFIX+dep.Name+constants.BUILD_PROPERTIES_RUNTIME_TOOLS_SUFFIX] =
+					"{" + constants.BUILD_PROPERTIES_RUNTIME_TOOLS_PREFIX + dep.Name + "-" + dep.Version + constants.BUILD_PROPERTIES_RUNTIME_TOOLS_SUFFIX + "}"
+				if dep.Packager != p.Name {
+					localProperties[constants.BUILD_PROPERTIES_RUNTIME_TOOLS_PREFIX+dep.Name+"-"+dep.Version+constants.BUILD_PROPERTIES_RUNTIME_TOOLS_SUFFIX] =
+						"{" + constants.BUILD_PROPERTIES_RUNTIME_TOOLS_PREFIX + dep.Name + "-" + dep.Packager + "-" + dep.Version + constants.BUILD_PROPERTIES_RUNTIME_TOOLS_SUFFIX + "}"
+				}
+			}
+			for _, coredep := range a.CoreDependencies {
+				// inherit all the tools from latest coredep
+				if err == nil {
+					coreDependencyMap[p.Name+":"+a.Architecture+":"+a.Version] =
+						coredep.Packager + ":" + a.Architecture
+				}
+			}
+			globalProperties[p.Name+":"+a.Architecture+":"+a.Version] = localProperties.Clone()
+		}
+	}
+
+	for idx, parentCore := range coreDependencyMap {
+		version, err := findLatestInstalledCore(data, strings.Split(parentCore, ":")[0], strings.Split(parentCore, ":")[1])
+		if err == nil {
+			globalProperties[idx] = globalProperties[parentCore+":"+version].Clone()
+		}
+	}
+
+	return globalProperties, err
+}
+
+func findLatestInstalledCore(data index, Packager string, Name string) (string, error) {
+	latest, _ := semver.Make("0.0.0")
+	for _, p := range data.Packages {
+		for _, a := range p.Platforms {
+			if p.Name == Packager && a.Architecture == Name {
+				test, _ := semver.Make(a.Version)
+				if test.GT(latest) && a.installed {
+					latest = test
+				}
+			}
+		}
+	}
+	var err error
+	test, _ := semver.Make("0.0.0")
+	if latest.EQ(test) {
+		err = errors.New("No such core available")
+	} else {
+		err = nil
+	}
+	return latest.String(), err
+}
+
+func PackageIndexesToGlobalIndex(packages *types.Packages, urls []string) (index, error) {
+
+	// first stub of arduino-pdpm
+	var data index
+	var err error
+
+	for _, url := range urls {
+
+		var body []byte
+		var localdata index
+		localpath, _ := filepath.Abs(url)
+		_, err := os.Stat(localpath)
+
+		if err != nil {
+			resp, err := http.Get(url)
+			if err == nil {
+				defer resp.Body.Close()
+				body, err = ioutil.ReadAll(resp.Body)
+				if err != nil {
+					break
+				}
+			}
+		} else {
+			body, err = ioutil.ReadFile(localpath)
+			if err != nil {
+				break
+			}
+		}
+		json.Unmarshal(body, &localdata)
+		for _, entry := range localdata.Packages {
+			data.Packages = append(data.Packages, entry)
+		}
+	}
+
+	for i, p := range data.Packages {
+		for j, a := range p.Platforms {
+			if packages != nil && packages.Packages[p.Name] != nil &&
+				packages.Packages[p.Name].Platforms[a.Architecture] != nil &&
+				packages.Packages[p.Name].Platforms[a.Architecture].Properties["version"] == a.Version {
+				data.Packages[i].Platforms[j].installed = true
+			}
+		}
+	}
+
+	return data, err
+}
+
+func CompareVersions(fv string, sv string) int {
+	v1, _ := semver.Make(fv)
+	v2, _ := semver.Make(sv)
+	if v1.EQ(v2) {
+		return 0
+	}
+	if v1.GT(v2) {
+		return 1
+	} else {
+		return -1
+	}
+}