Skip to content

Commit 4b3758b

Browse files
silvanocerzasilvanocerza
authored
Add support for file:// schema to set local additional URLs (#1098)
* Add support for file:// schema to set local additional URLs * Fix local path parsing on Windows * Move URL parsing in utility function
1 parent d0e904c commit 4b3758b

File tree

12 files changed

+145
-77
lines changed

12 files changed

+145
-77
lines changed

Diff for: ‎arduino/utils/url.go

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to [email protected].
15+
16+
package utils
17+
18+
import (
19+
"net/url"
20+
"runtime"
21+
)
22+
23+
// URLParse parses a raw URL string and handles local files URLs depending on the platform
24+
func URLParse(rawURL string) (*url.URL, error) {
25+
URL, err := url.Parse(rawURL)
26+
if err != nil {
27+
return nil, err
28+
}
29+
if URL.Scheme == "file" && runtime.GOOS == "windows" {
30+
// Parsed local file URLs on Windows are returned with a leading /
31+
// so we remove it
32+
URL.Path = URL.Path[1:]
33+
}
34+
return URL, nil
35+
}

Diff for: ‎arduino/utils/url_test.go

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to [email protected].
15+
16+
package utils
17+
18+
import (
19+
"fmt"
20+
"runtime"
21+
"testing"
22+
23+
"github.com/stretchr/testify/require"
24+
)
25+
26+
func TestURLParse(t *testing.T) {
27+
type test struct {
28+
URL string
29+
ExpectedHost string
30+
ExpectedPath string
31+
Skip bool
32+
}
33+
onWindows := runtime.GOOS == "windows"
34+
tests := []test{
35+
{"https://example.com", "example.com", "", false},
36+
{"https://example.com/some/path", "example.com", "/some/path", false},
37+
{"file:///home/user/some/path", "", "/home/user/some/path", onWindows},
38+
{"file:///C:/Users/me/some/path", "", "C:/Users/me/some/path", !onWindows},
39+
}
40+
41+
for i, test := range tests {
42+
t.Run(fmt.Sprintf("URLParseTest%02d", i), func(t *testing.T) {
43+
if test.Skip {
44+
t.Skip("Skipped")
45+
}
46+
res, err := URLParse(test.URL)
47+
require.NoError(t, err)
48+
require.Equal(t, test.ExpectedPath, res.Path)
49+
})
50+
}
51+
}

Diff for: ‎cli/cli.go

-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,6 @@ func createCliCommandTree(cmd *cobra.Command) {
104104
cmd.PersistentFlags().StringVar(&outputFormat, "format", "text", "The output format, can be {text|json}.")
105105
cmd.PersistentFlags().StringVar(&configFile, "config-file", "", "The custom config file (if not specified the default will be used).")
106106
cmd.PersistentFlags().StringSlice("additional-urls", []string{}, "Comma-separated list of additional URLs for the Boards Manager.")
107-
cmd.PersistentFlags().StringSlice("additional-paths", []string{}, "Comma-separated list of additional file paths for the Boards Manager.")
108107
configuration.BindFlags(cmd, configuration.Settings)
109108
}
110109

Diff for: ‎cli/config/validate.go

+12-13
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,18 @@ import (
2121
)
2222

2323
var validMap = map[string]reflect.Kind{
24-
"board_manager.additional_urls": reflect.Slice,
25-
"board_manager.additional_paths": reflect.Slice,
26-
"daemon.port": reflect.String,
27-
"directories.data": reflect.String,
28-
"directories.downloads": reflect.String,
29-
"directories.user": reflect.String,
30-
"library.enable_unsafe_install": reflect.Bool,
31-
"logging.file": reflect.String,
32-
"logging.format": reflect.String,
33-
"logging.level": reflect.String,
34-
"sketch.always_export_binaries": reflect.Bool,
35-
"telemetry.addr": reflect.String,
36-
"telemetry.enabled": reflect.Bool,
24+
"board_manager.additional_urls": reflect.Slice,
25+
"daemon.port": reflect.String,
26+
"directories.data": reflect.String,
27+
"directories.downloads": reflect.String,
28+
"directories.user": reflect.String,
29+
"library.enable_unsafe_install": reflect.Bool,
30+
"logging.file": reflect.String,
31+
"logging.format": reflect.String,
32+
"logging.level": reflect.String,
33+
"sketch.always_export_binaries": reflect.Bool,
34+
"telemetry.addr": reflect.String,
35+
"telemetry.enabled": reflect.Bool,
3736
}
3837

3938
func typeOf(key string) (reflect.Kind, error) {

Diff for: ‎commands/daemon/testdata/arduino-cli.yml

-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ board_manager:
22
additional_urls:
33
- http://foobar.com
44
- http://example.com
5-
additional_paths: []
65

76
daemon:
87
port: "50051"

Diff for: ‎commands/instances.go

+30-28
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"github.com/arduino/arduino-cli/arduino/libraries/librariesindex"
3333
"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
3434
"github.com/arduino/arduino-cli/arduino/security"
35+
"github.com/arduino/arduino-cli/arduino/utils"
3536
"github.com/arduino/arduino-cli/cli/globals"
3637
"github.com/arduino/arduino-cli/configuration"
3738
rpc "github.com/arduino/arduino-cli/rpc/commands"
@@ -202,36 +203,33 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo
202203

203204
indexpath := paths.New(configuration.Settings.GetString("directories.Data"))
204205

205-
for _, x := range configuration.Settings.GetStringSlice("board_manager.additional_paths") {
206-
logrus.Info("JSON PATH: ", x)
207-
208-
pathJSON, _ := paths.New(x).Abs()
209-
210-
if _, err := packageindex.LoadIndexNoSign(pathJSON); err != nil {
211-
return nil, fmt.Errorf("invalid package index in %s: %s", pathJSON, err)
212-
}
213-
214-
fi, _ := os.Stat(x)
215-
downloadCB(&rpc.DownloadProgress{
216-
File: "Updating index: " + pathJSON.Base(),
217-
TotalSize: fi.Size(),
218-
})
219-
downloadCB(&rpc.DownloadProgress{Completed: true})
220-
221-
}
222-
223206
urls := []string{globals.DefaultIndexURL}
224207
urls = append(urls, configuration.Settings.GetStringSlice("board_manager.additional_urls")...)
225208
for _, u := range urls {
226209
logrus.Info("URL: ", u)
227-
URL, err := url.Parse(u)
210+
URL, err := utils.URLParse(u)
228211
if err != nil {
229212
logrus.Warnf("unable to parse additional URL: %s", u)
230213
continue
231214
}
232215

233216
logrus.WithField("url", URL).Print("Updating index")
234217

218+
if URL.Scheme == "file" {
219+
path := paths.New(URL.Path)
220+
if _, err := packageindex.LoadIndexNoSign(path); err != nil {
221+
return nil, fmt.Errorf("invalid package index in %s: %s", path, err)
222+
}
223+
224+
fi, _ := os.Stat(path.String())
225+
downloadCB(&rpc.DownloadProgress{
226+
File: "Updating index: " + path.Base(),
227+
TotalSize: fi.Size(),
228+
})
229+
downloadCB(&rpc.DownloadProgress{Completed: true})
230+
continue
231+
}
232+
235233
var tmp *paths.Path
236234
if tmpFile, err := ioutil.TempFile("", ""); err != nil {
237235
return nil, fmt.Errorf("creating temp file for index download: %s", err)
@@ -659,22 +657,26 @@ func createInstance(ctx context.Context, getLibOnly bool) (*createInstanceResult
659657
urls := []string{globals.DefaultIndexURL}
660658
urls = append(urls, configuration.Settings.GetStringSlice("board_manager.additional_urls")...)
661659
for _, u := range urls {
662-
URL, err := url.Parse(u)
660+
URL, err := utils.URLParse(u)
663661
if err != nil {
664662
logrus.Warnf("Unable to parse index URL: %s, skip...", u)
665663
continue
666664
}
667665

668-
if err := res.Pm.LoadPackageIndex(URL); err != nil {
669-
res.PlatformIndexErrors = append(res.PlatformIndexErrors, err.Error())
670-
}
671-
}
666+
if URL.Scheme == "file" {
667+
path := paths.New(URL.Path)
668+
if err != nil {
669+
return nil, fmt.Errorf("can't get absolute path of %v: %w", path, err)
670+
}
672671

673-
for _, x := range configuration.Settings.GetStringSlice("board_manager.additional_paths") {
674-
pathJSON, _ := paths.New(x).Abs()
672+
_, err = res.Pm.LoadPackageIndexFromFile(path)
673+
if err != nil {
674+
res.PlatformIndexErrors = append(res.PlatformIndexErrors, err.Error())
675+
}
676+
continue
677+
}
675678

676-
_, err := res.Pm.LoadPackageIndexFromFile(pathJSON)
677-
if err != nil {
679+
if err := res.Pm.LoadPackageIndex(URL); err != nil {
678680
res.PlatformIndexErrors = append(res.PlatformIndexErrors, err.Error())
679681
}
680682
}

Diff for: ‎configuration/configuration.go

-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,6 @@ func BindFlags(cmd *cobra.Command, settings *viper.Viper) {
7676
settings.BindPFlag("logging.file", cmd.Flag("log-file"))
7777
settings.BindPFlag("logging.format", cmd.Flag("log-format"))
7878
settings.BindPFlag("board_manager.additional_urls", cmd.Flag("additional-urls"))
79-
settings.BindPFlag("board_manager.additional_paths", cmd.Flag("additional-paths"))
8079
}
8180

8281
// getDefaultArduinoDataDir returns the full path to the default arduino folder

Diff for: ‎configuration/configuration_test.go

-1
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,6 @@ func TestInit(t *testing.T) {
8989
require.Equal(t, "text", settings.GetString("logging.format"))
9090

9191
require.Empty(t, settings.GetStringSlice("board_manager.additional_urls"))
92-
require.Empty(t, settings.GetStringSlice("board_manager.additional_paths"))
9392

9493
require.NotEmpty(t, settings.GetString("directories.Data"))
9594
require.NotEmpty(t, settings.GetString("directories.Downloads"))

Diff for: ‎configuration/defaults.go

-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ func SetDefaults(settings *viper.Viper) {
3333

3434
// Boards Manager
3535
settings.SetDefault("board_manager.additional_urls", []string{})
36-
settings.SetDefault("board_manager.additional_paths", []string{})
3736

3837
// arduino directories
3938
settings.SetDefault("directories.Data", getDefaultArduinoDataDir())

Diff for: ‎docs/configuration.md

-23
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22

33
- `board_manager`
44
- `additional_urls` - the URLs to any additional Boards Manager package index files needed for your boards platforms.
5-
- `additional_paths` - the absolute file paths to any additional Boards Manager package index files needed for your
6-
boards platforms.
75
- `daemon` - options related to running Arduino CLI as a [gRPC] server.
86
- `port` - TCP port used for gRPC client connections.
97
- `directories` - directories used by Arduino CLI.
@@ -68,12 +66,6 @@ Setting an additional Boards Manager URL using the `ARDUINO_BOARD_MANAGER_ADDITI
6866
$ export ARDUINO_BOARD_MANAGER_ADDITIONAL_URLS=https://downloads.arduino.cc/packages/package_staging_index.json
6967
```
7068

71-
Setting an additional Boards Manager file using the `ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS` environment variable:
72-
73-
```sh
74-
$ export ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS=/path/to/your/package_staging_index.json
75-
```
76-
7769
### Configuration file
7870

7971
[`arduino-cli config init`][arduino-cli config init] creates or updates a configuration file with the current
@@ -136,21 +128,6 @@ Doing the same using a TOML format file:
136128
additional_urls = [ "https://downloads.arduino.cc/packages/package_staging_index.json" ]
137129
```
138130

139-
Setting an additional Boards Manager File using a YAML format configuration file:
140-
141-
```yaml
142-
board_manager:
143-
additional_paths:
144-
- /path/to/your/package_staging_index.json
145-
```
146-
147-
Doing the same using a TOML format file:
148-
149-
```toml
150-
[board_manager]
151-
additional_paths = [ "/path/to/your/package_staging_index.json" ]
152-
```
153-
154131
[grpc]: https://grpc.io
155132
[sketchbook directory]: sketch-specification.md#sketchbook
156133
[arduino cli lib install]: commands/arduino-cli_lib_install.md

Diff for: ‎docs/getting-started.md

+7-8
Original file line numberDiff line numberDiff line change
@@ -183,12 +183,12 @@ For example, to add the NRF52832 core, edit the configuration file and change th
183183

184184
```yaml
185185
board_manager:
186-
additional_paths:
187-
- /absolute/path/to/your/package_nrf52832_index.json
186+
additional_urls:
187+
- https://arduino.esp8266.com/stable/package_esp8266com_index.json
188+
- file:///absolute/path/to/your/package_nrf52832_index.json
188189
```
189190

190-
From now on, commands supporting custom cores will automatically use the additional URL and additional paths from the
191-
configuration file:
191+
From now on, commands supporting custom cores will automatically use the additional URL from the configuration file:
192192

193193
```sh
194194
$ arduino-cli core update-index
@@ -214,14 +214,13 @@ ID Version Name
214214
esp8266:esp8266 2.5.2 esp8266
215215
```
216216

217-
The same applies to the additional package index file provided by file paths. Use the `--additional-paths` option, that
218-
has to be specified every time and for every command that operates on a 3rd party platform core, for example:
217+
The same applies to the additional package index file provided by file paths:
219218

220219
```sh
221-
$ arduino-cli core update-index --additional-paths /absolute/path/to/your/package_esp8266com_index.json
220+
$ arduino-cli core update-index --additional-urls file:///absolute/path/to/your/package_esp8266com_index.json
222221
Updating index: package_esp8266com_index.json downloaded
223222
224-
$ arduino-cli core search esp8266 --additional-paths /absolute/path/to/your/package_esp8266com_index.json
223+
$ arduino-cli core search esp8266 --additional-urls file:///absolute/path/to/your/package_esp8266com_index.json
225224
ID Version Name
226225
esp8266:esp8266 2.5.2 esp8266
227226
```

Diff for: ‎test/test_core.py

+10
Original file line numberDiff line numberDiff line change
@@ -353,3 +353,13 @@ def ordered(obj):
353353
return obj
354354

355355
assert ordered(installed_json) == ordered(expected_installed_json)
356+
357+
358+
def test_core_update_with_local_url(run_command):
359+
test_index = str(Path(__file__).parent / "testdata" / "test_index.json")
360+
if platform.system() == "Windows":
361+
test_index = f"/{test_index}".replace("\\", "/")
362+
363+
res = run_command(f'core update-index --additional-urls="file://{test_index}"')
364+
assert res.ok
365+
assert "Updating index: test_index.json downloaded" in res.stdout

0 commit comments

Comments
 (0)