diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cb90e3998ae..9f93e13eb25 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,6 +60,7 @@ see following paragraph): task test-integration ``` ### Running only some tests + By default, all tests from all go packages are run. To run only unit tests from one or more specific packages, you can set the TARGETS environment variable, e.g.: @@ -75,11 +76,6 @@ Both can be combined as well, typically to run only a specific test: TEST_REGEX='^TestFindBoardWithFQBN$' TARGETS=./arduino/cores/packagemanager task test-unit -For integration test, the same options are supported. Note that when not -specified, `TEST_REGEX` defaults to "Integration" to select only -integration tests, so if you specify a broader regex, this could cause -non-integration tests to be run as well. - ### Integration tests Being a command line interface, Arduino CLI is heavily interactive and it has to @@ -95,7 +91,8 @@ assess the options are correctly understood and the output is what we expect. To run the full suite of integration tests you need an Arduino device attached to a serial port and a working Python environment. Chances are that you already have Python installed in your system, if this is not the case you can -[download][3] the official distribution or use the package manager provided by your Operating System. +[download][3] the official distribution or use the package manager provided by +your Operating System. Some dependencies need to be installed before running the tests and to avoid polluting your global Python enviroment with dependencies that might be only @@ -151,7 +148,8 @@ a list of items you can check before submitting a PR: * Maintain **clean commit history** and use **meaningful commit messages**. PRs with messy commit history are difficult to review and require a lot of work to be merged. -* Your PR must pass all CI tests before we will merge it. If you're seeing an error and don't think +* Your PR must pass all CI tests before we will merge it. If you're seeing an + error and don't think it's your fault, it may not be! The reviewer will help you if there are test failures that seem not related to the change you are making. diff --git a/Taskfile.yml b/Taskfile.yml index f4526b3ce4f..780f80f0b9b 100755 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -27,7 +27,6 @@ tasks: test-integration: desc: Run integration tests only cmds: - - go test -run '{{ default "Integration" .TEST_REGEX }}' {{ default "-v" .GOFLAGS }} -coverprofile=coverage_integ.txt {{ default .DEFAULT_TARGETS .TARGETS }} {{.TEST_LDFLAGS}} - pytest test test-legacy: diff --git a/cli/cli_test.go b/cli/cli_test.go index 0a8ed8a7087..72ee80a1eac 100644 --- a/cli/cli_test.go +++ b/cli/cli_test.go @@ -19,109 +19,15 @@ package cli import ( - "encoding/json" - "flag" "fmt" "io/ioutil" "os" "path/filepath" - "runtime" "testing" - "github.com/spf13/viper" - - "github.com/arduino/arduino-cli/cli/errorcodes" - "github.com/arduino/arduino-cli/cli/feedback" - - "bou.ke/monkey" - paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" - semver "go.bug.st/relaxed-semver" ) -var ( - // Redirecting stdOut so we can analyze output line by - // line and check with what we want. - stdOut = os.Stdout - stdErr = os.Stderr - - currDownloadDir string - currDataDir string - currUserDir string -) - -type outputRedirect struct { - tempFile *os.File -} - -func (grabber *outputRedirect) Open() { - tempFile, err := ioutil.TempFile(os.TempDir(), "test") - if err != nil { - panic("Opening temp output file") - } - os.Stdout = tempFile - os.Stderr = tempFile - grabber.tempFile = tempFile -} - -func (grabber *outputRedirect) GetOutput() []byte { - _, err := grabber.tempFile.Seek(0, 0) - if err != nil { - panic("Rewinding temp output file") - } - - output, err := ioutil.ReadAll(grabber.tempFile) - if err != nil { - panic("Reading temp output file") - } - - return output -} - -func (grabber *outputRedirect) Close() { - grabber.tempFile.Close() - err := os.Remove(grabber.tempFile.Name()) - if err != nil { - panic("Removing temp output file") - } - os.Stdout = stdOut - os.Stderr = stdErr -} - -func TestMain(m *testing.M) { - // all these tests perform actual operations, don't run in short mode - flag.Parse() - if testing.Short() { - fmt.Println("skip integration tests") - os.Exit(0) - } - - // SetUp - currDataDir = tmpDirOrDie() - os.MkdirAll(filepath.Join(currDataDir, "packages"), 0755) - os.Setenv("ARDUINO_DIRECTORIES_DATA", currDataDir) - currDownloadDir = tmpDirOrDie() - os.Setenv("ARDUINO_DIRECTORIES_DOWNLOADS", currDownloadDir) - currUserDir = filepath.Join("testdata", "custom_hardware") - // use ARDUINO_SKETCHBOOK_DIR instead of ARDUINO_DIRECTORIES_USER to - // ensure the backward compat code is working - os.Setenv("ARDUINO_SKETCHBOOK_DIR", currUserDir) - - // Run - res := m.Run() - - // TearDown - os.RemoveAll(currDataDir) - os.Unsetenv("ARDUINO_DIRECTORIES_DATA") - currDataDir = "" - os.RemoveAll(currDownloadDir) - os.Unsetenv("ARDUINO_DIRECTORIES_DOWNLOADS") - currDownloadDir = "" - os.Unsetenv("ARDUINO_SKETCHBOOK_DIR") - - os.Exit(res) -} - func tmpDirOrDie() string { dir, err := ioutil.TempDir(os.TempDir(), "cli_test") if err != nil { @@ -130,365 +36,6 @@ func tmpDirOrDie() string { return dir } -// executeWithArgs executes the Cobra Command with the given arguments -// and intercepts any errors (even `os.Exit()` ones), returning the exit code -func executeWithArgs(args ...string) (int, []byte) { - var output []byte - var exitCode int - fmt.Printf("RUNNING: %s\n", args) - viper.Reset() - - // This closure is here because we won't that the defer are executed after the end of the "executeWithArgs" method - func() { - redirect := &outputRedirect{} - redirect.Open() - // re-init feedback so it'll write to our grabber - feedback.SetDefaultFeedback(feedback.New(os.Stdout, os.Stdout, feedback.Text)) - defer func() { - output = redirect.GetOutput() - redirect.Close() - fmt.Print(string(output)) - fmt.Println() - }() - - // Mock the os.Exit function, so that we can use the - // error result for the test and prevent the test from exiting - fakeExitFired := false - fakeExit := func(code int) { - exitCode = code - fakeExitFired = true - - // use panic to exit and jump to deferred recover - panic(fmt.Errorf("os.Exit(%d)", code)) - } - patch := monkey.Patch(os.Exit, fakeExit) - defer patch.Unpatch() - defer func() { - if fakeExitFired { - recover() - } - }() - - // Execute the CLI command, start fresh every time - ArduinoCli.ResetCommands() - ArduinoCli.ResetFlags() - createCliCommandTree(ArduinoCli) - ArduinoCli.SetArgs(args) - if err := ArduinoCli.Execute(); err != nil { - exitCode = errorcodes.ErrGeneric - } - }() - - return exitCode, output -} - -func detectLatestAVRCore(t *testing.T) string { - jsonFile := filepath.Join(currDataDir, "package_index.json") - type index struct { - Packages []struct { - Name string - Platforms []struct { - Architecture string - Version string - } - } - } - var jsonIndex index - jsonData, err := ioutil.ReadFile(jsonFile) - require.NoError(t, err, "reading package_index.json") - err = json.Unmarshal(jsonData, &jsonIndex) - require.NoError(t, err, "parsing package_index.json") - latest := semver.MustParse("0.0.1") - for _, p := range jsonIndex.Packages { - if p.Name == "arduino" { - for _, pl := range p.Platforms { - ver, err := semver.Parse(pl.Version) - require.NoError(t, err, "version parsing") - if pl.Architecture == "avr" && ver.GreaterThan(latest) { - latest = ver - } - } - break - } - } - require.NotEmpty(t, latest, "latest avr core version") - return latest.String() -} - -// END -- Utility functions - -func TestUploadIntegration(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("This test runs only on Linux") - } - - exitCode, _ := executeWithArgs("core", "update-index") - require.Zero(t, exitCode) - - exitCode, _ = executeWithArgs("core", "install", "arduino:avr") - require.Zero(t, exitCode) - - // -i flag - exitCode, d := executeWithArgs("upload", "-i", filepath.Join(currUserDir, "test.hex"), "-b", "test:avr:testboard", "-p", "/dev/ttyACM0") - require.Zero(t, exitCode) - require.Contains(t, string(d), "QUIET") - require.Contains(t, string(d), "NOVERIFY") - require.Contains(t, string(d), "testdata/custom_hardware/test.hex") - - // -i flag with implicit extension - exitCode, d = executeWithArgs("upload", "-i", filepath.Join(currUserDir, "test"), "-b", "test:avr:testboard", "-p", "/dev/ttyACM0") - require.Zero(t, exitCode) - require.Contains(t, string(d), "QUIET") - require.Contains(t, string(d), "NOVERIFY") - require.Contains(t, string(d), "testdata/custom_hardware/test.hex") - - // -i with absolute path - fullPath, err := filepath.Abs(filepath.Join(currUserDir, "test.hex")) - require.NoError(t, err) - exitCode, d = executeWithArgs("upload", "-i", fullPath, "-b", "test:avr:testboard", "-p", "/dev/ttyACM0") - require.Zero(t, exitCode) - require.Contains(t, string(d), "QUIET") - require.Contains(t, string(d), "NOVERIFY") - require.Contains(t, string(d), "testdata/custom_hardware/test.hex") - - // -v verbose - exitCode, d = executeWithArgs("upload", "-v", "-t", "-i", filepath.Join(currUserDir, "test.hex"), "-b", "test:avr:testboard", "-p", "/dev/ttyACM0") - require.Zero(t, exitCode) - require.Contains(t, string(d), "VERBOSE") - require.Contains(t, string(d), "VERIFY") - require.Contains(t, string(d), "testdata/custom_hardware/test.hex") - - // -t verify - exitCode, d = executeWithArgs("upload", "-i", filepath.Join(currUserDir, "test.hex"), "-b", "test:avr:testboard", "-p", "/dev/ttyACM0") - require.Zero(t, exitCode) - require.Contains(t, string(d), "QUIET") - require.Contains(t, string(d), "NOVERIFY") - require.Contains(t, string(d), "testdata/custom_hardware/test.hex") - - // -v -t verbose verify - exitCode, d = executeWithArgs("upload", "-v", "-t", "-i", filepath.Join(currUserDir, "test.hex"), "-b", "test:avr:testboard", "-p", "/dev/ttyACM0") - require.Zero(t, exitCode) - require.Contains(t, string(d), "VERBOSE") - require.Contains(t, string(d), "VERIFY") - require.Contains(t, string(d), "testdata/custom_hardware/test.hex") - - // non-existent file - exitCode, _ = executeWithArgs("upload", "-i", filepath.Join(currUserDir, "test123.hex"), "-b", "test:avr:testboard", "-p", "/dev/ttyACM0") - require.NotZero(t, exitCode) - - // sketch - exitCode, d = executeWithArgs("upload", filepath.Join(currUserDir, "TestSketch"), "-b", "test:avr:testboard", "-p", "/dev/ttyACM0") - require.Zero(t, exitCode) - require.Contains(t, string(d), "QUIET") - require.Contains(t, string(d), "NOVERIFY") - require.Contains(t, string(d), "testdata/custom_hardware/TestSketch/TestSketch.test.avr.testboard.hex") - - // sketch without build - exitCode, _ = executeWithArgs("upload", filepath.Join(currUserDir, "TestSketch2"), "-b", "test:avr:testboard", "-p", "/dev/ttyACM0") - require.NotZero(t, exitCode) - - // platform without 'recipe.output.tmp_file' property - exitCode, _ = executeWithArgs("upload", "-i", filepath.Join(currUserDir, "test.hex"), "-b", "test2:avr:testboard", "-p", "/dev/ttyACM0") - require.NotZero(t, exitCode) -} - -func TestCompileCommandsIntegration(t *testing.T) { - // Set staging dir to a temporary dir - tmp := tmpDirOrDie() - defer os.RemoveAll(tmp) - - exitCode, _ := executeWithArgs("core", "update-index") - require.Zero(t, exitCode) - - // Download latest AVR - exitCode, _ = executeWithArgs("core", "install", "arduino:avr") - require.Zero(t, exitCode) - - // Create a test sketch - sketchPath := filepath.Join(tmp, "Test1") - exitCode, d := executeWithArgs("sketch", "new", sketchPath) - require.Zero(t, exitCode) - - // Build sketch without FQBN - exitCode, d = executeWithArgs("compile", sketchPath) - require.NotZero(t, exitCode) - require.Contains(t, string(d), "no FQBN provided") - - // Build sketch for arduino:avr:uno - exitCode, d = executeWithArgs("compile", "-b", "arduino:avr:uno", sketchPath) - require.Zero(t, exitCode) - require.Contains(t, string(d), "Sketch uses") - require.True(t, paths.New(sketchPath).Join("Test1.arduino.avr.uno.hex").Exist()) - - // Build sketch for arduino:avr:nano (without options) - exitCode, d = executeWithArgs("compile", "-b", "arduino:avr:nano", sketchPath) - require.Zero(t, exitCode) - require.Contains(t, string(d), "Sketch uses") - require.True(t, paths.New(sketchPath).Join("Test1.arduino.avr.nano.hex").Exist()) - - // Build sketch with --output path - pwd, err := os.Getwd() - require.NoError(t, err) - defer func() { require.NoError(t, os.Chdir(pwd)) }() - require.NoError(t, os.Chdir(tmp)) - - exitCode, d = executeWithArgs("compile", "-b", "arduino:avr:nano", "-o", "test", sketchPath) - require.Zero(t, exitCode) - require.Contains(t, string(d), "Sketch uses") - require.True(t, paths.New("test.hex").Exist()) - - exitCode, d = executeWithArgs("compile", "-b", "arduino:avr:nano", "-o", "test2.hex", sketchPath) - require.Zero(t, exitCode) - require.Contains(t, string(d), "Sketch uses") - require.True(t, paths.New("test2.hex").Exist()) - require.NoError(t, paths.New(tmp, "anothertest").MkdirAll()) - - exitCode, d = executeWithArgs("compile", "-b", "arduino:avr:nano", "-o", "anothertest/test", sketchPath) - require.Zero(t, exitCode) - require.Contains(t, string(d), "Sketch uses") - require.True(t, paths.New("anothertest", "test.hex").Exist()) - - exitCode, d = executeWithArgs("compile", "-b", "arduino:avr:nano", "-o", tmp+"/anothertest/test2", sketchPath) - require.Zero(t, exitCode) - require.Contains(t, string(d), "Sketch uses") - require.True(t, paths.New("anothertest", "test2.hex").Exist()) -} - -func TestInvalidCoreURLIntegration(t *testing.T) { - configFile := filepath.Join("testdata", t.Name()) - - // Dump config with cmd-line specific file - exitCode, d := executeWithArgs("--config-file", configFile, "config", "dump") - require.Zero(t, exitCode) - require.Contains(t, string(d), "- http://www.invalid-domain-asjkdakdhadjkh.com/package_example_index.json") - - // Update inexistent index - exitCode, _ = executeWithArgs("--config-file", configFile, "core", "update-index") - require.NotZero(t, exitCode) -} - -func Test3rdPartyCoreIntegration(t *testing.T) { - configFile := filepath.Join("testdata", t.Name()) - - // Update index and install esp32:esp32 - exitCode, _ := executeWithArgs("--config-file", configFile, "core", "update-index") - require.Zero(t, exitCode) - exitCode, d := executeWithArgs("--config-file", configFile, "core", "install", "esp32:esp32") - require.Zero(t, exitCode) - require.Contains(t, string(d), "installed") - - // Build a simple sketch and check if all artifacts are copied - tmp := tmpDirOrDie() - defer os.RemoveAll(tmp) - tmpSketch := paths.New(tmp).Join("Blink") - err := paths.New("testdata/Blink").CopyDirTo(tmpSketch) - require.NoError(t, err, "copying test sketch into temp dir") - exitCode, d = executeWithArgs("--config-file", configFile, "compile", "-b", "esp32:esp32:esp32", tmpSketch.String()) - require.Zero(t, exitCode) - require.True(t, tmpSketch.Join("Blink.esp32.esp32.esp32.bin").Exist()) - require.True(t, tmpSketch.Join("Blink.esp32.esp32.esp32.elf").Exist()) - require.True(t, tmpSketch.Join("Blink.esp32.esp32.esp32.partitions.bin").Exist()) // https://github.com/arduino/arduino-cli/issues/163 -} - -func TestCoreCommandsIntegration(t *testing.T) { - exitCode, _ := executeWithArgs("core", "update-index") - require.Zero(t, exitCode) - - AVR := "arduino:avr@" + detectLatestAVRCore(t) - - // Download a specific core version - exitCode, d := executeWithArgs("core", "download", "arduino:avr@1.6.16") - require.Zero(t, exitCode) - require.Regexp(t, "arduino:avr-gcc@4.9.2-atmel3.5.3-arduino2 (already )?downloaded", string(d)) - require.Regexp(t, "arduino:avrdude@6.3.0-arduino8 (already )?downloaded", string(d)) - require.Regexp(t, "arduino:arduinoOTA@1.0.0 (already )?downloaded", string(d)) - require.Regexp(t, "arduino:avr@1.6.16 (already )?downloaded", string(d)) - - // Download latest - exitCode, d = executeWithArgs("core", "download", "arduino:avr") - require.Zero(t, exitCode) - require.Regexp(t, AVR+" (already )?downloaded", string(d)) - - // Wrong downloads - exitCode, d = executeWithArgs("core", "download", "arduino:samd@1.2.3-notexisting") - require.NotZero(t, exitCode) - require.Contains(t, string(d), "required version 1.2.3-notexisting not found for platform arduino:samd") - - exitCode, d = executeWithArgs("core", "download", "arduino:notexistent") - require.NotZero(t, exitCode) - require.Contains(t, string(d), "not found") - - exitCode, d = executeWithArgs("core", "download", "wrongparameter") - require.NotZero(t, exitCode) - require.Contains(t, string(d), "invalid item") - - // Install avr 1.6.16 - exitCode, d = executeWithArgs("core", "install", "arduino:avr@1.6.16") - require.Zero(t, exitCode) - require.Contains(t, string(d), "arduino:avr@1.6.16 installed") - - exitCode, d = executeWithArgs("core", "list") - require.Zero(t, exitCode) - require.Contains(t, string(d), "arduino:avr") - require.Contains(t, string(d), "1.6.16") - - exitCode, d = executeWithArgs("core", "list", "--format", "json") - require.Zero(t, exitCode) - require.Contains(t, string(d), "arduino:avr") - require.Contains(t, string(d), "1.6.16") - - // Replace avr with 1.6.17 - exitCode, d = executeWithArgs("core", "install", "arduino:avr@1.6.17") - require.Zero(t, exitCode) - require.Contains(t, string(d), "Updating arduino:avr@1.6.16 with arduino:avr@1.6.17") - require.Contains(t, string(d), "arduino:avr@1.6.17 installed") - - exitCode, d = executeWithArgs("core", "list") - require.Zero(t, exitCode) - require.Contains(t, string(d), "arduino:avr") - require.Contains(t, string(d), "1.6.17") - - // List updatable cores - exitCode, d = executeWithArgs("core", "list", "--updatable") - require.Zero(t, exitCode) - require.Contains(t, string(d), "arduino:avr") - - exitCode, d = executeWithArgs("core", "list", "--updatable", "--format", "json") - require.Zero(t, exitCode) - require.Contains(t, string(d), "arduino:avr") - - // Upgrade platform - exitCode, d = executeWithArgs("core", "upgrade", "arduino:avr@1.6.18") - require.NotZero(t, exitCode) - require.Contains(t, string(d), "Invalid item arduino:avr@1.6.18") - - exitCode, d = executeWithArgs("core", "upgrade", "other:avr") - require.NotZero(t, exitCode) - require.Contains(t, string(d), "other:avr not found") - - exitCode, d = executeWithArgs("core", "upgrade", "arduino:samd") - require.NotZero(t, exitCode) - require.Contains(t, string(d), "arduino:samd is not installed") - - exitCode, d = executeWithArgs("core", "upgrade", "arduino:avr") - require.Zero(t, exitCode) - require.Contains(t, string(d), "Updating arduino:avr@1.6.17 with "+AVR) - - // List updatable cores - exitCode, d = executeWithArgs("core", "list", "--updatable") - require.Zero(t, exitCode) - require.NotContains(t, string(d), "arduino:avr") - - exitCode, d = executeWithArgs("core", "list") - require.Zero(t, exitCode) - require.Contains(t, string(d), "arduino:avr") - - // Uninstall arduino:avr - exitCode, d = executeWithArgs("core", "uninstall", "arduino:avr") - require.Zero(t, exitCode) - require.Contains(t, string(d), AVR+" uninstalled") -} - func TestSearchConfigTreeNotFound(t *testing.T) { tmp := tmpDirOrDie() require.Empty(t, searchConfigTree(tmp)) diff --git a/cli/testdata/Blink/Blink.ino b/cli/testdata/Blink/Blink.ino deleted file mode 100644 index 5054c040393..00000000000 --- a/cli/testdata/Blink/Blink.ino +++ /dev/null @@ -1,3 +0,0 @@ - -void setup() {} -void loop() {} diff --git a/cli/testdata/Test3rdPartyCoreIntegration/arduino-cli.yaml b/cli/testdata/Test3rdPartyCoreIntegration/arduino-cli.yaml deleted file mode 100644 index 16c07d44a1e..00000000000 --- a/cli/testdata/Test3rdPartyCoreIntegration/arduino-cli.yaml +++ /dev/null @@ -1,3 +0,0 @@ -board_manager: - additional_urls: - - https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json \ No newline at end of file diff --git a/cli/testdata/TestInvalidCoreURLIntegration/arduino-cli.yaml b/cli/testdata/TestInvalidCoreURLIntegration/arduino-cli.yaml deleted file mode 100644 index 1fbaa8b1c90..00000000000 --- a/cli/testdata/TestInvalidCoreURLIntegration/arduino-cli.yaml +++ /dev/null @@ -1,3 +0,0 @@ -board_manager: - additional_urls: - - http://www.invalid-domain-asjkdakdhadjkh.com/package_example_index.json \ No newline at end of file diff --git a/test/README.md b/test/README.md index 0ebe7434c84..25a2201d995 100644 --- a/test/README.md +++ b/test/README.md @@ -1,7 +1,7 @@ # Integration tests -This dir contains integration tests, the aim is to test the Command Line Interface and its output -from a pure user point of view. +This dir contains integration tests, aimed to test the Command Line Interface +and its output from a pure user point of view. ## Installation diff --git a/test/common.py b/test/common.py index 27a9b598a78..4f1e74db694 100644 --- a/test/common.py +++ b/test/common.py @@ -13,6 +13,10 @@ # software without disclosing the source code of your own applications. To purchase # a commercial license, send an email to license@arduino.cc. import os +import collections + + +Board = collections.namedtuple("Board", "address fqbn package architecture id core") def running_on_ci(): diff --git a/test/conftest.py b/test/conftest.py index e9e3dabd7f6..9007eed3488 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -16,6 +16,9 @@ import pytest from invoke.context import Context +import simplejson as json + +from .common import Board @pytest.fixture(scope="function") @@ -73,3 +76,35 @@ def _run(cmd_string): ) return _run + + +@pytest.fixture(scope="function") +def detected_boards(run_command): + """ + This fixture provides a list of all the boards attached to the host. + This fixture will parse the JSON output of `arduino-cli board list --format json` + to extract all the connected boards data. + + :returns a list `Board` objects. + """ + assert run_command("core update-index") + result = run_command("board list --format json") + assert result.ok + + detected_boards = [] + for port in json.loads(result.stdout): + for board in port.get("boards", []): + fqbn = board.get("FQBN") + package, architecture, _id = fqbn.split(":") + detected_boards.append( + Board( + address=port.get("address"), + fqbn=fqbn, + package=package, + architecture=architecture, + id=_id, + core="{}:{}".format(package, architecture), + ) + ) + + return detected_boards diff --git a/test/requirements.in b/test/requirements.in index d92cdb6b167..bf965b6a774 100644 --- a/test/requirements.in +++ b/test/requirements.in @@ -1,4 +1,4 @@ -pytest==5.3.1 +pytest==5.3.4 simplejson==3.17.0 semver==2.9.0 pyserial==3.4 diff --git a/test/requirements.txt b/test/requirements.txt index a945122fc13..12155b5fe1e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -12,7 +12,7 @@ pluggy==0.13.1 # via pytest py==1.8.0 # via pytest pyparsing==2.4.0 # via packaging pyserial==3.4 -pytest==5.3.1 +pytest==5.3.4 semver==2.9.0 simplejson==3.17.0 six==1.12.0 # via packaging diff --git a/test/test_compile.py b/test/test_compile.py index cefba17b0fe..9a05a191124 100644 --- a/test/test_compile.py +++ b/test/test_compile.py @@ -14,6 +14,7 @@ # a commercial license, send an email to license@arduino.cc. import json import os +import platform import pytest @@ -34,7 +35,7 @@ def test_compile_without_fqbn(run_command): assert result.failed -def test_compile_with_simple_sketch(run_command, data_dir): +def test_compile_with_simple_sketch(run_command, data_dir, working_dir): # Init the environment explicitly result = run_command("core update-index") assert result.ok @@ -73,6 +74,54 @@ def test_compile_with_simple_sketch(run_command, data_dir): expected_trace_sequence, json_log_lines ) + # Test the --output flag with absolute path + target = os.path.join(data_dir, "test.hex") + result = run_command( + "compile -b {fqbn} {sketch_path} -o {target}".format( + fqbn=fqbn, sketch_path=sketch_path, target=target + ) + ) + assert result.ok + assert os.path.exists(target) + + +@pytest.mark.skipif( + running_on_ci() and platform.system() == "Windows", + reason="Test disabled on Github Actions Win VM until tmpdir inconsistent behavior bug is fixed", +) +def test_output_flag_default_path(run_command, data_dir, working_dir): + # Init the environment explicitly + result = run_command("core update-index") + assert result.ok + + # Download latest AVR + result = run_command("core install arduino:avr") + assert result.ok + + # Create a test sketch + sketch_path = os.path.join(data_dir, "test_output_flag_default_path") + fqbn = "arduino:avr:uno" + result = run_command("sketch new {}".format(sketch_path)) + assert result.ok + + # Test the --output flag defaulting to current working dir + result = run_command( + "compile -b {fqbn} {sketch_path} -o test".format( + fqbn=fqbn, sketch_path=sketch_path + ) + ) + assert result.ok + assert os.path.exists(os.path.join(working_dir, "test.hex")) + + # Test extension won't be added if already present + result = run_command( + "compile -b {fqbn} {sketch_path} -o test2.hex".format( + fqbn=fqbn, sketch_path=sketch_path + ) + ) + assert result.ok + assert os.path.exists(os.path.join(working_dir, "test2.hex")) + def test_compile_with_sketch_with_symlink_selfloop(run_command, data_dir): # Init the environment explicitly @@ -131,7 +180,7 @@ def test_compile_with_sketch_with_symlink_selfloop(run_command, data_dir): @pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") -def test_compile_and_compile_combo(run_command, data_dir): +def test_compile_and_compile_combo(run_command, data_dir, detected_boards): # Init the environment explicitly result = run_command("core update-index") assert result.ok @@ -148,58 +197,17 @@ def test_compile_and_compile_combo(run_command, data_dir): assert result.ok assert "Sketch created in: {}".format(sketch_path) in result.stdout - # - # Build a list of detected boards to test, if any. - # - result = run_command("board list --format json") - assert result.ok - - # - # The `board list --format json` returns a JSON that looks like to the following: - # - # [ - # { - # "address": "/dev/cu.usbmodem14201", - # "protocol": "serial", - # "protocol_label": "Serial Port (USB)", - # "boards": [ - # { - # "name": "Arduino NANO 33 IoT", - # "FQBN": "arduino:samd:nano_33_iot" - # } - # ] - # } - # ] - - detected_boards = [] - - ports = json.loads(result.stdout) - assert isinstance(ports, list) - for port in ports: - boards = port.get('boards') - if boards is None: - continue - assert isinstance(boards, list) - for board in boards: - detected_boards.append( - dict(address=port.get("address"), fqbn=board.get("FQBN")) - ) - - assert len(detected_boards) >= 1, "There are no boards available for testing" - # Build sketch for each detected board for board in detected_boards: - log_file_name = "{fqbn}-compile.log".format( - fqbn=board.get("fqbn").replace(":", "-") - ) + log_file_name = "{fqbn}-compile.log".format(fqbn=board.fqbn.replace(":", "-")) log_file_path = os.path.join(data_dir, log_file_name) command_log_flags = "--log-format json --log-file {} --log-level trace".format( log_file_path ) result = run_command( "compile -b {fqbn} --upload -p {address} {sketch_path} {log_flags}".format( - fqbn=board.get("fqbn"), - address=board.get("address"), + fqbn=board.fqbn, + address=board.address, sketch_path=sketch_path, log_flags=command_log_flags, ) @@ -210,16 +218,16 @@ def test_compile_and_compile_combo(run_command, data_dir): json_log_lines = log_json.readlines() expected_trace_sequence = [ "Compile {sketch} for {fqbn} started".format( - sketch=sketch_path, fqbn=board.get("fqbn") + sketch=sketch_path, fqbn=board.fqbn ), "Compile {sketch} for {fqbn} successful".format( - sketch=sketch_name, fqbn=board.get("fqbn") + sketch=sketch_name, fqbn=board.fqbn ), "Upload {sketch} on {fqbn} started".format( - sketch=sketch_path, fqbn=board.get("fqbn") + sketch=sketch_path, fqbn=board.fqbn ), "Upload {sketch} on {fqbn} successful".format( - sketch=sketch_name, fqbn=board.get("fqbn") + sketch=sketch_name, fqbn=board.fqbn ), ] assert is_message_sequence_in_json_log_traces( diff --git a/test/test_core.py b/test/test_core.py index f0ff54d82a5..da7f21e53aa 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -12,6 +12,8 @@ # otherwise use the software for commercial activities involving the Arduino # software without disclosing the source code of your own applications. To purchase # a commercial license, send an email to license@arduino.cc. +import os +import platform import pytest import simplejson as json @@ -95,3 +97,101 @@ def test_core_search_no_args(run_command): break assert found assert len(platforms) == num_platforms + + +def test_core_updateindex_invalid_url(run_command): + url = "http://www.invalid-domain-asjkdakdhadjkh.com/package_example_index.json" + result = run_command("core update-index --additional-urls={}".format(url)) + assert result.failed + + +@pytest.mark.skipif( + platform.system() == "Windows", + reason="core fails with fatal error: bits/c++config.h: No such file or directory", +) +def test_core_install_esp32(run_command, data_dir): + # update index + url = "https://dl.espressif.com/dl/package_esp32_index.json" + assert run_command("core update-index --additional-urls={}".format(url)) + # install 3rd-party core + assert run_command("core install esp32:esp32 --additional-urls={}".format(url)) + # create a sketch and compile to double check the core was successfully installed + sketch_path = os.path.join(data_dir, "test_core_install_esp32") + assert run_command("sketch new {}".format(sketch_path)) + assert run_command("compile -b esp32:esp32:esp32 {}".format(sketch_path)) + # prevent regressions for https://github.com/arduino/arduino-cli/issues/163 + assert os.path.exists( + os.path.join( + sketch_path, "test_core_install_esp32.esp32.esp32.esp32.partitions.bin" + ) + ) + + +def test_core_download(run_command, downloads_dir): + assert run_command("core update-index") + + # Download a specific core version + assert run_command("core download arduino:avr@1.6.16") + assert os.path.exists(os.path.join(downloads_dir, "packages", "avr-1.6.16.tar.bz2")) + + # Wrong core version + result = run_command("core download arduino:avr@69.42.0") + assert result.failed + + # Wrong core + result = run_command("core download bananas:avr") + assert result.failed + + +def _in(jsondata, name, version=None): + installed_cores = json.loads(jsondata) + for c in installed_cores: + if name == c.get("ID"): + if version is None: + return True + elif version == c.get("Installed"): + return True + return False + + +def test_core_install(run_command): + assert run_command("core update-index") + + # Install a specific core version + assert run_command("core install arduino:avr@1.6.16") + result = run_command("core list --format json") + assert result.ok + assert _in(result.stdout, "arduino:avr", "1.6.16") + + # Replace it with a more recent one + assert run_command("core install arduino:avr@1.6.17") + result = run_command("core list --format json") + assert result.ok + assert _in(result.stdout, "arduino:avr", "1.6.17") + + # Confirm core is listed as "updatable" + result = run_command("core list --updatable --format json") + assert result.ok + assert _in(result.stdout, "arduino:avr", "1.6.17") + + # Upgrade the core to latest version + assert run_command("core upgrade arduino:avr") + result = run_command("core list --format json") + assert result.ok + assert not _in(result.stdout, "arduino:avr", "1.6.17") + # double check the code isn't updatable anymore + result = run_command("core list --updatable --format json") + assert result.ok + assert not _in(result.stdout, "arduino:avr") + + +def test_core_uninstall(run_command): + assert run_command("core update-index") + assert run_command("core install arduino:avr") + result = run_command("core list --format json") + assert result.ok + assert _in(result.stdout, "arduino:avr") + assert run_command("core uninstall arduino:avr") + result = run_command("core list --format json") + assert result.ok + assert not _in(result.stdout, "arduino:avr") diff --git a/test/test_upload.py b/test/test_upload.py new file mode 100644 index 00000000000..8a8125a4528 --- /dev/null +++ b/test/test_upload.py @@ -0,0 +1,46 @@ +# This file is part of arduino-cli. +# +# Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +# +# This software is released under the GNU General Public License version 3, +# which covers the main part of arduino-cli. +# The terms of this license can be found at: +# https://www.gnu.org/licenses/gpl-3.0.en.html +# +# You can be released from the requirements of the above licenses by purchasing +# a commercial license. Buying such a license is mandatory if you want to modify or +# otherwise use the software for commercial activities involving the Arduino +# software without disclosing the source code of your own applications. To purchase +# a commercial license, send an email to license@arduino.cc. +import os + +import pytest + +from .common import running_on_ci + +# Skip this module when running in CI environments +pytestmark = pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") + + +def test_upload(run_command, data_dir, detected_boards): + # Init the environment explicitly + assert run_command("core update-index") + + for board in detected_boards: + # Download core + assert run_command("core install {}".format(board.core)) + # Create a sketch + sketch_path = os.path.join(data_dir, "foo") + assert run_command("sketch new {}".format(sketch_path)) + # Build sketch + assert run_command( + "compile -b {fqbn} {sketch_path}".format( + fqbn=board.fqbn, sketch_path=sketch_path + ) + ) + # Upload + assert run_command( + "upload -b {fqbn} -p {port} {sketch_path}".format( + sketch_path=sketch_path, fqbn=board.fqbn, port=board.address + ) + )