From e7d499c13ccc885e50b8541dc2c9f11f14aa3f14 Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Tue, 23 Jul 2019 08:40:09 +0200 Subject: [PATCH 1/4] add tmp folder fixture --- test/conftest.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++ test/test_main.py | 4 ++-- 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 test/conftest.py diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000000..fc6c2214f74 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,48 @@ +# This file is part of arduino-cli. + +# Copyright 2019 ARDUINO SA (http://www.arduino.cc/) + +# This software is released under the GNU General Public License version 3, +# which covers the main part of arduino-cli. +# The terms of this license can be found at: +# https://www.gnu.org/licenses/gpl-3.0.en.html + +# You can be released from the requirements of the above licenses by purchasing +# a commercial license. Buying such a license is mandatory if you want to modify or +# otherwise use the software for commercial activities involving the Arduino +# software without disclosing the source code of your own applications. To purchase +# a commercial license, send an email to license@arduino.cc. +import os + +import pytest +from invoke import run + + +@pytest.fixture(scope="session") +def sketchbook_path(tmpdir_factory): + """ + A tmp folder will be created before running + the tests and deleted at the end. + """ + fn = tmpdir_factory.mktemp('ArduinoTest') + return fn + + +@pytest.fixture(scope="session") +def runner(sketchbook_path): + return Runner(sketchbook_path) + + +class Runner: + def __init__(self, sketchbook_path): + self.sketchbook_path = None + self.cli_path = os.path.join(pytest.config.rootdir, '..', 'arduino-cli') + + def _cli_line(self, *args): + # Accept a list of arguments cli_line('lib list --format json') + # Return a full command line string e.g. 'arduino-cli help --format json' + cli_full_line = ' '.join([self.cli_path, ' '.join(str(arg) for arg in args), "--sketchbook-path {}".format(self.sketchbook_path)]) + return cli_full_line + + def run(self, *args): + return run(self._cli_line(*args), echo=False, hide=True, warn=True) diff --git a/test/test_main.py b/test/test_main.py index 976b780deab..ac045cab706 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -25,8 +25,8 @@ def run_command(*args): return result -def test_command_help(): - result = run_command('help') +def test_command_help(runner): + result = runner.run('help') assert result.ok assert result.stderr == '' assert 'Usage' in result.stdout From c72dd09bbbffecdc49ac34d4229ea79721531c87 Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Tue, 23 Jul 2019 11:33:51 +0200 Subject: [PATCH 2/4] exit at first failure since tests are inter-dependant --- test/pytest.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pytest.ini b/test/pytest.ini index bde27d789f4..957be6d9fae 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -7,4 +7,5 @@ filterwarnings = markers = slow: marks tests as slow (deselect with '-m "not slow"') -addopts = -s --verbose --tb=short \ No newline at end of file +# atm some tests depend on each other, better to exit at first failure (-x) +addopts = -x -s --verbose --tb=short \ No newline at end of file From 8df3d026d839298380df8b424ede65269ad08f3d Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Tue, 23 Jul 2019 11:34:04 +0200 Subject: [PATCH 3/4] use fixtures to run tests in isolation --- test/conftest.py | 35 ++++++++++++++++++++--------------- test/test_main.py | 39 ++++++++++----------------------------- 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index fc6c2214f74..15c99e557a3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -19,7 +19,7 @@ @pytest.fixture(scope="session") -def sketchbook_path(tmpdir_factory): +def data_dir(tmpdir_factory): """ A tmp folder will be created before running the tests and deleted at the end. @@ -29,20 +29,25 @@ def sketchbook_path(tmpdir_factory): @pytest.fixture(scope="session") -def runner(sketchbook_path): - return Runner(sketchbook_path) - - -class Runner: - def __init__(self, sketchbook_path): - self.sketchbook_path = None - self.cli_path = os.path.join(pytest.config.rootdir, '..', 'arduino-cli') +def run_command(data_dir): + """ + Provide a wrapper around invoke's `run` API so that every test + will work in the same temporary folder. - def _cli_line(self, *args): + Useful reference: + http://docs.pyinvoke.org/en/1.2/api/runners.html#invoke.runners.Result + """ + cli_path = os.path.join(pytest.config.rootdir, '..', 'arduino-cli') + env = { + "ARDUINO_DATA_DIR": data_dir, + "ARDUINO_DOWNLOADS_DIR": data_dir, + "ARDUINO_SKETCHBOOK_DIR": data_dir + } + + def _run(*args): # Accept a list of arguments cli_line('lib list --format json') # Return a full command line string e.g. 'arduino-cli help --format json' - cli_full_line = ' '.join([self.cli_path, ' '.join(str(arg) for arg in args), "--sketchbook-path {}".format(self.sketchbook_path)]) - return cli_full_line - - def run(self, *args): - return run(self._cli_line(*args), echo=False, hide=True, warn=True) + cli_full_line = ' '.join([cli_path, ' '.join(str(arg) for arg in args)]) + return run(cli_full_line, echo=False, hide=True, warn=True, env=env) + + return _run diff --git a/test/test_main.py b/test/test_main.py index ac045cab706..e6c3d6c40f3 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -5,34 +5,15 @@ import semver from datetime import datetime -this_test_path = os.path.dirname(os.path.realpath(__file__)) -# Calculate absolute path of the CLI -cli_path = os.path.join(this_test_path, '..', 'arduino-cli') -# Useful reference: -# http://docs.pyinvoke.org/en/1.2/api/runners.html#invoke.runners.Result - - -def cli_line(*args): - # Accept a list of arguments cli_line('lib list --format json') - # Return a full command line string e.g. 'arduino-cli help --format json' - cli_full_line = ' '.join([cli_path, ' '.join(str(arg) for arg in args)]) - return cli_full_line - - -def run_command(*args): - result = run(cli_line(*args), echo=False, hide=True, warn=True) - return result - - -def test_command_help(runner): - result = runner.run('help') +def test_command_help(run_command): + result = run_command('help') assert result.ok assert result.stderr == '' assert 'Usage' in result.stdout -def test_command_lib_list(): +def test_command_lib_list(run_command): """ When ouput is empty, nothing is printed out, no matter the output format """ @@ -43,7 +24,7 @@ def test_command_lib_list(): assert '' == result.stdout -def test_command_lib_install(): +def test_command_lib_install(run_command): libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"'] # Should be safe to run install multiple times result_1 = run_command('lib install {}'.format(' '.join(libs))) @@ -51,18 +32,18 @@ def test_command_lib_install(): result_2 = run_command('lib install {}'.format(' '.join(libs))) assert result_2.ok -def test_command_lib_update_index(): +def test_command_lib_update_index(run_command): result = run_command('lib update-index') assert result.ok assert 'Updating index: library_index.json downloaded' == result.stdout.splitlines()[-1].strip() -def test_command_lib_remove(): +def test_command_lib_remove(run_command): libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"'] result = run_command('lib uninstall {}'.format(' '.join(libs))) assert result.ok @pytest.mark.slow -def test_command_lib_search(): +def test_command_lib_search(run_command): result = run_command('lib search') assert result.ok out_lines = result.stdout.splitlines() @@ -81,7 +62,7 @@ def test_command_lib_search(): assert number_of_libs == number_of_libs_from_json -def test_command_board_list(): +def test_command_board_list(run_command): result = run_command('board list --format json') assert result.ok # check is a valid json and contains a list of ports @@ -92,13 +73,13 @@ def test_command_board_list(): assert 'protocol_label' in port -def test_command_board_listall(): +def test_command_board_listall(run_command): result = run_command('board listall') assert result.ok assert ['Board', 'Name', 'FQBN'] == result.stdout.splitlines()[0].strip().split() -def test_command_version(): +def test_command_version(run_command): result = run_command('version --format json') assert result.ok parsed_out = json.loads(result.stdout) From f5de81c09f90213ec826b169e1a5657e266c0dae Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Tue, 23 Jul 2019 11:40:21 +0200 Subject: [PATCH 4/4] consistent use of the run function --- test/conftest.py | 6 ++---- test/test_main.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 15c99e557a3..afc1d5e2695 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -44,10 +44,8 @@ def run_command(data_dir): "ARDUINO_SKETCHBOOK_DIR": data_dir } - def _run(*args): - # Accept a list of arguments cli_line('lib list --format json') - # Return a full command line string e.g. 'arduino-cli help --format json' - cli_full_line = ' '.join([cli_path, ' '.join(str(arg) for arg in args)]) + def _run(cmd_string): + cli_full_line = "{} {}".format(cli_path, cmd_string) return run(cli_full_line, echo=False, hide=True, warn=True, env=env) return _run diff --git a/test/test_main.py b/test/test_main.py index e6c3d6c40f3..aa3a1cb5115 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -20,7 +20,7 @@ def test_command_lib_list(run_command): result = run_command('lib list') assert result.ok assert '' == result.stderr - result = run_command('lib list', '--format json') + result = run_command('lib list --format json') assert '' == result.stdout