From 7406bd0f1c6371d5e623cd8eca88a9116d12385a Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Thu, 1 Aug 2019 14:36:32 +0200 Subject: [PATCH] make tests independent from each other --- test/.flake8 | 2 + test/README.md | 40 +++++++++++---- test/__init__.py | 0 test/common.py | 23 +++++++++ test/conftest.py | 26 +++++++--- test/requirements.txt | 1 + test/test_board.py | 41 ++++++++++++++++ test/test_lib.py | 109 ++++++++++++++++++++++++++++++++++++++++ test/test_main.py | 112 ++++++++++-------------------------------- 9 files changed, 251 insertions(+), 103 deletions(-) create mode 100644 test/.flake8 create mode 100644 test/__init__.py create mode 100644 test/common.py create mode 100644 test/test_board.py create mode 100644 test/test_lib.py diff --git a/test/.flake8 b/test/.flake8 new file mode 100644 index 00000000000..79a16af7eeb --- /dev/null +++ b/test/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 \ No newline at end of file diff --git a/test/README.md b/test/README.md index 576dc51458f..0ebe7434c84 100644 --- a/test/README.md +++ b/test/README.md @@ -1,15 +1,37 @@ -## Integration tests +# Integration tests This dir contains integration tests, the aim is to test the Command Line Interface and its output from a pure user point of view. -### Installation +## Installation - cd test - virtualenv --python=python3 venv - source venv/bin/activate - pip install -r requirements.txt - -### Running tests +See also [Contributing][0]. - pytest \ No newline at end of file +```shell +cd test +virtualenv --python=python3 venv +source venv/bin/activate +pip install -r requirements.txt +``` + +## Running tests + +To run all the tests: + +```shell +pytest +``` + +To run specific modules: + +```shell +pytest test_lib.py +``` + +To run very specific test functions: + +```shell +pytest test_lib.py::test_list +``` + +[0]: ../CONTRIBUTING.md diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/common.py b/test/common.py new file mode 100644 index 00000000000..eeb82456c9d --- /dev/null +++ b/test/common.py @@ -0,0 +1,23 @@ +# This file is part of arduino-cli. + +# Copyright 2019 ARDUINO SA (http://www.arduino.cc/) + +# This software is released under the GNU General Public License version 3, +# which covers the main part of arduino-cli. +# The terms of this license can be found at: +# https://www.gnu.org/licenses/gpl-3.0.en.html + +# You can be released from the requirements of the above licenses by purchasing +# a commercial license. Buying such a license is mandatory if you want to modify or +# otherwise use the software for commercial activities involving the Arduino +# software without disclosing the source code of your own applications. To purchase +# a commercial license, send an email to license@arduino.cc. +import os + + +def running_on_ci(): + """ + Returns whether the program is running on a CI environment + """ + val = os.getenv("APPVEYOR") or os.getenv("DRONE") + return val is not None diff --git a/test/conftest.py b/test/conftest.py index d8cd4a51534..b03bd20609a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -18,17 +18,27 @@ from invoke import run -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") def data_dir(tmpdir_factory): """ A tmp folder will be created before running - the tests and deleted at the end. + each test and deleted at the end, this way all the + tests work in isolation. """ - return str(tmpdir_factory.mktemp('ArduinoTest')) + return str(tmpdir_factory.mktemp("ArduinoTest")) @pytest.fixture(scope="session") -def run_command(data_dir): +def downloads_dir(tmpdir_factory): + """ + To save time and bandwidth, all the tests will access + the same download cache folder. + """ + return str(tmpdir_factory.mktemp("ArduinoTest")) + + +@pytest.fixture(scope="function") +def run_command(data_dir, downloads_dir): """ Provide a wrapper around invoke's `run` API so that every test will work in the same temporary folder. @@ -36,11 +46,11 @@ def run_command(data_dir): Useful reference: http://docs.pyinvoke.org/en/1.2/api/runners.html#invoke.runners.Result """ - cli_path = os.path.join(pytest.config.rootdir, '..', 'arduino-cli') + cli_path = os.path.join(pytest.config.rootdir, "..", "arduino-cli") env = { - "ARDUINO_DATA_DIR": data_dir, - "ARDUINO_DOWNLOADS_DIR": data_dir, - "ARDUINO_SKETCHBOOK_DIR": data_dir + "ARDUINO_DATA_DIR": data_dir, + "ARDUINO_DOWNLOADS_DIR": downloads_dir, + "ARDUINO_SKETCHBOOK_DIR": data_dir, } def _run(cmd_string): diff --git a/test/requirements.txt b/test/requirements.txt index 435c582dadc..f83a20e3198 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -15,6 +15,7 @@ pylint==2.3.1 pyparsing==2.4.0 pytest==5.0.1 semver==2.8.1 +simplejson==3.16.0 six==1.12.0 typed-ast==1.4.0 wcwidth==0.1.7 diff --git a/test/test_board.py b/test/test_board.py new file mode 100644 index 00000000000..76351e1f4b3 --- /dev/null +++ b/test/test_board.py @@ -0,0 +1,41 @@ +# This file is part of arduino-cli. + +# Copyright 2019 ARDUINO SA (http://www.arduino.cc/) + +# This software is released under the GNU General Public License version 3, +# which covers the main part of arduino-cli. +# The terms of this license can be found at: +# https://www.gnu.org/licenses/gpl-3.0.en.html + +# You can be released from the requirements of the above licenses by purchasing +# a commercial license. Buying such a license is mandatory if you want to modify or +# otherwise use the software for commercial activities involving the Arduino +# software without disclosing the source code of your own applications. To purchase +# a commercial license, send an email to license@arduino.cc. +import pytest +import simplejson as json + +from .common import running_on_ci + + +@pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") +def test_board_list(run_command): + result = run_command("core update-index") + assert result.ok + result = run_command("board list --format json") + assert result.ok + # check is a valid json and contains a list of ports + ports = json.loads(result.stdout).get("ports") + assert isinstance(ports, list) + for port in ports: + assert "protocol" in port + assert "protocol_label" in port + + +@pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") +def test_board_listall(run_command): + assert run_command("core update-index") + result = run_command("board listall") + print(result.stderr, result.stdout) + assert result.ok + assert ["Board", "Name", "FQBN"] == result.stdout.splitlines()[0].strip().split() diff --git a/test/test_lib.py b/test/test_lib.py new file mode 100644 index 00000000000..df8d94a8726 --- /dev/null +++ b/test/test_lib.py @@ -0,0 +1,109 @@ +# This file is part of arduino-cli. + +# Copyright 2019 ARDUINO SA (http://www.arduino.cc/) + +# This software is released under the GNU General Public License version 3, +# which covers the main part of arduino-cli. +# The terms of this license can be found at: +# https://www.gnu.org/licenses/gpl-3.0.en.html + +# You can be released from the requirements of the above licenses by purchasing +# a commercial license. Buying such a license is mandatory if you want to modify or +# otherwise use the software for commercial activities involving the Arduino +# software without disclosing the source code of your own applications. To purchase +# a commercial license, send an email to license@arduino.cc. +import pytest +import simplejson as json + + +def test_list(run_command): + # Init the environment explicitly + assert run_command("core update-index") + + # When ouput is empty, nothing is printed out, no matter the output format + result = run_command("lib list") + assert result.ok + assert "" == result.stderr + assert "" == result.stdout + result = run_command("lib list --format json") + assert result.ok + assert "" == result.stderr + assert "" == result.stdout + + # Install something we can list at a version older than latest + result = run_command("lib install ArduinoJson@6.11.0") + assert result.ok + + # Look at the plain text output + result = run_command("lib list") + assert result.ok + assert "" == result.stderr + lines = result.stdout.strip().splitlines() + assert 2 == len(lines) + toks = lines[1].split("\t") + # be sure line contain the current version AND the available version + assert "" != toks[1] + assert "" != toks[2] + + # Look at the JSON output + result = run_command("lib list --format json") + assert result.ok + assert "" == result.stderr + data = json.loads(result.stdout) + assert 1 == len(data) + # be sure data contains the available version + assert "" != data[0]["release"]["version"] + + +def test_install(run_command): + libs = ['"AzureIoTProtocol_MQTT"', '"CMMC MQTT Connector"', '"WiFiNINA"'] + # Should be safe to run install multiple times + assert run_command("lib install {}".format(" ".join(libs))) + assert run_command("lib install {}".format(" ".join(libs))) + + +def test_update_index(run_command): + result = run_command("lib update-index") + assert result.ok + assert ( + "Updating index: library_index.json downloaded" + == result.stdout.splitlines()[-1].strip() + ) + + +def test_remove(run_command): + libs = ['"AzureIoTProtocol_MQTT"', '"CMMC MQTT Connector"', '"WiFiNINA"'] + assert run_command("lib install {}".format(" ".join(libs))) + + result = run_command("lib uninstall {}".format(" ".join(libs))) + assert result.ok + + +@pytest.mark.slow +def test_search(run_command): + result = run_command("lib search") + assert result.ok + out_lines = result.stdout.splitlines() + # Create an array with just the name of the vars + libs = [] + for line in out_lines: + if line.startswith("Name: "): + start = line.find('"') + 1 + libs.append(line[start:-1]) + + expected = {"WiFi101", "WiFi101OTA", "Firebase Arduino based on WiFi101"} + assert expected == {lib for lib in libs if "WiFi101" in lib} + + result = run_command("lib search --format json") + assert result.ok + libs_json = json.loads(result.stdout) + assert len(libs) == len(libs_json.get("libraries")) + + result = run_command("lib search") + assert result.ok + + # Search for a specific target + result = run_command("lib search ArduinoJson --format json") + assert result.ok + libs_json = json.loads(result.stdout) + assert 1 == len(libs_json.get("libraries")) diff --git a/test/test_main.py b/test/test_main.py index 66b165d041c..3612b4e2f3a 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,98 +1,38 @@ -from invoke import run, Responder, exceptions -import os -import json -import pytest -import semver -from datetime import datetime - +# This file is part of arduino-cli. -def running_on_ci(): - val = os.getenv('APPVEYOR') or os.getenv('DRONE') - return val is not None - - -def test_command_help(run_command): - result = run_command('help') - assert result.ok - assert result.stderr == '' - assert 'Usage' in result.stdout +# Copyright 2019 ARDUINO SA (http://www.arduino.cc/) +# This software is released under the GNU General Public License version 3, +# which covers the main part of arduino-cli. +# The terms of this license can be found at: +# https://www.gnu.org/licenses/gpl-3.0.en.html -def test_command_lib_list(run_command): - """ - When ouput is empty, nothing is printed out, no matter the output format - """ - result = run_command('lib list') - assert result.ok - assert '' == result.stderr - result = run_command('lib list --format json') - assert '' == result.stdout - - -def test_command_lib_install(run_command): - libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"'] - # Should be safe to run install multiple times - result_1 = run_command('lib install {}'.format(' '.join(libs))) - assert result_1.ok - result_2 = run_command('lib install {}'.format(' '.join(libs))) - assert result_2.ok - -def test_command_lib_update_index(run_command): - result = run_command('lib update-index') - assert result.ok - assert 'Updating index: library_index.json downloaded' == result.stdout.splitlines()[-1].strip() - -def test_command_lib_remove(run_command): - libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"'] - result = run_command('lib uninstall {}'.format(' '.join(libs))) - assert result.ok - -@pytest.mark.slow -def test_command_lib_search(run_command): - result = run_command('lib search') - assert result.ok - out_lines = result.stdout.splitlines() - libs = [] - # Create an array with just the name of the vars - for line in out_lines: - if 'Name: ' in line: - libs.append(line.split()[1].strip('\"')) - number_of_libs = len(libs) - assert sorted(libs) == libs - assert ['WiFi101', 'WiFi101OTA'] == [lib for lib in libs if 'WiFi101' in lib] - result = run_command('lib search --format json') - assert result.ok - libs_found_from_json = json.loads(result.stdout) - number_of_libs_from_json = len(libs_found_from_json.get('libraries')) - assert number_of_libs == number_of_libs_from_json +# You can be released from the requirements of the above licenses by purchasing +# a commercial license. Buying such a license is mandatory if you want to modify or +# otherwise use the software for commercial activities involving the Arduino +# software without disclosing the source code of your own applications. To purchase +# a commercial license, send an email to license@arduino.cc. +import json +import semver -@pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") -def test_command_board_list(run_command): - result = run_command('core update-index') - assert result.ok - result = run_command('board list --format json') +def test_help(run_command): + result = run_command("help") assert result.ok - # check is a valid json and contains a list of ports - ports = json.loads(result.stdout).get('ports') - assert isinstance(ports, list) - for port in ports: - assert 'protocol' in port - assert 'protocol_label' in port + assert result.stderr == "" + assert "Usage" in result.stdout -@pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") -def test_command_board_listall(run_command): - result = run_command('board listall') +def test_version(run_command): + result = run_command("version") assert result.ok - assert ['Board', 'Name', 'FQBN'] == result.stdout.splitlines()[0].strip().split() + assert "Version:" in result.stdout + assert "Commit:" in result.stdout + assert "" == result.stderr - -def test_command_version(run_command): - result = run_command('version --format json') + result = run_command("version --format json") assert result.ok parsed_out = json.loads(result.stdout) - - assert parsed_out.get('Application', False) == 'arduino-cli' - assert isinstance(semver.parse(parsed_out.get('VersionString', False)), dict) - assert isinstance(parsed_out.get('Commit', False), str) + assert parsed_out.get("Application", False) == "arduino-cli" + assert isinstance(semver.parse(parsed_out.get("VersionString", False)), dict) + assert isinstance(parsed_out.get("Commit", False), str)