From 929d57942beb531d1983a93b776621e46cfa3617 Mon Sep 17 00:00:00 2001 From: Abdelatif Guettouche Date: Tue, 4 Jan 2022 21:51:06 +0100 Subject: [PATCH 1/5] Add initial infrastructure to build and run tests on hardware. Signed-off-by: Abdelatif Guettouche --- .github/workflows/hil.yml | 113 ++++++++++++++++++++++++++ tests/.gitignore | 3 + tests/hello_world/hello_world.ino | 12 +++ tests/hello_world/test_hello_world.py | 2 + tests/pytest.ini | 13 +++ tests/requirements.txt | 13 +++ tests/serial/serial.ino | 12 +++ tests/serial/test_serial.py | 2 + 8 files changed, 170 insertions(+) create mode 100644 .github/workflows/hil.yml create mode 100644 tests/.gitignore create mode 100644 tests/hello_world/hello_world.ino create mode 100644 tests/hello_world/test_hello_world.py create mode 100644 tests/pytest.ini create mode 100644 tests/requirements.txt create mode 100644 tests/serial/serial.ino create mode 100644 tests/serial/test_serial.py diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml new file mode 100644 index 00000000000..470309e842f --- /dev/null +++ b/.github/workflows/hil.yml @@ -0,0 +1,113 @@ +name: Run tests in hardware + +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + +# schedule: +# - cron: '0 2 * * *' + +concurrency: + group: build-${{github.event.pull_request.number || github.ref}} + cancel-in-progress: true + +jobs: + prep_sketches: + if: contains(github.event.pull_request.labels.*.name, 'hil_test') + name: Prepare Sketches + runs-on: ubuntu-latest + outputs: + sketches: ${{ steps.prep-sketches.outputs.sketches }} + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Prepare Sketches + id: prep-sketches + run: | + SKETCHES=$(jq -c -n '{sketches: $ARGS.positional}' --args `find tests -mindepth 1 -type d | cut -d"/" -f2`) + echo "::set-output name=sketches::${SKETCHES}" + + Build: + needs: prep_sketches + name: Build ${{matrix.sketches}} + runs-on: ubuntu-latest + strategy: + matrix: ${{fromJson(needs.prep_sketches.outputs.sketches)}} + + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Build sketches + uses: arduino/compile-sketches@v1 + with: + fqbn: 'esp32:esp32:esp32' + platforms: | + - name: esp32:esp32 + source-url: https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json + cli-compile-flags: | + - --build-path + - tests/${{matrix.sketches}}/build + sketch-paths: | + - tests/${{matrix.sketches}} + + - name: Upload ${{matrix.sketches}} artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{matrix.sketches}}.artifacts + path: | + tests/${{matrix.sketches}}/build/*.bin + tests/${{matrix.sketches}}/build/*.json + + Test: + needs: [prep_sketches, Build] + name: Test ${{matrix.sketches}} + runs-on: ESP32 + env: + PYTHON_VERSION: 3.10.1 + PYENV_VERSION: v2.2.3 + strategy: + matrix: ${{fromJson(needs.prep_sketches.outputs.sketches)}} + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download ${{matrix.sketches}} artifacts + uses: actions/download-artifact@v2 + with: + name: ${{matrix.sketches}}.artifacts + path: tests/${{matrix.sketches}}/build + + # Our self-hosted runners run on an RPI, the setup-python action doesn't + # support an ARM architecture. + + - name: Cache Python + id: cache-python + uses: actions/cache@v2 + with: + path: ~/.pyenv + key: pyenv-${{env.PYENV_VERSION}}-${{env.PYTHON_VERSION}} + + - name: Install Python + if: steps.cache-python.outputs.cache-hit != 'true' + run: | + rm -rf ~/.pyenv + git clone https://github.com/pyenv/pyenv.git -b ${{env.PYENV_VERSION}} ~/.pyenv + export PYENV_ROOT=~/.pyenv + export PATH=$PYENV_ROOT/bin:$PATH + pyenv install ${{env.PYTHON_VERSION}} + + - name: Install dependencies + run: | + ~/.pyenv/versions/${{env.PYTHON_VERSION}}/bin/python -m venv test_venv + source test_venv/bin/activate + pip install -U pip + pip install -r tests/requirements.txt + + - name: Run Tests + run: | + ~/.pyenv/versions/${{env.PYTHON_VERSION}}/bin/python -m venv test_venv + source test_venv/bin/activate + pytest tests -k test_${{matrix.sketches}} diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 00000000000..8290ecc0df5 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,3 @@ +build/ +__pycache__/ +*.log diff --git a/tests/hello_world/hello_world.ino b/tests/hello_world/hello_world.ino new file mode 100644 index 00000000000..0cc110d42a5 --- /dev/null +++ b/tests/hello_world/hello_world.ino @@ -0,0 +1,12 @@ +void setup(){ + // Open serial communications and wait for port to open: + Serial.begin(115200); + while (!Serial) { + ; + } + + Serial.println("Hello Arduino!"); +} + +void loop(){ +} diff --git a/tests/hello_world/test_hello_world.py b/tests/hello_world/test_hello_world.py new file mode 100644 index 00000000000..0e7cea81597 --- /dev/null +++ b/tests/hello_world/test_hello_world.py @@ -0,0 +1,2 @@ +def test_hello_arduino(dut): + dut.expect('Hello Arduino!') diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 00000000000..ef7e6d7c5ae --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,13 @@ +[pytest] +addopts = --embedded-services esp,arduino + +# log related +log_cli = True +log_cli_level = INFO +log_cli_format = %(asctime)s %(levelname)s %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S + +log_file = test.log +log_file_level = INFO +log_file_format = %(asctime)s %(levelname)s %(message)s +log_file_date_format = %Y-%m-%d %H:%M:%S diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000000..b4e3916f37f --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,13 @@ +pyserial>=3.0 +esptool>=3.1 +pytest-cov +cryptography<3.4; platform_machine == "armv7l" + +pytest>=6.2.0 +pexpect>=4.4 + +pytest-embedded==0.5.0rc0 +pytest-embedded-serial==0.5.0rc0 +pytest-embedded-serial-esp==0.5.0rc0 +pytest-embedded-arduino==0.5.0rc0 + diff --git a/tests/serial/serial.ino b/tests/serial/serial.ino new file mode 100644 index 00000000000..7baec2ed0ba --- /dev/null +++ b/tests/serial/serial.ino @@ -0,0 +1,12 @@ +void setup(){ + // Open serial communications and wait for port to open: + Serial.begin(115200); + while (!Serial) { + ; + } + + Serial.println("Hello Serial!"); +} + +void loop(){ +} diff --git a/tests/serial/test_serial.py b/tests/serial/test_serial.py new file mode 100644 index 00000000000..75422e0b934 --- /dev/null +++ b/tests/serial/test_serial.py @@ -0,0 +1,2 @@ +def test_serial(dut): + dut.expect('Hello Serial!') From 5ef94ec8cb9c161a510555d4c563f2beaf4fbfc0 Mon Sep 17 00:00:00 2001 From: Abdelatif Guettouche Date: Fri, 7 Jan 2022 00:50:43 +0100 Subject: [PATCH 2/5] .github/workflows: Remove the slefhost runner test. Signed-off-by: Abdelatif Guettouche --- .github/workflows/test_selfhosted_runner.yml | 28 -------------------- 1 file changed, 28 deletions(-) delete mode 100644 .github/workflows/test_selfhosted_runner.yml diff --git a/.github/workflows/test_selfhosted_runner.yml b/.github/workflows/test_selfhosted_runner.yml deleted file mode 100644 index ddac78abbc3..00000000000 --- a/.github/workflows/test_selfhosted_runner.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Test Github action on self hosted RPI runnes - -on: - push: - branches: - - master - pull_request: - -jobs: - build: - name: Dummy test - self hosted GHR - runs-on: self-hosted - if: github.repository == 'espressif/arduino-esp32' - steps: - - name: Check out repo - uses: actions/checkout@v2 - - name: Test message 1 - run: echo "This is test message" - - name: Test message 2 - run: echo "This is test message2" - - name: List directory - run: ls - - name: Create copy of README - run: cp README.md README2.md - - name: Read README2 - run: cat README2.md - - name: Delete README2 - run: rm README2.md From d17e28e061e08f55273d54e0e62e1abe8db5a943 Mon Sep 17 00:00:00 2001 From: Abdelatif Guettouche Date: Fri, 7 Jan 2022 11:27:12 +0100 Subject: [PATCH 3/5] hil.yml: Add the other chips. Signed-off-by: Abdelatif Guettouche --- .github/workflows/hil.yml | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml index 470309e842f..b22ce83f357 100644 --- a/.github/workflows/hil.yml +++ b/.github/workflows/hil.yml @@ -25,24 +25,29 @@ jobs: - name: Prepare Sketches id: prep-sketches run: | - SKETCHES=$(jq -c -n '{sketches: $ARGS.positional}' --args `find tests -mindepth 1 -type d | cut -d"/" -f2`) + SKETCHES=$(jq -c -n '$ARGS.positional' --args `find tests -mindepth 1 -maxdepth 1 -type d | cut -d"/" -f2`) echo "::set-output name=sketches::${SKETCHES}" Build: needs: prep_sketches - name: Build ${{matrix.sketches}} + name: Build ${{matrix.sketches}} for ${{matrix.fqbn}} runs-on: ubuntu-latest strategy: - matrix: ${{fromJson(needs.prep_sketches.outputs.sketches)}} + matrix: + fqbn: ['esp32:esp32:esp32', 'esp32:esp32:esp32s2', 'esp32:esp32:esp32c3'] + sketches: ${{fromJson(needs.prep_sketches.outputs.sketches)}} steps: - name: Checkout Repository uses: actions/checkout@v2 + - name: Get target + run: echo "chip=$(echo ${{matrix.fqbn}} | cut -d":" -f3)" >> $GITHUB_ENV + - name: Build sketches uses: arduino/compile-sketches@v1 with: - fqbn: 'esp32:esp32:esp32' + fqbn: ${{matrix.fqbn}} platforms: | - name: esp32:esp32 source-url: https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json @@ -52,34 +57,42 @@ jobs: sketch-paths: | - tests/${{matrix.sketches}} - - name: Upload ${{matrix.sketches}} artifacts + - name: Upload ${{matrix.sketches}}-${{env.chip}} artifacts uses: actions/upload-artifact@v2 with: - name: ${{matrix.sketches}}.artifacts + name: ${{matrix.sketches}}-${{env.chip}}.artifacts path: | tests/${{matrix.sketches}}/build/*.bin tests/${{matrix.sketches}}/build/*.json Test: needs: [prep_sketches, Build] - name: Test ${{matrix.sketches}} + name: Test ${{matrix.sketches}} for ${{matrix.fqbn}} runs-on: ESP32 env: PYTHON_VERSION: 3.10.1 PYENV_VERSION: v2.2.3 strategy: - matrix: ${{fromJson(needs.prep_sketches.outputs.sketches)}} + matrix: + fqbn: ['esp32:esp32:esp32', 'esp32:esp32:esp32s2', 'esp32:esp32:esp32c3'] + sketches: ${{fromJson(needs.prep_sketches.outputs.sketches)}} steps: - name: Checkout repository uses: actions/checkout@v2 - - name: Download ${{matrix.sketches}} artifacts + - name: Get target + run: echo "chip=$(echo ${{matrix.fqbn}} | cut -d":" -f3)" >> $GITHUB_ENV + + - name: Download ${{matrix.sketches}}-${{env.chip}} artifacts uses: actions/download-artifact@v2 with: - name: ${{matrix.sketches}}.artifacts + name: ${{matrix.sketches}}-${{env.chip}}.artifacts path: tests/${{matrix.sketches}}/build + - name: Check chip + run: cat tests/${{matrix.sketches}}/build/build.options.json + # Our self-hosted runners run on an RPI, the setup-python action doesn't # support an ARM architecture. From 0faec9aeccb358b0c0aff05acc409eaa2bdc2060 Mon Sep 17 00:00:00 2001 From: Abdelatif Guettouche Date: Fri, 7 Jan 2022 13:40:49 +0100 Subject: [PATCH 4/5] hil.yml: Disable fast failing for test jobs. --- .github/workflows/hil.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml index b22ce83f357..39a83e51305 100644 --- a/.github/workflows/hil.yml +++ b/.github/workflows/hil.yml @@ -73,6 +73,7 @@ jobs: PYTHON_VERSION: 3.10.1 PYENV_VERSION: v2.2.3 strategy: + fail-fast: false matrix: fqbn: ['esp32:esp32:esp32', 'esp32:esp32:esp32s2', 'esp32:esp32:esp32c3'] sketches: ${{fromJson(needs.prep_sketches.outputs.sketches)}} From d7d6606dd728d044ce22d88abded1d0f0392cefd Mon Sep 17 00:00:00 2001 From: Abdelatif Guettouche Date: Tue, 18 Jan 2022 14:56:09 +0100 Subject: [PATCH 5/5] hil.yml: Pyenv is already installed. --- .github/workflows/hil.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml index 39a83e51305..9699fe60c13 100644 --- a/.github/workflows/hil.yml +++ b/.github/workflows/hil.yml @@ -94,25 +94,6 @@ jobs: - name: Check chip run: cat tests/${{matrix.sketches}}/build/build.options.json - # Our self-hosted runners run on an RPI, the setup-python action doesn't - # support an ARM architecture. - - - name: Cache Python - id: cache-python - uses: actions/cache@v2 - with: - path: ~/.pyenv - key: pyenv-${{env.PYENV_VERSION}}-${{env.PYTHON_VERSION}} - - - name: Install Python - if: steps.cache-python.outputs.cache-hit != 'true' - run: | - rm -rf ~/.pyenv - git clone https://github.com/pyenv/pyenv.git -b ${{env.PYENV_VERSION}} ~/.pyenv - export PYENV_ROOT=~/.pyenv - export PATH=$PYENV_ROOT/bin:$PATH - pyenv install ${{env.PYTHON_VERSION}} - - name: Install dependencies run: | ~/.pyenv/versions/${{env.PYTHON_VERSION}}/bin/python -m venv test_venv