Add workflow to run tests on hardware.

.github/workflows/hil.yml

@@ -0,0 +1,108 @@
+name: Run tests in hardware
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize, labeled]
+
+# schedule:
+#   - cron: '0 2 * * *'
+
+concurrency:
+  group: build-${{github.event.pull_request.number || github.ref}}
+  cancel-in-progress: true
+
+jobs:
+  prep_sketches:
+    if: contains(github.event.pull_request.labels.*.name, 'hil_test')
+    name: Prepare Sketches
+    runs-on: ubuntu-latest
+    outputs:
+      sketches: ${{ steps.prep-sketches.outputs.sketches }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Prepare Sketches
+        id: prep-sketches
+        run: |
+          SKETCHES=$(jq -c -n '$ARGS.positional' --args `find tests -mindepth 1 -maxdepth 1 -type d | cut -d"/" -f2`)
+          echo "::set-output name=sketches::${SKETCHES}"
+
+  Build:
+    needs: prep_sketches
+    name: Build ${{matrix.sketches}} for ${{matrix.fqbn}}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        fqbn: ['esp32:esp32:esp32', 'esp32:esp32:esp32s2', 'esp32:esp32:esp32c3']
+        sketches: ${{fromJson(needs.prep_sketches.outputs.sketches)}}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Get target
+        run: echo "chip=$(echo ${{matrix.fqbn}} | cut -d":" -f3)" >> $GITHUB_ENV
+
+      - name: Build sketches
+        uses: arduino/compile-sketches@v1
+        with:
+          fqbn: ${{matrix.fqbn}}
+          platforms: |
+            - name: esp32:esp32
+              source-url: https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json
+          cli-compile-flags: |
+            - --build-path
+            - tests/${{matrix.sketches}}/build
+          sketch-paths: |
+            - tests/${{matrix.sketches}}
+
+      - name: Upload ${{matrix.sketches}}-${{env.chip}} artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{matrix.sketches}}-${{env.chip}}.artifacts
+          path: |
+            tests/${{matrix.sketches}}/build/*.bin
+            tests/${{matrix.sketches}}/build/*.json
+
+  Test:
+    needs: [prep_sketches, Build]
+    name: Test ${{matrix.sketches}} for ${{matrix.fqbn}}
+    runs-on: ESP32
+    env:
+      PYTHON_VERSION: 3.10.1
+      PYENV_VERSION: v2.2.3
+    strategy:
+      fail-fast: false
+      matrix:
+        fqbn: ['esp32:esp32:esp32', 'esp32:esp32:esp32s2', 'esp32:esp32:esp32c3']
+        sketches: ${{fromJson(needs.prep_sketches.outputs.sketches)}}
+
+    steps:
+       - name: Checkout repository
+         uses: actions/checkout@v2
+
+       - name: Get target
+         run: echo "chip=$(echo ${{matrix.fqbn}} | cut -d":" -f3)" >> $GITHUB_ENV
+
+       - name: Download ${{matrix.sketches}}-${{env.chip}} artifacts
+         uses: actions/download-artifact@v2
+         with:
+           name: ${{matrix.sketches}}-${{env.chip}}.artifacts
+           path: tests/${{matrix.sketches}}/build
+
+       - name: Check chip
+         run: cat tests/${{matrix.sketches}}/build/build.options.json
+
+       - name: Install dependencies
+         run: |
+           ~/.pyenv/versions/${{env.PYTHON_VERSION}}/bin/python -m venv test_venv
+           source test_venv/bin/activate
+           pip install -U pip
+           pip install -r tests/requirements.txt
+
+       - name: Run Tests
+         run: |
+           ~/.pyenv/versions/${{env.PYTHON_VERSION}}/bin/python -m venv test_venv
+           source test_venv/bin/activate
+           pytest tests -k test_${{matrix.sketches}}

tests/.gitignore

@@ -0,0 +1,3 @@
+build/
+__pycache__/
+*.log

tests/hello_world/hello_world.ino

@@ -0,0 +1,12 @@
+void setup(){
+  // Open serial communications and wait for port to open:
+  Serial.begin(115200);
+  while (!Serial) {
+    ;
+  }
+
+  Serial.println("Hello Arduino!");
+}
+
+void loop(){
+}

tests/hello_world/test_hello_world.py

@@ -0,0 +1,2 @@
+def test_hello_arduino(dut):
+    dut.expect('Hello Arduino!')

tests/pytest.ini

@@ -0,0 +1,13 @@
+[pytest]
+addopts = --embedded-services esp,arduino
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

tests/requirements.txt

@@ -0,0 +1,13 @@
+pyserial>=3.0
+esptool>=3.1
+pytest-cov
+cryptography<3.4; platform_machine == "armv7l"
+
+pytest>=6.2.0
+pexpect>=4.4
+
+pytest-embedded==0.5.0rc0
+pytest-embedded-serial==0.5.0rc0
+pytest-embedded-serial-esp==0.5.0rc0
+pytest-embedded-arduino==0.5.0rc0
+

tests/serial/serial.ino

@@ -0,0 +1,12 @@
+void setup(){
+  // Open serial communications and wait for port to open:
+  Serial.begin(115200);
+  while (!Serial) {
+    ;
+  }
+
+  Serial.println("Hello Serial!");
+}
+
+void loop(){
+}

tests/serial/test_serial.py

@@ -0,0 +1,2 @@
+def test_serial(dut):
+    dut.expect('Hello Serial!')