Merge branch 'ci/linux_host_test_with_pytest' into 'master'

CI: Linux host test with pytest

Closes IDFCI-1571

See merge request espressif/esp-idf!21907

Author: hfudev

.gitlab/ci/build.yml

@@ -176,7 +176,7 @@ build_pytest_examples_esp32:
   extends:
     - .build_pytest_no_jtag_template
     - .rules:build:example_test-esp32
-  parallel: 4
+  parallel: 6
   variables:
     IDF_TARGET: esp32
     TEST_DIR: examples
@@ -203,7 +203,7 @@ build_pytest_examples_esp32c3:
   extends:
     - .build_pytest_no_jtag_template
     - .rules:build:example_test-esp32c3
-  parallel: 3
+  parallel: 4
   variables:
     IDF_TARGET: esp32c3
     TEST_DIR: examples
@@ -212,6 +212,7 @@ build_pytest_examples_esp32c2:
   extends:
     - .build_pytest_no_jtag_template
     - .rules:build:example_test-esp32c2
+  parallel: 2
   variables:
     IDF_TARGET: esp32c2
     TEST_DIR: examples
@@ -244,6 +245,7 @@ build_pytest_examples_esp32c6:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32c6
+  parallel: 2
   variables:
     IDF_TARGET: esp32c6
     TEST_DIR: examples
@@ -252,7 +254,7 @@ build_pytest_components_esp32:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32
-  parallel: 2
+  parallel: 5
   variables:
     IDF_TARGET: esp32
     TEST_DIR: components
@@ -261,7 +263,7 @@ build_pytest_components_esp32s2:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32s2
-  parallel: 2
+  parallel: 4
   variables:
     IDF_TARGET: esp32s2
     TEST_DIR: components
@@ -270,7 +272,7 @@ build_pytest_components_esp32s3:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32s3
-  parallel: 2
+  parallel: 4
   variables:
     IDF_TARGET: esp32s3
     TEST_DIR: components
@@ -279,7 +281,7 @@ build_pytest_components_esp32c3:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32c3
-  parallel: 2
+  parallel: 4
   variables:
     IDF_TARGET: esp32c3
     TEST_DIR: components
@@ -288,6 +290,7 @@ build_pytest_components_esp32c2:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32c2
+  parallel: 3
   variables:
     IDF_TARGET: esp32c2
     TEST_DIR: components
@@ -296,6 +299,7 @@ build_pytest_components_esp32c6:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32c6
+  parallel: 3
   variables:
     IDF_TARGET: esp32c6
     TEST_DIR: components
@@ -304,7 +308,7 @@ build_non_test_components_apps:
   extends:
     - .build_cmake_template
     - .rules:build:component_ut
-  parallel: 2
+  parallel: 5
   script:
     - set_component_ut_vars
     # CI specific options start from "--collect-size-info xxx". could ignore when running locally
@@ -358,6 +362,7 @@ build_pytest_test_apps_esp32s3:
   extends:
     - .build_pytest_test_apps_template
     - .rules:build:custom_test-esp32s3
+  parallel: 2
   variables:
     IDF_TARGET: esp32s3
     TEST_DIR: tools/test_apps
@@ -519,15 +524,13 @@ build_esp_idf_tests_cmake_esp32:
   extends:
     - .build_esp_idf_tests_cmake_template
     - .rules:build:unit_test-esp32
-  parallel: 2
   variables:
     IDF_TARGET: esp32
 
 build_esp_idf_tests_cmake_esp32s2:
   extends:
     - .build_esp_idf_tests_cmake_template
     - .rules:build:unit_test-esp32s2
-  parallel: 2
   variables:
     IDF_TARGET: esp32s2
 
@@ -563,7 +566,7 @@ build_examples_cmake_esp32:
   extends:
     - .build_cmake_template
     - .rules:build:example_test-esp32
-  parallel: 9
+  parallel: 8
   variables:
     IDF_TARGET: esp32
     TEST_DIR: examples
@@ -572,7 +575,7 @@ build_examples_cmake_esp32s2:
   extends:
     - .build_cmake_template
     - .rules:build:example_test-esp32s2
-  parallel: 6
+  parallel: 7
   variables:
     IDF_TARGET: esp32s2
     TEST_DIR: examples
@@ -590,7 +593,7 @@ build_examples_cmake_esp32c2:
   extends:
     - .build_cmake_template
     - .rules:build:example_test-esp32c2
-  parallel: 4
+  parallel: 6
   variables:
     IDF_TARGET: esp32c2
     TEST_DIR: examples
@@ -616,7 +619,7 @@ build_examples_cmake_esp32c6:
   extends:
     - .build_cmake_template
     - .rules:build:example_test-esp32c6
-  parallel: 4
+  parallel: 6
   variables:
     IDF_TARGET: esp32c6
     TEST_DIR: examples

.gitlab/ci/host-test.yml

@@ -20,13 +20,6 @@ test_nvs_on_host:
     - cd components/nvs_flash/test_nvs_host
     - make test
 
-test_nvs_on_host_cmake:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/nvs_flash/host_test/nvs_host_test
-    - idf.py build
-    - build/nvs_host_test.elf
-
 test_nvs_coverage:
   extends:
     - .host_test_template
@@ -340,132 +333,20 @@ test_split_path_by_spaces:
     - cd ${IDF_PATH}/tools
     - python -m unittest split_paths_by_spaces.py
 
-test_nvs_page:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/nvs_flash/host_test/nvs_page_test
-    - idf.py build
-    - build/test_nvs_page_host.elf
-
 test_mqtt_on_host:
   extends: .host_test_template
   script:
     - cd ${IDF_PATH}/components/mqtt/esp-mqtt/host_test
     - idf.py build
     - LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_mqtt_client_test.elf
 
-test_log:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/log/host_test/log_test
-    - idf.py build
-    - build/test_log_host.elf
-
-test_esp_event:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/esp_event/host_test/esp_event_unit_test
-    - idf.py build
-    - build/test_esp_event_host.elf
-
-test_hello_world_linux_compatible_example:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/tools/test_apps/linux_compatible/hello_world_linux_compatible
-    - idf.py --preview set-target linux
-    - idf.py build
-    - timeout 15 build/hello_world.elf | tee test.txt
-    - grep "Hello world!" test.txt
-
-test_esp_system:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/esp_system/host_test/test_esp_system/
-    - idf.py build
-    - echo "*" | timeout 5 build/test_esp_system.elf | tee log.txt || true
-    - grep "6 Tests 0 Failures 0 Ignored" log.txt
-
-test_heap_linux:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/heap/host_test/host_test_linux/
-    - idf.py build
-    - echo "*" | timeout 5 build/test_heap.elf | tee log.txt || true
-    - grep "4 Tests 0 Failures 0 Ignored" log.txt
-
-test_esp_hw_support_linux:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/esp_hw_support/host_test/host_test_linux/
-    - idf.py build
-    - echo "*" | timeout 5 build/test_hw_support_linux.elf | tee log.txt || true
-    - grep "2 Tests 0 Failures 0 Ignored" log.txt
-
-test_esp_timer_cxx:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/esp_timer
-    - idf.py build
-    - build/test_esp_timer_cxx_host.elf
-
 test_eh_frame_parser:
   extends: .host_test_template
   script:
     - cd ${IDF_PATH}/components/esp_system/test_eh_frame_parser
     - make
     - ./eh_frame_test
 
-test_rom_on_linux_works:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/esp_rom/host_test/rom_test
-    - idf.py build
-    - build/test_rom_host.elf
-
-test_cxx_gpio:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/gpio
-    - idf.py build
-    - build/test_gpio_cxx_host.elf
-
-test_i2c_cxx:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/i2c
-    - idf.py build
-    - build/test_i2c_cxx_host.elf
-
-test_spi_cxx:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/spi
-    - idf.py build
-    - build/test_spi_cxx_host.elf
-
-test_system_cxx:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/system
-    - idf.py build
-    - build/test_system_cxx_host.elf
-
-test_partition_api_host:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/esp_partition/host_test/partition_api_test
-    - idf.py build
-    - timeout 5 ./build/partition_api_test.elf | tee test.txt
-    - grep " 0 Failures" test.txt
-
-test_spiffs_host:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/spiffs/host_test
-    - idf.py build
-    - timeout 5 ./build/host_test_spiffs.elf | tee test.txt
-    - grep " 0 Failures" test.txt
-
 test_gen_soc_caps_kconfig:
   extends: .host_test_template
   script:
@@ -497,3 +378,26 @@ test_pytest_qemu:
       --embedded-services idf,qemu
       --junitxml=XUNIT_RESULT.xml
       --known-failure-cases-file known_failure_cases/known_failure_cases.txt
+
+test_pytest_linux:
+  extends:
+    - .host_test_template
+  artifacts:
+    when: always
+    paths:
+      - XUNIT_RESULT.xml
+      - pytest_embedded_log/
+    reports:
+      junit: XUNIT_RESULT.xml
+    expire_in: 1 week
+  script:
+    # TODO: fix the warnings in build flags and ignore-warning-str: IDF-6637
+    # Record the warnings regexes in file tools/ci/ignore_build_warnings_linux.txt to avoid using parentheses and
+    # doublequotes with bash.
+    # Please remove that file while fixing all the warnings.
+    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv
+      --target linux
+      --pytest-apps
+      -m host_test
+      --ignore-warning-file tools/ci/ignore_build_warnings_linux.txt
+    - run_cmd pytest --target linux -m host_test --junitxml=XUNIT_RESULT.xml

components/esp_event/host_test/esp_event_unit_test/pytest_esp_event_linux.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_event_linux(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

components/esp_hw_support/host_test/host_test_linux/pytest_esp_hw_support_linux.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_hw_support_linux(dut: Dut) -> None:
+    dut.expect_exact('Press ENTER to see the list of tests.')
+    dut.write('*')
+    dut.expect_unity_test_output(timeout=5)

components/esp_partition/host_test/partition_api_test/pytest_esp_partition_linux.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_partition_linux(dut: Dut) -> None:
+    dut.expect_unity_test_output(timeout=5)

components/esp_rom/host_test/rom_test/pytest_esp_rom_linux.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_rom_linux(dut: Dut) -> None:
+    dut.expect_exact('All tests passed', timeout=5)

components/esp_system/host_test/test_esp_system/pytest_esp_system_linux.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.linux
+@pytest.mark.host_test
+def test_esp_system_linux(dut: Dut) -> None:
+    dut.expect_exact('Press ENTER to see the list of tests.')
+    dut.write('*')
+    dut.expect_unity_test_output(timeout=10)