Merge branch 'test/unittest_with_config_file' into 'main'

unit_test_app: Added env config file with USB PORT info and improved test report generation

See merge request app-frameworks/esp-insights!152

Author: shahpiyushv

.gitlab-ci.yml

@@ -24,7 +24,7 @@ before_script:
   - echo Checking git version
   - git --version
   - echo Cloning esp-idf - Started
-  - git clone --depth 1 --shallow-submodules --recursive --single-branch -b ${IDF_GIT_BRANCH} --reference-if-able /Users/espressif/local_reference/esp-idf https://gitlab-ci-token:${BOT_TOKEN}@gitlab.espressif.cn:6688/espressif/esp-idf.git
+  - git clone --depth 1 --shallow-submodules --recursive --single-branch -b ${IDF_GIT_BRANCH} --reference-if-able /Users/espressif/local_reference/esp-idf https://github.com/espressif/esp-idf.git
   - cd esp-idf
   - echo Cloning esp-idf - Done
   - echo Running install.sh - Started
@@ -56,7 +56,6 @@ before_script:
   - echo Building diagnostics_smoke_test app - Started
   - ls
   - cd $CI_PROJECT_DIR/examples/diagnostics_smoke_test
-
   # Build this example for HTTPS transport, insights_auth_key.txt file is required
   - touch main/insights_auth_key.txt
   - for TARGET in $EXAMPLE_TARGETS; do
@@ -146,13 +145,17 @@ build_idf_v4.1.2:
 
 unit_tests:
   before_script: []
+  variables:
+    TEST_CONFIG_PATH: $CI_PROJECT_DIR/components/esp_diag_data_store/test/python_tests/env_config.yml
   stage: test
   timeout: 2 hours
   tags:
     - idf_unit_test_runner
   script:
       - *setup_idf
-      - echo IDF PATH is $IDF_PATH
+      # update value of usb port in env config file
+      - cd $CI_PROJECT_DIR/components/esp_diag_data_store/test/python_tests
+      - yq eval ".default.unit-test-app = \"$UT_USB_PORT\"" -i "env_config.yml"
       - cd $IDF_PATH/tools/unit-test-app
       # Clean any previous configuration and builds
       - rm -rf build sdkconfig $IDF_PATH/TEST_LOGS
@@ -164,26 +167,25 @@ unit_tests:
       # test application will be flashed in following step only
       - ./unit_test.py "data store init deinit" "data store write" "data store write read release_all"
         "data store wrapped_read write_till_exact_full" "data store write read release_zero read release_zero release_all"
-        "data store validate data in RTC after crash"
-      # Run python test cases
+        "data store validate data in RTC after crash" --env_config_file $TEST_CONFIG_PATH
+      # Flash diagnostics smoke test example and run the tests
+      - cd $CI_PROJECT_DIR/examples/diagnostics_smoke_test
+      # Clean any previous configuration and builds
+      - rm -rf build sdkconfig
+      # update WIFI details
+      - echo 'CONFIG_EXAMPLE_WIFI_SSID="ES_Staff"' >> sdkconfig.defaults
+      - echo 'CONFIG_EXAMPLE_WIFI_PASSWORD="ES@puba45"' >> sdkconfig.defaults
+      - echo CONFIG_ESP_INSIGHTS_TRANSPORT_MQTT=y >> sdkconfig.defaults
+      - echo $InsightsAuthKeyToken > main/insights_auth_key.txt || true
+      - idf.py build flash -p $UT_USB_PORT
       - cd $CI_PROJECT_DIR/components/esp_diag_data_store/test/python_tests
+       # Run python test cases
       - python3 stress_tests.py --password=${PASSWORD} --username=${USERNAME} --base_uri=${BASE_URI} --diag_uri=${DIAG_URI}
-      # report generation
-      - echo ******* Report Generation Started **********
-      - cd $CI_PROJECT_DIR
-      # Delete existing test_report directory if any and create new one
-      - rm -rf test_report
-      - mkdir test_report
-      # Convert xml reports to html format
-      - junit2html components/esp_diag_data_store/test/python_tests/*.xml test_report/python_test_results.html
-      - cd $IDF_PATH/TEST_LOGS
-      - result_xml=$(find . -type f -name '*.xml')
-      - junit2html $result_xml $CI_PROJECT_DIR/test_report/result.html
-      - echo ******* Report Generation Finished **********
+
   artifacts:
     name: "Test_Reports"
     paths:
-      - $CI_PROJECT_DIR/test_report/*.html
+      - $CI_PROJECT_DIR/components/esp_diag_data_store/test/python_tests/*html
 
 push_master_to_github:
   stage: deploy

components/esp_diag_data_store/test/python_tests/env_config.yml

@@ -0,0 +1,3 @@
+default:
+  unit-test-app: "port"
+  # port value is updated run time during gitlab CI for local use update the port value manually

components/esp_diag_data_store/test/python_tests/stress_tests.py

@@ -7,6 +7,9 @@
 import sys
 import argparse
 import json
+import os
+import glob
+import xml.etree.ElementTree as ET
 
 from datetime import datetime, timedelta
 
@@ -50,34 +53,33 @@ def test_100_get_crash_count(self):
         - Expected : crash count shall not be more than 5.
         """
         # Arrange ------------------------------------------------------------------------
-        from_ts = round(int(datetime.timestamp(datetime.now() - timedelta(seconds=5))))
+        from_ts = round(int(datetime.timestamp(datetime.now() - timedelta(minutes=1))))
         header = {"Authorization": self.access_token}
-        node_id = 'BiPVyKJVAenRJaBMTZ7HzE'
+        node_id = 'eZzk4U2Xn4Aqwmitw4x6gc'
 
         curr_ts = round(int(datetime.timestamp(datetime.now())))
-        max_time = 10*60    # duration in seconds
+        max_time = 10*60    # duration in seconds (10 minutes)
         crash_count = 0
         filt = '[{"f":"Node.ID","o":"keyword","v":["%s"]},{"f":"Type","o":"keyword","v":["crash"]}]' % node_id
         encoded_filter = urllib.parse.quote(filt, safe='~@#$&()*!+=:;,?/\'')
+
         while (curr_ts-from_ts) < max_time:
-            print("\n completed {} seconds out of {} crash count is {} ".format(curr_ts-from_ts, max_time, crash_count))
             url = self.diag_uri + "/query/filters/suggest?from_ts={}&to_ts={}&filters={}&fieldname=Type".\
                 format(from_ts, curr_ts, encoded_filter)
             # Act ----------------------------------------------------------------------------
             resp_crash_filter = requests.get(url=url, verify=False, headers=header, cookies=None)
             json_resp_crash_filter = json.loads(resp_crash_filter.text)["list"]
-
             # Assert -------------------------------------------------------------------------
             self.assertEqual(resp_crash_filter.status_code, 200, "S100.1 Unexpected status code for {} \n".format(node_id))
             # Check if any crash is present
             if len(json.loads(resp_crash_filter.text)["list"]):
-                self.assertEqual(json_resp_crash_filter["key"], "crash", "S100.2 key is not present for {}".format(node_id))
-                self.assertLessEqual(json_resp_crash_filter["count"], 5, "S100.3 Crash count is not same as expected for {}". format(node_id))
-                crash_count = json_resp_crash_filter["count"]
-
+                self.assertEqual(json_resp_crash_filter[0]["key"], "crash", "S100.2 key is not present for {}".format(node_id))
+                self.assertLessEqual(json_resp_crash_filter[0]["count"], 5, "S100.3 Crash count is not same as expected for {}". format(node_id))
+                crash_count = json_resp_crash_filter[0]["count"]
             # update variable curr_ts after every one minute
             time.sleep(60)
             curr_ts = round(int(datetime.timestamp(datetime.now())))
+        print("\n Final crash count after 10 minutes is {} ".format(crash_count))
 
 
 if __name__ == '__main__':
@@ -103,4 +105,35 @@ def test_100_get_crash_count(self):
     suite = unittest.TestSuite()
     suite.addTest(TestCaseInsights('test_100_get_crash_count', **kwargs))
     xmlrunner.XMLTestRunner(verbosity=2).run(unittest.TestSuite(suite))
+
+    # Test report generation
+    for ele in (os.listdir()):
+        if ele.endswith("xml"):
+            os.system("junit2html {} {}.html".format(ele, os.path.splitext(ele)[0]))
+            # Parse the XML report
+            xml_report = ET.parse(ele)
+            failures = xml_report.findall('.//failure')
+            errors = xml_report.findall('.//error')
+            if len(failures) > 0 or len(errors) > 0:
+                print("Tests failed. see console log for details")
+                sys.exit(1)
+
+    # PATH of test reports generated by IDF unit tests
+    IDF_DIR = '../../../../esp-idf/TEST_LOGS'
+    xml_reports = glob.glob(os.path.join(IDF_DIR, '**/*.xml'), recursive=True)
+    xml_file_path = os.path.abspath(xml_reports[0])
+
+    if not xml_file_path:
+        print("XML reports of IDF unit tests not generated")
+        sys.exit(1)
+    else:
+        os.system("junit2html {} {}.html".format(xml_file_path, "unit_test_report"))
+        # Parse the XML report
+        xml_report = ET.parse(xml_file_path)
+        failures = xml_report.findall('.//failure')
+        errors = xml_report.findall('.//error')
+        if len(failures) > 0 or len(errors) > 0:
+            print("Tests failed. see console log for details")
+            sys.exit(1)
+
     sys.exit(0)