test: Allow per-unit-test timeouts

Work around for "(WL) write/read speed test" taking >30s in some configs

diff --git a/components/fatfs/test/test_fatfs_sdmmc.c b/components/fatfs/test/test_fatfs_sdmmc.c
index 6dc1ec3fc064daba2c170147997c2fe7af6f9008..c81302207122b8d3e350f594ee2484352601334c 100644 (file)
--- a/components/fatfs/test/test_fatfs_sdmmc.c
+++ b/components/fatfs/test/test_fatfs_sdmmc.c
@@ -154,7 +154,7 @@ TEST_CASE("(SD) multiple tasks can use same volume", "[fatfs][test_env=UT_T1_SDM
 
 static void speed_test(void* buf, size_t buf_size, size_t file_size, bool write);
 
-TEST_CASE("(SD) write/read speed test", "[fatfs][sd][test_env=UT_T1_SDMODE]")
+TEST_CASE("(SD) write/read speed test", "[fatfs][sd][test_env=UT_T1_SDMODE][timeout=60]")
 {
     size_t heap_size;
     HEAP_SIZE_CAPTURE(heap_size);

diff --git a/components/fatfs/test/test_fatfs_spiflash.c b/components/fatfs/test/test_fatfs_spiflash.c
index 55b3b68788bb33c9a17c1f0587723d7e6772d44d..70f73f0e539b69db5e3fbe9b1693f0f3af2b1035 100644 (file)
--- a/components/fatfs/test/test_fatfs_spiflash.c
+++ b/components/fatfs/test/test_fatfs_spiflash.c
@@ -145,7 +145,7 @@ TEST_CASE("(WL) multiple tasks can use same volume", "[fatfs][wear_levelling]")
     test_teardown();
 }
 
-TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling]")
+TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling][timeout=60]")
 {
     /* Erase partition before running the test to get consistent results */
     const esp_partition_t* part = get_test_data_partition();

diff --git a/tools/tiny-test-fw/CIAssignUnitTest.py b/tools/tiny-test-fw/CIAssignUnitTest.py
index 03768dbf8653ba8a4005ddab000ce3fb912f4b47..3c31514589d2fa8aae106614795bb16875e100a3 100644 (file)
--- a/tools/tiny-test-fw/CIAssignUnitTest.py
+++ b/tools/tiny-test-fw/CIAssignUnitTest.py
@@ -38,12 +38,17 @@ class Group(CIAssignTest.Group):
         return test_app[3:]
 
     def _create_extra_data(self, test_function):
+        """
+        For unit test case, we need to copy some attributes of test cases into config file.
+        So unit test function knows how to run the case.
+        """
         case_data = []
         for case in self.case_list:
             one_case_data = {
                 "config": self._get_ut_config(self._get_case_attr(case, "Test App")),
                 "name": self._get_case_attr(case, "summary"),
                 "reset": self._get_case_attr(case, "reset"),
+                "timeout": self._get_case_attr(case, "timeout"),
             }
 
             if test_function in ["run_multiple_devices_cases", "run_multiple_stage_cases"]:

diff --git a/tools/unit-test-app/tools/TagDefinition.yml b/tools/unit-test-app/tools/TagDefinition.yml
index 293208d7b25016ba44c2e44e22b928c1391dbb72..f84d9a702417e9e57cdb84cd3a98413b69cc3d4c 100644 (file)
--- a/tools/unit-test-app/tools/TagDefinition.yml
+++ b/tools/unit-test-app/tools/TagDefinition.yml
@@ -15,3 +15,6 @@ multi_device:
 multi_stage:
   default: "Yes"
   omitted: "No"
+timeout:
+  default: 30
+  omitted: 30

diff --git a/tools/unit-test-app/tools/UnitTestParser.py b/tools/unit-test-app/tools/UnitTestParser.py
index db9a94119a51ea90572ee56d598f8a40e5bb86f3..688e4ce37d5fdbc299f8a536eb1f33d7a74f779a 100644 (file)
--- a/tools/unit-test-app/tools/UnitTestParser.py
+++ b/tools/unit-test-app/tools/UnitTestParser.py
@@ -8,7 +8,6 @@ import hashlib
 from copy import deepcopy
 import CreateSectionTable
 
-
 TEST_CASE_PATTERN = {
     "initial condition": "UTINIT1",
     "SDK": "ESP32_IDF",
@@ -21,7 +20,7 @@ TEST_CASE_PATTERN = {
     "test environment": "UT_T1_1",
     "reset": "",
     "expected result": "1. set succeed",
-    "cmd set": "test_unit_test_case",
+    "cmd set": "test_unit_test_case"
 }
 
 CONFIG_FILE_PATTERN = {
@@ -191,7 +190,8 @@ class Parser(object):
                           "sub module": self.module_map[prop["module"]]['sub module'],
                           "summary": name,
                           "multi_device": prop["multi_device"],
-                          "multi_stage": prop["multi_stage"]})
+                          "multi_stage": prop["multi_stage"],
+                          "timeout": int(prop["timeout"])})
         return test_case
 
     def dump_test_cases(self, test_cases):

diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py
index 594a71becfed2fce9fff9410e595ed94a074db99..e049e7505e5e8358b82c6c8f3101807af7f52288 100644 (file)
--- a/tools/unit-test-app/unit_test.py
+++ b/tools/unit-test-app/unit_test.py
@@ -27,7 +27,8 @@ RESET_PATTERN = re.compile(r"(ets [\w]{3}\s+[\d]{1,2} [\d]{4} [\d]{2}:[\d]{2}:[\
 EXCEPTION_PATTERN = re.compile(r"(Guru Meditation Error: Core\s+\d panic'ed \([\w].*?\))")
 ABORT_PATTERN = re.compile(r"(abort\(\) was called at PC 0x[a-eA-E\d]{8} on core \d)")
 FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored")
-UT_TIMEOUT = 30
+
+STARTUP_TIMEOUT=10
 
 
 def format_test_case_config(test_case_data):
@@ -142,7 +143,7 @@ def run_unit_test_cases(env, extra_data):
             # to determine if DUT is ready to test.
             dut.write("-", flush=False)
             dut.expect_any(UT_APP_BOOT_UP_DONE,
-                           "0 Tests 0 Failures 0 Ignored", timeout=UT_TIMEOUT)
+                           "0 Tests 0 Failures 0 Ignored", timeout=STARTUP_TIMEOUT)
 
             # run test case
             dut.write("\"{}\"".format(one_case["name"]))
@@ -203,7 +204,7 @@ def run_unit_test_cases(env, extra_data):
                                    (ABORT_PATTERN, handle_exception_reset),
                                    (FINISH_PATTERN, handle_test_finish),
                                    (UT_APP_BOOT_UP_DONE, handle_reset_finish),
-                                   timeout=UT_TIMEOUT)
+                                   timeout=one_case["timeout"])
                 except ExpectTimeout:
                     Utility.console_log("Timeout in expect", color="orange")
                     one_case_finish(False)
@@ -223,7 +224,7 @@ class Handler(threading.Thread):
     SEND_SIGNAL_PATTERN = re.compile(r'Send signal: \[(.+)\]!')
     FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored")
 
-    def __init__(self, dut, sent_signal_list, lock, parent_case_name, child_case_index, timeout=30):
+    def __init__(self, dut, sent_signal_list, lock, parent_case_name, child_case_index, timeout):
         self.dut = dut
         self.sent_signal_list = sent_signal_list
         self.lock = lock
@@ -288,7 +289,7 @@ class Handler(threading.Thread):
                                     (self.WAIT_SIGNAL_PATTERN, device_wait_action),  # wait signal pattern
                                     (self.SEND_SIGNAL_PATTERN, device_send_action),  # send signal pattern
                                     (self.FINISH_PATTERN, handle_device_test_finish),  # test finish pattern
-                                    timeout=UT_TIMEOUT)
+                                    timeout=self.timeout)
             except ExpectTimeout:
                 Utility.console_log("Timeout in expect", color="orange")
                 one_device_case_finish(False)
@@ -321,7 +322,7 @@ def case_run(duts, ut_config, env, one_case, failed_cases):
     for i in range(case_num):
         dut = get_dut(duts, env, "dut%d" % i, ut_config)
         threads.append(Handler(dut, send_signal_list, lock,
-                               parent_case, i))
+                               parent_case, i, one_case["timeout"]))
     for thread in threads:
         thread.setDaemon(True)
         thread.start()
@@ -487,7 +488,7 @@ def run_multiple_stage_cases(env, extra_data):
                                        (ABORT_PATTERN, handle_exception_reset),
                                        (FINISH_PATTERN, handle_test_finish),
                                        (UT_APP_BOOT_UP_DONE, handle_next_stage),
-                                       timeout=UT_TIMEOUT)
+                                       timeout=one_case["timeout"])
                     except ExpectTimeout:
                         Utility.console_log("Timeout in expect", color="orange")
                         one_case_finish(False)