unit-test: fix 2 issues in unit test script:

1. Gitlab will only regard failure (not error) as failed case. Use log
failure instead of log error, to display all error msg in test result
2. fix failed case not recognized by test script when failed during
reset DUT

diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py
index 4f4a8148229583ca27389f8dde97d64886890dad..86e352d7f31b8cd3168cce25479db20d8f8ab8f2 100755 (executable)
--- a/tools/unit-test-app/unit_test.py
+++ b/tools/unit-test-app/unit_test.py
@@ -64,7 +64,11 @@ MULTI_DEVICE_ID = 2
 DEFAULT_TIMEOUT = 20
 
 DUT_STARTUP_CHECK_RETRY_COUNT = 5
-TEST_HISTROY_CHECK_TIMEOUT = 1
+TEST_HISTORY_CHECK_TIMEOUT = 1
+
+
+class TestCaseFailed(AssertionError):
+    pass
 
 
 def format_test_case_config(test_case_data):
@@ -163,7 +167,7 @@ def reset_dut(dut):
     for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT):
         dut.write("-")
         try:
-            dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT)
+            dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT)
             break
         except ExpectTimeout:
             pass
@@ -171,7 +175,7 @@ def reset_dut(dut):
         raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port))
 
 
-def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
+def run_one_normal_case(dut, one_case, junit_test_case):
 
     reset_dut(dut)
 
@@ -194,9 +198,9 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
         if result:
             Utility.console_log("Success: " + one_case["name"], color="green")
         else:
-            failed_cases.append(one_case["name"])
             Utility.console_log("Failed: " + one_case["name"], color="red")
             junit_test_case.add_failure_info(output)
+            raise TestCaseFailed()
 
     def handle_exception_reset(data):
         """
@@ -242,7 +246,7 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
                            timeout=one_case["timeout"])
         except ExpectTimeout:
             Utility.console_log("Timeout in expect", color="orange")
-            junit_test_case.add_error_info("timeout")
+            junit_test_case.add_failure_info("timeout")
             one_case_finish(False)
             break
 
@@ -284,10 +288,12 @@ def run_unit_test_cases(env, extra_data):
             # create junit report test case
             junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
             try:
-                run_one_normal_case(dut, one_case, junit_test_case, failed_cases)
-                TinyFW.JunitReport.test_case_finish(junit_test_case)
+                run_one_normal_case(dut, one_case, junit_test_case)
+            except TestCaseFailed:
+                failed_cases.append(one_case["name"])
             except Exception as e:
-                junit_test_case.add_error_info("Unexpected exception: " + str(e))
+                junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+            finally:
                 TinyFW.JunitReport.test_case_finish(junit_test_case)
 
     # raise exception if any case fails
@@ -476,9 +482,9 @@ def run_multiple_devices_cases(env, extra_data):
             try:
                 run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases,
                                               one_case.get('app_bin'), junit_test_case)
-                TinyFW.JunitReport.test_case_finish(junit_test_case)
             except Exception as e:
-                junit_test_case.add_error_info("Unexpected exception: " + str(e))
+                junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+            finally:
                 TinyFW.JunitReport.test_case_finish(junit_test_case)
 
     if failed_cases:
@@ -488,7 +494,7 @@ def run_multiple_devices_cases(env, extra_data):
         raise AssertionError("Unit Test Failed")
 
 
-def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
+def run_one_multiple_stage_case(dut, one_case, junit_test_case):
     reset_dut(dut)
 
     dut.start_capture_raw_data()
@@ -523,7 +529,7 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
                     err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"],
                                                                                            exception_reset_list)
                     Utility.console_log(err_msg, color="orange")
-                    junit_test_case.add_error_info(err_msg)
+                    junit_test_case.add_failure_info(err_msg)
             else:
                 # we allow omit reset in multi stage cases
                 result = True
@@ -538,9 +544,9 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
             if result:
                 Utility.console_log("Success: " + one_case["name"], color="green")
             else:
-                failed_cases.append(one_case["name"])
                 Utility.console_log("Failed: " + one_case["name"], color="red")
                 junit_test_case.add_failure_info(output)
+                raise TestCaseFailed()
             stage_finish.append("break")
 
         def handle_exception_reset(data):
@@ -620,10 +626,12 @@ def run_multiple_stage_cases(env, extra_data):
         for one_case in case_config[ut_config]:
             junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
             try:
-                run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case)
-                TinyFW.JunitReport.test_case_finish(junit_test_case)
+                run_one_multiple_stage_case(dut, one_case, junit_test_case)
+            except TestCaseFailed:
+                failed_cases.append(one_case["name"])
             except Exception as e:
-                junit_test_case.add_error_info("Unexpected exception: " + str(e))
+                junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+            finally:
                 TinyFW.JunitReport.test_case_finish(junit_test_case)
 
     # raise exception if any case fails