From: He Yin Ling Date: Thu, 20 Dec 2018 09:00:12 +0000 (+0800) Subject: unit-test: fix 2 issues in unit test script: X-Git-Tag: v3.3-beta2~114^2 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=431653bba87e69ba3d854ef34d450263ee1921db;p=esp-idf unit-test: fix 2 issues in unit test script: 1. Gitlab will only regard failure (not error) as failed case. Use log failure instead of log error, to display all error msg in test result 2. fix failed case not recognized by test script when failed during reset DUT --- diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index 4f4a814822..86e352d7f3 100755 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -64,7 +64,11 @@ MULTI_DEVICE_ID = 2 DEFAULT_TIMEOUT = 20 DUT_STARTUP_CHECK_RETRY_COUNT = 5 -TEST_HISTROY_CHECK_TIMEOUT = 1 +TEST_HISTORY_CHECK_TIMEOUT = 1 + + +class TestCaseFailed(AssertionError): + pass def format_test_case_config(test_case_data): @@ -163,7 +167,7 @@ def reset_dut(dut): for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT): dut.write("-") try: - dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT) + dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT) break except ExpectTimeout: pass @@ -171,7 +175,7 @@ def reset_dut(dut): raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port)) -def run_one_normal_case(dut, one_case, junit_test_case, failed_cases): +def run_one_normal_case(dut, one_case, junit_test_case): reset_dut(dut) @@ -194,9 +198,9 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases): if result: Utility.console_log("Success: " + one_case["name"], color="green") else: - failed_cases.append(one_case["name"]) Utility.console_log("Failed: " + one_case["name"], color="red") junit_test_case.add_failure_info(output) + raise TestCaseFailed() def handle_exception_reset(data): """ @@ -242,7 +246,7 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases): timeout=one_case["timeout"]) except ExpectTimeout: Utility.console_log("Timeout in expect", color="orange") - junit_test_case.add_error_info("timeout") + junit_test_case.add_failure_info("timeout") one_case_finish(False) break @@ -284,10 +288,12 @@ def run_unit_test_cases(env, extra_data): # create junit report test case junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"])) try: - run_one_normal_case(dut, one_case, junit_test_case, failed_cases) - TinyFW.JunitReport.test_case_finish(junit_test_case) + run_one_normal_case(dut, one_case, junit_test_case) + except TestCaseFailed: + failed_cases.append(one_case["name"]) except Exception as e: - junit_test_case.add_error_info("Unexpected exception: " + str(e)) + junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + finally: TinyFW.JunitReport.test_case_finish(junit_test_case) # raise exception if any case fails @@ -476,9 +482,9 @@ def run_multiple_devices_cases(env, extra_data): try: run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases, one_case.get('app_bin'), junit_test_case) - TinyFW.JunitReport.test_case_finish(junit_test_case) except Exception as e: - junit_test_case.add_error_info("Unexpected exception: " + str(e)) + junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + finally: TinyFW.JunitReport.test_case_finish(junit_test_case) if failed_cases: @@ -488,7 +494,7 @@ def run_multiple_devices_cases(env, extra_data): raise AssertionError("Unit Test Failed") -def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case): +def run_one_multiple_stage_case(dut, one_case, junit_test_case): reset_dut(dut) dut.start_capture_raw_data() @@ -523,7 +529,7 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case): err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"], exception_reset_list) Utility.console_log(err_msg, color="orange") - junit_test_case.add_error_info(err_msg) + junit_test_case.add_failure_info(err_msg) else: # we allow omit reset in multi stage cases result = True @@ -538,9 +544,9 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case): if result: Utility.console_log("Success: " + one_case["name"], color="green") else: - failed_cases.append(one_case["name"]) Utility.console_log("Failed: " + one_case["name"], color="red") junit_test_case.add_failure_info(output) + raise TestCaseFailed() stage_finish.append("break") def handle_exception_reset(data): @@ -620,10 +626,12 @@ def run_multiple_stage_cases(env, extra_data): for one_case in case_config[ut_config]: junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"])) try: - run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case) - TinyFW.JunitReport.test_case_finish(junit_test_case) + run_one_multiple_stage_case(dut, one_case, junit_test_case) + except TestCaseFailed: + failed_cases.append(one_case["name"]) except Exception as e: - junit_test_case.add_error_info("Unexpected exception: " + str(e)) + junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + finally: TinyFW.JunitReport.test_case_finish(junit_test_case) # raise exception if any case fails