From 2b068f3cebcf31c2f93fcae6314d0813bccf1d84 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Tue, 16 Jan 2018 22:16:03 +0800 Subject: [PATCH] tiny-test-fw: fix misc bugs: 1. configs are not functional * decorator will be executed when search case, need to set default config before search case. 2. fix DUT encode/decode errors * python3 serial don't support write string/unicode, need to convert to bytes first. * python2 string could failed to encode/decode non-acsii chars 3. fix bug that log folder not created 4. fix bug that test suite name is not correctly passed: * the keyward arg should be `test_suite_name` not `test_name` 5. fix bug that test stopped on failed case 6. fix DUT `read` don't return all data cache * `_DataCache.get_data` should first copy all data from queue to data cache and then return to user. 7. fix bug that `expect_all` failed even all expected item passed 8. optimize error info for expect * print pattern for regular expression when match failed 9. fix bug that set default config doesn't work --- tools/tiny-test-fw/App.py | 9 ++++-- tools/tiny-test-fw/DUT.py | 46 ++++++++++++++++++++++++------- tools/tiny-test-fw/Env.py | 4 +-- tools/tiny-test-fw/EnvConfig.py | 2 +- tools/tiny-test-fw/Runner.py | 14 ++++++---- tools/tiny-test-fw/TinyFW.py | 12 ++++---- tools/tiny-test-fw/docs/index.rst | 2 +- tools/tiny-test-fw/example.py | 2 +- 8 files changed, 62 insertions(+), 29 deletions(-) diff --git a/tools/tiny-test-fw/App.py b/tools/tiny-test-fw/App.py index 84e8716a17..1dbadf85af 100644 --- a/tools/tiny-test-fw/App.py +++ b/tools/tiny-test-fw/App.py @@ -78,9 +78,12 @@ class BaseApp(object): if not test_suite_name: test_suite_name = os.path.splitext(os.path.basename(sys.modules['__main__'].__file__))[0] sdk_path = cls.get_sdk_path() - return os.path.join(sdk_path, "TEST_LOGS", - test_suite_name + - time.strftime("_%m%d_%H_%M_%S", time.localtime(LOG_FOLDER_TIMESTAMP))) + log_folder = os.path.join(sdk_path, "TEST_LOGS", + test_suite_name + + time.strftime("_%m%d_%H_%M_%S", time.localtime(LOG_FOLDER_TIMESTAMP))) + if not os.path.exists(log_folder): + os.makedirs(log_folder) + return log_folder def process_app_info(self): """ diff --git a/tools/tiny-test-fw/DUT.py b/tools/tiny-test-fw/DUT.py index 1cfd2c7d53..882afa1f99 100644 --- a/tools/tiny-test-fw/DUT.py +++ b/tools/tiny-test-fw/DUT.py @@ -85,6 +85,14 @@ def _decode_data(data): return data +def _pattern_to_string(pattern): + try: + ret = "RegEx: " + pattern.pattern + except AttributeError: + ret = pattern + return ret + + class _DataCache(_queue.Queue): """ Data cache based on Queue. Allow users to process data cache based on bytes instead of Queue." @@ -94,6 +102,21 @@ class _DataCache(_queue.Queue): _queue.Queue.__init__(self, maxsize=maxsize) self.data_cache = str() + def _move_from_queue_to_cache(self): + """ + move all of the available data in the queue to cache + + :return: True if moved any item from queue to data cache, else False + """ + ret = False + while True: + try: + self.data_cache += _decode_data(self.get(0)) + ret = True + except _queue.Empty: + break + return ret + def get_data(self, timeout=0): """ get a copy of data from cache. @@ -105,12 +128,16 @@ class _DataCache(_queue.Queue): if timeout < 0: timeout = 0 - try: - data = self.get(timeout=timeout) - self.data_cache += _decode_data(data) - except _queue.Empty: - # don't do anything when on update for cache - pass + ret = self._move_from_queue_to_cache() + + if not ret: + # we only wait for new data if we can't provide a new data_cache + try: + data = self.get(timeout=timeout) + self.data_cache += _decode_data(data) + except _queue.Empty: + # don't do anything when on update for cache + pass return copy.deepcopy(self.data_cache) def flush(self, index=0xFFFFFFFF): @@ -417,7 +444,7 @@ class BaseDUT(object): data = self.data_cache.get_data(time.time() + timeout - start_time) if ret is None: - raise ExpectTimeout(self.name + ": " + str(pattern)) + raise ExpectTimeout(self.name + ": " + _pattern_to_string(pattern)) return ret def _expect_multi(self, expect_all, expect_item_list, timeout): @@ -457,12 +484,11 @@ class BaseDUT(object): if expect_item["ret"] is not None: # match succeed for one item matched_expect_items.append(expect_item) - break # if expect all, then all items need to be matched, # else only one item need to matched if expect_all: - match_succeed = (matched_expect_items == expect_items) + match_succeed = len(matched_expect_items) == len(expect_items) else: match_succeed = True if matched_expect_items else False @@ -482,7 +508,7 @@ class BaseDUT(object): # flush already matched data self.data_cache.flush(slice_index) else: - raise ExpectTimeout(self.name + ": " + str(expect_items)) + raise ExpectTimeout(self.name + ": " + str([_pattern_to_string(x) for x in expect_items])) @_expect_lock def expect_any(self, *expect_items, **timeout): diff --git a/tools/tiny-test-fw/Env.py b/tools/tiny-test-fw/Env.py index fa194d10c9..b623847edb 100644 --- a/tools/tiny-test-fw/Env.py +++ b/tools/tiny-test-fw/Env.py @@ -49,12 +49,12 @@ class Env(object): dut=None, env_tag=None, env_config_file=None, - test_name=None, + test_suite_name=None, **kwargs): self.app_cls = app self.default_dut_cls = dut self.config = EnvConfig.Config(env_config_file, env_tag) - self.log_path = self.app_cls.get_log_folder(test_name) + self.log_path = self.app_cls.get_log_folder(test_suite_name) if not os.path.exists(self.log_path): os.makedirs(self.log_path) diff --git a/tools/tiny-test-fw/EnvConfig.py b/tools/tiny-test-fw/EnvConfig.py index 2ce28d811e..79de6bd3f8 100644 --- a/tools/tiny-test-fw/EnvConfig.py +++ b/tools/tiny-test-fw/EnvConfig.py @@ -53,7 +53,7 @@ class Config(object): try: with open(config_file) as f: configs = yaml.load(f)[env_name] - except (OSError, TypeError): + except (OSError, TypeError, IOError): configs = dict() return configs diff --git a/tools/tiny-test-fw/Runner.py b/tools/tiny-test-fw/Runner.py index 0adf441fe0..70f60eb354 100644 --- a/tools/tiny-test-fw/Runner.py +++ b/tools/tiny-test-fw/Runner.py @@ -40,18 +40,22 @@ class Runner(threading.Thread): def __init__(self, test_case, case_config, env_config_file=None): super(Runner, self).__init__() self.setDaemon(True) - test_methods = SearchCases.Search.search_test_cases(test_case) - self.test_cases = CaseConfig.Parser.apply_config(test_methods, case_config) - self.test_result = True if case_config: test_suite_name = os.path.splitext(os.path.basename(case_config))[0] else: test_suite_name = "TestRunner" TinyFW.set_default_config(env_config_file=env_config_file, test_suite_name=test_suite_name) + test_methods = SearchCases.Search.search_test_cases(test_case) + self.test_cases = CaseConfig.Parser.apply_config(test_methods, case_config) + self.test_result = [] def run(self): for case in self.test_cases: - self.test_result = self.test_result and case.run() + result = case.run() + self.test_result.append(result) + + def get_test_result(self): + return self.test_result and all(self.test_result) if __name__ == '__main__': @@ -76,5 +80,5 @@ if __name__ == '__main__': except KeyboardInterrupt: print("exit by Ctrl-C") break - if not runner.test_result: + if not runner.get_test_result(): sys.exit(1) diff --git a/tools/tiny-test-fw/TinyFW.py b/tools/tiny-test-fw/TinyFW.py index 09b950c583..b227cf7aad 100644 --- a/tools/tiny-test-fw/TinyFW.py +++ b/tools/tiny-test-fw/TinyFW.py @@ -132,12 +132,6 @@ def test_method(**kwargs): case_info["name"] = test_func.__name__ case_info.update(kwargs) - # create env instance - env_config = DefaultEnvConfig.get_default_config() - for key in kwargs: - if key in env_config: - env_config[key] = kwargs[key] - @functools.wraps(test_func) def handle_test(extra_data=None, **overwrite): """ @@ -147,6 +141,12 @@ def test_method(**kwargs): :param overwrite: args that runner or main want to overwrite :return: None """ + # create env instance + env_config = DefaultEnvConfig.get_default_config() + for key in kwargs: + if key in env_config: + env_config[key] = kwargs[key] + env_config.update(overwrite) env_inst = Env.Env(**env_config) # prepare for xunit test results diff --git a/tools/tiny-test-fw/docs/index.rst b/tools/tiny-test-fw/docs/index.rst index bc3a3f5f82..b83254cb18 100644 --- a/tools/tiny-test-fw/docs/index.rst +++ b/tools/tiny-test-fw/docs/index.rst @@ -69,7 +69,7 @@ Let's first check a simple simple:: if __name__ == '__main__': - TinyFW.set_default_config(config_file="EnvConfigTemplate.yml") + TinyFW.set_default_config(env_config_file="EnvConfigTemplate.yml") test_examples_protocol_https_request() diff --git a/tools/tiny-test-fw/example.py b/tools/tiny-test-fw/example.py index df1b25576e..324c904383 100644 --- a/tools/tiny-test-fw/example.py +++ b/tools/tiny-test-fw/example.py @@ -47,5 +47,5 @@ def test_examples_protocol_https_request(env, extra_data): if __name__ == '__main__': - TinyFW.set_default_config(config_file="EnvConfigTemplate.yml", dut=IDF.IDFDUT) + TinyFW.set_default_config(env_config_file="EnvConfigTemplate.yml", dut=IDF.IDFDUT) test_examples_protocol_https_request() -- 2.40.0