CI: support test UT with different config by CI

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 80ad945481a149633b6c5a2403c52db168cee95d..adcfdb6a462e859ac4baa6ddd4fb8ed28c99f3ed 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -412,6 +412,9 @@ assign_test:
   dependencies:
     - build_esp_idf_tests
     - build_ssc
+  variables:
+    UT_BIN_PATH: "tools/unit-test-app/output"
+    OUTPUT_BIN_PATH: "test_bins/ESP32_IDF"
   artifacts:
     paths:
       - test_bins
@@ -421,9 +424,10 @@ assign_test:
   before_script: *add_gitlab_key_before
   script:
     # first move test bins together: test_bins/CHIP_SDK/TestApp/bin_files
-    - mkdir -p test_bins/ESP32_IDF/UT
-    - cp -r tools/unit-test-app/build/* test_bins/ESP32_IDF/UT
-    - cp -r SSC/ssc_bin/* test_bins/ESP32_IDF
+    - mkdir -p $OUTPUT_BIN_PATH
+    # copy and rename folder name to "UT_config"
+    - for CONFIG in $(ls $UT_BIN_PATH); do cp -r "$UT_BIN_PATH/$CONFIG" "$OUTPUT_BIN_PATH/UT_$CONFIG"; done
+    - cp -r SSC/ssc_bin/* $OUTPUT_BIN_PATH
     # clone test script to assign tests
     - git clone $TEST_SCRIPT_REPOSITORY
     - cd auto_test_script
@@ -508,42 +512,161 @@ UT_001_01:
   tags:
     - ESP32_IDF
     - UT_T1_1
+    - UT_default
 
 UT_001_02:
   <<: *unit_test_template
   tags:
     - ESP32_IDF
     - UT_T1_1
+    - UT_default
 
 UT_001_03:
   <<: *unit_test_template
   tags:
     - ESP32_IDF
     - UT_T1_1
+    - UT_default
 
 UT_001_04:
   <<: *unit_test_template
   tags:
     - ESP32_IDF
     - UT_T1_1
+    - UT_default
 
 UT_001_05:
   <<: *unit_test_template
   tags:
     - ESP32_IDF
     - UT_T1_SDMODE
+    - UT_default
 
 UT_001_06:
   <<: *unit_test_template
   tags:
     - ESP32_IDF
     - UT_T1_SPIMODE
+    - UT_default
 
 UT_001_07:
   <<: *unit_test_template
   tags:
     - ESP32_IDF
     - UT_T1_1
+    - UT_default
+
+UT_001_08:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_default
+
+UT_002_01:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_release
+
+UT_002_02:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_release
+
+UT_002_03:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_release
+
+UT_002_04:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_release
+
+UT_002_05:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_SDMODE
+    - UT_release
+
+UT_002_06:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_SPIMODE
+    - UT_release
+
+UT_002_07:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_release
+
+UT_002_08:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_release
+
+UT_003_01:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_single_core
+
+UT_003_02:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_single_core
+
+UT_003_03:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_single_core
+
+UT_003_04:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_single_core
+
+UT_003_05:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_SDMODE
+    - UT_single_core
+
+UT_003_06:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_SPIMODE
+    - UT_single_core
+
+UT_003_07:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - UT_single_core
 
 IT_001_01:
   <<: *test_template

diff --git a/tools/unit-test-app/tools/UnitTestParser.py b/tools/unit-test-app/tools/UnitTestParser.py
index dd7f4417c31d7bed1a7d64a804976febebaac8fb..286468654c9ed651ba359466d9b1c40c4fbfdbf8 100644 (file)
--- a/tools/unit-test-app/tools/UnitTestParser.py
+++ b/tools/unit-test-app/tools/UnitTestParser.py
@@ -14,7 +14,6 @@ TEST_CASE_PATTERN = {
     "SDK": "ESP32_IDF",
     "level": "Unit",
     "execution time": 0,
-    "Test App": "UT",
     "auto test": "Yes",
     "category": "Function",
     "test point 1": "basic function",
@@ -36,20 +35,31 @@ class Parser(object):
     TAG_PATTERN = re.compile("([^=]+)(=)?(.+)?")
     DESCRIPTION_PATTERN = re.compile("\[([^]\[]+)\]")
 
+    # file path (relative to idf path)
+    TAG_DEF_FILE = os.path.join("tools", "unit-test-app", "tools", "TagDefinition.yml")
+    MODULE_DEF_FILE = os.path.join("tools", "unit-test-app", "tools", "ModuleDefinition.yml")
+    MODULE_ARTIFACT_FILE = os.path.join("components", "idf_test", "ModuleDefinition.yml")
+    TEST_CASE_FILE = os.path.join("components", "idf_test", "unit_test", "TestCaseAll.yml")
+    UT_BIN_FOLDER = os.path.join("tools", "unit-test-app", "builds")
+    ELF_FILE = "unit-test-app.elf"
+    APP_NAME_PREFIX = "UT_"
+
     def __init__(self, idf_path=os.getenv("IDF_PATH")):
         self.test_env_tags = {}
         self.unit_jobs = {}
         self.file_name_cache = {}
         self.idf_path = idf_path
-        self.tag_def = yaml.load(open(os.path.join(idf_path, "tools", "unit-test-app", "tools",
-                                                   "TagDefinition.yml"), "r"))
-        self.module_map = yaml.load(open(os.path.join(idf_path, "tools", "unit-test-app", "tools",
-                                                      "ModuleDefinition.yml"), "r"))
+        self.tag_def = yaml.load(open(os.path.join(idf_path, self.TAG_DEF_FILE), "r"))
+        self.module_map = yaml.load(open(os.path.join(idf_path, self.MODULE_DEF_FILE), "r"))
+        # used to check if duplicated test case names
+        self.test_case_names = set()
+        self.parsing_errors = []
 
-    def parse_test_cases_from_elf(self, elf_file):
+    def parse_test_cases_from_elf(self, elf_file, app_name):
         """
-        parse test cases from elf and save test cases to unit test folder
+        parse test cases from elf and save test cases need to be executed to unit test folder
         :param elf_file: elf file path
+        :param app_name: built unit test app name
         """
         subprocess.check_output('xtensa-esp32-elf-objdump -t {} | grep \ test_desc > case_address.tmp'.format(elf_file),
                                 shell=True)
@@ -71,25 +81,37 @@ class Parser(object):
                 desc = table.get_string("any", desc_addr)
                 file_name = table.get_string("any", file_name_addr)
 
-                tc = self.parse_one_test_case(name, desc, file_name)
+                tc = self.parse_one_test_case(name, desc, file_name, app_name)
+
+                # check if duplicated case names
+                # we need to use it to select case,
+                # if duplicated IDs, Unity could select incorrect case to run
+                # and we need to check all cases no matter if it's going te be executed by CI
+                # also add app_name here, we allow same case for different apps
+                if (tc["summary"] + app_name) in self.test_case_names:
+                    self.parsing_errors.append("duplicated test case ID: " + tc["summary"])
+                else:
+                    self.test_case_names.add(tc["summary"] + app_name)
+
                 if tc["CI ready"] == "Yes":
                     # update test env list and the cases of same env list
                     if tc["test environment"] in self.test_env_tags:
                         self.test_env_tags[tc["test environment"]].append(tc["ID"])
                     else:
                         self.test_env_tags.update({tc["test environment"]: [tc["ID"]]})
-                test_cases.append(tc)
+                    # only add cases need to be executed
+                    test_cases.append(tc)
 
         os.remove("section_table.tmp")
         os.remove("case_address.tmp")
 
-        self.dump_test_cases(test_cases)
+        return test_cases
 
     def parse_case_properities(self, tags_raw):
         """
         parse test case tags (properities) with the following rules:
             * first tag is always group of test cases, it's mandatory
-            * the rest tags should be [type=value]. 
+            * the rest tags should be [type=value].
                 * if the type have default value, then [type] equal to [type=default_value].
                 * if the type don't don't exist, then equal to [type=omitted_value]
             default_value and omitted_value are defined in TagDefinition.yml
@@ -123,21 +145,22 @@ class Parser(object):
                 pass
         return p
 
-    def parse_one_test_case(self, name, description, file_name):
+    def parse_one_test_case(self, name, description, file_name, app_name):
         """
         parse one test case
         :param name: test case name (summary)
         :param description: test case description (tag string)
         :param file_name: the file defines this test case
+        :param app_name: built unit test app name
         :return: parsed test case
         """
         prop = self.parse_case_properities(description)
-        
+
         idf_path = os.getenv("IDF_PATH")
-        
+
         # use relative file path to IDF_PATH, to make sure file path is consist
         relative_file_path = os.path.relpath(file_name, idf_path)
-        
+
         file_name_hash = int(hashlib.sha256(relative_file_path).hexdigest(), base=16) % 1000
 
         if file_name_hash in self.file_name_cache:
@@ -149,8 +172,10 @@ class Parser(object):
                                        self.module_map[prop["module"]]['sub module abbr'],
                                        file_name_hash,
                                        self.file_name_cache[file_name_hash])
+
         test_case = deepcopy(TEST_CASE_PATTERN)
-        test_case.update({"module": self.module_map[prop["module"]]['module'],
+        test_case.update({"Test App": self.APP_NAME_PREFIX + app_name,
+                          "module": self.module_map[prop["module"]]['module'],
                           "CI ready": "No" if prop["ignore"] == "Yes" else "Yes",
                           "cmd set": ["IDFUnitTest/UnitTest", [name]],
                           "ID": tc_id,
@@ -166,15 +191,28 @@ class Parser(object):
         dump parsed test cases to YAML file for test bench input
         :param test_cases: parsed test cases
         """
-        with open(os.path.join(self.idf_path, "components", "idf_test", "unit_test", "TestCaseAll.yml"), "wb+") as f:
+        with open(os.path.join(self.idf_path, self.TEST_CASE_FILE), "wb+") as f:
             yaml.dump({"test cases": test_cases}, f, allow_unicode=True, default_flow_style=False)
 
     def copy_module_def_file(self):
         """ copy module def file to artifact path """
-        src = os.path.join(self.idf_path, "tools", "unit-test-app", "tools", "ModuleDefinition.yml")
-        dst = os.path.join(self.idf_path, "components", "idf_test")
+        src = os.path.join(self.idf_path, self.MODULE_DEF_FILE)
+        dst = os.path.join(self.idf_path, self.MODULE_ARTIFACT_FILE)
         shutil.copy(src, dst)
 
+    def parse_test_cases(self):
+        """ parse test cases from multiple built unit test apps """
+        test_cases = []
+
+        test_app_folder = os.path.join(self.idf_path, self.UT_BIN_FOLDER)
+        test_apps = os.listdir(test_app_folder)
+        for app in test_apps:
+            elf_file = os.path.join(test_app_folder, app, self.ELF_FILE)
+            if os.path.exists(elf_file):
+                test_cases.extend(self.parse_test_cases_from_elf(elf_file, app))
+
+        self.dump_test_cases(test_cases)
+
 
 def test_parser():
     parser = Parser()
@@ -210,12 +248,16 @@ def main():
     test_parser()
 
     idf_path = os.getenv("IDF_PATH")
-    elf_path = os.path.join(idf_path, "tools", "unit-test-app", "build", "unit-test-app.elf")
 
     parser = Parser(idf_path)
-    parser.parse_test_cases_from_elf(elf_path)
+    parser.parse_test_cases()
     parser.copy_module_def_file()
+    if len(parser.parsing_errors) > 0:
+        for error in parser.parsing_errors:
+            print error
+        exit(-1)
 
 
 if __name__ == '__main__':
     main()
+