From 7b0086f280de3f9379988c62a0756cb4451fdd46 Mon Sep 17 00:00:00 2001 From: Crispin Dent-Young Date: Wed, 29 Jun 2016 09:49:38 +0100 Subject: [PATCH] Support arbitrary tagging and selection of testcases. A testcase can optionally have a list of tags associated with it. Srunner can be run with an optional include list of tags and an optional exclude list of tags. These will have the effect of filtering testcases that would otherwise be run. --- doc/check.texi | 110 +++++- src/check.c | 67 +++- src/check.h.in | 75 +++- src/check_impl.h | 4 + src/check_run.c | 48 ++- tests/CMakeLists.txt | 1 + tests/Makefile.am | 5 +- tests/check_check.h | 1 + tests/check_check_main.c | 1 + tests/check_check_tags.c | 763 +++++++++++++++++++++++++++++++++++++++ tests/check_mem_leaks.c | 2 +- 11 files changed, 1057 insertions(+), 20 deletions(-) create mode 100644 tests/check_check_tags.c diff --git a/doc/check.texi b/doc/check.texi index b59b4e4..91f4f98 100644 --- a/doc/check.texi +++ b/doc/check.texi @@ -974,6 +974,8 @@ easier for the developer to write, run, and analyze tests. * Test Fixtures:: * Multiple Suites in one SRunner:: * Selective Running of Tests:: +* Selecting Tests by Suite or Test Case:: +* Selecting Tests Based on Arbitrary Tags:: * Testing Signal Handling and Exit Values:: * Looping Tests:: * Test Timeouts:: @@ -1302,19 +1304,105 @@ srunner_add_suite (sr, make_pack_suite ()); @end example @node Selective Running of Tests, Testing Signal Handling and Exit Values, Multiple Suites in one SRunner, Advanced Features + @section Selective Running of Tests +After adding a couple of suites and some test cases in each, it is +sometimes practical to be able to run only one suite, or one specific +test case, without recompiling the test code. Check provides two ways +to accomplish this, either by specifying a suite or test case by name +or by assigning tags to test cases and specifying one or more tags to +run. + +@menu +* Selecting Tests by Suite or Test Case:: +* Selecting Tests Based on Arbitrary Tags:: +@end menu + +@node Selecting Tests by Suite or Test Case, Selecting Tests Based on Arbitrary Tags, Selective Running of Tests, Selective Running of Tests +@subsection Selecting Tests by Suite or Test Case + @vindex CK_RUN_SUITE @vindex CK_RUN_CASE -After adding a couple of suites and some test cases in each, it is -sometimes practical to be able to run only one suite, or one -specific test case, without recompiling the test code. There are -two environment variables available that offers this ability, -@code{CK_RUN_SUITE} and @code{CK_RUN_CASE}. Just set the value to -the name of the suite and/or test case you want to run. These -environment variables can also be a good integration tool for -running specific tests from within another tool, e.g. an IDE. - + +There are two environment variables available that offer this +ability, @code{CK_RUN_SUITE} and @code{CK_RUN_CASE}. Just set the +value to the name of the suite and/or test case you want to run. These +environment variables can also be a good integration tool for running +specific tests from within another tool, e.g. an IDE. + +@node Selecting Tests Based on Arbitrary Tags, ,Selecting Tests by Suite or Test Case, Selective Running of Tests +@subsection Selecting Tests Based on Arbitrary Tags + +@vindex CK_INCLUDE_TAGS +@vindex CK_EXCLUDE_TAGS + +It can be useful to dynamically include or exclude groups of tests to +be run based on criteria other than the suite or test case name. For +example, one or more tags can be assigned to test cases. The tags +could indicate if a test runs for a long time, so such tests could be +excluded in order to run quicker tests for a sanity +check. Alternately, tags may be used to indicate which functional +areas test cover. Tests can then be run that include all test cases +for a given set of functional areas. + +In Check, a tag is a string of characters without white space. One or +more tags can be assigned to a test case by using the +@code{tcase_set_tags} function. This function accepts a string, and +multiple tags can be specified by delimiting them with spaces. For +example: + +@example +@verbatim + Suite *s; + + TCase *red, *blue, *purple, *yellow, *black; + + s = suite_create("Check Tag Filtering"); + + red = tcase_create("Red"); + tcase_set_tags(red, "Red"); + suite_add_tcase (s, red); + tcase_add_test(red, red_test1); + + blue = tcase_create("Blue"); + tcase_set_tags(blue, "Blue"); + suite_add_tcase (s, blue); + tcase_add_test(blue, blue_test1); + + purple = tcase_create("Purple"); + tcase_set_tags(purple, "Red Blue"); + suite_add_tcase (s, purple); + tcase_add_test(purple, purple_test1); + +@end verbatim +@end example + +Once test cases are tagged they may be selectively run in one of two ways: + +a) Using Environment Variables + +There are two environment variables available for selecting test cases +based on tags: @code{CK_INCLUDE_TAGS} and +@code{CK_EXCLUDE_TAGS}. These can be set to a space separated list of +tag names. If @code{CK_INCLUDE_TAGS} is set then test cases which +include at least one tag in common with @code{CK_INCLUDE_TAGS} will be +run. If @code{CK_EXCLUDE_TAGS} is set then test cases with one tag in +common with @code{CK_EXCLUDE_TAGS} will not be run. In cases where +both @code{CK_INCLUDE_TAGS} and @code{CK_EXCLUDE_TAGS} match a tag for +a test case the test will be excluded. + +Both @code{CK_INCLUDE_TAGS} and @code{CK_EXCLUDE_TAGS} can be +specified in conjunction with @code{CK_RUN_SUITE} or even +@code{CK_RUN_CASE} in which case they will have the effect of further +narrowing the selection. + +b) Programmatically + +The @code{srunner_run_tagged} function allows one to specify which +tags to run or exclude from a suite runner. This can be used to +programmatically control which test cases may run. + @node Testing Signal Handling and Exit Values, Looping Tests, Selective Running of Tests, Advanced Features @section Testing Signal Handling and Exit Values @@ -1975,6 +2063,10 @@ CK_RUN_CASE: Name of a test case, runs only that test. See section @ref{Selectiv CK_RUN_SUITE: Name of a test suite, runs only that suite. See section @ref{Selective Running of Tests}. +CK_INCLUDE_TAGS: String of space separated tags, runs only test cases associated with at least one of the tags, See section @ref{Selecting Tests Based on Arbitrary Tags}. + +CK_EXCLUDE_TAGS: String of space separated tags, runs only test cases not associated with any of the tags, See section @ref{Selecting Tests Based on Arbitrary Tags}. + CK_VERBOSITY: How much output to emit, accepts: ``silent'', ``minimal'', ``normal'', ``subunit'', or ``verbose''. See section @ref{SRunner Output}. CK_FORK: Set to ``no'' to disable using fork() to run unit tests in their own process. This is useful for debugging segmentation faults. See section @ref{No Fork Mode}. diff --git a/src/check.c b/src/check.c index 020fdfc..f49ed40 100644 --- a/src/check.c +++ b/src/check.c @@ -103,6 +103,7 @@ static void suite_free(Suite * s) free(s); } + TCase *tcase_create(const char *name) { char *env; @@ -149,10 +150,49 @@ TCase *tcase_create(const char *name) tc->ch_sflst = check_list_create(); tc->unch_tflst = check_list_create(); tc->ch_tflst = check_list_create(); + tc->tags = check_list_create(); return tc; } +/* + * Helper function to create a list of tags from + * a space separated string. + */ +List *tag_string_to_list(const char *tags_string) +{ + List *list; + char *tags; + char *tag; + + list = check_list_create(); + + if (NULL == tags_string) + { + return list; + } + + tags = strdup(tags_string); + tag = strtok(tags, " "); + while (tag) + { + check_list_add_end(list, strdup(tag)); + tag = strtok(NULL, " "); + } + free(tags); + return list; +} + +void tcase_set_tags(TCase * tc, const char *tags_orig) +{ + /* replace any pre-existing list */ + if (tc->tags) + { + check_list_apply(tc->tags, free); + check_list_free(tc->tags); + } + tc->tags = tag_string_to_list(tags_orig); +} static void tcase_free(TCase * tc) { @@ -161,15 +201,40 @@ static void tcase_free(TCase * tc) check_list_apply(tc->ch_sflst, free); check_list_apply(tc->unch_tflst, free); check_list_apply(tc->ch_tflst, free); + check_list_apply(tc->tags, free); check_list_free(tc->tflst); check_list_free(tc->unch_sflst); check_list_free(tc->ch_sflst); check_list_free(tc->unch_tflst); check_list_free(tc->ch_tflst); - + check_list_free(tc->tags); free(tc); } +unsigned int tcase_matching_tag(TCase *tc, List *check_for) +{ + + if (NULL == check_for) + { + return 0; + } + + for(check_list_front(check_for); !check_list_at_end(check_for); + check_list_advance(check_for)) + { + for(check_list_front(tc->tags); !check_list_at_end(tc->tags); + check_list_advance(tc->tags)) + { + if (0 == strcmp((const char *)check_list_val(tc->tags), + (const char *)check_list_val(check_for))) + { + return 1; + } + } + } + return 0; +} + void suite_add_tcase(Suite * s, TCase * tc) { if(s == NULL || tc == NULL || check_list_contains(s->tclst, tc)) diff --git a/src/check.h.in b/src/check.h.in index b956cf9..e9e5de7 100644 --- a/src/check.h.in +++ b/src/check.h.in @@ -172,6 +172,19 @@ CK_DLL_EXP void CK_EXPORT suite_add_tcase(Suite * s, TCase * tc); * */ CK_DLL_EXP TCase *CK_EXPORT tcase_create(const char *name); +/** + * Associate a test case with certain tags. + * Replaces any existing tags with the new set. + * + * @param tc the test case + * + * @param tags string containing arbitrary tags separated by spaces. + * This will be copied. Passing NULL clears all tags. + * + * @since 0.11.0 + * */ +CK_DLL_EXP void CK_EXPORT tcase_set_tags(TCase * tc, + const char *tags); /** * Add a test function to a test case * @@ -955,8 +968,9 @@ CK_DLL_EXP void CK_EXPORT srunner_free(SRunner * sr); * In addition to running all suites, if the suite runner has been * configured to output to a log, that is also performed. * - * Note that if the CK_RUN_CASE and/or CK_RUN_SUITE environment variables - * are defined, then only the named suite and/or test case is run. + * Note that if the CK_RUN_CASE, CK_RUN_SUITE, CK_INCLUDE_TAGS and/or + * CK_EXCLUDE_TAGS environment variables are defined, then only the + * named suites or test cases will run. * * @param sr suite runner to run all suites from * @param print_mode the verbosity in which to report results to stdout @@ -974,9 +988,22 @@ CK_DLL_EXP void CK_EXPORT srunner_run_all(SRunner * sr, * suite runner has been configured to output to a log, that is also * performed. * + * Note that if the sname and tcname parameters are passed as null + * then the function will fallback to using the environment variables + * CK_RUN_SUITE and CK_RUN_CASE respectively in order to select the + * suite/cases. + * + * Similarly if the CK_INCLUDE_TAGS and/or CK_EXCLUDE_TAGS environment + * variables are defined then these will further filter the test cases + * (see srunner_run_tagged, below). + * * @param sr suite runner where the given suite or test case must be - * @param sname suite name to run. A NULL means "any suite". - * @param tcname test case name to run. A NULL means "any test case" + * @param sname suite name to run. A NULL means use the value of the + * environment variable CK_RUN_SUITE if set, otherwise run "any/every + * suite". + * @param tcname test case name to run. A NULL means use the value of + * the environment variable CK_RUN_CASE if set, otherwise run + * "any/every case". * @param print_mode the verbosity in which to report results to stdout * * @since 0.9.9 @@ -986,6 +1013,46 @@ CK_DLL_EXP void CK_EXPORT srunner_run(SRunner * sr, const char *sname, enum print_output print_mode); +/** + * Run a specific suite or test case or testcases with specific tags + * from a suite runner, printing results to stdout as specified by the + * print_mode. + * + * In addition to running any applicable suites or test cases, if the + * suite runner has been configured to output to a log, that is also + * performed. + * + * Note that if sname, tcname, include_tags, exclude_tags parameters + * are passed as NULL then if the environment variables CK_RUN_SUITE, + * CK_RUN_CASE, CK_INCLUDE_TAGS, CK_EXCLUDE_TAGS are defined then these + * values will be used instead. + * + * @param sr suite runner where the given suite or test case must be + * @param sname suite name to run. A NULL means use the value of the + * environment variable CK_RUN_SUITE if set, otherwise run "any/every + * suite". + * @param tcname test case name to run. A NULL means use the value of + * the environment variable CK_RUN_CASE if set, otherwise run + * "any/every case". + * @param include_tags space separate list of tags. Only run test + * cases that share one of these tags. A NULL means use the value of + * the environment variable CK_INCLUDE_TAGS if set, otherwise run + * "any/every test case". + * @param exclude_tags space separate list of tags. Only run test + * cases that do not share one of these tags even if they are selected + * by an included tag. A NULL means use the value of the environment + * variable CK_EXCLUDE_TAGS if set, otherwise run "any/every test + * case". + * @param print_mode the verbosity in which to report results to stdout + * + * @since 0.11.0 + */ +CK_DLL_EXP void CK_EXPORT srunner_run_tagged(SRunner * sr, const char *sname, + const char *tcname, + const char *include_tags, + const char *exclude_tags, + enum print_output print_mode); + /** * Retrieve the number of failed tests executed by a suite runner. * diff --git a/src/check_impl.h b/src/check_impl.h index 15b14f1..bddd186 100644 --- a/src/check_impl.h +++ b/src/check_impl.h @@ -65,6 +65,7 @@ struct TCase List *unch_tflst; List *ch_sflst; List *ch_tflst; + List *tags; }; typedef struct TestStats @@ -134,4 +135,7 @@ enum fork_status cur_fork_status(void); clockid_t check_get_clockid(void); +unsigned int tcase_matching_tag(TCase *tc, List *check_for); +List *tag_string_to_list(const char *tags_string); + #endif /* CHECK_IMPL_H */ diff --git a/src/check_run.c b/src/check_run.c index fe09e08..3714765 100644 --- a/src/check_run.c +++ b/src/check_run.c @@ -60,6 +60,8 @@ static void srunner_run_init(SRunner * sr, enum print_output print_mode); static void srunner_run_end(SRunner * sr, enum print_output print_mode); static void srunner_iterate_suites(SRunner * sr, const char *sname, const char *tcname, + const char *include_tags, + const char *exclude_tags, enum print_output print_mode); static void srunner_iterate_tcase_tfuns(SRunner * sr, TCase * tc); static void srunner_add_failure(SRunner * sr, TestResult * tf); @@ -160,15 +162,22 @@ static void srunner_run_end(SRunner * sr, static void srunner_iterate_suites(SRunner * sr, const char *sname, const char *tcname, + const char *include_tags, + const char *exclude_tags, enum print_output CK_ATTRIBUTE_UNUSED print_mode) { + List *include_tag_lst; + List *exclude_tag_lst; List *slst; List *tcl; TCase *tc; slst = sr->slst; + include_tag_lst = tag_string_to_list(include_tags); + exclude_tag_lst = tag_string_to_list(exclude_tags); + for(check_list_front(slst); !check_list_at_end(slst); check_list_advance(slst)) { @@ -191,12 +200,31 @@ static void srunner_iterate_suites(SRunner * sr, { continue; } + if (include_tags != NULL) + { + if (!tcase_matching_tag(tc, include_tag_lst)) + { + continue; + } + } + if (exclude_tags != NULL) + { + if (tcase_matching_tag(tc, exclude_tag_lst)) + { + continue; + } + } srunner_run_tcase(sr, tc); } log_suite_end(sr, s); } + + check_list_apply(include_tag_lst, free); + check_list_apply(exclude_tag_lst, free); + check_list_free(include_tag_lst); + check_list_free(exclude_tag_lst); } static void srunner_iterate_tcase_tfuns(SRunner * sr, TCase * tc) @@ -741,8 +769,9 @@ void srunner_run_all(SRunner * sr, enum print_output print_mode) print_mode); } -void srunner_run(SRunner * sr, const char *sname, const char *tcname, - enum print_output print_mode) +void srunner_run_tagged(SRunner * sr, const char *sname, const char *tcname, + const char *include_tags, const char *exclude_tags, + enum print_output print_mode) { #if defined(HAVE_SIGACTION) && defined(HAVE_FORK) static struct sigaction sigalarm_old_action; @@ -756,7 +785,11 @@ void srunner_run(SRunner * sr, const char *sname, const char *tcname, if(!tcname) tcname = getenv("CK_RUN_CASE"); if(!sname) - sname = getenv("CK_RUN_SUITE"); + sname = getenv("CK_RUN_SUITE"); + if(!include_tags) + include_tags = getenv("CK_INCLUDE_TAGS"); + if(!exclude_tags) + exclude_tags = getenv("CK_EXCLUDE_TAGS"); if(sr == NULL) return; @@ -779,7 +812,8 @@ void srunner_run(SRunner * sr, const char *sname, const char *tcname, sigaction(SIGTERM, &sigterm_new_action, &sigterm_old_action); #endif /* HAVE_SIGACTION && HAVE_FORK */ srunner_run_init(sr, print_mode); - srunner_iterate_suites(sr, sname, tcname, print_mode); + srunner_iterate_suites(sr, sname, tcname, include_tags, exclude_tags, + print_mode); srunner_run_end(sr, print_mode); #if defined(HAVE_SIGACTION) && defined(HAVE_FORK) sigaction(SIGALRM, &sigalarm_old_action, NULL); @@ -788,6 +822,12 @@ void srunner_run(SRunner * sr, const char *sname, const char *tcname, #endif /* HAVE_SIGACTION && HAVE_FORK */ } +void srunner_run(SRunner * sr, const char *sname, const char *tcname, + enum print_output print_mode) +{ + srunner_run_tagged(sr, sname, tcname, NULL, NULL, print_mode); +} + pid_t check_fork(void) { #if defined(HAVE_FORK) && HAVE_FORK==1 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 02f406e..6e80679 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -53,6 +53,7 @@ set(CHECK_CHECK_SOURCES check_check_pack.c check_check_selective.c check_check_sub.c + check_check_tags.c check_list.c) set(CHECK_CHECK_HEADERS check_check.h) add_executable(check_check ${CHECK_CHECK_HEADERS} ${CHECK_CHECK_SOURCES}) diff --git a/tests/Makefile.am b/tests/Makefile.am index 4995a74..06bf29d 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -41,6 +41,7 @@ check_check_export_SOURCES = \ check_check_master.c \ check_check_log.c \ check_check_fork.c \ + check_check_tags.c \ check_check_export_main.c check_check_export_LDADD = $(top_builddir)/src/libcheck.la $(top_builddir)/lib/libcompat.la @@ -57,7 +58,8 @@ check_check_SOURCES = \ check_check_fixture.c \ check_check_pack.c \ check_check_exit.c \ - check_check_selective.c \ + check_check_selective.c \ + check_check_tags.c \ check_check_main.c check_check_LDADD = $(top_builddir)/src/libcheckinternal.la $(top_builddir)/lib/libcompat.la @@ -67,6 +69,7 @@ check_mem_leaks_SOURCES = \ check_check_fork.c \ check_check_exit.c \ check_check_selective.c \ + check_check_tags.c \ check_check_sub.c \ check_check_master.c check_mem_leaks_LDADD = $(top_builddir)/src/libcheck.la $(top_builddir)/lib/libcompat.la diff --git a/tests/check_check.h b/tests/check_check.h index ba17492..32c23d1 100644 --- a/tests/check_check.h +++ b/tests/check_check.h @@ -55,6 +55,7 @@ Suite *make_fixture_suite(void); Suite *make_pack_suite(void); Suite *make_exit_suite(void); Suite *make_selective_suite(void); +Suite *make_tag_suite(void); extern int master_tests_lineno[]; void init_master_tests_lineno(int num_master_tests); diff --git a/tests/check_check_main.c b/tests/check_check_main.c index 758aaf2..3391493 100644 --- a/tests/check_check_main.c +++ b/tests/check_check_main.c @@ -43,6 +43,7 @@ int main (void) srunner_add_suite(sr, make_fork_suite()); srunner_add_suite(sr, make_fixture_suite()); srunner_add_suite(sr, make_pack_suite()); + srunner_add_suite(sr, make_tag_suite()); #if defined(HAVE_FORK) && HAVE_FORK==1 srunner_add_suite(sr, make_exit_suite()); diff --git a/tests/check_check_tags.c b/tests/check_check_tags.c new file mode 100644 index 0000000..e4f485a --- /dev/null +++ b/tests/check_check_tags.c @@ -0,0 +1,763 @@ +/* + * Check: a unit test framework for C + * Copyright (C) 2001, 2002 Arien Malec + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ +#include "../lib/libcompat.h" + +#include +#include +#include +#include +#include +#include "check_check.h" + +enum { + RED, + BLUE, + PURPLE, + YELLOW, + BLACK, + MAX_TESTS +} test_ids; + +/* + * Flag per test which is set true when that test runs + */ +unsigned int test_executed[MAX_TESTS]; + +static void reset_executed(void) +{ + memset(test_executed, 0, sizeof(test_executed)); +} + +START_TEST(red_test1) +{ + test_executed[RED] = 1; +} +END_TEST + +START_TEST(blue_test1) +{ + test_executed[BLUE] = 1; +} +END_TEST + +START_TEST(purple_test1) +{ + test_executed[PURPLE] = 1; +} +END_TEST + +START_TEST(yellow_test1) +{ + test_executed[YELLOW] = 1; +} +END_TEST + +START_TEST(black_test1) +{ + test_executed[BLACK] = 1; +} +END_TEST + + +static SRunner *make_tagged_testrunner(void) +{ + + SRunner *sr; + Suite *s1, *s2; + + TCase *red, *blue, *purple, *yellow, *black; + + s1 = suite_create("RedBlue Suite"); + + red = tcase_create("Red"); + tcase_set_tags(red, "Red"); + suite_add_tcase (s1, red); + tcase_add_test(red, red_test1); + + blue = tcase_create("Blue"); + tcase_set_tags(blue, "Blue"); + suite_add_tcase (s1, blue); + tcase_add_test(blue, blue_test1); + + s2 = suite_create("PurpleYellowBlack Suite"); + + purple = tcase_create("Purple"); + tcase_set_tags(purple, "Red Blue"); + suite_add_tcase (s2, purple); + tcase_add_test(purple, purple_test1); + + yellow = tcase_create("Yellow"); + tcase_set_tags(yellow, "Yellow"); + suite_add_tcase (s2, yellow); + tcase_add_test(yellow, yellow_test1); + + black = tcase_create("Black"); + suite_add_tcase (s2, black); + tcase_add_test(black, black_test1); + + sr = srunner_create(s1); + srunner_add_suite(sr, s2); + + return sr; +} + +static SRunner *sr = NULL; + +static void tag_test_setup(void) +{ + sr = make_tagged_testrunner(); + srunner_set_fork_status(sr, CK_NOFORK); +} + +static void tag_test_teardown (void) +{ + srunner_free (sr); +} + +/* + * Show that with no filter we run all the tests + */ +START_TEST(null_filter) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, NULL, NULL, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + +} END_TEST + +START_TEST(include_nothing) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, "", NULL, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + +} END_TEST + +START_TEST(exclude_nothing) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, NULL, "", CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + +} END_TEST + +START_TEST(include_nothing_exclude_nothing) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, "", "", CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + +} END_TEST + +START_TEST(include_yellow) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, "Yellow", NULL, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + +} END_TEST + + +START_TEST(include_red) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, "Red", NULL, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + +} END_TEST + +START_TEST(include_red_blue) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, "Red Blue", NULL, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + +} END_TEST + +START_TEST(include_red_blue_yellow) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, "Red Blue Yellow", NULL, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + +} END_TEST + +START_TEST(exclude_yellow) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, NULL, "Yellow", CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + +} END_TEST + +START_TEST(exclude_red) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, NULL, "Red", CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + +} END_TEST + +START_TEST(exclude_red_blue) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, NULL, "Red Blue", CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + +} END_TEST + +START_TEST(exclude_red_blue_yellow) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, NULL, "Red Blue Yellow", CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + +} END_TEST + +#if HAVE_DECL_SETENV + +/* env var driven tests */ + +START_TEST(include_yellow_env) +{ + reset_executed(); + + setenv ("CK_INCLUDE_TAGS", "Yellow", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_INCLUDE_TAGS"); + +} END_TEST + + +START_TEST(include_red_env) +{ + reset_executed(); + + setenv ("CK_INCLUDE_TAGS", "Red", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_INCLUDE_TAGS"); + +} END_TEST + +START_TEST(include_red_blue_env) +{ + reset_executed(); + + setenv ("CK_INCLUDE_TAGS", "Red Blue", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_INCLUDE_TAGS"); + +} END_TEST + +START_TEST(include_red_blue_yellow_env) +{ + reset_executed(); + + setenv ("CK_INCLUDE_TAGS", "Red Blue Yellow", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_INCLUDE_TAGS"); + +} END_TEST + +START_TEST(exclude_yellow_env) +{ + reset_executed(); + + setenv ("CK_EXCLUDE_TAGS", "Yellow", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + + unsetenv ("CK_EXCLUDE_TAGS"); + +} END_TEST + +START_TEST(exclude_red_env) +{ + reset_executed(); + + setenv ("CK_EXCLUDE_TAGS", "Red", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + + unsetenv ("CK_EXCLUDE_TAGS"); + +} END_TEST + +START_TEST(exclude_red_blue_env) +{ + reset_executed(); + + setenv ("CK_EXCLUDE_TAGS", "Red Blue", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + + unsetenv ("CK_EXCLUDE_TAGS"); + +} END_TEST + +START_TEST(exclude_red_blue_yellow_env) +{ + reset_executed(); + + setenv ("CK_EXCLUDE_TAGS", "Red Blue Yellow", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + + unsetenv ("CK_EXCLUDE_TAGS"); + +} END_TEST + +START_TEST(include_red_case_red_env) +{ + reset_executed(); + + setenv ("CK_INCLUDE_TAGS", "Red Yellow", 1); + setenv ("CK_RUN_CASE", "Red", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_INCLUDE_TAGS"); + unsetenv ("CK_RUN_CASE"); + +} END_TEST + +START_TEST(include_red_case_blue_env) +{ + reset_executed(); + + setenv ("CK_INCLUDE_TAGS", "Red Yellow", 1); + setenv ("CK_RUN_CASE", "Blue", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_INCLUDE_TAGS"); + unsetenv ("CK_RUN_CASE"); + +} END_TEST + +START_TEST(exclude_red_case_red_env) +{ + reset_executed(); + + setenv ("CK_EXCLUDE_TAGS", "Red Yellow", 1); + setenv ("CK_RUN_CASE", "Red", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_EXCLUDE_TAGS"); + unsetenv ("CK_RUN_CASE"); + +} END_TEST + +START_TEST(exclude_red_case_blue_env) +{ + reset_executed(); + + setenv ("CK_EXCLUDE_TAGS", "Red Yellow", 1); + setenv ("CK_RUN_CASE", "Blue", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_EXCLUDE_TAGS"); + unsetenv ("CK_RUN_CASE"); + +} END_TEST + +START_TEST(include_red_suite_redblue_env) +{ + reset_executed(); + + setenv ("CK_INCLUDE_TAGS", "Red Yellow", 1); + setenv ("CK_RUN_SUITE", "RedBlue Suite", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_INCLUDE_TAGS"); + unsetenv ("CK_RUN_SUITE"); + +} END_TEST + +START_TEST(include_red_suite_purpleyellowblack_env) +{ + reset_executed(); + + setenv ("CK_INCLUDE_TAGS", "Red Yellow", 1); + setenv ("CK_RUN_SUITE", "PurpleYellowBlack Suite", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_INCLUDE_TAGS"); + unsetenv ("CK_RUN_SUITE"); + +} END_TEST + +START_TEST(exclude_red_suite_redblue_env) +{ + reset_executed(); + + setenv ("CK_EXCLUDE_TAGS", "Red Yellow", 1); + setenv ("CK_RUN_SUITE", "RedBlue Suite", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + + unsetenv ("CK_EXCLUDE_TAGS"); + unsetenv ("CK_RUN_SUITE"); + +} END_TEST + +START_TEST(exclude_red_suite_purpleyellowblack_env) +{ + reset_executed(); + + setenv ("CK_EXCLUDE_TAGS", "Red Yellow", 1); + setenv ("CK_RUN_SUITE", "PurpleYellowBlack Suite", 1); + srunner_run_all(sr, CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(test_executed[BLACK]); + + unsetenv ("CK_EXCLUDE_TAGS"); + unsetenv ("CK_RUN_SUITE"); + +} END_TEST + + +#endif /* HAVE_DECL_SETENV */ + + +START_TEST(include_red_exclude_red) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, "Red", "Red", CK_VERBOSE); + + ck_assert(!test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + +} END_TEST + +START_TEST(include_red_exclude_blue) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, "Red", "Blue", CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(!test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + +} END_TEST + +START_TEST(include_red_include_red) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, "Red Red Red", NULL, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(!test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + +} END_TEST + +START_TEST(include_w_spaces) +{ + reset_executed(); + + srunner_run_tagged(sr, NULL, NULL, " Red Blue ", NULL, CK_VERBOSE); + + ck_assert(test_executed[RED]); + ck_assert(test_executed[BLUE]); + ck_assert(test_executed[PURPLE]); + ck_assert(!test_executed[YELLOW]); + ck_assert(!test_executed[BLACK]); + +} END_TEST + +Suite *make_tag_suite(void) +{ + TCase *set_get_tags, *no_filters; + TCase *include_filters, *exclude_filters; +#if HAVE_DECL_SETENV + TCase *include_filters_env, *exclude_filters_env; +#endif /* HAVE_DECL_SETENV */ + TCase *include_exclude_filters, *strange_filters; + TCase *tag_plus_suite_env, *tag_plus_case_env; + Suite *s; + + s = suite_create("Check Tag Filtering"); + + no_filters = tcase_create("no tag filters"); + suite_add_tcase (s, no_filters); + tcase_add_test(no_filters, null_filter); + tcase_add_test(no_filters, include_nothing); + tcase_add_test(no_filters, exclude_nothing); + tcase_add_unchecked_fixture (no_filters, + tag_test_setup, + tag_test_teardown); + + include_filters = tcase_create("include tags"); + suite_add_tcase (s, include_filters); + tcase_add_test(include_filters, include_yellow); + tcase_add_test(include_filters, include_red); + tcase_add_test(include_filters, include_red_blue); + tcase_add_test(include_filters, include_red_blue_yellow); + tcase_add_unchecked_fixture (include_filters, + tag_test_setup, + tag_test_teardown); + + exclude_filters = tcase_create("exclude tags"); + suite_add_tcase (s, exclude_filters); + tcase_add_test(exclude_filters, exclude_yellow); + tcase_add_test(exclude_filters, exclude_red); + tcase_add_test(exclude_filters, exclude_red_blue); + tcase_add_test(exclude_filters, exclude_red_blue_yellow); + tcase_add_unchecked_fixture (exclude_filters, + tag_test_setup, + tag_test_teardown); + +#if HAVE_DECL_SETENV + + include_filters_env = tcase_create("include tags via env"); + suite_add_tcase (s, include_filters_env); + tcase_add_test(include_filters_env, include_yellow_env); + tcase_add_test(include_filters_env, include_red_env); + tcase_add_test(include_filters_env, include_red_blue_env); + tcase_add_test(include_filters_env, include_red_blue_yellow_env); + tcase_add_unchecked_fixture (include_filters_env, + tag_test_setup, + tag_test_teardown); + + exclude_filters_env = tcase_create("exclude tags via env"); + suite_add_tcase (s, exclude_filters_env); + tcase_add_test(exclude_filters_env, exclude_yellow_env); + tcase_add_test(exclude_filters_env, exclude_red_env); + tcase_add_test(exclude_filters_env, exclude_red_blue_env); + tcase_add_test(exclude_filters_env, exclude_red_blue_yellow_env); + tcase_add_unchecked_fixture (exclude_filters_env, + tag_test_setup, + tag_test_teardown); + + tag_plus_suite_env = tcase_create("combining tag filters with suite selection"); + suite_add_tcase (s, tag_plus_suite_env); + tcase_add_test(tag_plus_suite_env, include_red_suite_redblue_env); + tcase_add_test(tag_plus_suite_env, include_red_suite_purpleyellowblack_env); + tcase_add_test(tag_plus_suite_env, exclude_red_suite_redblue_env); + tcase_add_test(tag_plus_suite_env, exclude_red_suite_purpleyellowblack_env); + tcase_add_unchecked_fixture (tag_plus_suite_env, + tag_test_setup, + tag_test_teardown); + + tag_plus_case_env = tcase_create("combining tag filters with case selection"); + suite_add_tcase (s, tag_plus_case_env); + tcase_add_test(tag_plus_case_env, include_red_case_red_env); + tcase_add_test(tag_plus_case_env, include_red_case_blue_env); + tcase_add_test(tag_plus_case_env, exclude_red_case_red_env); + tcase_add_test(tag_plus_case_env, exclude_red_case_blue_env); + tcase_add_unchecked_fixture (tag_plus_case_env, + tag_test_setup, + tag_test_teardown); + +#endif /* HAVE_DECL_SETENV */ + + include_exclude_filters = tcase_create("include and exclude tags"); + suite_add_tcase (s, include_exclude_filters); + tcase_add_test(include_exclude_filters, include_nothing_exclude_nothing); + tcase_add_test(include_exclude_filters, include_red_exclude_blue); + tcase_add_test(include_exclude_filters, include_red_exclude_red); + tcase_add_unchecked_fixture (include_exclude_filters, + tag_test_setup, + tag_test_teardown); + + strange_filters = tcase_create("strange tag filters"); + suite_add_tcase (s, strange_filters); + tcase_add_test(strange_filters, include_red_include_red); + tcase_add_test(strange_filters, include_w_spaces); + tcase_add_unchecked_fixture (strange_filters, + tag_test_setup, + tag_test_teardown); + + return s; +} diff --git a/tests/check_mem_leaks.c b/tests/check_mem_leaks.c index 72f6647..4528fb0 100644 --- a/tests/check_mem_leaks.c +++ b/tests/check_mem_leaks.c @@ -67,7 +67,7 @@ int main () #if defined(HAVE_FORK) && HAVE_FORK==1 srunner_add_suite(sr, make_exit_suite()); #endif - + srunner_add_suite(sr, make_tag_suite()); srunner_add_suite(sr, make_selective_suite()); /* -- 2.40.0