Add a facility to skip running some test suites
With the build option SKIP_TEST_SUITES=..., the specified test suites
are built, but skipped when running tests. Usage:
make check SKIP_TEST_SUITES=timing,gcm
or
cmake -D SKIP_TEST_SUITES=timing,gcm ...
The list can be separated by any of space, comma or semicolon, and each
element can be a regular expression in ERE syntax except that "." stands
for itself. Skipping "foo" skips not only "foo" itself but also
any "foo.bar", but does not skip "foobar".
diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl
index 7ffbd74..1c9dc1d 100755
--- a/tests/scripts/run-test-suites.pl
+++ b/tests/scripts/run-test-suites.pl
@@ -10,13 +10,16 @@
Execute all the test suites and print a summary of the results.
- run-test-suites.pl [[-v|--verbose] [VERBOSITY]]
+ run-test-suites.pl [[-v|--verbose] [VERBOSITY]] [--skip=SUITE[...]]
Options:
-v|--verbose Print detailed failure information.
-v 2|--verbose=2 Print detailed failure information and summary messages.
-v 3|--verbose=3 Print detailed information about every test case.
+ --skip=SUITE[,SUITE...]
+ Skip the specified SUITE(s). This option can be used
+ multiple times.
=cut
@@ -26,11 +29,13 @@
use utf8;
use open qw(:std utf8);
-use Getopt::Long qw(:config auto_help);
+use Getopt::Long qw(:config auto_help gnu_compat);
use Pod::Usage;
my $verbose = 0;
+my @skip_patterns = ();
GetOptions(
+ 'skip=s' => \@skip_patterns,
'verbose|v:1' => \$verbose,
) or die;
@@ -41,6 +46,17 @@
@suites = grep { !/\.c$/ && !/\.data$/ && -f } @suites;
die "$0: no test suite found\n" unless @suites;
+# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
+# but not "test_suite_foobar".
+my $skip_re =
+ ( '\Atest_suite_(' .
+ join('|', map {
+ s/[ ,;]/|/g; # allow any of " ,;|" as separators
+ s/\./\./g; # "." in the input means ".", not "any character"
+ $_
+ } @skip_patterns) .
+ ')(\z|\.)' );
+
# in case test suites are linked dynamically
$ENV{'LD_LIBRARY_PATH'} = '../library';
$ENV{'DYLD_LIBRARY_PATH'} = '../library';
@@ -50,6 +66,7 @@
my ($failed_suites, $total_tests_run, $failed, $suite_cases_passed,
$suite_cases_failed, $suite_cases_skipped, $total_cases_passed,
$total_cases_failed, $total_cases_skipped );
+my $suites_skipped = 0;
sub pad_print_center {
my( $width, $padchar, $string ) = @_;
@@ -60,6 +77,12 @@
for my $suite (@suites)
{
print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " ";
+ if( $suite =~ /$skip_re/o ) {
+ print "SKIP\n";
+ ++$suites_skipped;
+ next;
+ }
+
my $command = "$prefix$suite";
if( $verbose ) {
$command .= ' -v';
@@ -106,7 +129,10 @@
print "-" x 72, "\n";
print $failed_suites ? "FAILED" : "PASSED";
-printf " (%d suites, %d tests run)\n", scalar @suites, $total_tests_run;
+printf( " (%d suites, %d tests run%s)\n",
+ scalar(@suites) - $suites_skipped,
+ $total_tests_run,
+ $suites_skipped ? ", $suites_skipped suites skipped" : "" );
if( $verbose > 1 ) {
print " test cases passed :", $total_cases_passed, "\n";
@@ -116,8 +142,11 @@
"\n";
print " of available tests :",
( $total_cases_passed + $total_cases_failed + $total_cases_skipped ),
- "\n"
+ "\n";
+ if( $suites_skipped != 0 ) {
+ print "Note: $suites_skipped suites were skipped.\n";
}
+}
exit( $failed_suites ? 1 : 0 );