fbrosson | 533407a | 2018-04-04 21:44:29 +0000 | [diff] [blame] | 1 | #!/usr/bin/env perl |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 2 | |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 3 | # run-test-suites.pl |
| 4 | # |
Bence Szépkúti | 1e14827 | 2020-08-07 13:07:28 +0200 | [diff] [blame] | 5 | # Copyright The Mbed TLS Contributors |
Bence Szépkúti | c7da1fe | 2020-05-26 01:54:15 +0200 | [diff] [blame] | 6 | # SPDX-License-Identifier: Apache-2.0 |
| 7 | # |
| 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 9 | # not use this file except in compliance with the License. |
| 10 | # You may obtain a copy of the License at |
| 11 | # |
| 12 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 13 | # |
| 14 | # Unless required by applicable law or agreed to in writing, software |
| 15 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 16 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 17 | # See the License for the specific language governing permissions and |
| 18 | # limitations under the License. |
Gilles Peskine | 15db850 | 2018-12-14 18:23:13 +0100 | [diff] [blame] | 19 | |
| 20 | =head1 SYNOPSIS |
| 21 | |
| 22 | Execute all the test suites and print a summary of the results. |
| 23 | |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 24 | run-test-suites.pl [[-v|--verbose] [VERBOSITY]] [--skip=SUITE[...]] |
Gilles Peskine | 15db850 | 2018-12-14 18:23:13 +0100 | [diff] [blame] | 25 | |
| 26 | Options: |
| 27 | |
| 28 | -v|--verbose Print detailed failure information. |
| 29 | -v 2|--verbose=2 Print detailed failure information and summary messages. |
| 30 | -v 3|--verbose=3 Print detailed information about every test case. |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 31 | --skip=SUITE[,SUITE...] |
| 32 | Skip the specified SUITE(s). This option can be used |
| 33 | multiple times. |
Gilles Peskine | 15db850 | 2018-12-14 18:23:13 +0100 | [diff] [blame] | 34 | |
| 35 | =cut |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 36 | |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 37 | use warnings; |
| 38 | use strict; |
| 39 | |
| 40 | use utf8; |
| 41 | use open qw(:std utf8); |
| 42 | |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 43 | use Getopt::Long qw(:config auto_help gnu_compat); |
Gilles Peskine | 15db850 | 2018-12-14 18:23:13 +0100 | [diff] [blame] | 44 | use Pod::Usage; |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 45 | |
Jaeden Amero | 8396a71 | 2018-10-05 13:23:35 +0100 | [diff] [blame] | 46 | my $verbose = 0; |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 47 | my @skip_patterns = (); |
Gilles Peskine | 15db850 | 2018-12-14 18:23:13 +0100 | [diff] [blame] | 48 | GetOptions( |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 49 | 'skip=s' => \@skip_patterns, |
Gilles Peskine | 15db850 | 2018-12-14 18:23:13 +0100 | [diff] [blame] | 50 | 'verbose|v:1' => \$verbose, |
| 51 | ) or die; |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 52 | |
Gilles Peskine | 071db41 | 2017-05-03 16:26:47 +0200 | [diff] [blame] | 53 | # All test suites = executable files, excluding source files, debug |
| 54 | # and profiling information, etc. We can't just grep {! /\./} because |
Simon Butcher | 597dbf8 | 2018-06-27 16:16:39 +0100 | [diff] [blame] | 55 | # some of our test cases' base names contain a dot. |
Gilles Peskine | 071db41 | 2017-05-03 16:26:47 +0200 | [diff] [blame] | 56 | my @suites = grep { -x $_ || /\.exe$/ } glob 'test_suite_*'; |
Simon Butcher | 6e3606e | 2018-09-30 21:53:16 +0100 | [diff] [blame] | 57 | @suites = grep { !/\.c$/ && !/\.data$/ && -f } @suites; |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 58 | die "$0: no test suite found\n" unless @suites; |
| 59 | |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 60 | # "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar" |
| 61 | # but not "test_suite_foobar". |
| 62 | my $skip_re = |
| 63 | ( '\Atest_suite_(' . |
| 64 | join('|', map { |
| 65 | s/[ ,;]/|/g; # allow any of " ,;|" as separators |
| 66 | s/\./\./g; # "." in the input means ".", not "any character" |
| 67 | $_ |
| 68 | } @skip_patterns) . |
| 69 | ')(\z|\.)' ); |
| 70 | |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 71 | # in case test suites are linked dynamically |
| 72 | $ENV{'LD_LIBRARY_PATH'} = '../library'; |
Andres Amaya Garcia | 79db933 | 2018-03-27 19:57:58 +0100 | [diff] [blame] | 73 | $ENV{'DYLD_LIBRARY_PATH'} = '../library'; |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 74 | |
| 75 | my $prefix = $^O eq "MSWin32" ? '' : './'; |
| 76 | |
Manuel Pégourié-Gonnard | ac6c670 | 2022-10-11 10:48:32 +0200 | [diff] [blame] | 77 | my (@failed_suites, $total_tests_run, $failed, $suite_cases_passed, |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 78 | $suite_cases_failed, $suite_cases_skipped, $total_cases_passed, |
| 79 | $total_cases_failed, $total_cases_skipped ); |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 80 | my $suites_skipped = 0; |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 81 | |
Jaeden Amero | 79e4f4e | 2018-10-05 12:21:15 +0100 | [diff] [blame] | 82 | sub pad_print_center { |
| 83 | my( $width, $padchar, $string ) = @_; |
| 84 | my $padlen = ( $width - length( $string ) - 2 ) / 2; |
| 85 | print $padchar x( $padlen ), " $string ", $padchar x( $padlen ), "\n"; |
| 86 | } |
| 87 | |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 88 | for my $suite (@suites) |
| 89 | { |
| 90 | print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " "; |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 91 | if( $suite =~ /$skip_re/o ) { |
| 92 | print "SKIP\n"; |
| 93 | ++$suites_skipped; |
| 94 | next; |
| 95 | } |
| 96 | |
Jaeden Amero | 79e4f4e | 2018-10-05 12:21:15 +0100 | [diff] [blame] | 97 | my $command = "$prefix$suite"; |
| 98 | if( $verbose ) { |
| 99 | $command .= ' -v'; |
| 100 | } |
| 101 | my $result = `$command`; |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 102 | |
| 103 | $suite_cases_passed = () = $result =~ /.. PASS/g; |
| 104 | $suite_cases_failed = () = $result =~ /.. FAILED/g; |
| 105 | $suite_cases_skipped = () = $result =~ /.. ----/g; |
| 106 | |
Gilles Peskine | 8b5389f | 2019-10-21 19:08:07 +0200 | [diff] [blame] | 107 | if( $? == 0 ) { |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 108 | print "PASS\n"; |
Jaeden Amero | 8396a71 | 2018-10-05 13:23:35 +0100 | [diff] [blame] | 109 | if( $verbose > 2 ) { |
| 110 | pad_print_center( 72, '-', "Begin $suite" ); |
| 111 | print $result; |
| 112 | pad_print_center( 72, '-', "End $suite" ); |
| 113 | } |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 114 | } else { |
Manuel Pégourié-Gonnard | ac6c670 | 2022-10-11 10:48:32 +0200 | [diff] [blame] | 115 | push @failed_suites, $suite; |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 116 | print "FAIL\n"; |
Jaeden Amero | 79e4f4e | 2018-10-05 12:21:15 +0100 | [diff] [blame] | 117 | if( $verbose ) { |
| 118 | pad_print_center( 72, '-', "Begin $suite" ); |
| 119 | print $result; |
| 120 | pad_print_center( 72, '-', "End $suite" ); |
| 121 | } |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 122 | } |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 123 | |
| 124 | my ($passed, $tests, $skipped) = $result =~ /([0-9]*) \/ ([0-9]*) tests.*?([0-9]*) skipped/; |
| 125 | $total_tests_run += $tests - $skipped; |
| 126 | |
Jaeden Amero | 8396a71 | 2018-10-05 13:23:35 +0100 | [diff] [blame] | 127 | if( $verbose > 1 ) { |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 128 | print "(test cases passed:", $suite_cases_passed, |
| 129 | " failed:", $suite_cases_failed, |
| 130 | " skipped:", $suite_cases_skipped, |
SimonB | 8ca7bc4 | 2016-04-17 23:24:50 +0100 | [diff] [blame] | 131 | " of total:", ($suite_cases_passed + $suite_cases_failed + |
| 132 | $suite_cases_skipped), |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 133 | ")\n" |
| 134 | } |
| 135 | |
| 136 | $total_cases_passed += $suite_cases_passed; |
| 137 | $total_cases_failed += $suite_cases_failed; |
| 138 | $total_cases_skipped += $suite_cases_skipped; |
Manuel Pégourié-Gonnard | 8511384 | 2015-07-08 21:20:03 +0100 | [diff] [blame] | 139 | } |
| 140 | |
| 141 | print "-" x 72, "\n"; |
Manuel Pégourié-Gonnard | ac6c670 | 2022-10-11 10:48:32 +0200 | [diff] [blame] | 142 | print @failed_suites ? "FAILED" : "PASSED"; |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 143 | printf( " (%d suites, %d tests run%s)\n", |
| 144 | scalar(@suites) - $suites_skipped, |
| 145 | $total_tests_run, |
| 146 | $suites_skipped ? ", $suites_skipped suites skipped" : "" ); |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 147 | |
Manuel Pégourié-Gonnard | ac6c670 | 2022-10-11 10:48:32 +0200 | [diff] [blame] | 148 | if( $verbose && @failed_suites ) { |
| 149 | # the output can be very long, so provide a summary of which suites failed |
| 150 | print " failed suites : @failed_suites\n"; |
| 151 | } |
| 152 | |
Jaeden Amero | 8396a71 | 2018-10-05 13:23:35 +0100 | [diff] [blame] | 153 | if( $verbose > 1 ) { |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 154 | print " test cases passed :", $total_cases_passed, "\n"; |
| 155 | print " failed :", $total_cases_failed, "\n"; |
| 156 | print " skipped :", $total_cases_skipped, "\n"; |
| 157 | print " of tests executed :", ( $total_cases_passed + $total_cases_failed ), |
| 158 | "\n"; |
| 159 | print " of available tests :", |
| 160 | ( $total_cases_passed + $total_cases_failed + $total_cases_skipped ), |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 161 | "\n"; |
| 162 | if( $suites_skipped != 0 ) { |
| 163 | print "Note: $suites_skipped suites were skipped.\n"; |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 164 | } |
Gilles Peskine | ac372cc | 2018-11-29 10:15:06 +0000 | [diff] [blame] | 165 | } |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 166 | |
Manuel Pégourié-Gonnard | ac6c670 | 2022-10-11 10:48:32 +0200 | [diff] [blame] | 167 | exit( @failed_suites ? 1 : 0 ); |
SimonB | ad8fbc0 | 2016-03-11 17:33:39 +0000 | [diff] [blame] | 168 | |