blob: 22eadd180543b9f09877b7e88d4a4ebc2cdb2d4e [file] [log] [blame]
fbrosson533407a2018-04-04 21:44:29 +00001#!/usr/bin/env perl
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +01002
SimonBad8fbc02016-03-11 17:33:39 +00003# run-test-suites.pl
4#
Bence Szépkúti1e148272020-08-07 13:07:28 +02005# Copyright The Mbed TLS Contributors
Bence Szépkútic7da1fe2020-05-26 01:54:15 +02006# SPDX-License-Identifier: Apache-2.0
7#
8# Licensed under the Apache License, Version 2.0 (the "License"); you may
9# not use this file except in compliance with the License.
10# You may obtain a copy of the License at
11#
12# http://www.apache.org/licenses/LICENSE-2.0
13#
14# Unless required by applicable law or agreed to in writing, software
15# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17# See the License for the specific language governing permissions and
18# limitations under the License.
Gilles Peskine15db8502018-12-14 18:23:13 +010019
20=head1 SYNOPSIS
21
22Execute all the test suites and print a summary of the results.
23
Gilles Peskineac372cc2018-11-29 10:15:06 +000024 run-test-suites.pl [[-v|--verbose] [VERBOSITY]] [--skip=SUITE[...]]
Gilles Peskine15db8502018-12-14 18:23:13 +010025
26Options:
27
28 -v|--verbose Print detailed failure information.
29 -v 2|--verbose=2 Print detailed failure information and summary messages.
30 -v 3|--verbose=3 Print detailed information about every test case.
Gilles Peskineac372cc2018-11-29 10:15:06 +000031 --skip=SUITE[,SUITE...]
32 Skip the specified SUITE(s). This option can be used
33 multiple times.
Gilles Peskine15db8502018-12-14 18:23:13 +010034
35=cut
SimonBad8fbc02016-03-11 17:33:39 +000036
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +010037use warnings;
38use strict;
39
40use utf8;
41use open qw(:std utf8);
42
Gilles Peskineac372cc2018-11-29 10:15:06 +000043use Getopt::Long qw(:config auto_help gnu_compat);
Gilles Peskine15db8502018-12-14 18:23:13 +010044use Pod::Usage;
SimonBad8fbc02016-03-11 17:33:39 +000045
Jaeden Amero8396a712018-10-05 13:23:35 +010046my $verbose = 0;
Gilles Peskineac372cc2018-11-29 10:15:06 +000047my @skip_patterns = ();
Gilles Peskine15db8502018-12-14 18:23:13 +010048GetOptions(
Gilles Peskineac372cc2018-11-29 10:15:06 +000049 'skip=s' => \@skip_patterns,
Gilles Peskine15db8502018-12-14 18:23:13 +010050 'verbose|v:1' => \$verbose,
51 ) or die;
SimonBad8fbc02016-03-11 17:33:39 +000052
Gilles Peskine071db412017-05-03 16:26:47 +020053# All test suites = executable files, excluding source files, debug
54# and profiling information, etc. We can't just grep {! /\./} because
Simon Butcher597dbf82018-06-27 16:16:39 +010055# some of our test cases' base names contain a dot.
Gilles Peskine071db412017-05-03 16:26:47 +020056my @suites = grep { -x $_ || /\.exe$/ } glob 'test_suite_*';
Simon Butcher6e3606e2018-09-30 21:53:16 +010057@suites = grep { !/\.c$/ && !/\.data$/ && -f } @suites;
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +010058die "$0: no test suite found\n" unless @suites;
59
Gilles Peskineac372cc2018-11-29 10:15:06 +000060# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
61# but not "test_suite_foobar".
62my $skip_re =
63 ( '\Atest_suite_(' .
64 join('|', map {
65 s/[ ,;]/|/g; # allow any of " ,;|" as separators
66 s/\./\./g; # "." in the input means ".", not "any character"
67 $_
68 } @skip_patterns) .
69 ')(\z|\.)' );
70
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +010071# in case test suites are linked dynamically
72$ENV{'LD_LIBRARY_PATH'} = '../library';
Andres Amaya Garcia79db9332018-03-27 19:57:58 +010073$ENV{'DYLD_LIBRARY_PATH'} = '../library';
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +010074
75my $prefix = $^O eq "MSWin32" ? '' : './';
76
Manuel Pégourié-Gonnardac6c6702022-10-11 10:48:32 +020077my (@failed_suites, $total_tests_run, $failed, $suite_cases_passed,
SimonBad8fbc02016-03-11 17:33:39 +000078 $suite_cases_failed, $suite_cases_skipped, $total_cases_passed,
79 $total_cases_failed, $total_cases_skipped );
Gilles Peskineac372cc2018-11-29 10:15:06 +000080my $suites_skipped = 0;
SimonBad8fbc02016-03-11 17:33:39 +000081
Jaeden Amero79e4f4e2018-10-05 12:21:15 +010082sub pad_print_center {
83 my( $width, $padchar, $string ) = @_;
84 my $padlen = ( $width - length( $string ) - 2 ) / 2;
85 print $padchar x( $padlen ), " $string ", $padchar x( $padlen ), "\n";
86}
87
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +010088for my $suite (@suites)
89{
90 print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " ";
Gilles Peskineac372cc2018-11-29 10:15:06 +000091 if( $suite =~ /$skip_re/o ) {
92 print "SKIP\n";
93 ++$suites_skipped;
94 next;
95 }
96
Jaeden Amero79e4f4e2018-10-05 12:21:15 +010097 my $command = "$prefix$suite";
98 if( $verbose ) {
99 $command .= ' -v';
100 }
101 my $result = `$command`;
SimonBad8fbc02016-03-11 17:33:39 +0000102
103 $suite_cases_passed = () = $result =~ /.. PASS/g;
104 $suite_cases_failed = () = $result =~ /.. FAILED/g;
105 $suite_cases_skipped = () = $result =~ /.. ----/g;
106
Gilles Peskine8b5389f2019-10-21 19:08:07 +0200107 if( $? == 0 ) {
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +0100108 print "PASS\n";
Jaeden Amero8396a712018-10-05 13:23:35 +0100109 if( $verbose > 2 ) {
110 pad_print_center( 72, '-', "Begin $suite" );
111 print $result;
112 pad_print_center( 72, '-', "End $suite" );
113 }
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +0100114 } else {
Manuel Pégourié-Gonnardac6c6702022-10-11 10:48:32 +0200115 push @failed_suites, $suite;
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +0100116 print "FAIL\n";
Jaeden Amero79e4f4e2018-10-05 12:21:15 +0100117 if( $verbose ) {
118 pad_print_center( 72, '-', "Begin $suite" );
119 print $result;
120 pad_print_center( 72, '-', "End $suite" );
121 }
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +0100122 }
SimonBad8fbc02016-03-11 17:33:39 +0000123
124 my ($passed, $tests, $skipped) = $result =~ /([0-9]*) \/ ([0-9]*) tests.*?([0-9]*) skipped/;
125 $total_tests_run += $tests - $skipped;
126
Jaeden Amero8396a712018-10-05 13:23:35 +0100127 if( $verbose > 1 ) {
SimonBad8fbc02016-03-11 17:33:39 +0000128 print "(test cases passed:", $suite_cases_passed,
129 " failed:", $suite_cases_failed,
130 " skipped:", $suite_cases_skipped,
SimonB8ca7bc42016-04-17 23:24:50 +0100131 " of total:", ($suite_cases_passed + $suite_cases_failed +
132 $suite_cases_skipped),
SimonBad8fbc02016-03-11 17:33:39 +0000133 ")\n"
134 }
135
136 $total_cases_passed += $suite_cases_passed;
137 $total_cases_failed += $suite_cases_failed;
138 $total_cases_skipped += $suite_cases_skipped;
Manuel Pégourié-Gonnard85113842015-07-08 21:20:03 +0100139}
140
141print "-" x 72, "\n";
Manuel Pégourié-Gonnardac6c6702022-10-11 10:48:32 +0200142print @failed_suites ? "FAILED" : "PASSED";
Gilles Peskineac372cc2018-11-29 10:15:06 +0000143printf( " (%d suites, %d tests run%s)\n",
144 scalar(@suites) - $suites_skipped,
145 $total_tests_run,
146 $suites_skipped ? ", $suites_skipped suites skipped" : "" );
SimonBad8fbc02016-03-11 17:33:39 +0000147
Manuel Pégourié-Gonnardac6c6702022-10-11 10:48:32 +0200148if( $verbose && @failed_suites ) {
149 # the output can be very long, so provide a summary of which suites failed
150 print " failed suites : @failed_suites\n";
151}
152
Jaeden Amero8396a712018-10-05 13:23:35 +0100153if( $verbose > 1 ) {
SimonBad8fbc02016-03-11 17:33:39 +0000154 print " test cases passed :", $total_cases_passed, "\n";
155 print " failed :", $total_cases_failed, "\n";
156 print " skipped :", $total_cases_skipped, "\n";
157 print " of tests executed :", ( $total_cases_passed + $total_cases_failed ),
158 "\n";
159 print " of available tests :",
160 ( $total_cases_passed + $total_cases_failed + $total_cases_skipped ),
Gilles Peskineac372cc2018-11-29 10:15:06 +0000161 "\n";
162 if( $suites_skipped != 0 ) {
163 print "Note: $suites_skipped suites were skipped.\n";
SimonBad8fbc02016-03-11 17:33:39 +0000164 }
Gilles Peskineac372cc2018-11-29 10:15:06 +0000165}
SimonBad8fbc02016-03-11 17:33:39 +0000166
Manuel Pégourié-Gonnardac6c6702022-10-11 10:48:32 +0200167exit( @failed_suites ? 1 : 0 );
SimonBad8fbc02016-03-11 17:33:39 +0000168