Adds verbose mode to the test suites
Added a verbose option to the generated test suites which can list the
dependencies not met for skipped test cases.
Also clarifies internal interfaces between the main_test.function and test code,
and fixed a bug on calculating available tests in run-test-suites.pl.
diff --git a/tests/scripts/generate_code.pl b/tests/scripts/generate_code.pl
index 5c623f8..5892f7b 100755
--- a/tests/scripts/generate_code.pl
+++ b/tests/scripts/generate_code.pl
@@ -2,6 +2,8 @@
# generate_code.pl
#
+# This file is part of mbed TLS (https://tls.mbed.org)
+#
# Copyright (c) 2009-2016, ARM Limited, All Rights Reserved
#
# Purpose
@@ -202,7 +204,7 @@
if( substr($def, 0, 4) eq "int " )
{
$param_defs .= " int param$i;\n";
- $param_checks .= " if( verify_int( params[$i], ¶m$i ) != 0 ) return( 2 );\n";
+ $param_checks .= " if( verify_int( params[$i], ¶m$i ) != 0 ) return( DISPATCH_INVALID_TEST_DATA );\n";
push @dispatch_params, "param$i";
$mapping_regex .= ":([\\d\\w |\\+\\-\\(\\)]+)";
@@ -211,7 +213,7 @@
elsif( substr($def, 0, 6) eq "char *" )
{
$param_defs .= " char *param$i = params[$i];\n";
- $param_checks .= " if( verify_string( ¶m$i ) != 0 ) return( 2 );\n";
+ $param_checks .= " if( verify_string( ¶m$i ) != 0 ) return( DISPATCH_INVALID_TEST_DATA );\n";
push @dispatch_params, "param$i";
$mapping_regex .= ":[^:\n]+";
}
@@ -248,14 +250,14 @@
if( cnt != $param_count )
{
mbedtls_fprintf( stderr, "\\nIncorrect argument count (%d != %d)\\n", cnt, $param_count );
- return( 2 );
+ return( DISPATCH_INVALID_TEST_DATA );
}
$param_checks
test_suite_$function_name( $call_params );
- return ( 0 );
+ return ( DISPATCH_TEST_SUCCESS );
$function_post_code
- return ( 3 );
+ return ( DISPATCH_UNSUPPORTED_SUITE );
}
else
END
@@ -283,9 +285,9 @@
if( strcmp( str, "$key" ) == 0 )
{
#if defined($key)
- return( 0 );
+ return( DEPENDENCY_SUPPORTED );
#else
- return( 1 );
+ return( DEPENDENCY_NOT_SUPPORTED );
#endif
}
END
@@ -298,7 +300,7 @@
if( strcmp( str, "$key" ) == 0 )
{
*value = ( $key );
- return( 0 );
+ return( KEY_VALUE_MAPPING_FOUND );
}
END
@@ -315,7 +317,7 @@
$dispatch_code =~ s/^(.+)/ $1/mg;
-$test_main =~ s/TEST_FILENAME/$test_case_data/;
+$test_main =~ s/TEST_FILENAME/$test_case_data/g;
$test_main =~ s/FUNCTION_CODE//;
$test_main =~ s/DEP_CHECK_CODE/$dep_check_code/;
$test_main =~ s/DISPATCH_FUNCTION/$dispatch_code/;
diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl
index fb77e15..58f827c 100755
--- a/tests/scripts/run-test-suites.pl
+++ b/tests/scripts/run-test-suites.pl
@@ -2,6 +2,8 @@
# run-test-suites.pl
#
+# This file is part of mbed TLS (https://tls.mbed.org)
+#
# Copyright (c) 2015-2016, ARM Limited, All Rights Reserved
#
# Purpose
@@ -66,7 +68,8 @@
print "(test cases passed:", $suite_cases_passed,
" failed:", $suite_cases_failed,
" skipped:", $suite_cases_skipped,
- " of total:", ( $suite_cases_passed + $suite_cases_failed ),
+ " of total:", ($suite_cases_passed + $suite_cases_failed +
+ $suite_cases_skipped),
")\n"
}
diff --git a/tests/suites/helpers.function b/tests/suites/helpers.function
index c18eed8..2eff043 100644
--- a/tests/suites/helpers.function
+++ b/tests/suites/helpers.function
@@ -32,9 +32,18 @@
/*----------------------------------------------------------------------------*/
-/* Global variables */
+/* Constants */
-static int test_errors = 0;
+#define DEPENDENCY_SUPPORTED 0
+#define DEPENDENCY_NOT_SUPPORTED 1
+
+#define KEY_VALUE_MAPPING_FOUND 0
+#define KEY_VALUE_MAPPING_NOT_FOUND -1
+
+#define DISPATCH_TEST_SUCCESS 0
+#define DISPATCH_TEST_FN_NOT_FOUND 1
+#define DISPATCH_INVALID_TEST_DATA 2
+#define DISPATCH_UNSUPPORTED_SUITE 3
/*----------------------------------------------------------------------------*/
@@ -81,6 +90,12 @@
/*----------------------------------------------------------------------------*/
+/* Global variables */
+
+static int test_errors = 0;
+
+
+/*----------------------------------------------------------------------------*/
/* Helper Functions */
static int unhexify( unsigned char *obuf, const char *ibuf )
diff --git a/tests/suites/main_test.function b/tests/suites/main_test.function
index 7ec69b4..525df5b 100644
--- a/tests/suites/main_test.function
+++ b/tests/suites/main_test.function
@@ -60,7 +60,7 @@
MAPPING_CODE
mbedtls_printf( "Expected integer for parameter and got: %s\n", str );
- return( -1 );
+ return( KEY_VALUE_MAPPING_NOT_FOUND );
}
@@ -81,7 +81,7 @@
DEP_CHECK_CODE
- return( 1 );
+ return( DEPENDENCY_NOT_SUPPORTED );
}
int dispatch_test(int cnt, char *params[50])
@@ -91,14 +91,18 @@
((void) params);
#if defined(TEST_SUITE_ACTIVE)
+ ret = DISPATCH_TEST_SUCCESS;
+
DISPATCH_FUNCTION
{
- mbedtls_fprintf( stdout, "FAILED\nSkipping unknown test function '%s'\n", params[0] );
+ mbedtls_fprintf( stdout,
+ "FAILED\nSkipping unknown test function '%s'\n",
+ params[0] );
fflush( stdout );
- return( 1 );
+ ret = DISPATCH_TEST_FN_NOT_FOUND;
}
#else
- return( 3 );
+ ret = DISPATCH_UNSUPPORTED_SUITE;
#endif
return( ret );
}
@@ -107,6 +111,19 @@
/*----------------------------------------------------------------------------*/
/* Main Test code */
+#define USAGE \
+ "Usage: %s [OPTIONS] files...\n\n" \
+ " Command line arguments:\n" \
+ " files... One or more test data file. If no file is specified\n" \
+ " the followimg default test case is used:\n" \
+ " %s\n\n" \
+ " Options:\n" \
+ " -v | --verbose Display full information about each test\n" \
+ " -h | --help Display this information\n\n", \
+ argv[0], \
+ "TEST_FILENAME"
+
+
int get_line( FILE *f, char *buf, size_t len )
{
char *ret;
@@ -216,11 +233,18 @@
int main(int argc, const char *argv[])
{
- int testfile_index, testfile_count, ret, i, cnt;
- int total_errors = 0, total_tests = 0, total_skipped = 0;
+ /* Local Configurations and options */
const char *default_filename = "TEST_FILENAME";
const char *test_filename = NULL;
const char **test_files = NULL;
+ int testfile_count = 0;
+ int option_verbose = 0;
+
+ /* Other Local variables */
+ int arg_index = 1;
+ const char *next_arg;
+ int testfile_index, ret, i, cnt;
+ int total_errors = 0, total_tests = 0, total_skipped = 0;
FILE *file;
char buf[5000];
char *params[50];
@@ -253,17 +277,41 @@
return( 0 );
}
- if ( argc <= 1 )
+ while( arg_index < argc)
+ {
+ next_arg = argv[ arg_index ];
+
+ if( strcmp(next_arg, "--verbose" ) == 0 ||
+ strcmp(next_arg, "-v" ) == 0 )
+ {
+ option_verbose = 1;
+ }
+ else if( strcmp(next_arg, "--help" ) == 0 ||
+ strcmp(next_arg, "-h" ) == 0 )
+ {
+ mbedtls_fprintf( stdout, USAGE );
+ mbedtls_exit( EXIT_SUCCESS );
+ }
+ else
+ {
+ /* Not an option, therefore treat all further arguments as the file
+ * list.
+ */
+ test_files = &argv[ arg_index ];
+ testfile_count = argc - arg_index;
+ }
+
+ arg_index++;
+ }
+
+ /* If no files were specified, assume a default */
+ if ( test_files == NULL || testfile_count == 0 )
{
test_files = &default_filename;
testfile_count = 1;
}
- else
- {
- test_files = &argv[1];
- testfile_count = argc - 1;
- }
+ /* Now begin to execute the tests in the testfiles */
for ( testfile_index = 0;
testfile_index < testfile_count;
testfile_index++ )
@@ -280,7 +328,8 @@
while( !feof( file ) )
{
- int skip = 0;
+ int unmet_dep_count = 0;
+ char *unmet_dependencies[20];
if( ( ret = get_line( file, buf, sizeof(buf) ) ) != 0 )
break;
@@ -300,32 +349,61 @@
if( strcmp( params[0], "depends_on" ) == 0 )
{
for( i = 1; i < cnt; i++ )
- if( dep_check( params[i] ) != 0 )
- skip = 1;
+ {
+ if( dep_check( params[i] ) != DEPENDENCY_SUPPORTED )
+ {
+ unmet_dependencies[ i-1 ] = strdup(params[i]);
+ if( unmet_dependencies[ i-1 ] == NULL )
+ {
+ mbedtls_printf("FATAL: Out of memory\n");
+ mbedtls_exit( MBEDTLS_PLATFORM_STD_EXIT_FAILURE );
+ }
+ unmet_dep_count++;
+ }
+ }
if( ( ret = get_line( file, buf, sizeof(buf) ) ) != 0 )
break;
cnt = parse_arguments( buf, strlen(buf), params );
}
-
- if( skip == 0 )
+
+ // If there are no unmet dependencies execute the test
+ if( unmet_dep_count == 0 )
{
test_errors = 0;
ret = dispatch_test( cnt, params );
}
- if( skip == 1 || ret == 3 )
+ if( unmet_dep_count > 0 || ret == DISPATCH_UNSUPPORTED_SUITE )
{
total_skipped++;
mbedtls_fprintf( stdout, "----\n" );
+
+ if( 1 == option_verbose && ret == DISPATCH_UNSUPPORTED_SUITE )
+ {
+ mbedtls_fprintf( stdout, " Test Suite not enabled" );
+ }
+
+ if( 1 == option_verbose && unmet_dep_count > 0 )
+ {
+ mbedtls_fprintf( stdout, " Unmet dependencies: " );
+ while( unmet_dep_count > 0)
+ {
+ mbedtls_fprintf(stdout, "%s ",
+ unmet_dependencies[unmet_dep_count - 1]);
+ free(unmet_dependencies[unmet_dep_count - 1]);
+ unmet_dep_count--;
+ }
+ mbedtls_fprintf( stdout, "\n" );
+ }
fflush( stdout );
}
- else if( ret == 0 && test_errors == 0 )
+ else if( ret == DISPATCH_TEST_SUCCESS && test_errors == 0 )
{
mbedtls_fprintf( stdout, "PASS\n" );
fflush( stdout );
}
- else if( ret == 2 )
+ else if( ret == DISPATCH_INVALID_TEST_DATA )
{
mbedtls_fprintf( stderr, "FAILED: FATAL PARSE ERROR\n" );
fclose(file);