Merge pull request #4932 from tom-daubney-arm/all.sh-subshells-2.x
Backport 2.x: Run all.sh components in a subshell
diff --git a/programs/.gitignore b/programs/.gitignore
index 353a92d..5d3c5bd 100644
--- a/programs/.gitignore
+++ b/programs/.gitignore
@@ -53,6 +53,7 @@
ssl/ssl_server2
test/benchmark
test/cpp_dummy_build
+test/cpp_dummy_build.cpp
test/ecp-bench
test/query_compile_time_config
test/selftest
diff --git a/programs/Makefile b/programs/Makefile
index 35ef7ad..550887a 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -310,6 +310,10 @@
echo " CC test/benchmark.c"
$(CC) $(LOCAL_CFLAGS) $(CFLAGS) test/benchmark.c $(LOCAL_LDFLAGS) $(LDFLAGS) -o $@
+test/cpp_dummy_build.cpp: test/generate_cpp_dummy_build.sh
+ echo " Gen test/cpp_dummy_build.cpp"
+ test/generate_cpp_dummy_build.sh
+
test/cpp_dummy_build$(EXEXT): test/cpp_dummy_build.cpp $(DEP)
echo " CXX test/cpp_dummy_build.cpp"
$(CXX) $(LOCAL_CXXFLAGS) $(CXXFLAGS) test/cpp_dummy_build.cpp $(LOCAL_LDFLAGS) $(LDFLAGS) -o $@
@@ -374,10 +378,11 @@
ifndef WINDOWS
rm -f $(APPS)
-rm -f ssl/ssl_pthread_server$(EXEXT)
- -rm -f test/cpp_dummy_build$(EXEXT)
+ -rm -f test/cpp_dummy_build.cpp test/cpp_dummy_build$(EXEXT)
else
if exist *.o del /Q /F *.o
if exist *.exe del /Q /F *.exe
+ if exist test\cpp_dummy_build.cpp del /Q /F test\cpp_dummy_build.cpp
endif
$(MAKE) -C fuzz clean
diff --git a/programs/test/CMakeLists.txt b/programs/test/CMakeLists.txt
index 2b1e61e..04ec7fc 100644
--- a/programs/test/CMakeLists.txt
+++ b/programs/test/CMakeLists.txt
@@ -22,7 +22,17 @@
)
if(TEST_CPP)
- list(APPEND executables_mbedcrypto cpp_dummy_build)
+ set(cpp_dummy_build_cpp "${CMAKE_CURRENT_BINARY_DIR}/cpp_dummy_build.cpp")
+ set(generate_cpp_dummy_build "${CMAKE_CURRENT_SOURCE_DIR}/generate_cpp_dummy_build.sh")
+ add_custom_command(
+ OUTPUT "${cpp_dummy_build_cpp}"
+ COMMAND "${generate_cpp_dummy_build}" "${cpp_dummy_build_cpp}"
+ DEPENDS "${generate_cpp_dummy_build}"
+ WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+ )
+ add_executable(cpp_dummy_build "${cpp_dummy_build_cpp}")
+ target_include_directories(cpp_dummy_build PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../include)
+ target_link_libraries(cpp_dummy_build ${mbedcrypto_target})
endif()
foreach(exe IN LISTS executables_libs executables_mbedcrypto)
diff --git a/programs/test/cpp_dummy_build.cpp b/programs/test/cpp_dummy_build.cpp
deleted file mode 100644
index d052682..0000000
--- a/programs/test/cpp_dummy_build.cpp
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * This program is a dummy C++ program to ensure Mbed TLS library header files
- * can be included and built with a C++ compiler.
- *
- * Copyright The Mbed TLS Contributors
- * SPDX-License-Identifier: Apache-2.0
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#if !defined(MBEDTLS_CONFIG_FILE)
-#include "mbedtls/config.h"
-#else
-#include MBEDTLS_CONFIG_FILE
-#endif
-
-#include "mbedtls/aes.h"
-#include "mbedtls/aesni.h"
-#include "mbedtls/arc4.h"
-#include "mbedtls/aria.h"
-#include "mbedtls/asn1.h"
-#include "mbedtls/asn1write.h"
-#include "mbedtls/base64.h"
-#include "mbedtls/bignum.h"
-#include "mbedtls/blowfish.h"
-#include "mbedtls/bn_mul.h"
-#include "mbedtls/camellia.h"
-#include "mbedtls/ccm.h"
-#include "mbedtls/certs.h"
-#include "mbedtls/chacha20.h"
-#include "mbedtls/chachapoly.h"
-#include "mbedtls/check_config.h"
-#include "mbedtls/cipher.h"
-#include "mbedtls/cipher_internal.h"
-#include "mbedtls/cmac.h"
-#include "mbedtls/compat-1.3.h"
-#include "mbedtls/config_psa.h"
-#include "mbedtls/ctr_drbg.h"
-#include "mbedtls/debug.h"
-#include "mbedtls/des.h"
-#include "mbedtls/dhm.h"
-#include "mbedtls/ecdh.h"
-#include "mbedtls/ecdsa.h"
-#include "mbedtls/ecjpake.h"
-#include "mbedtls/ecp.h"
-#include "mbedtls/ecp_internal.h"
-#include "mbedtls/entropy.h"
-#include "mbedtls/entropy_poll.h"
-#include "mbedtls/error.h"
-#include "mbedtls/gcm.h"
-#include "mbedtls/havege.h"
-#include "mbedtls/hkdf.h"
-#include "mbedtls/hmac_drbg.h"
-#include "mbedtls/md.h"
-#include "mbedtls/md2.h"
-#include "mbedtls/md4.h"
-#include "mbedtls/md5.h"
-#include "mbedtls/md_internal.h"
-#include "mbedtls/net.h"
-#include "mbedtls/net_sockets.h"
-#include "mbedtls/nist_kw.h"
-#include "mbedtls/oid.h"
-#include "mbedtls/padlock.h"
-#include "mbedtls/pem.h"
-#include "mbedtls/pk.h"
-#include "mbedtls/pk_internal.h"
-#include "mbedtls/pkcs11.h"
-#include "mbedtls/pkcs12.h"
-#include "mbedtls/pkcs5.h"
-#include "mbedtls/platform_time.h"
-#include "mbedtls/platform_util.h"
-#include "mbedtls/poly1305.h"
-#include "mbedtls/psa_util.h"
-#include "mbedtls/ripemd160.h"
-#include "mbedtls/rsa.h"
-#include "mbedtls/rsa_internal.h"
-#include "mbedtls/sha1.h"
-#include "mbedtls/sha256.h"
-#include "mbedtls/sha512.h"
-#include "mbedtls/ssl.h"
-#include "mbedtls/ssl_cache.h"
-#include "mbedtls/ssl_ciphersuites.h"
-#include "mbedtls/ssl_cookie.h"
-#include "mbedtls/ssl_internal.h"
-#include "mbedtls/ssl_ticket.h"
-#include "mbedtls/threading.h"
-#include "mbedtls/timing.h"
-#include "mbedtls/version.h"
-#include "mbedtls/x509.h"
-#include "mbedtls/x509_crl.h"
-#include "mbedtls/x509_crt.h"
-#include "mbedtls/x509_csr.h"
-#include "mbedtls/xtea.h"
-
-#if defined(MBEDTLS_PLATFORM_C)
-#include "mbedtls/platform.h"
-#endif
-
-#if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C)
-#include "mbedtls/memory_buffer_alloc.h"
-#endif
-
-#include "psa/crypto.h"
-#include "psa/crypto_se_driver.h"
-#include "../library/psa_crypto_its.h"
-
-int main()
-{
- mbedtls_platform_context *ctx = NULL;
- mbedtls_platform_setup(ctx);
- mbedtls_printf("CPP Build test\n");
- mbedtls_platform_teardown(ctx);
-}
diff --git a/programs/test/generate_cpp_dummy_build.sh b/programs/test/generate_cpp_dummy_build.sh
new file mode 100755
index 0000000..90a181d
--- /dev/null
+++ b/programs/test/generate_cpp_dummy_build.sh
@@ -0,0 +1,97 @@
+#!/bin/sh
+
+DEFAULT_OUTPUT_FILE=programs/test/cpp_dummy_build.cpp
+
+if [ "$1" = "--help" ]; then
+ cat <<EOF
+Usage: $0 [OUTPUT]
+Generate a C++ dummy build program that includes all the headers.
+OUTPUT defaults to "programs/test/cpp_dummy_build.cpp".
+Run this program from the root of an Mbed TLS directory tree or from
+its "programs" or "programs/test" subdirectory.
+EOF
+ exit
+fi
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Ensure a reproducible order for *.h
+export LC_ALL=C
+
+print_cpp () {
+ cat <<'EOF'
+/* Automatically generated file. Do not edit.
+ *
+ * This program is a dummy C++ program to ensure Mbed TLS library header files
+ * can be included and built with a C++ compiler.
+ *
+ * Copyright The Mbed TLS Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "mbedtls/config.h"
+
+EOF
+
+ for header in include/mbedtls/*.h include/psa/*.h; do
+ case ${header#include/} in
+ psa/crypto_config.h) :;; # not meant for direct inclusion
+ # Some of the psa/crypto_*.h headers are not meant to be included directly.
+ # They do have include guards that make them no-ops if psa/crypto.h
+ # has been included before. Since psa/crypto.h comes before psa/crypto_*.h
+ # in the wildcard enumeration, we don't need to skip those headers.
+ *) echo "#include \"${header#include/}\"";;
+ esac
+ done
+
+ cat <<'EOF'
+
+int main()
+{
+ mbedtls_platform_context *ctx = NULL;
+ mbedtls_platform_setup(ctx);
+ mbedtls_printf("CPP Build test passed\n");
+ mbedtls_platform_teardown(ctx);
+}
+EOF
+}
+
+if [ -d include/mbedtls ]; then
+ :
+elif [ -d ../include/mbedtls ]; then
+ cd ..
+elif [ -d ../../include/mbedtls ]; then
+ cd ../..
+else
+ echo >&2 "This script must be run from an Mbed TLS source tree."
+ exit 3
+fi
+
+print_cpp >"${1:-$DEFAULT_OUTPUT_FILE}"
diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh
index 98950c0..07503a2 100755
--- a/tests/scripts/all.sh
+++ b/tests/scripts/all.sh
@@ -1,4 +1,4 @@
-#! /usr/bin/env sh
+#! /usr/bin/env bash
# all.sh
#
@@ -59,6 +59,15 @@
# This script must be invoked from the toplevel directory of a git
# working copy of Mbed TLS.
#
+# The behavior on an error depends on whether --keep-going (alias -k)
+# is in effect.
+# * Without --keep-going: the script stops on the first error without
+# cleaning up. This lets you work in the configuration of the failing
+# component.
+# * With --keep-going: the script runs all requested components and
+# reports failures at the end. In particular the script always cleans
+# up on exit.
+#
# Note that the output is not saved. You may want to run
# script -c tests/scripts/all.sh
# or
@@ -81,6 +90,12 @@
#
# Each component must start by invoking `msg` with a short informative message.
#
+# Warning: due to the way bash detects errors, the failure of a command
+# inside 'if' or '!' is not detected. Use the 'not' function instead of '!'.
+#
+# Each component is executed in a separate shell process. The component
+# fails if any command in it returns a non-zero status.
+#
# The framework performs some cleanup tasks after each component. This
# means that components can assume that the working directory is in a
# cleaned-up state, and don't need to perform the cleanup themselves.
@@ -91,19 +106,6 @@
# `tests/Makefile` and `programs/fuzz/Makefile` from git.
# This cleans up after an in-tree use of CMake.
#
-# Any command that is expected to fail must be protected so that the
-# script keeps running in --keep-going mode despite `set -e`. In keep-going
-# mode, if a protected command fails, this is logged as a failure and the
-# script will exit with a failure status once it has run all components.
-# Commands can be protected in any of the following ways:
-# * `make` is a function which runs the `make` command with protection.
-# Note that you must write `make VAR=value`, not `VAR=value make`,
-# because the `VAR=value make` syntax doesn't work with functions.
-# * Put `report_status` before the command to protect it.
-# * Put `if_build_successful` before a command. This protects it, and
-# additionally skips it if a prior invocation of `make` in the same
-# component failed.
-#
# The tests are roughly in order from fastest to slowest. This doesn't
# have to be exact, but in general you should add slower tests towards
# the end and fast checks near the beginning.
@@ -114,8 +116,9 @@
#### Initialization and command line parsing
################################################################
-# Abort on errors (and uninitialised variables)
-set -eu
+# Abort on errors (even on the left-hand side of a pipe).
+# Treat uninitialised variables as errors.
+set -e -o pipefail -u
pre_check_environment () {
if [ -d library -a -d include -a -d tests ]; then :; else
@@ -126,9 +129,16 @@
pre_initialize_variables () {
CONFIG_H='include/mbedtls/config.h'
- CONFIG_BAK="$CONFIG_H.bak"
CRYPTO_CONFIG_H='include/psa/crypto_config.h'
- CRYPTO_CONFIG_BAK="$CRYPTO_CONFIG_H.bak"
+
+ # Files that are clobbered by some jobs will be backed up. Use a different
+ # suffix from auxiliary scripts so that all.sh and auxiliary scripts can
+ # independently decide when to remove the backup file.
+ backup_suffix='.all.bak'
+ # Files clobbered by config.py
+ files_to_back_up="$CONFIG_H $CRYPTO_CONFIG_H"
+ # Files clobbered by in-tree cmake
+ files_to_back_up="$files_to_back_up Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile"
append_outcome=0
MEMORY=0
@@ -176,8 +186,8 @@
# Gather the list of available components. These are the functions
# defined in this script whose name starts with "component_".
- # Parse the script with sed, because in sh there is no way to list
- # defined functions.
+ # Parse the script with sed. This way we get the functions in the order
+ # they are defined.
ALL_COMPONENTS=$(sed -n 's/^ *component_\([0-9A-Z_a-z]*\) *().*/\1/p' <"$0")
# Exclude components that are not supported on this platform.
@@ -194,6 +204,8 @@
# Test whether the component $1 is included in the command line patterns.
is_component_included()
{
+ # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS
+ # only does word splitting.
set -f
for pattern in $COMMAND_LINE_COMPONENTS; do
set +f
@@ -235,6 +247,11 @@
Prefix for a cross-compiler for arm-linux-gnueabi
(default: "${ARM_LINUX_GNUEABI_GCC_PREFIX}")
--armcc Run ARM Compiler builds (on by default).
+ --restore First clean up the build tree, restoring backed up
+ files. Do not run any components unless they are
+ explicitly specified.
+ --error-test Error test mode: run a failing function in addition
+ to any specified component. May be repeated.
--except Exclude the COMPONENTs listed on the command line,
instead of running only those.
--no-append-outcome Write a new outcome file and analyze it (default).
@@ -263,13 +280,11 @@
EOF
}
-# remove built files as well as the cmake cache/config
+# Cleanup before/after running a component.
+# Remove built files as well as the cmake cache/config.
+# Does not remove generated source files.
cleanup()
{
- if [ -n "${MBEDTLS_ROOT_DIR+set}" ]; then
- cd "$MBEDTLS_ROOT_DIR"
- fi
-
command make clean
# Remove CMake artefacts
@@ -280,21 +295,26 @@
-iname CMakeCache.txt \) -exec rm {} \+
# Recover files overwritten by in-tree CMake builds
rm -f include/Makefile include/mbedtls/Makefile programs/*/Makefile
- git update-index --no-skip-worktree Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile
- git checkout -- Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile
# Remove any artifacts from the component_test_cmake_as_subdirectory test.
rm -rf programs/test/cmake_subproject/build
rm -f programs/test/cmake_subproject/Makefile
rm -f programs/test/cmake_subproject/cmake_subproject
- if [ -f "$CONFIG_BAK" ]; then
- mv "$CONFIG_BAK" "$CONFIG_H"
- fi
+ # Restore files that may have been clobbered by the job
+ for x in $files_to_back_up; do
+ cp -p "$x$backup_suffix" "$x"
+ done
+}
- if [ -f "$CRYPTO_CONFIG_BAK" ]; then
- mv "$CRYPTO_CONFIG_BAK" "$CRYPTO_CONFIG_H"
- fi
+# Final cleanup when this script exits (except when exiting on a failure
+# in non-keep-going mode).
+final_cleanup () {
+ cleanup
+
+ for x in $files_to_back_up; do
+ rm -f "$x$backup_suffix"
+ done
}
# Executed on exit. May be redefined depending on command line options.
@@ -303,7 +323,7 @@
}
fatal_signal () {
- cleanup
+ final_cleanup
final_report $1
trap - $1
kill -$1 $$
@@ -373,17 +393,11 @@
done
}
-check_headers_in_cpp () {
- ls include/mbedtls | grep "\.h$" >headers.txt
- <programs/test/cpp_dummy_build.cpp sed -n 's/"$//; s!^#include "mbedtls/!!p' |
- sort |
- diff headers.txt -
- rm headers.txt
-}
-
pre_parse_command_line () {
COMMAND_LINE_COMPONENTS=
all_except=0
+ error_test=0
+ restore_first=0
no_armcc=
# Note that legacy options are ignored instead of being omitted from this
@@ -397,6 +411,7 @@
--armcc) no_armcc=;;
--armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";;
--armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";;
+ --error-test) error_test=$((error_test + 1));;
--except) all_except=1;;
--force|-f) FORCE=1;;
--gnutls-cli) shift; GNUTLS_CLI="$1";;
@@ -422,6 +437,7 @@
--quiet|-q) QUIET=1;;
--random-seed) unset SEED;;
--release-test|-r) SEED=$RELEASE_SEED;;
+ --restore) restore_first=1;;
--seed|-s) shift; SEED="$1";;
-*)
echo >&2 "Unknown option: $1"
@@ -434,7 +450,7 @@
done
# With no list of components, run everything.
- if [ -z "$COMMAND_LINE_COMPONENTS" ]; then
+ if [ -z "$COMMAND_LINE_COMPONENTS" ] && [ $restore_first -eq 0 ]; then
all_except=1
fi
@@ -444,6 +460,32 @@
COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS *_armcc*"
fi
+ # Error out if an explicitly requested component doesn't exist.
+ if [ $all_except -eq 0 ]; then
+ unsupported=0
+ # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS
+ # only does word splitting.
+ set -f
+ for component in $COMMAND_LINE_COMPONENTS; do
+ set +f
+ # If the requested name includes a wildcard character, don't
+ # check it. Accept wildcard patterns that don't match anything.
+ case $component in
+ *[*?\[]*) continue;;
+ esac
+ case " $SUPPORTED_COMPONENTS " in
+ *" $component "*) :;;
+ *)
+ echo >&2 "Component $component was explicitly requested, but is not known or not supported."
+ unsupported=$((unsupported + 1));;
+ esac
+ done
+ set +f
+ if [ $unsupported -ne 0 ]; then
+ exit 2
+ fi
+ fi
+
# Build the list of components to run.
RUN_COMPONENTS=
for component in $SUPPORTED_COMPONENTS; do
@@ -479,9 +521,36 @@
fi
}
+pre_restore_files () {
+ # If the makefiles have been generated by a framework such as cmake,
+ # restore them from git. If the makefiles look like modifications from
+ # the ones checked into git, take care not to modify them. Whatever
+ # this function leaves behind is what the script will restore before
+ # each component.
+ case "$(head -n1 Makefile)" in
+ *[Gg]enerated*)
+ git update-index --no-skip-worktree Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile
+ git checkout -- Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile
+ ;;
+ esac
+}
+
+pre_back_up () {
+ for x in $files_to_back_up; do
+ cp -p "$x" "$x$backup_suffix"
+ done
+}
+
pre_setup_keep_going () {
- failure_summary=
- failure_count=0
+ failure_count=0 # Number of failed components
+ last_failure_status=0 # Last failure status in this component
+
+ # See err_trap
+ previous_failure_status=0
+ previous_failed_command=
+ previous_failure_funcall_depth=0
+ unset report_failed_command
+
start_red=
end_color=
if [ -t 1 ]; then
@@ -492,76 +561,106 @@
;;
esac
fi
- record_status () {
- if "$@"; then
- last_status=0
- else
- last_status=$?
- text="$current_section: $* -> $last_status"
- failure_summary="$failure_summary
-$text"
- failure_count=$((failure_count + 1))
- echo "${start_red}^^^^$text^^^^${end_color}" >&2
- fi
- }
- make () {
- case "$*" in
- *test|*check)
- if [ $build_status -eq 0 ]; then
- record_status command make "$@"
- else
- echo "(skipped because the build failed)"
- fi
- ;;
- *)
- record_status command make "$@"
- build_status=$last_status
- ;;
+
+ # Keep a summary of failures in a file. We'll print it out at the end.
+ failure_summary_file=$PWD/all-sh-failures-$$.log
+ : >"$failure_summary_file"
+
+ # Whether it makes sense to keep a component going after the specified
+ # command fails (test command) or not (configure or build).
+ # This function normally receives the failing simple command
+ # ($BASH_COMMAND) as an argument, but if $report_failed_command is set,
+ # this is passed instead.
+ # This doesn't have to be 100% accurate: all failures are recorded anyway.
+ # False positives result in running things that can't be expected to
+ # work. False negatives result in things not running after something else
+ # failed even though they might have given useful feedback.
+ can_keep_going_after_failure () {
+ case "$1" in
+ "msg "*) false;;
+ "cd "*) false;;
+ *make*[\ /]tests*) false;; # make tests, make CFLAGS=-I../tests, ...
+ *test*) true;; # make test, tests/stuff, env V=v tests/stuff, ...
+ *make*check*) true;;
+ "grep "*) true;;
+ "[ "*) true;;
+ "! "*) true;;
+ *) false;;
esac
}
+
+ # This function runs if there is any error in a component.
+ # It must either exit with a nonzero status, or set
+ # last_failure_status to a nonzero value.
+ err_trap () {
+ # Save $? (status of the failing command). This must be the very
+ # first thing, before $? is overridden.
+ last_failure_status=$?
+ failed_command=${report_failed_command-$BASH_COMMAND}
+
+ if [[ $last_failure_status -eq $previous_failure_status &&
+ "$failed_command" == "$previous_failed_command" &&
+ ${#FUNCNAME[@]} == $((previous_failure_funcall_depth - 1)) ]]
+ then
+ # The same command failed twice in a row, but this time one level
+ # less deep in the function call stack. This happens when the last
+ # command of a function returns a nonzero status, and the function
+ # returns that same status. Ignore the second failure.
+ previous_failure_funcall_depth=${#FUNCNAME[@]}
+ return
+ fi
+ previous_failure_status=$last_failure_status
+ previous_failed_command=$failed_command
+ previous_failure_funcall_depth=${#FUNCNAME[@]}
+
+ text="$current_section: $failed_command -> $last_failure_status"
+ echo "${start_red}^^^^$text^^^^${end_color}" >&2
+ echo "$text" >>"$failure_summary_file"
+
+ # If the command is fatal (configure or build command), stop this
+ # component. Otherwise (test command) keep the component running
+ # (run more tests from the same build).
+ if ! can_keep_going_after_failure "$failed_command"; then
+ exit $last_failure_status
+ fi
+ }
+
final_report () {
if [ $failure_count -gt 0 ]; then
echo
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
- echo "${start_red}FAILED: $failure_count${end_color}$failure_summary"
+ echo "${start_red}FAILED: $failure_count components${end_color}"
+ cat "$failure_summary_file"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
- exit 1
elif [ -z "${1-}" ]; then
echo "SUCCESS :)"
fi
if [ -n "${1-}" ]; then
echo "Killed by SIG$1."
fi
+ rm -f "$failure_summary_file"
+ if [ $failure_count -gt 0 ]; then
+ exit 1
+ fi
}
}
+# record_status() and if_build_succeeded() are kept temporarily for backward
+# compatibility. Don't use them in new components.
+record_status () {
+ "$@"
+}
if_build_succeeded () {
- if [ $build_status -eq 0 ]; then
- record_status "$@"
- fi
+ "$@"
}
-# to be used instead of ! for commands run with
-# record_status or if_build_succeeded
-not() {
- ! "$@"
-}
-
-pre_setup_quiet_redirect () {
- if [ $QUIET -ne 1 ]; then
- redirect_out () {
- "$@"
- }
- redirect_err () {
- "$@"
- }
- else
- redirect_out () {
- "$@" >/dev/null
- }
- redirect_err () {
- "$@" 2>/dev/null
- }
+# '! true' does not trigger the ERR trap. Arrange to trigger it, with
+# a reasonably informative error message (not just "$@").
+not () {
+ if "$@"; then
+ report_failed_command="! $*"
+ false
+ unset report_failed_command
fi
}
@@ -686,28 +785,28 @@
component_check_recursion () {
msg "Check: recursion.pl" # < 1s
- record_status tests/scripts/recursion.pl library/*.c
+ tests/scripts/recursion.pl library/*.c
}
component_check_generated_files () {
msg "Check: freshness of generated source files" # < 1s
- record_status tests/scripts/check-generated-files.sh
+ tests/scripts/check-generated-files.sh
}
component_check_doxy_blocks () {
msg "Check: doxygen markup outside doxygen blocks" # < 1s
- record_status tests/scripts/check-doxy-blocks.pl
+ tests/scripts/check-doxy-blocks.pl
}
component_check_files () {
msg "Check: file sanity checks (permissions, encodings)" # < 1s
- record_status tests/scripts/check_files.py
+ tests/scripts/check_files.py
}
component_check_changelog () {
msg "Check: changelog entries" # < 1s
rm -f ChangeLog.new
- record_status scripts/assemble_changelog.py -o ChangeLog.new
+ scripts/assemble_changelog.py -o ChangeLog.new
if [ -e ChangeLog.new ]; then
# Show the diff for information. It isn't an error if the diff is
# non-empty.
@@ -718,7 +817,7 @@
component_check_names () {
msg "Check: declared and exported names (builds the library)" # < 3s
- record_status tests/scripts/check_names.py -v
+ tests/scripts/check_names.py -v
}
component_check_test_cases () {
@@ -728,13 +827,13 @@
else
opt=''
fi
- record_status tests/scripts/check_test_cases.py $opt
+ tests/scripts/check_test_cases.py $opt
unset opt
}
component_check_doxygen_warnings () {
msg "Check: doxygen warnings (builds the documentation)" # ~ 3s
- record_status tests/scripts/doxygen.sh
+ tests/scripts/doxygen.sh
}
@@ -754,7 +853,7 @@
make test
msg "selftest: make, default config (out-of-box)" # ~10s
- if_build_succeeded programs/test/selftest
+ programs/test/selftest
export MBEDTLS_TEST_OUTCOME_FILE="$SAVE_MBEDTLS_TEST_OUTCOME_FILE"
unset SAVE_MBEDTLS_TEST_OUTCOME_FILE
@@ -769,16 +868,16 @@
make test
msg "test: selftest (ASan build)" # ~ 10s
- if_build_succeeded programs/test/selftest
+ programs/test/selftest
msg "test: ssl-opt.sh (ASan build)" # ~ 1 min
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
msg "test: compat.sh (ASan build)" # ~ 6 min
- if_build_succeeded tests/compat.sh
+ tests/compat.sh
msg "test: context-info.sh (ASan build)" # ~ 15 sec
- if_build_succeeded tests/context-info.sh
+ tests/context-info.sh
}
component_test_full_cmake_gcc_asan () {
@@ -791,16 +890,16 @@
make test
msg "test: selftest (ASan build)" # ~ 10s
- if_build_succeeded programs/test/selftest
+ programs/test/selftest
msg "test: ssl-opt.sh (full config, ASan build)"
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
msg "test: compat.sh (full config, ASan build)"
- if_build_succeeded tests/compat.sh
+ tests/compat.sh
msg "test: context-info.sh (full config, ASan build)" # ~ 15 sec
- if_build_succeeded tests/context-info.sh
+ tests/context-info.sh
}
component_test_psa_crypto_key_id_encodes_owner () {
@@ -838,7 +937,7 @@
# Check that if a symbol is renamed by crypto_spe.h, the non-renamed
# version is not present.
echo "Checking for renamed symbols in the library"
- if_build_succeeded check_renamed_symbols tests/include/spe/crypto_spe.h library/libmbedcrypto.a
+ check_renamed_symbols tests/include/spe/crypto_spe.h library/libmbedcrypto.a
}
component_test_psa_crypto_client () {
@@ -861,7 +960,7 @@
make test
msg "test: ssl-opt.sh (zlib, make)"
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
}
support_test_zlib_make () {
base=support_test_zlib_$$
@@ -885,7 +984,7 @@
make test
msg "test: ssl-opt.sh (zlib, cmake)"
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
}
support_test_zlib_cmake () {
support_test_zlib_make "$@"
@@ -903,7 +1002,7 @@
component_test_ref_configs () {
msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s
CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan .
- record_status tests/scripts/test-ref-configs.pl
+ tests/scripts/test-ref-configs.pl
}
component_test_sslv3 () {
@@ -916,14 +1015,14 @@
make test
msg "build: SSLv3 - compat.sh (ASan build)" # ~ 6 min
- if_build_succeeded tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2'
- if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3'
+ tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2'
+ env OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3'
msg "build: SSLv3 - ssl-opt.sh (ASan build)" # ~ 6 min
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
msg "build: SSLv3 - context-info.sh (ASan build)" # ~ 15 sec
- if_build_succeeded tests/context-info.sh
+ tests/context-info.sh
}
component_test_no_renegotiation () {
@@ -936,7 +1035,7 @@
make test
msg "test: !MBEDTLS_SSL_RENEGOTIATION - ssl-opt.sh (ASan build)" # ~ 6 min
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
}
component_test_no_pem_no_fs () {
@@ -952,7 +1051,7 @@
make test
msg "test: !MBEDTLS_PEM_PARSE_C !MBEDTLS_FS_IO - ssl-opt.sh (ASan build)" # ~ 6 min
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
}
component_test_rsa_no_crt () {
@@ -965,13 +1064,13 @@
make test
msg "test: RSA_NO_CRT - RSA-related part of ssl-opt.sh (ASan build)" # ~ 5s
- if_build_succeeded tests/ssl-opt.sh -f RSA
+ tests/ssl-opt.sh -f RSA
msg "test: RSA_NO_CRT - RSA-related part of compat.sh (ASan build)" # ~ 3 min
- if_build_succeeded tests/compat.sh -t RSA
+ tests/compat.sh -t RSA
msg "test: RSA_NO_CRT - RSA-related part of context-info.sh (ASan build)" # ~ 15 sec
- if_build_succeeded tests/context-info.sh
+ tests/context-info.sh
}
component_test_no_ctr_drbg_classic () {
@@ -990,10 +1089,10 @@
# The SSL tests are slow, so run a small subset, just enough to get
# confidence that the SSL code copes with HMAC_DRBG.
msg "test: Full minus CTR_DRBG, classic crypto - ssl-opt.sh (subset)"
- if_build_succeeded tests/ssl-opt.sh -f 'Default\|SSL async private.*delay=\|tickets enabled on server'
+ tests/ssl-opt.sh -f 'Default\|SSL async private.*delay=\|tickets enabled on server'
msg "test: Full minus CTR_DRBG, classic crypto - compat.sh (subset)"
- if_build_succeeded tests/compat.sh -m tls1_2 -t 'ECDSA PSK' -V NO -p OpenSSL
+ tests/compat.sh -m tls1_2 -t 'ECDSA PSK' -V NO -p OpenSSL
}
component_test_no_ctr_drbg_use_psa () {
@@ -1012,10 +1111,10 @@
# The SSL tests are slow, so run a small subset, just enough to get
# confidence that the SSL code copes with HMAC_DRBG.
msg "test: Full minus CTR_DRBG, USE_PSA_CRYPTO - ssl-opt.sh (subset)"
- if_build_succeeded tests/ssl-opt.sh -f 'Default\|SSL async private.*delay=\|tickets enabled on server'
+ tests/ssl-opt.sh -f 'Default\|SSL async private.*delay=\|tickets enabled on server'
msg "test: Full minus CTR_DRBG, USE_PSA_CRYPTO - compat.sh (subset)"
- if_build_succeeded tests/compat.sh -m tls1_2 -t 'ECDSA PSK' -V NO -p OpenSSL
+ tests/compat.sh -m tls1_2 -t 'ECDSA PSK' -V NO -p OpenSSL
}
component_test_no_hmac_drbg_classic () {
@@ -1037,12 +1136,12 @@
# Test SSL with non-deterministic ECDSA. Only test features that
# might be affected by how ECDSA signature is performed.
msg "test: Full minus HMAC_DRBG, classic crypto - ssl-opt.sh (subset)"
- if_build_succeeded tests/ssl-opt.sh -f 'Default\|SSL async private: sign'
+ tests/ssl-opt.sh -f 'Default\|SSL async private: sign'
# To save time, only test one protocol version, since this part of
# the protocol is identical in (D)TLS up to 1.2.
msg "test: Full minus HMAC_DRBG, classic crypto - compat.sh (ECDSA)"
- if_build_succeeded tests/compat.sh -m tls1_2 -t 'ECDSA'
+ tests/compat.sh -m tls1_2 -t 'ECDSA'
}
component_test_no_hmac_drbg_use_psa () {
@@ -1064,12 +1163,12 @@
# Test SSL with non-deterministic ECDSA. Only test features that
# might be affected by how ECDSA signature is performed.
msg "test: Full minus HMAC_DRBG, USE_PSA_CRYPTO - ssl-opt.sh (subset)"
- if_build_succeeded tests/ssl-opt.sh -f 'Default\|SSL async private: sign'
+ tests/ssl-opt.sh -f 'Default\|SSL async private: sign'
# To save time, only test one protocol version, since this part of
# the protocol is identical in (D)TLS up to 1.2.
msg "test: Full minus HMAC_DRBG, USE_PSA_CRYPTO - compat.sh (ECDSA)"
- if_build_succeeded tests/compat.sh -m tls1_2 -t 'ECDSA'
+ tests/compat.sh -m tls1_2 -t 'ECDSA'
}
component_test_psa_external_rng_no_drbg_classic () {
@@ -1093,7 +1192,7 @@
make test
msg "test: PSA_CRYPTO_EXTERNAL_RNG minus *_DRBG, classic crypto - ssl-opt.sh (subset)"
- if_build_succeeded tests/ssl-opt.sh -f 'Default'
+ tests/ssl-opt.sh -f 'Default'
}
component_test_psa_external_rng_no_drbg_use_psa () {
@@ -1113,7 +1212,7 @@
make test
msg "test: PSA_CRYPTO_EXTERNAL_RNG minus *_DRBG, PSA crypto - ssl-opt.sh (subset)"
- if_build_succeeded tests/ssl-opt.sh -f 'Default\|opaque'
+ tests/ssl-opt.sh -f 'Default\|opaque'
}
component_test_psa_external_rng_use_psa_crypto () {
@@ -1128,7 +1227,7 @@
make test
msg "test: full + PSA_CRYPTO_EXTERNAL_RNG + USE_PSA_CRYPTO minus CTR_DRBG"
- if_build_succeeded tests/ssl-opt.sh -f 'Default\|opaque'
+ tests/ssl-opt.sh -f 'Default\|opaque'
}
component_test_ecp_no_internal_rng () {
@@ -1178,11 +1277,11 @@
make test
msg "test: new ECDH context - ECDH-related part of ssl-opt.sh (ASan build)" # ~ 5s
- if_build_succeeded tests/ssl-opt.sh -f ECDH
+ tests/ssl-opt.sh -f ECDH
msg "test: new ECDH context - compat.sh with some ECDH ciphersuites (ASan build)" # ~ 3 min
# Exclude some symmetric ciphers that are redundant here to gain time.
- if_build_succeeded tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4'
+ tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4'
}
component_test_everest () {
@@ -1196,11 +1295,11 @@
make test
msg "test: Everest ECDH context - ECDH-related part of ssl-opt.sh (ASan build)" # ~ 5s
- if_build_succeeded tests/ssl-opt.sh -f ECDH
+ tests/ssl-opt.sh -f ECDH
msg "test: Everest ECDH context - compat.sh with some ECDH ciphersuites (ASan build)" # ~ 3 min
# Exclude some symmetric ciphers that are redundant here to gain time.
- if_build_succeeded tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4'
+ tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4'
}
component_test_everest_curve25519_only () {
@@ -1230,7 +1329,7 @@
make
msg "test: small SSL_OUT_CONTENT_LEN - ssl-opt.sh MFL and large packet tests"
- if_build_succeeded tests/ssl-opt.sh -f "Max fragment\|Large packet"
+ tests/ssl-opt.sh -f "Max fragment\|Large packet"
}
component_test_small_ssl_in_content_len () {
@@ -1241,7 +1340,7 @@
make
msg "test: small SSL_IN_CONTENT_LEN - ssl-opt.sh MFL tests"
- if_build_succeeded tests/ssl-opt.sh -f "Max fragment"
+ tests/ssl-opt.sh -f "Max fragment"
}
component_test_small_ssl_dtls_max_buffering () {
@@ -1251,7 +1350,7 @@
make
msg "test: small MBEDTLS_SSL_DTLS_MAX_BUFFERING #0 - ssl-opt.sh specific reordering test"
- if_build_succeeded tests/ssl-opt.sh -f "DTLS reordering: Buffer out-of-order hs msg before reassembling next, free buffered msg"
+ tests/ssl-opt.sh -f "DTLS reordering: Buffer out-of-order hs msg before reassembling next, free buffered msg"
}
component_test_small_mbedtls_ssl_dtls_max_buffering () {
@@ -1261,15 +1360,15 @@
make
msg "test: small MBEDTLS_SSL_DTLS_MAX_BUFFERING #1 - ssl-opt.sh specific reordering test"
- if_build_succeeded tests/ssl-opt.sh -f "DTLS reordering: Buffer encrypted Finished message, drop for fragmented NewSessionTicket"
+ tests/ssl-opt.sh -f "DTLS reordering: Buffer encrypted Finished message, drop for fragmented NewSessionTicket"
}
component_test_psa_collect_statuses () {
msg "build+test: psa_collect_statuses" # ~30s
scripts/config.py full
- record_status tests/scripts/psa_collect_statuses.py
+ tests/scripts/psa_collect_statuses.py
# Check that psa_crypto_init() succeeded at least once
- record_status grep -q '^0:psa_crypto_init:' tests/statuses.log
+ grep -q '^0:psa_crypto_init:' tests/statuses.log
rm -f tests/statuses.log
}
@@ -1283,16 +1382,16 @@
make test
msg "test: psa_constant_names (full config, clang)" # ~ 1s
- record_status tests/scripts/test_psa_constant_names.py
+ tests/scripts/test_psa_constant_names.py
msg "test: ssl-opt.sh default, ECJPAKE, SSL async (full config)" # ~ 1s
- if_build_succeeded tests/ssl-opt.sh -f 'Default\|ECJPAKE\|SSL async private'
+ tests/ssl-opt.sh -f 'Default\|ECJPAKE\|SSL async private'
msg "test: compat.sh RC4, DES, 3DES & NULL (full config)" # ~ 2 min
- if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '^$' -f 'NULL\|DES\|RC4\|ARCFOUR'
+ env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '^$' -f 'NULL\|DES\|RC4\|ARCFOUR'
msg "test: compat.sh ARIA + ChachaPoly"
- if_build_succeeded env OPENSSL_CMD="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA'
+ env OPENSSL_CMD="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA'
}
component_test_memsan_constant_flow () {
@@ -1399,70 +1498,68 @@
msg "build: make, crypto only"
scripts/config.py crypto
make CFLAGS='-O1 -Werror'
- if_build_succeeded are_empty_libraries library/libmbedx509.* library/libmbedtls.*
+ are_empty_libraries library/libmbedx509.* library/libmbedtls.*
}
component_build_crypto_full () {
msg "build: make, crypto only, full config"
scripts/config.py crypto_full
make CFLAGS='-O1 -Werror'
- if_build_succeeded are_empty_libraries library/libmbedx509.* library/libmbedtls.*
+ are_empty_libraries library/libmbedx509.* library/libmbedtls.*
}
component_build_crypto_baremetal () {
msg "build: make, crypto only, baremetal config"
scripts/config.py crypto_baremetal
make CFLAGS='-O1 -Werror'
- if_build_succeeded are_empty_libraries library/libmbedx509.* library/libmbedtls.*
+ are_empty_libraries library/libmbedx509.* library/libmbedtls.*
}
component_test_depends_curves () {
msg "test/build: curves.pl (gcc)" # ~ 4 min
- record_status tests/scripts/curves.pl
+ tests/scripts/curves.pl
}
component_test_depends_curves_psa () {
msg "test/build: curves.pl with MBEDTLS_USE_PSA_CRYPTO defined (gcc)"
scripts/config.py set MBEDTLS_USE_PSA_CRYPTO
- record_status tests/scripts/curves.pl
+ tests/scripts/curves.pl
}
component_test_depends_hashes () {
msg "test/build: depends-hashes.pl (gcc)" # ~ 2 min
- record_status tests/scripts/depends-hashes.pl
+ tests/scripts/depends-hashes.pl
}
component_test_depends_hashes_psa () {
msg "test/build: depends-hashes.pl with MBEDTLS_USE_PSA_CRYPTO defined (gcc)"
scripts/config.py set MBEDTLS_USE_PSA_CRYPTO
- record_status tests/scripts/depends-hashes.pl
+ tests/scripts/depends-hashes.pl
}
component_test_depends_pkalgs () {
msg "test/build: depends-pkalgs.pl (gcc)" # ~ 2 min
- record_status tests/scripts/depends-pkalgs.pl
+ tests/scripts/depends-pkalgs.pl
}
component_test_depends_pkalgs_psa () {
msg "test/build: depends-pkalgs.pl with MBEDTLS_USE_PSA_CRYPTO defined (gcc)"
scripts/config.py set MBEDTLS_USE_PSA_CRYPTO
- record_status tests/scripts/depends-pkalgs.pl
+ tests/scripts/depends-pkalgs.pl
}
component_build_key_exchanges () {
msg "test/build: key-exchanges (gcc)" # ~ 1 min
- record_status tests/scripts/key-exchanges.pl
+ tests/scripts/key-exchanges.pl
}
-component_build_default_make_gcc_and_cxx () {
- msg "build: Unix make, -Os (gcc)" # ~ 30s
- make CC=gcc CFLAGS='-Werror -Wall -Wextra -Os'
+component_test_make_cxx () {
+ msg "build: Unix make, full, gcc + g++"
+ scripts/config.py full
+ make TEST_CPP=1 lib programs
- msg "test: verify header list in cpp_dummy_build.cpp"
- record_status check_headers_in_cpp
-
- msg "build: Unix make, incremental g++"
- make TEST_CPP=1
+ msg "test: cpp_dummy_build"
+ programs/test/cpp_dummy_build
}
component_test_no_use_psa_crypto_full_cmake_asan() {
@@ -1482,16 +1579,16 @@
make test
msg "test: ssl-opt.sh (full minus MBEDTLS_USE_PSA_CRYPTO)"
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
msg "test: compat.sh default (full minus MBEDTLS_USE_PSA_CRYPTO)"
- if_build_succeeded tests/compat.sh
+ tests/compat.sh
msg "test: compat.sh RC4, DES & NULL (full minus MBEDTLS_USE_PSA_CRYPTO)"
- if_build_succeeded env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR'
+ env OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR'
msg "test: compat.sh ARIA + ChachaPoly (full minus MBEDTLS_USE_PSA_CRYPTO)"
- if_build_succeeded env OPENSSL_CMD="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA'
+ env OPENSSL_CMD="$OPENSSL_NEXT" tests/compat.sh -e '^$' -f 'ARIA\|CHACHA'
}
component_test_psa_crypto_config_basic() {
@@ -2040,7 +2137,7 @@
msg "test: ssl-opt.sh, MBEDTLS_MEMORY_BUFFER_ALLOC_C"
# MBEDTLS_MEMORY_BUFFER_ALLOC is slow. Skip tests that tend to time out.
- if_build_succeeded tests/ssl-opt.sh -e '^DTLS proxy'
+ tests/ssl-opt.sh -e '^DTLS proxy'
}
component_test_no_max_fragment_length () {
@@ -2051,7 +2148,7 @@
make
msg "test: ssl-opt.sh, MFL-related tests"
- if_build_succeeded tests/ssl-opt.sh -f "Max fragment length"
+ tests/ssl-opt.sh -f "Max fragment length"
}
component_test_asan_remove_peer_certificate () {
@@ -2064,13 +2161,13 @@
make test
msg "test: ssl-opt.sh, !MBEDTLS_SSL_KEEP_PEER_CERTIFICATE"
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
msg "test: compat.sh, !MBEDTLS_SSL_KEEP_PEER_CERTIFICATE"
- if_build_succeeded tests/compat.sh
+ tests/compat.sh
msg "test: context-info.sh, !MBEDTLS_SSL_KEEP_PEER_CERTIFICATE"
- if_build_succeeded tests/context-info.sh
+ tests/context-info.sh
}
component_test_no_max_fragment_length_small_ssl_out_content_len () {
@@ -2082,10 +2179,10 @@
make
msg "test: MFL tests (disabled MFL extension case) & large packet tests"
- if_build_succeeded tests/ssl-opt.sh -f "Max fragment length\|Large buffer"
+ tests/ssl-opt.sh -f "Max fragment length\|Large buffer"
msg "test: context-info.sh (disabled MFL extension case)"
- if_build_succeeded tests/context-info.sh
+ tests/context-info.sh
}
component_test_variable_ssl_in_out_buffer_len () {
@@ -2098,10 +2195,10 @@
make test
msg "test: ssl-opt.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH enabled"
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
msg "test: compat.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH enabled"
- if_build_succeeded tests/compat.sh
+ tests/compat.sh
}
component_test_variable_ssl_in_out_buffer_len_CID () {
@@ -2116,10 +2213,10 @@
make test
msg "test: ssl-opt.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH and MBEDTLS_SSL_DTLS_CONNECTION_ID enabled"
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
msg "test: compat.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH and MBEDTLS_SSL_DTLS_CONNECTION_ID enabled"
- if_build_succeeded tests/compat.sh
+ tests/compat.sh
}
component_test_variable_ssl_in_out_buffer_len_record_splitting () {
@@ -2134,10 +2231,10 @@
make test
msg "test: ssl-opt.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH and MBEDTLS_SSL_CBC_RECORD_SPLITTING enabled"
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
msg "test: compat.sh, MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH and MBEDTLS_SSL_CBC_RECORD_SPLITTING enabled"
- if_build_succeeded tests/compat.sh
+ tests/compat.sh
}
component_test_ssl_alloc_buffer_and_mfl () {
@@ -2154,7 +2251,7 @@
make test
msg "test: MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH, MBEDTLS_MEMORY_BUFFER_ALLOC_C, MBEDTLS_MEMORY_DEBUG and MBEDTLS_SSL_MAX_FRAGMENT_LENGTH"
- if_build_succeeded tests/ssl-opt.sh -f "Handshake memory usage"
+ tests/ssl-opt.sh -f "Handshake memory usage"
}
component_test_when_no_ciphersuites_have_mac () {
@@ -2168,7 +2265,7 @@
make test
msg "test ssl-opt.sh: !MBEDTLS_SSL_SOME_MODES_USE_MAC"
- if_build_succeeded tests/ssl-opt.sh -f 'Default\|EtM' -e 'without EtM'
+ tests/ssl-opt.sh -f 'Default\|EtM' -e 'without EtM'
}
component_test_null_entropy () {
@@ -2220,7 +2317,7 @@
msg "selftest: malloc(0) returns NULL (ASan+UBSan build)"
# Just the calloc selftest. "make test" ran the others as part of the
# test suites.
- if_build_succeeded programs/test/selftest calloc
+ programs/test/selftest calloc
msg "test ssl-opt.sh: malloc(0) returns NULL (ASan+UBSan build)"
# Run a subset of the tests. The choice is a balance between coverage
@@ -2228,7 +2325,7 @@
# The current choice is to skip tests whose description includes
# "proxy", which is an approximation of skipping tests that use the
# UDP proxy, which tend to be slower and flakier.
- if_build_succeeded tests/ssl-opt.sh -e 'proxy'
+ tests/ssl-opt.sh -e 'proxy'
}
component_test_aes_fewer_tables () {
@@ -2396,7 +2493,7 @@
make test
msg "test ssl-opt.sh, i386, make, gcc-O1"
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
}
support_test_m32_o1 () {
support_test_m32_o0 "$@"
@@ -2412,11 +2509,11 @@
make test
msg "test: i386, Everest ECDH context - ECDH-related part of ssl-opt.sh (ASan build)" # ~ 5s
- if_build_succeeded tests/ssl-opt.sh -f ECDH
+ tests/ssl-opt.sh -f ECDH
msg "test: i386, Everest ECDH context - compat.sh with some ECDH ciphersuites (ASan build)" # ~ 3 min
# Exclude some symmetric ciphers that are redundant here to gain time.
- if_build_succeeded tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4'
+ tests/compat.sh -f ECDH -V NO -e 'ARCFOUR\|ARIA\|CAMELLIA\|CHACHA\|DES\|RC4'
}
support_test_m32_everest () {
support_test_m32_o0 "$@"
@@ -2556,7 +2653,7 @@
scripts/config.py set MBEDTLS_NO_UDBL_DIVISION
make CC="${ARM_NONE_EABI_GCC_PREFIX}gcc" AR="${ARM_NONE_EABI_GCC_PREFIX}ar" LD="${ARM_NONE_EABI_GCC_PREFIX}ld" CFLAGS='-std=c99 -Werror -Wall -Wextra' lib
echo "Checking that software 64-bit division is not required"
- if_build_succeeded not grep __aeabi_uldiv library/*.o
+ not grep __aeabi_uldiv library/*.o
}
component_build_arm_none_eabi_gcc_no_64bit_multiplication () {
@@ -2565,7 +2662,7 @@
scripts/config.py set MBEDTLS_NO_64BIT_MULTIPLICATION
make CC="${ARM_NONE_EABI_GCC_PREFIX}gcc" AR="${ARM_NONE_EABI_GCC_PREFIX}ar" LD="${ARM_NONE_EABI_GCC_PREFIX}ld" CFLAGS='-std=c99 -Werror -O1 -march=armv6-m -mthumb' lib
echo "Checking that software 64-bit multiplication is not required"
- if_build_succeeded not grep __aeabi_lmul library/*.o
+ not grep __aeabi_lmul library/*.o
}
component_build_armcc () {
@@ -2639,13 +2736,13 @@
make test
msg "test: ssl-opt.sh (MSan)" # ~ 1 min
- if_build_succeeded tests/ssl-opt.sh
+ tests/ssl-opt.sh
# Optional part(s)
if [ "$MEMORY" -gt 0 ]; then
msg "test: compat.sh (MSan)" # ~ 6 min 20s
- if_build_succeeded tests/compat.sh
+ tests/compat.sh
fi
}
@@ -2661,17 +2758,17 @@
# seem to receive signals under valgrind on OS X).
if [ "$MEMORY" -gt 0 ]; then
msg "test: ssl-opt.sh --memcheck (Release)"
- if_build_succeeded tests/ssl-opt.sh --memcheck
+ tests/ssl-opt.sh --memcheck
fi
if [ "$MEMORY" -gt 1 ]; then
msg "test: compat.sh --memcheck (Release)"
- if_build_succeeded tests/compat.sh --memcheck
+ tests/compat.sh --memcheck
fi
if [ "$MEMORY" -gt 0 ]; then
msg "test: context-info.sh --memcheck (Release)"
- if_build_succeeded tests/context-info.sh --memcheck
+ tests/context-info.sh --memcheck
fi
}
@@ -2690,15 +2787,13 @@
# "No such file or directory", which would indicate that some required
# file is missing (ssl-opt.sh tolerates the absence of some files so
# may exit with status 0 but emit errors).
- if_build_succeeded ./tests/ssl-opt.sh -f 'Fallback SCSV: beginning of list' 2>ssl-opt.err
- if [ -s ssl-opt.err ]; then
- cat ssl-opt.err >&2
- record_status [ ! -s ssl-opt.err ]
- rm ssl-opt.err
- fi
+ ./tests/ssl-opt.sh -f 'Fallback SCSV: beginning of list' 2>ssl-opt.err
+ cat ssl-opt.err >&2
+ # If ssl-opt.err is non-empty, record an error and keep going.
+ [ ! -s ssl-opt.err ]
+ rm ssl-opt.err
cd "$MBEDTLS_ROOT_DIR"
rm -rf "$OUT_OF_SOURCE_DIR"
- unset MBEDTLS_ROOT_DIR
}
component_test_cmake_as_subdirectory () {
@@ -2708,7 +2803,7 @@
cd programs/test/cmake_subproject
cmake .
make
- if_build_succeeded ./cmake_subproject
+ ./cmake_subproject
cd "$MBEDTLS_ROOT_DIR"
unset MBEDTLS_ROOT_DIR
@@ -2733,9 +2828,9 @@
for compiler in clang gcc; do
msg "test: $compiler $optimization_flag, mbedtls_platform_zeroize()"
make programs CC="$compiler" DEBUG=1 CFLAGS="$optimization_flag"
- if_build_succeeded gdb -ex "$gdb_disable_aslr" -x tests/scripts/test_zeroize.gdb -nw -batch -nx 2>&1 | tee test_zeroize.log
- if_build_succeeded grep "The buffer was correctly zeroized" test_zeroize.log
- if_build_succeeded not grep -i "error" test_zeroize.log
+ gdb -ex "$gdb_disable_aslr" -x tests/scripts/test_zeroize.gdb -nw -batch -nx 2>&1 | tee test_zeroize.log
+ grep "The buffer was correctly zeroized" test_zeroize.log
+ not grep -i "error" test_zeroize.log
rm -f test_zeroize.log
make clean
done
@@ -2746,7 +2841,7 @@
component_check_python_files () {
msg "Lint: Python scripts"
- record_status tests/scripts/check-python-files.sh
+ tests/scripts/check-python-files.sh
}
component_check_generate_test_code () {
@@ -2754,7 +2849,7 @@
# unittest writes out mundane stuff like number or tests run on stderr.
# Our convention is to reserve stderr for actual errors, and write
# harmless info on stdout so it can be suppress with --quiet.
- record_status ./tests/scripts/test_generate_test_code.py 2>&1
+ ./tests/scripts/test_generate_test_code.py 2>&1
}
################################################################
@@ -2763,7 +2858,7 @@
post_report () {
msg "Done, cleaning up"
- cleanup
+ final_cleanup
final_report
}
@@ -2774,26 +2869,71 @@
#### Run all the things
################################################################
+# Function invoked by --error-test to test error reporting.
+pseudo_component_error_test () {
+ msg "Testing error reporting $error_test_i"
+ if [ $KEEP_GOING -ne 0 ]; then
+ echo "Expect three failing commands."
+ fi
+ # If the component doesn't run in a subshell, changing error_test_i to an
+ # invalid integer will cause an error in the loop that runs this function.
+ error_test_i=this_should_not_be_used_since_the_component_runs_in_a_subshell
+ # Expected error: 'grep non_existent /dev/null -> 1'
+ grep non_existent /dev/null
+ # Expected error: '! grep -q . tests/scripts/all.sh -> 1'
+ not grep -q . "$0"
+ # Expected error: 'make unknown_target -> 2'
+ make unknown_target
+ false "this should not be executed"
+}
+
# Run one component and clean up afterwards.
run_component () {
- # Back up the configuration in case the component modifies it.
- # The cleanup function will restore it.
- cp -p "$CONFIG_H" "$CONFIG_BAK"
- cp -p "$CRYPTO_CONFIG_H" "$CRYPTO_CONFIG_BAK"
current_component="$1"
export MBEDTLS_TEST_CONFIGURATION="$current_component"
# Unconditionally create a seedfile that's sufficiently long.
# Do this before each component, because a previous component may
# have messed it up or shortened it.
- redirect_err dd if=/dev/urandom of=./tests/seedfile bs=64 count=1
+ local dd_cmd
+ dd_cmd=(dd if=/dev/urandom of=./tests/seedfile bs=64 count=1)
+ case $OSTYPE in
+ linux*|freebsd*|openbsd*|darwin*) dd_cmd+=(status=none)
+ esac
+ "${dd_cmd[@]}"
- # Run the component code.
- if [ $QUIET -eq 1 ]; then
- # msg() is silenced, so just print the component name here
- echo "${current_component#component_}"
+ # Run the component in a subshell, with error trapping and output
+ # redirection set up based on the relevant options.
+ if [ $KEEP_GOING -eq 1 ]; then
+ # We want to keep running if the subshell fails, so 'set -e' must
+ # be off when the subshell runs.
+ set +e
fi
- redirect_out "$@"
+ (
+ if [ $QUIET -eq 1 ]; then
+ # msg() will be silenced, so just print the component name here.
+ echo "${current_component#component_}"
+ exec >/dev/null
+ fi
+ if [ $KEEP_GOING -eq 1 ]; then
+ # Keep "set -e" off, and run an ERR trap instead to record failures.
+ set -E
+ trap err_trap ERR
+ fi
+ # The next line is what runs the component
+ "$@"
+ if [ $KEEP_GOING -eq 1 ]; then
+ trap - ERR
+ exit $last_failure_status
+ fi
+ )
+ component_status=$?
+ if [ $KEEP_GOING -eq 1 ]; then
+ set -e
+ if [ $component_status -ne 0 ]; then
+ failure_count=$((failure_count + 1))
+ fi
+ fi
# Restore the build tree to a clean state.
cleanup
@@ -2806,22 +2946,23 @@
pre_parse_command_line "$@"
pre_check_git
+pre_restore_files
+pre_back_up
build_status=0
if [ $KEEP_GOING -eq 1 ]; then
pre_setup_keep_going
-else
- record_status () {
- "$@"
- }
fi
-pre_setup_quiet_redirect
pre_prepare_outcome_file
pre_print_configuration
pre_check_tools
cleanup
# Run the requested tests.
+for ((error_test_i=1; error_test_i <= error_test; error_test_i++)); do
+ run_component pseudo_component_error_test
+done
+unset error_test_i
for component in $RUN_COMPONENTS; do
run_component "component_$component"
done