Add support for marking tests long-running
Tests such as vcpu_state.concurrent_save_restore run for a long
period of time in the hope that they will catch an issue which does
not manifest deterministically. This patch adds a property to the
hftest JSON indicating that the test requires a longer time limit.
To aid development, hftest.py will skip such tests when the
environment contains HAFNIUM_SKIP_LONG_RUNNING_TESTS=true (default for
local builds). 'SKIP <test_name>' is printed to inform the user that
a test was skipped.
Change-Id: I2f1b8c36f5aa7df30ac964d6c1bc11f0d82e727d
diff --git a/build/run_in_container.sh b/build/run_in_container.sh
index ae0850e..cfb8629 100755
--- a/build/run_in_container.sh
+++ b/build/run_in_container.sh
@@ -81,6 +81,21 @@
echo "WARNING: Docker seccomp profile is disabled!" 1>&2
ARGS+=(--cap-add=SYS_PTRACE --security-opt seccomp=unconfined)
fi
+# Propagate "HAFNIUM_*" environment variables.
+# Note: Cannot use `env | while` because the loop would run inside a child
+# process and would not have any effect on variables in the parent.
+while read -r ENV_LINE
+do
+ VAR_NAME="$(echo ${ENV_LINE} | cut -d= -f1)"
+ case "${VAR_NAME}" in
+ HAFNIUM_HERMETIC_BUILD)
+ # Skip this one. It will be overridden below.
+ ;;
+ HAFNIUM_*)
+ ARGS+=(-e "${ENV_LINE}")
+ ;;
+ esac
+done <<< "$(env)"
# Set environment variable informing the build that we are running inside
# a container.
ARGS+=(-e HAFNIUM_HERMETIC_BUILD=inside)
diff --git a/kokoro/ubuntu/build.sh b/kokoro/ubuntu/build.sh
index 0214a1f..4277d17 100755
--- a/kokoro/ubuntu/build.sh
+++ b/kokoro/ubuntu/build.sh
@@ -41,42 +41,52 @@
return $?
}
-# Default value of HAFNIUM_HERMETIC_BUILD is "true" for Kokoro builds.
-if [ -v KOKORO_JOB_NAME -a ! -v HAFNIUM_HERMETIC_BUILD ]
+# Assigns value (second arg) of a variable (first arg) if it is not set already.
+function default_value {
+ local var_name=$1
+ local value=$2
+ export ${var_name}=${!var_name:-${value}}
+}
+
+# Assign default values to variables.
+if [ -v KOKORO_JOB_NAME ]
then
- HAFNIUM_HERMETIC_BUILD=true
+ # Default config for Kokoro builds.
+ default_value HAFNIUM_HERMETIC_BUILD true
+ default_value HAFNIUM_SKIP_LONG_RUNNING_TESTS false
+else
+ # Default config for local builds.
+ default_value HAFNIUM_HERMETIC_BUILD false
+ default_value HAFNIUM_SKIP_LONG_RUNNING_TESTS true
fi
-# If HAFNIUM_HERMETIC_BUILD is "true" (not default), relaunch this script inside
-# a container. The 'run_in_container.sh' script will set the variable value to
-# 'inside' to avoid recursion.
-if [ "${HAFNIUM_HERMETIC_BUILD:-}" == "true" ]
+# If HAFNIUM_HERMETIC_BUILD is "true", relaunch this script inside a container.
+# The 'run_in_container.sh' script will set the variable value to 'inside' to
+# avoid recursion.
+if [ "${HAFNIUM_HERMETIC_BUILD}" == "true" ]
then
exec "${ROOT_DIR}/build/run_in_container.sh" ${SCRIPT_NAME} $@
fi
-USE_FVP=0
+USE_FVP=false
while test $# -gt 0
do
- case "$1" in
- --fvp) USE_FVP=1
- ;;
- *) echo "Unexpected argument $1"
- exit 1
- ;;
- esac
- shift
+ case "$1" in
+ --fvp)
+ USE_FVP=true
+ ;;
+ --skip-long-running-tests)
+ HAFNIUM_SKIP_LONG_RUNNING_TESTS=true
+ ;;
+ *)
+ echo "Unexpected argument $1"
+ exit 1
+ ;;
+ esac
+ shift
done
-# Detect server vs local run. Local run should be from the project's root
-# directory.
-if [ -v KOKORO_JOB_NAME ]
-then
- # Server
- cd git/hafnium
-fi
-
CLANG=${PWD}/prebuilts/linux-x64/clang/bin/clang
# Kokoro does something weird that makes all files look dirty to git diff-index;
@@ -101,12 +111,16 @@
# Step 2: make sure it works.
#
-if [ $USE_FVP == 1 ]
+TEST_ARGS=()
+if [ $USE_FVP == true ]
then
- ./kokoro/ubuntu/test.sh --fvp
-else
- ./kokoro/ubuntu/test.sh
+ TEST_ARGS+=(--fvp)
fi
+if [ "${HAFNIUM_SKIP_LONG_RUNNING_TESTS}" == "true" ]
+then
+ TEST_ARGS+=(--skip-long-running-tests)
+fi
+./kokoro/ubuntu/test.sh ${TEST_ARGS[@]}
#
# Step 3: static analysis.
diff --git a/kokoro/ubuntu/test.sh b/kokoro/ubuntu/test.sh
index 632d4f4..80e5b6f 100755
--- a/kokoro/ubuntu/test.sh
+++ b/kokoro/ubuntu/test.sh
@@ -26,12 +26,15 @@
# Display commands being run.
set -x
-USE_FVP=0
+USE_FVP=false
+SKIP_LONG_RUNNING_TESTS=false
while test $# -gt 0
do
case "$1" in
- --fvp) USE_FVP=1
+ --fvp) USE_FVP=true
+ ;;
+ --skip-long-running-tests) SKIP_LONG_RUNNING_TESTS=true
;;
*) echo "Unexpected argument $1"
exit 1
@@ -40,30 +43,38 @@
shift
done
-TIMEOUT="timeout --foreground"
+TIMEOUT=(timeout --foreground)
PROJECT="${PROJECT:-reference}"
OUT="out/${PROJECT}"
# Run the tests with a timeout so they can't loop forever.
-if [ $USE_FVP == 1 ]
+HFTEST=(${TIMEOUT[@]} 300s ./test/hftest/hftest.py --log "$OUT/kokoro_log")
+if [ $USE_FVP == true ]
then
- HFTEST="$TIMEOUT 300s ./test/hftest/hftest.py --fvp=true --out $OUT/aem_v8a_fvp_clang --out_initrd $OUT/aem_v8a_fvp_vm_clang --log $OUT/kokoro_log"
+ HFTEST+=(--fvp)
+ HFTEST+=(--out "$OUT/aem_v8a_fvp_clang")
+ HFTEST+=(--out_initrd "$OUT/aem_v8a_fvp_vm_clang")
else
- HFTEST="$TIMEOUT 30s ./test/hftest/hftest.py --out $OUT/qemu_aarch64_clang --out_initrd $OUT/qemu_aarch64_vm_clang --log $OUT/kokoro_log"
+ HFTEST+=(--out "$OUT/qemu_aarch64_clang")
+ HFTEST+=(--out_initrd "$OUT/qemu_aarch64_vm_clang")
+fi
+if [ $SKIP_LONG_RUNNING_TESTS == true ]
+then
+ HFTEST+=(--skip-long-running-tests)
fi
# Add prebuilt libc++ to the path.
-export LD_LIBRARY_PATH=$PWD/prebuilts/linux-x64/clang/lib64
+export LD_LIBRARY_PATH="$PWD/prebuilts/linux-x64/clang/lib64"
# Run the host unit tests.
-mkdir -p $OUT/kokoro_log/unit_tests
-$TIMEOUT 30s $OUT/host_fake_clang/unit_tests \
+mkdir -p "$OUT/kokoro_log/unit_tests"
+${TIMEOUT[@]} 30s "$OUT/host_fake_clang/unit_tests" \
--gtest_output="xml:$OUT/kokoro_log/unit_tests/sponge_log.xml" \
- | tee $OUT/kokoro_log/unit_tests/sponge_log.log
+ | tee "$OUT/kokoro_log/unit_tests/sponge_log.log"
-$HFTEST arch_test
-$HFTEST hafnium --initrd test/vmapi/arch/aarch64/aarch64_test
-$HFTEST hafnium --initrd test/vmapi/arch/aarch64/gicv3/gicv3_test
-$HFTEST hafnium --initrd test/vmapi/primary_only/primary_only_test
-$HFTEST hafnium --initrd test/vmapi/primary_with_secondaries/primary_with_secondaries_test
-$HFTEST hafnium --initrd test/linux/linux_test --vm_args "rdinit=/test_binary --"
+${HFTEST[@]} arch_test
+${HFTEST[@]} hafnium --initrd test/vmapi/arch/aarch64/aarch64_test
+${HFTEST[@]} hafnium --initrd test/vmapi/arch/aarch64/gicv3/gicv3_test
+${HFTEST[@]} hafnium --initrd test/vmapi/primary_only/primary_only_test
+${HFTEST[@]} hafnium --initrd test/vmapi/primary_with_secondaries/primary_with_secondaries_test
+${HFTEST[@]} hafnium --initrd test/linux/linux_test --vm_args "rdinit=/test_binary --"
diff --git a/test/hftest/common.c b/test/hftest/common.c
index 81fc516..cc2062d 100644
--- a/test/hftest/common.c
+++ b/test/hftest/common.c
@@ -109,8 +109,11 @@
* It's easier to put the comma at the start of the line
* than the end even though the JSON looks a bit funky.
*/
- HFTEST_LOG(" %c\"%s\"",
- tests_in_suite ? ',' : ' ', test->name);
+ HFTEST_LOG(" %c{", tests_in_suite ? ',' : ' ');
+ HFTEST_LOG(" \"name\": \"%s\",", test->name);
+ HFTEST_LOG(" \"is_long_running\": %s",
+ test->is_long_running ? "true" : "false");
+ HFTEST_LOG(" }");
++tests_in_suite;
}
}
diff --git a/test/hftest/hftest.py b/test/hftest/hftest.py
index 1da311b..4ee0709 100755
--- a/test/hftest/hftest.py
+++ b/test/hftest/hftest.py
@@ -197,10 +197,12 @@
def __init__(self, args):
Driver.__init__(self, args)
- def gen_exec_args(self, test_args, dtb_path=None, dumpdtb_path=None):
+ def gen_exec_args(self, test_args, is_long_running, dtb_path=None,
+ dumpdtb_path=None):
"""Generate command line arguments for QEMU."""
+ time_limit = "120s" if is_long_running else "10s"
exec_args = [
- "timeout", "--foreground", "10s",
+ "timeout", "--foreground", time_limit,
"./prebuilts/linux-x64/qemu/qemu-system-aarch64",
"-machine", "virt,virtualization=on,gic_version=3",
"-cpu", "cortex-a57", "-smp", "4", "-m", "64M",
@@ -224,10 +226,10 @@
return exec_args
def dump_dtb(self, run_state, test_args, path):
- dumpdtb_args = self.gen_exec_args(test_args, dumpdtb_path=path)
+ dumpdtb_args = self.gen_exec_args(test_args, False, dumpdtb_path=path)
self.exec_logged(run_state, dumpdtb_args)
- def run(self, run_name, test_args):
+ def run(self, run_name, test_args, is_long_running):
"""Run test given by `test_args` in QEMU."""
run_state = self.start_run(run_name)
@@ -244,7 +246,8 @@
run_state, base_dtb_path, self.args.manifest, dtb_path)
# Execute test in QEMU..
- exec_args = self.gen_exec_args(test_args, dtb_path=dtb_path)
+ exec_args = self.gen_exec_args(test_args, is_long_running,
+ dtb_path=dtb_path)
self.exec_logged(run_state, exec_args)
except DriverRunException:
pass
@@ -319,7 +322,7 @@
return fvp_args
- def run(self, run_name, test_args):
+ def run(self, run_name, test_args, is_long_running):
run_state = self.start_run(run_name)
base_dts_path = self.args.artifacts.create_file(run_name, ".base.dts")
@@ -374,10 +377,12 @@
"""Class which communicates with a test platform to obtain a list of
available tests and driving their execution."""
- def __init__(self, artifacts, driver, image_name, suite_regex, test_regex):
+ def __init__(self, artifacts, driver, image_name, suite_regex, test_regex,
+ skip_long_running_tests):
self.artifacts = artifacts
self.driver = driver
self.image_name = image_name
+ self.skip_long_running_tests = skip_long_running_tests
self.suite_re = re.compile(suite_regex or ".*")
self.test_re = re.compile(test_regex or ".*")
@@ -396,7 +401,7 @@
def get_test_json(self):
"""Invoke the test platform and request a JSON of available test and
test suites."""
- out = self.driver.run("json", "json")
+ out = self.driver.run("json", "json", False)
hf_out = "\n".join(self.extract_hftest_lines(out))
try:
return json.loads(hf_out)
@@ -430,19 +435,24 @@
"""Invoke the test platform and request to run a given `test` in given
`suite`. Create a new XML node with results under `suite_xml`.
Test only invoked if it matches the regex given to constructor."""
- if not self.test_re.match(test):
+ if not self.test_re.match(test["name"]):
return TestRunnerResult(tests_run=0, tests_failed=0)
- print(" RUN", test)
- log_name = suite["name"] + "." + test
+ if self.skip_long_running_tests and test["is_long_running"]:
+ print(" SKIP", test["name"])
+ return TestRunnerResult(tests_run=0, tests_failed=0)
+
+ print(" RUN", test["name"])
+ log_name = suite["name"] + "." + test["name"]
test_xml = ET.SubElement(suite_xml, "testcase")
- test_xml.set("name", test)
- test_xml.set("classname", suite['name'])
+ test_xml.set("name", test["name"])
+ test_xml.set("classname", suite["name"])
test_xml.set("status", "run")
out = self.extract_hftest_lines(self.driver.run(
- log_name, "run {} {}".format(suite["name"], test)))
+ log_name, "run {} {}".format(suite["name"], test["name"]),
+ test["is_long_running"]))
if self.is_passed_test(out):
print(" PASS")
@@ -510,7 +520,8 @@
parser.add_argument("--suite")
parser.add_argument("--test")
parser.add_argument("--vm_args")
- parser.add_argument("--fvp", type=bool)
+ parser.add_argument("--fvp", action="store_true")
+ parser.add_argument("--skip-long-running-tests", action="store_true")
args = parser.parse_args()
# Resolve some paths.
@@ -536,7 +547,8 @@
driver = QemuDriver(driver_args)
# Create class which will drive test execution.
- runner = TestRunner(artifacts, driver, image_name, args.suite, args.test)
+ runner = TestRunner(artifacts, driver, image_name, args.suite, args.test,
+ args.skip_long_running_tests)
# Run tests.
runner_result = runner.run_tests()
diff --git a/test/hftest/inc/hftest.h b/test/hftest/inc/hftest.h
index f8d388a..af581e1 100644
--- a/test/hftest/inc/hftest.h
+++ b/test/hftest/inc/hftest.h
@@ -35,7 +35,12 @@
/*
* Define a test as part of a test suite.
*/
-#define TEST(suite, test) HFTEST_TEST(suite, test)
+#define TEST(suite, test) HFTEST_TEST(suite, test, false)
+
+/*
+ * Define a test as part of a test suite and mark it long-running.
+ */
+#define TEST_LONG_RUNNING(suite, test) HFTEST_TEST(suite, test, true)
/*
* Define a test service.
diff --git a/test/hftest/inc/hftest_impl.h b/test/hftest/inc/hftest_impl.h
index 9598b12..f3c9ffa 100644
--- a/test/hftest/inc/hftest_impl.h
+++ b/test/hftest/inc/hftest_impl.h
@@ -103,7 +103,7 @@
} \
static void HFTEST_TEAR_DOWN_FN(suite_name)(void)
-#define HFTEST_TEST(suite_name, test_name) \
+#define HFTEST_TEST(suite_name, test_name, long_running) \
static void HFTEST_TEST_FN(suite_name, test_name)(void); \
const struct hftest_test __attribute__((used)) __attribute__( \
(section(HFTEST_TEST_SECTION(suite_name, test_name)))) \
@@ -111,6 +111,7 @@
.suite = #suite_name, \
.kind = HFTEST_KIND_TEST, \
.name = #test_name, \
+ .is_long_running = long_running, \
.fn = HFTEST_TEST_FN(suite_name, test_name), \
}; \
static void __attribute__((constructor)) \
@@ -166,6 +167,7 @@
const char *suite;
enum hftest_kind kind;
const char *name;
+ bool is_long_running;
hftest_test_fn fn;
};
diff --git a/test/vmapi/primary_with_secondaries/run_race.c b/test/vmapi/primary_with_secondaries/run_race.c
index 3689825..4c10e86 100644
--- a/test/vmapi/primary_with_secondaries/run_race.c
+++ b/test/vmapi/primary_with_secondaries/run_race.c
@@ -78,7 +78,7 @@
* CPUs concurrently. The vCPU checks that the state is ok while it bounces
* between the physical CPUs.
*/
-TEST(vcpu_state, concurrent_save_restore)
+TEST_LONG_RUNNING(vcpu_state, concurrent_save_restore)
{
alignas(4096) static char stack[4096];
static struct mailbox_buffers mb;