Add more LAVA fvp tests, fix regex and sanitise job timeouts
Add more FVP test jobs to increase code coverage
I had taken the old LAVA tests from MPS2 and as it is trying to match colour codes, it meant some different coloured tests were being missed. Remove the colour codes from the regex.
Shorten the MPS2 job timeouts as the jobs are much shorter, and increase the FVP timeouts as some of the regression tests take a bit longer than expected.
Change-Id: I528a261593ee5b15019b8ee44269cf3ce07e1177
diff --git a/build_helper/build_helper_configs.py b/build_helper/build_helper_configs.py
index d56c4c3..ba6e056 100755
--- a/build_helper/build_helper_configs.py
+++ b/build_helper/build_helper_configs.py
@@ -594,11 +594,11 @@
config_lava_debug = {
"seed_params": {
- "target_platform": ["AN521"],
+ "target_platform": ["AN521", "AN519"],
"compiler": ["GNUARM"],
- "proj_config": ["ConfigCoreIPC", "ConfigCoreIPCTfmLevel2", "ConfigRegression"],
+ "proj_config": ["ConfigRegressionIPC", "ConfigRegressionIPCTfmLevel2", "ConfigRegression"],
"cmake_build_type": ["Release"],
- "with_mcuboot": [True],
+ "with_mcuboot": [True, False],
},
"common_params": _common_tfm_builder_cfg,
"invalid": [
diff --git a/jenkins/lava-submit.jpl b/jenkins/lava-submit.jpl
index 73d5b78..4be24b5 100644
--- a/jenkins/lava-submit.jpl
+++ b/jenkins/lava-submit.jpl
@@ -17,14 +17,19 @@
withCredentials([usernamePassword(credentialsId: 'LAVA_CREDENTIALS', passwordVariable: 'LAVA_TOKEN', usernameVariable: 'LAVA_USER')]) {
print("Generating LAVA jobs...")
def bl2_string = ""
+ def psa_string = ""
if (env.BL2.equals("True")) {
bl2_string = "--bl2"
}
+ // work around this string containing quotes?
+ if (!env.PSA_API_SUITE.replace('\'', '').isEmpty()) {
+ psa_string = "--psa-api-suite ${env.PSA_API_SUITE}"
+ }
dir("tf-m-ci-scripts") {
def res = sh(script: """./lava_helper/lava_create_jobs.py \
--build-number ${env.BUILD_NUMBER} --output-dir lava_jobs \
--compiler ${env.COMPILER} --platform ${env.TARGET_PLATFORM} \
-${bl2_string} --build-type ${env.CMAKE_BUILD_TYPE} \
+${bl2_string} ${psa_string} --build-type ${env.CMAKE_BUILD_TYPE} \
--jenkins-build-url ${env.BUILD_URL} --proj-config ${env.PROJ_CONFIG} \
--docker-prefix ${env.DOCKER_PREFIX} --license-variable "${env.LICENSE_VARIABLE}"
""", returnStdout: true).trim()
diff --git a/lava_helper/jinja2_templates/template_tfm_mps2_fvp.jinja2 b/lava_helper/jinja2_templates/fvp_mps2.jinja2
similarity index 94%
rename from lava_helper/jinja2_templates/template_tfm_mps2_fvp.jinja2
rename to lava_helper/jinja2_templates/fvp_mps2.jinja2
index c90a0e1..e2b70eb 100644
--- a/lava_helper/jinja2_templates/template_tfm_mps2_fvp.jinja2
+++ b/lava_helper/jinja2_templates/fvp_mps2.jinja2
@@ -42,8 +42,8 @@
- "--application cpu0={NS}"
- "--data cpu0={S}@{{ data_bin_offset }}"
- "--parameter fvp_mps2.platform_type=2"
- - "--parameter cpu0.baseline=0"
- - "--parameter cpu0.INITVTOR_S=0x10000000"
+ - "--parameter cpu0.baseline={{ cpu0_baseline }}"
+ - "--parameter cpu0.INITVTOR_S={{ cpu0_initvtor_s }}"
- "--parameter cpu0.semihosting-enable=0"
- "--parameter fvp_mps2.DISABLE_GATING=0"
- "--parameter fvp_mps2.telnetterminal0.start_telnet=1"
diff --git a/lava_helper/jinja2_templates/template_tfm_mps2_sse_200.jinja2 b/lava_helper/jinja2_templates/mps2.jinja2
similarity index 100%
rename from lava_helper/jinja2_templates/template_tfm_mps2_sse_200.jinja2
rename to lava_helper/jinja2_templates/mps2.jinja2
diff --git a/lava_helper/lava_create_jobs.py b/lava_helper/lava_create_jobs.py
index 24bf487..c4f9279 100755
--- a/lava_helper/lava_create_jobs.py
+++ b/lava_helper/lava_create_jobs.py
@@ -137,6 +137,8 @@
"docker_prefix": vars(user_args).get('docker_prefix', ''),
"license_variable": vars(user_args).get('license_variable', ''),
"build_job_url": artifact_store_url,
+ "cpu0_baseline": config.get("cpu0_baseline", 0),
+ "cpu0_initvtor_s": config.get("cpu0_initvtor_s", "0x10000000")
}
params.update(
{
@@ -179,7 +181,6 @@
# Only test this platform
platform = user_args.platform
config["platforms"] = {platform: config["platforms"][platform]}
-
# Generate the ouptut definition
definitions = generate_test_definitions(config, work_dir, user_args)
@@ -269,6 +270,9 @@
"--license-variable", dest="license_variable", action="store", help="License string for Fastmodels"
)
cmdargs.add_argument("--bl2", dest="bl2", action="store_true", help="BL2")
+ cmdargs.add_argument(
+ "--psa-api-suite", dest="psa_suite", action="store", help="PSA API Suite name"
+ )
return parser.parse_args()
diff --git a/lava_helper/lava_helper_configs.py b/lava_helper/lava_helper_configs.py
index 86181dd..a3f507f 100644
--- a/lava_helper/lava_helper_configs.py
+++ b/lava_helper/lava_helper_configs.py
@@ -46,20 +46,17 @@
tfm_mps2_sse_200 = {
- "templ": "template_tfm_mps2_sse_200.jinja2",
- "job_name": "mps2plus-arm-tfm",
+ "templ": "mps2.jinja2",
+ "job_name": "mps2_an521_bl2",
"device_type": "mps",
- "job_timeout": 180,
- "action_timeout": 90,
- "monitor_timeout": 90,
- "poweroff_timeout": 5,
- "recovery_store_url": "%(jenkins_url)s/"
- "job/%(jenkins_job)s",
- "artifact_store_url": "%(jenkins_url)s/"
- "job/%(jenkins_job)s",
- "platforms": {"AN521": "mps2_an521_v3.0.tar.gz"},
- "compilers": ["GNUARM"],
- "build_types": ["Debug", "Release"],
+ "job_timeout": 15,
+ "action_timeout": 10,
+ "monitor_timeout": 10,
+ "poweroff_timeout": 1,
+ "recovery_store_url": "https://ci.trustedfirmware.org/userContent/",
+ "platforms": {"AN521": "mps2_sse200_an512.tar.gz"},
+ "compilers": ["GNUARM", "ARMCLANG"],
+ "build_types": ["Debug", "Release", "Minsizerel"],
"boot_types": ["BL2"],
"tests": {
'Default': {
@@ -90,8 +87,8 @@
'name': 'Secure_Test_Suites_Summary',
'start': 'Secure test suites summary',
'end': 'End of Secure test suites',
- 'pattern': r"[\x1b]\\[37mTest suite '(?P<"
- r"test_case_id>[^\n]+)' has [\x1b]\\[32m "
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
r"(?P<result>PASSED|FAILED)",
'fixup': {"pass": "PASSED", "fail": "FAILED"},
'required': [
@@ -113,8 +110,114 @@
'name': 'Non_Secure_Test_Suites_Summary',
'start': 'Non-secure test suites summary',
'end': r'End of Non-secure test suites',
- 'pattern': r"[\x1b]\\[37mTest suite '(?P"
- r"<test_case_id>[^\n]+)' has [\x1b]\\[32m "
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # Regression
+ 'RegressionIPC': {
+ "binaries": {
+ "firmware": "tfm_sign.bin",
+ "bootloader": "mcuboot.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # Regression
+ 'RegressionIPCTfmLevel2': {
+ "binaries": {
+ "firmware": "tfm_sign.bin",
+ "bootloader": "mcuboot.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
r"(?P<result>PASSED|FAILED)",
'fixup': {"pass": "PASSED", "fail": "FAILED"},
'required': [
@@ -173,13 +276,13 @@
}
-tfm_mps2_fvp_bl2 = {
- "templ": "template_tfm_mps2_fvp.jinja2",
- "job_name": "mps2plus-arm-tfm-fvp",
+fvp_mps2_an521_bl2 = {
+ "templ": "fvp_mps2.jinja2",
+ "job_name": "fvp_mps2_an521_bl2",
"device_type": "fvp",
- "job_timeout": 5,
- "action_timeout": 2,
- "monitor_timeout": 2,
+ "job_timeout": 15,
+ "action_timeout": 10,
+ "monitor_timeout": 10,
"poweroff_timeout": 1,
"recovery_store_url": "%(jenkins_url)s/"
"job/%(jenkins_job)s",
@@ -187,7 +290,7 @@
"job/%(jenkins_job)s",
"platforms": {"AN521": "mps2_an521_v3.0.tar.gz"},
"compilers": ["GNUARM", "ARMCLANG"],
- "build_types": ["Debug", "Release"],
+ "build_types": ["Debug", "Release", "Minsizerel"],
"boot_types": ["BL2"],
"data_bin_offset": "0x10080000",
"tests": {
@@ -219,8 +322,8 @@
'name': 'Secure_Test_Suites_Summary',
'start': 'Secure test suites summary',
'end': 'End of Secure test suites',
- 'pattern': r"[\x1b]\\[37mTest suite '(?P<"
- r"test_case_id>[^\n]+)' has [\x1b]\\[32m "
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
r"(?P<result>PASSED|FAILED)",
'fixup': {"pass": "PASSED", "fail": "FAILED"},
'required': [
@@ -242,8 +345,114 @@
'name': 'Non_Secure_Test_Suites_Summary',
'start': 'Non-secure test suites summary',
'end': r'End of Non-secure test suites',
- 'pattern': r"[\x1b]\\[37mTest suite '(?P"
- r"<test_case_id>[^\n]+)' has [\x1b]\\[32m "
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # Regression
+ 'RegressionIPC': {
+ "binaries": {
+ "firmware": "mcuboot.axf",
+ "bootloader": "tfm_s_ns_signed.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # Regression
+ 'RegressionIPCTfmLevel2': {
+ "binaries": {
+ "firmware": "mcuboot.axf",
+ "bootloader": "tfm_s_ns_signed.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
r"(?P<result>PASSED|FAILED)",
'fixup': {"pass": "PASSED", "fail": "FAILED"},
'required': [
@@ -302,13 +511,13 @@
}
-tfm_mps2_fvp_nobl2 = {
- "templ": "template_tfm_mps2_fvp.jinja2",
- "job_name": "mps2plus-arm-tfm-fvp",
+fvp_mps2_an521_nobl2 = {
+ "templ": "fvp_mps2.jinja2",
+ "job_name": "fvp_mps2_an521_nobl2",
"device_type": "fvp",
- "job_timeout": 5,
- "action_timeout": 2,
- "monitor_timeout": 2,
+ "job_timeout": 15,
+ "action_timeout": 10,
+ "monitor_timeout": 10,
"poweroff_timeout": 1,
"recovery_store_url": "%(jenkins_url)s/"
"job/%(jenkins_job)s",
@@ -316,7 +525,7 @@
"job/%(jenkins_job)s",
"platforms": {"AN521": "mps2_an521_v3.0.tar.gz"},
"compilers": ["GNUARM", "ARMCLANG"],
- "build_types": ["Debug", "Release"],
+ "build_types": ["Debug", "Release", "Minsizerel"],
"boot_types": ["NOBL2"],
"data_bin_offset": "0x00100000",
"tests": {
@@ -348,8 +557,8 @@
'name': 'Secure_Test_Suites_Summary',
'start': 'Secure test suites summary',
'end': 'End of Secure test suites',
- 'pattern': r"[\x1b]\\[37mTest suite '(?P<"
- r"test_case_id>[^\n]+)' has [\x1b]\\[32m "
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
r"(?P<result>PASSED|FAILED)",
'fixup': {"pass": "PASSED", "fail": "FAILED"},
'required': [
@@ -371,8 +580,8 @@
'name': 'Non_Secure_Test_Suites_Summary',
'start': 'Non-secure test suites summary',
'end': r'End of Non-secure test suites',
- 'pattern': r"[\x1b]\\[37mTest suite '(?P"
- r"<test_case_id>[^\n]+)' has [\x1b]\\[32m "
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
r"(?P<result>PASSED|FAILED)",
'fixup': {"pass": "PASSED", "fail": "FAILED"},
'required': [
@@ -391,6 +600,112 @@
}
] # Monitors
}, # Regression
+ 'RegressionIPC': {
+ "binaries": {
+ "firmware": "tfm_s.axf",
+ "bootloader": "tfm_ns.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # RegressionIPC
+ 'RegressionIPCTfmLevel2': {
+ "binaries": {
+ "firmware": "tfm_s.axf",
+ "bootloader": "tfm_ns.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # RegressionIPCTfmLevel2
'CoreIPC': {
"binaries": {
"firmware": "tfm_s.axf",
@@ -430,10 +745,478 @@
} # Tests
}
+
+fvp_mps2_an519_bl2 = {
+ "templ": "fvp_mps2.jinja2",
+ "job_name": "fvp_mps2_an519_bl2",
+ "device_type": "fvp",
+ "job_timeout": 15,
+ "action_timeout": 10,
+ "monitor_timeout": 10,
+ "poweroff_timeout": 1,
+ "platforms": {"AN519": ""},
+ "compilers": ["GNUARM", "ARMCLANG"],
+ "build_types": ["Debug", "Release", "Minsizerel"],
+ "boot_types": ["BL2"],
+ "data_bin_offset": "0x10080000",
+ "cpu0_baseline": 1,
+ "tests": {
+ 'Default': {
+ "binaries": {
+ "firmware": "mcuboot.axf",
+ "bootloader": "tfm_s_ns_signed.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': r'[Sec Thread]',
+ 'end': r'system starting',
+ 'pattern': r'\x1b\\[1;34m\\[Sec Thread\\] '
+ r'(?P<test_case_id>Secure image '
+ r'initializing)(?P<result>!)',
+ 'fixup': {"pass": "!", "fail": ""},
+ 'required': ["secure_image_initializing"]
+ } # Monitors
+ ]
+ }, # Default
+ 'Regression': {
+ "binaries": {
+ "firmware": "mcuboot.axf",
+ "bootloader": "tfm_s_ns_signed.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # Regression
+ 'RegressionIPC': {
+ "binaries": {
+ "firmware": "mcuboot.axf",
+ "bootloader": "tfm_s_ns_signed.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # Regression
+ 'RegressionIPCTfmLevel2': {
+ "binaries": {
+ "firmware": "mcuboot.axf",
+ "bootloader": "tfm_s_ns_signed.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # Regression
+ 'CoreIPC': {
+ "binaries": {
+ "firmware": "mcuboot.axf",
+ "bootloader": "tfm_s_ns_signed.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': r'[Sec Thread]',
+ 'end': r'system starting',
+ 'pattern': r'\x1b\\[1;34m\\[Sec Thread\\] '
+ r'(?P<test_case_id>Secure image '
+ r'initializing)(?P<result>!)',
+ 'fixup': {"pass": "!", "fail": ""},
+ 'required': ["secure_image_initializing"]
+ } # Monitors
+ ]
+ }, # CoreIPC
+ 'CoreIPCTfmLevel2': {
+ "binaries": {
+ "firmware": "mcuboot.axf",
+ "bootloader": "tfm_s_ns_signed.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': r'[Sec Thread]',
+ 'end': r'system starting',
+ 'pattern': r'\x1b\\[1;34m\\[Sec Thread\\] '
+ r'(?P<test_case_id>Secure image '
+ r'initializing)(?P<result>!)',
+ 'fixup': {"pass": "!", "fail": ""},
+ 'required': ["secure_image_initializing"]
+ } # Monitors
+ ]
+ }, # CoreIPCTfmLevel2
+ } # Tests
+}
+
+
+fvp_mps2_an519_nobl2 = {
+ "templ": "fvp_mps2.jinja2",
+ "job_name": "fvp_mps2_an519_nobl2",
+ "device_type": "fvp",
+ "job_timeout": 15,
+ "action_timeout": 10,
+ "monitor_timeout": 10,
+ "poweroff_timeout": 1,
+ "platforms": {"AN519": ""},
+ "compilers": ["GNUARM", "ARMCLANG"],
+ "build_types": ["Debug", "Release", "Minsizerel"],
+ "boot_types": ["NOBL2"],
+ "data_bin_offset": "0x00100000",
+ "cpu0_baseline": 1,
+ "tests": {
+ 'Default': {
+ "binaries": {
+ "firmware": "tfm_s.axf",
+ "bootloader": "tfm_ns.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': r'[Sec Thread]',
+ 'end': r'system starting',
+ 'pattern': r'\x1b\\[1;34m\\[Sec Thread\\] '
+ r'(?P<test_case_id>Secure image '
+ r'initializing)(?P<result>!)',
+ 'fixup': {"pass": "!", "fail": ""},
+ 'required': ["secure_image_initializing"]
+ }
+ ]
+ }, # Default
+ 'Regression': {
+ "binaries": {
+ "firmware": "tfm_s.axf",
+ "bootloader": "tfm_ns.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # Regression
+ 'RegressionIPC': {
+ "binaries": {
+ "firmware": "tfm_s.axf",
+ "bootloader": "tfm_ns.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # RegressionIPC
+ 'RegressionIPCTfmLevel2': {
+ "binaries": {
+ "firmware": "tfm_s.axf",
+ "bootloader": "tfm_ns.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': 'Secure test suites summary',
+ 'end': 'End of Secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage_"
+ "s_interface_tests_tfm_sst_test_2xxx_"),
+ "sst_reliability_tests_tfm_sst_test_3xxx_",
+ "sst_rollback_protection_tests_tfm_sst_test_4xxx_",
+ ("psa_internal_trusted_storage_"
+ "s_interface_tests_tfm_its_test_2xxx_"),
+ "its_reliability_tests_tfm_its_test_3xxx_",
+ ("audit_"
+ "logging_secure_interface_test_tfm_audit_test_1xxx_"),
+ "crypto_secure_interface_tests_tfm_crypto_test_5xxx_",
+ ("initial_attestation_service_"
+ "secure_interface_tests_tfm_attest_test_1xxx_"),
+ ]
+ },
+ {
+ 'name': 'Non_Secure_Test_Suites_Summary',
+ 'start': 'Non-secure test suites summary',
+ 'end': r'End of Non-secure test suites',
+ 'pattern': r"Test suite '(?P<"
+ r"test_case_id>[^\n]+)' has (.*) "
+ r"(?P<result>PASSED|FAILED)",
+ 'fixup': {"pass": "PASSED", "fail": "FAILED"},
+ 'required': [
+ ("psa_protected_storage"
+ "_ns_interface_tests_tfm_sst_test_1xxx_"),
+ ("psa_internal_trusted_storage"
+ "_ns_interface_tests_tfm_its_test_1xxx_"),
+ ("auditlog_"
+ "non_secure_interface_test_tfm_audit_test_1xxx_"),
+ ("crypto_"
+ "non_secure_interface_test_tfm_crypto_test_6xxx_"),
+ ("initial_attestation_service_"
+ "non_secure_interface_tests_tfm_attest_test_2xxx_"),
+ "core_non_secure_positive_tests_tfm_core_test_1xxx_"
+ ]
+ }
+ ] # Monitors
+ }, # RegressionIPCTfmLevel2
+ 'CoreIPC': {
+ "binaries": {
+ "firmware": "tfm_s.axf",
+ "bootloader": "tfm_ns.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': r'[Sec Thread]',
+ 'end': r'system starting',
+ 'pattern': r'\x1b\\[1;34m\\[Sec Thread\\] '
+ r'(?P<test_case_id>Secure image '
+ r'initializing)(?P<result>!)',
+ 'fixup': {"pass": "!", "fail": ""},
+ 'required': ["secure_image_initializing"]
+ } # Monitors
+ ]
+ }, # CoreIPC
+ 'CoreIPCTfmLevel2': {
+ "binaries": {
+ "firmware": "tfm_s.axf",
+ "bootloader": "tfm_ns.bin"
+ },
+ "monitors": [
+ {
+ 'name': 'Secure_Test_Suites_Summary',
+ 'start': r'[Sec Thread]',
+ 'end': r'system starting',
+ 'pattern': r'\x1b\\[1;34m\\[Sec Thread\\] '
+ r'(?P<test_case_id>Secure image '
+ r'initializing)(?P<result>!)',
+ 'fixup': {"pass": "!", "fail": ""},
+ 'required': ["secure_image_initializing"]
+ } # Monitors
+ ]
+ }, # CoreIPCTfmLevel2
+ } # Tests
+}
+
+
# All configurations should be mapped here
-lava_gen_config_map = {"tfm_mps2_sse_200": tfm_mps2_sse_200,
- "tfm_mps2_fvp_bl2": tfm_mps2_fvp_bl2,
- "tfm_mps2_fvp_nobl2": tfm_mps2_fvp_nobl2}
+lava_gen_config_map = {"mps2_an521_bl2": tfm_mps2_sse_200,
+ "fvp_mps2_an521_bl2": fvp_mps2_an521_bl2,
+ "fvp_mps2_an521_nobl2": fvp_mps2_an521_nobl2,
+ "fvp_mps2_an519_bl2": fvp_mps2_an519_bl2,
+ "fvp_mps2_an519_nobl2": fvp_mps2_an519_nobl2}
+
lavagen_config_sort_order = [
"templ",
"job_name",
diff --git a/tfm_ci_pylib/lava_rpc_connector.py b/tfm_ci_pylib/lava_rpc_connector.py
index 5db23b1..a0255d8 100644
--- a/tfm_ci_pylib/lava_rpc_connector.py
+++ b/tfm_ci_pylib/lava_rpc_connector.py
@@ -118,11 +118,14 @@
return job_info
def get_error_reason(self, job_id):
- lava_res = self.results.get_testsuite_results_yaml(job_id, 'lava')
- results = yaml.load(lava_res)
- for test in results:
- if test['name'] == 'job':
- return(test.get('metadata', {}).get('error_type', ''))
+ try:
+ lava_res = self.results.get_testsuite_results_yaml(job_id, 'lava')
+ results = yaml.load(lava_res)
+ for test in results:
+ if test['name'] == 'job':
+ return(test.get('metadata', {}).get('error_type', ''))
+ except Exception:
+ return("Unknown")
def get_job_state(self, job_id):
return self.scheduler.job_state(job_id)["job_state"]