| #!/usr/bin/env groovy |
| //------------------------------------------------------------------------------- |
| // Copyright (c) 2020, Arm Limited and Contributors. All rights reserved. |
| // |
| // SPDX-License-Identifier: BSD-3-Clause |
| // |
| //------------------------------------------------------------------------------- |
| |
| @Library('trustedfirmware') _ |
| import org.trustedfirmware.Gerrit |
| import org.trustedfirmware.Summary |
| |
| mapPlatform = ["cypress/psoc64": "psoc64", |
| "mps2/an519": "AN519", |
| "mps2/an521": "AN521", |
| "mps2/an539": "AN539", |
| "mps2/sse-200_aws": "SSE-200_AWS", |
| "mps3/an524": "AN524", |
| "musca_a": "MUSCA_A", |
| "musca_b1": "MUSCA_B1", |
| "musca_s1": "MUSCA_S1"] |
| |
| mapCompiler = ["toolchain_GNUARM.cmake": "GNUARM", |
| "toolchain_ARMCLANG.cmake": "ARMCLANG"] |
| |
| mapBL2 = ["True": "--bl2", |
| "False": ""] |
| |
| mapTestPsaApi = ["OFF": "", |
| "INTERNAL_TRUSTED_STORAGE": "ITS", |
| "PROTECTED_STORAGE": "PS", |
| "CRYPTO": "Crypto", |
| "INITIAL_ATTESTATION": "Attest", |
| "IPC": "FF"] |
| |
| // BL2, NS, PSA_API, ISOLATION_LEVEL, TEST_REG, TEST_PSA_API, PROFILE, CONFIG_NAME |
| mapConfigs = [ |
| ["True", "True", "False", "1", "False", "OFF", "N.A", "Default"], |
| ["True", "True", "True", "1", "False", "OFF", "N.A", "CoreIPC"], |
| ["True", "True", "True", "2", "False", "OFF", "N.A", "CoreIPCTfmLevel2"], |
| ["True", "True", "True", "3", "False", "OFF", "N.A", "CoreIPCTfmLevel3"], |
| ["True", "True", "False", "1", "False", "OFF", "profile_small", "DefaultProfileS"], |
| ["True", "True", "True", "2", "False", "OFF", "profile_medium", "DefaultProfileM"], |
| ["True", "True", "False", "1", "True", "OFF", "N.A", "Regression"], |
| ["True", "True", "True", "1", "True", "OFF", "N.A", "RegressionIPC"], |
| ["True", "True", "True", "2", "True", "OFF", "N.A", "RegressionIPCTfmLevel2"], |
| ["True", "True", "True", "3", "True", "OFF", "N.A", "RegressionIPCTfmLevel3"], |
| ["True", "True", "False", "1", "True", "OFF", "profile_small", "RegressionProfileS"], |
| ["True", "True", "True", "2", "True", "OFF", "profile_medium", "RegressionProfileM"], |
| ["True", "True", "False", "1", "False", "INTERNAL_TRUSTED_STORAGE", "N.A", "PsaApiTest (ITS)"], |
| ["True", "True", "False", "1", "False", "PROTECTED_STORAGE", "N.A", "PsaApiTest (PS)"], |
| ["True", "True", "False", "1", "False", "CRYPTO", "N.A", "PsaApiTest (Crypto)"], |
| ["True", "True", "False", "1", "False", "INITIAL_ATTESTATION", "N.A", "PsaApiTest (Attest)"], |
| ["True", "True", "False", "1", "False", "IPC", "N.A", "PsaApiTest (FF)"], |
| ["True", "True", "True", "1", "False", "INTERNAL_TRUSTED_STORAGE", "N.A", "PsaApiTestIPC (ITS)"], |
| ["True", "True", "True", "1", "False", "PROTECTED_STORAGE", "N.A", "PsaApiTestIPC (PS)"], |
| ["True", "True", "True", "1", "False", "CRYPTO", "N.A", "PsaApiTestIPC (Crypto)"], |
| ["True", "True", "True", "1", "False", "INITIAL_ATTESTATION", "N.A", "PsaApiTestIPC (Attest)"], |
| ["True", "True", "True", "1", "False", "IPC", "N.A", "PsaApiTestIPC (FF)"], |
| ["True", "True", "True", "2", "False", "INTERNAL_TRUSTED_STORAGE", "N.A", "PsaApiTestIPCTfmLevel2 (ITS)"], |
| ["True", "True", "True", "2", "False", "PROTECTED_STORAGE", "N.A", "PsaApiTestIPCTfmLevel2 (PS)"], |
| ["True", "True", "True", "2", "False", "CRYPTO", "N.A", "PsaApiTestIPCTfmLevel2 (Crypto)"], |
| ["True", "True", "True", "2", "False", "INITIAL_ATTESTATION", "N.A", "PsaApiTestIPCTfmLevel2 (Attest)"], |
| ["True", "True", "True", "2", "False", "IPC", "N.A", "PsaApiTestIPCTfmLevel2 (FF)"], |
| ["True", "True", "True", "3", "False", "INTERNAL_TRUSTED_STORAGE", "N.A", "PsaApiTestIPCTfmLevel3 (ITS)"], |
| ["True", "True", "True", "3", "False", "PROTECTED_STORAGE", "N.A", "PsaApiTestIPCTfmLevel3 (PS)"], |
| ["True", "True", "True", "3", "False", "CRYPTO", "N.A", "PsaApiTestIPCTfmLevel3 (Crypto)"], |
| ["True", "True", "True", "3", "False", "INITIAL_ATTESTATION", "N.A", "PsaApiTestIPCTfmLevel3 (Attest)"], |
| ["True", "True", "True", "3", "False", "IPC", "N.A", "PsaApiTestIPCTfmLevel3 (FF)"], |
| ] |
| |
| cfgs = ["Default", "CoreIPC", "CoreIPCTfmLevel2", "CoreIPCTfmLevel3", |
| "Regression", "RegressionIPC", |
| "RegressionIPCTfmLevel2", "RegressionIPCTfmLevel3", |
| "DefaultProfileS", "RegressionProfileS", |
| "DefaultProfileM", "RegressionProfileM", "RegressionProfileM PSOFF", |
| "PsaApiTest (Attest)", "PsaApiTestIPC (Attest)", |
| "PsaApiTestIPCTfmLevel2 (Attest)", |
| "PsaApiTest (Crypto)", "PsaApiTestIPC (Crypto)", |
| "PsaApiTestIPCTfmLevel2 (Crypto)", |
| "PsaApiTest (PS)", "PsaApiTestIPC (PS)", |
| "PsaApiTestIPCTfmLevel2 (PS)", |
| "PsaApiTest (ITS)", "PsaApiTestIPC (ITS)", |
| "PsaApiTestIPCTfmLevel2 (ITS)", |
| "PsaApiTestIPC (FF)", |
| "PsaApiTestIPCTfmLevel2 (FF)", |
| "PsaApiTestIPCTfmLevel3 (ITS)", "PsaApiTestIPCTfmLevel3 (PS)", |
| "PsaApiTestIPCTfmLevel3 (Crypto)", "PsaApiTestIPCTfmLevel3 (Attest)", |
| "PsaApiTestIPCTfmLevel3 (FF)"] |
| |
| def generateLavaParam(build_params) { |
| def params = [] |
| params += string(name: "TARGET_PLATFORM", \ |
| value: mapPlatform[build_params["TFM_PLATFORM"]]) |
| params += string(name: "COMPILER", \ |
| value: mapCompiler[build_params["TOOLCHAIN_FILE"]]) |
| params += string(name: "PSA_API_SUITE", \ |
| value: mapTestPsaApi[build_params["TEST_PSA_API"]]) |
| |
| configName = "Config" |
| config_params = [build_params["BL2"], build_params["NS"], \ |
| build_params["PSA_API"], build_params["ISOLATION_LEVEL"], \ |
| build_params["TEST_REGRESSION"], build_params["TEST_PSA_API"], \ |
| build_params["PROFILE"]] |
| for (config in mapConfigs) { |
| if (config_params == config[0..6]) { |
| configName += config[7].split(' ')[0] |
| break |
| } |
| } |
| if (configName == "Config") { |
| configName = "ConfigDefault" |
| } |
| params += string(name: "PROJ_CONFIG", value: configName) |
| |
| return params |
| } |
| |
| def listConfigs(ci_scripts_dir, config_list, filter_group) { |
| dir(ci_scripts_dir) { |
| echo "Obtaining list of configs." |
| echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}" |
| def build_config_list_raw = sh(script: """\ |
| python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} |
| """, returnStdout: true).trim() |
| def build_config_list = build_config_list_raw.tokenize('\n') |
| config_list.addAll(build_config_list) |
| } |
| } |
| |
| def buildConfig(ci_scripts_dir, config, filter_group, results) { |
| def params = [] |
| def params_collection = [:] |
| def build_config_params |
| dir(ci_scripts_dir) { |
| echo "Obtaining build configuration for config ${config}" |
| echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}" |
| build_config_params = sh(script: """\ |
| python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config} |
| """, returnStdout: true).trim() |
| } |
| def lines = build_config_params.tokenize('\n') |
| for (String line : lines) { |
| def key, value |
| (key, value) = line.tokenize('=') |
| params += string(name: key, value: value) |
| params_collection[key] = value |
| } |
| params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH) |
| params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST) |
| params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER) |
| params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION) |
| params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC) |
| params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION) |
| params += string(name: 'CODE_REPO', value: env.CODE_REPO) |
| params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN) |
| params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH) |
| return { -> results |
| def build_res = build(job: 'tf-m-build-config', parameters: params, propagate: false) |
| def build_info = [build_res, config, params_collection] |
| results['builds'][build_res.number] = build_info |
| def build_url = build_res.getAbsoluteUrl() |
| print("${build_res.number}: ${config} ${build_res.result} ${build_url}") |
| failure_states = ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"] |
| if (build_res.result in failure_states) { |
| error("Build failed at ${build_url}") |
| } |
| else if (params_collection["NS"] == "False" || \ |
| (params_collection["PROFILE"] == "profile_medium" && \ |
| params_collection["PARTITION_PS"] == "OFF")) { |
| print("LAVA is not needed for ${build_url}") |
| } |
| else { |
| print("Doing LAVA stuff for ${build_url}") |
| params += generateLavaParam(params_collection) |
| params += string(name: 'BUILD_NUMBER', value: "${build_res.number}") |
| params += string(name: 'BUILD_URL', value: build_url) |
| params += string(name: 'LAVA_URL', value: env.LAVA_URL) |
| params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH) |
| params += string(name: 'LAVA_CREDENTIALS', value: env.LAVA_CREDENTIALS) |
| def lava_res = build(job: 'tf-m-lava-submit', parameters: params, propagate: false) |
| if (lava_res.result in failure_states) { |
| error("LAVA Create and Submit failed at ${lava_res.getAbsoluteUrl()}") |
| } |
| else { |
| results['lava_jobs'] += lava_res.getDescription() |
| } |
| } |
| } |
| } |
| |
| def buildDocs(results) { |
| def params = [] |
| params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH) |
| params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST) |
| params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER) |
| params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION) |
| params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC) |
| params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION) |
| params += string(name: 'CODE_REPO', value: env.CODE_REPO) |
| params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH) |
| return { -> results |
| def res = build(job: 'tf-m-build-docs', parameters: params, propagate:false) |
| print("${res.number}: Docs ${res.result} ${res.getAbsoluteUrl()}") |
| results['docs'] = [res.number, res.result, params] |
| if (res.result in ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"]) { |
| error("Build failed at ${res.getAbsoluteUrl()}") |
| } |
| } |
| } |
| |
| def generateEmailBody(stage, failed_jobs) { |
| body = "Check console output at ${env.BUILD_URL} \n\n" |
| |
| body += "Failed Jobs:\n" |
| failed_jobs.each { job -> |
| body += "${job.key} ${job.value}\n" |
| } |
| |
| body += "\nFor detailed ${stage} results please refer to \ |
| ${env.BUILD_URL}artifact/${stage}_results.csv \n" |
| return body |
| } |
| |
| def emailNotification(results, stage, failed_jobs) { |
| script { |
| if (env.JOB_NAME.equals("tf-m-nightly") && !env.EMAIL_NOTIFICATION.equals('')) { |
| def result = "Fail." |
| if (results == true) { |
| result = "Success." |
| print("Skip sending as ${result} for ${stage}") |
| } |
| else { |
| emailext ( |
| subject: ("Job ${env.JOB_NAME} ${stage} ${env.BUILD_NUMBER} ${result}"), |
| body: generateEmailBody(stage, failed_jobs), |
| to: "${EMAIL_NOTIFICATION}" |
| ) |
| } |
| } |
| } /* script */ |
| } |
| |
| def filterFailedBuild(results) { |
| def failed_builds = [:] |
| results.each { result -> |
| if (result.value[0].getResult() == "FAILURE") { |
| failed_builds[result.value[1]] = result.value[0].getAbsoluteUrl() |
| } |
| } |
| return failed_builds |
| } |
| |
| def filterFailedTest(string) { |
| def failed_tests = [:] |
| line = lineInString(string, "FAILURE_TESTS:") |
| a = line.split(' ') |
| if (a.size() > 1) { |
| a = line.split(' ')[1..-1] |
| a.each { fail_test -> |
| config_link = fail_test.split(':') |
| failed_tests[config_link[0]] = config_link[1..-1].join(':') |
| } |
| } |
| return failed_tests |
| } |
| |
| @NonCPS |
| def generateCsvContent(results) { |
| def resultsParam = [] |
| results.each { result -> |
| if (result.value[2]['BL2'] == "True") { |
| resultsParam.add([result.value[1], \ |
| result.value[0].getResult(), \ |
| result.value[2]['TFM_PLATFORM'], \ |
| result.value[2]['TOOLCHAIN_FILE'], \ |
| result.value[2]['CMAKE_BUILD_TYPE'], \ |
| result.value[2]['BL2'], \ |
| result.value[2]['NS'], \ |
| result.value[2]['PSA_API'], \ |
| result.value[2]['ISOLATION_LEVEL'], \ |
| result.value[2]['TEST_REGRESSION'], \ |
| result.value[2]['TEST_PSA_API'], \ |
| result.value[2]['PROFILE'], \ |
| result.value[2]['PARTITION_PS'], \ |
| result.value[2]['OTP']]) |
| } |
| } |
| resultsParam.each { result -> |
| if (result[2] == 'musca_b1') { |
| if (result[13] != 'off') { |
| result[2] = 'musca_b1_OTP' |
| } |
| } |
| result[3] = mapCompiler[result[3]] |
| build_params = result[5..12] |
| configName = "" |
| for (map_cfg in mapConfigs) { |
| if (build_params[0..6] == map_cfg[0..6]) { |
| configName = map_cfg[7] |
| break |
| } |
| } |
| if (configName == "") { |
| configName = "Default" |
| } |
| else if (configName == "RegressionProfileM") { |
| if (build_params[7] == "OFF") { |
| configName = "RegressionProfileM PSOFF" |
| } |
| } |
| result.add(configName) |
| } |
| def csvContent = [] |
| resultsParam.each { result -> |
| current_row = result[2..4] |
| cfgs.each {cfg -> |
| if (cfg == result[14]) { |
| current_row.add(cfg) |
| current_row.add(result[1]) |
| } |
| } |
| csvContent.add(current_row) |
| } |
| csvContent.sort{a,b -> a[0] <=> b[0] ?: a[1] <=> b[1] ?: a[2] <=> b[2]} |
| build_summary = [] |
| current_platform = "" |
| current_compiler = "" |
| current_build_type = "" |
| csvContent.each { build_cfg -> |
| if (current_platform != build_cfg[0] || \ |
| current_compiler != build_cfg[1] || \ |
| current_build_type != build_cfg[2]) { |
| current_platform = build_cfg[0] |
| current_compiler = build_cfg[1] |
| current_build_type = build_cfg[2] |
| csv_line = [current_platform, current_compiler, current_build_type] |
| cfgs.each { |
| csv_line.add("N.A.") |
| } |
| build_summary.add(csv_line) |
| } |
| i = 0 |
| cfgs.each { cfg -> |
| if (cfg == build_cfg[3]) { |
| build_summary[-1][3+i] = build_cfg[4] |
| } |
| i += 1 |
| } |
| } |
| build_summary.add(0, ['Platform', 'Compiler', 'Cmake Build Type']) |
| build_summary[0] += cfgs |
| return build_summary |
| } |
| |
| def generateBuildCsv(results) { |
| def csvContent = generateCsvContent(results) |
| node("master") { |
| writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL |
| archiveArtifacts 'build_results.csv' |
| } |
| } |
| |
| def buildCsv(results) { |
| def summary = new Summary(); |
| def csvContent = summary.getBuildCsv(results) |
| node("master") { |
| writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL |
| archiveArtifacts 'build_results.csv' |
| } |
| } |
| |
| def writeSummary(results) { |
| def summary = new Summary(); |
| def buildLinks = summary.getLinks(results) |
| node("master") { |
| writeFile file: "build_links.html", text: buildLinks |
| archiveArtifacts 'build_links.html' |
| } |
| } |
| |
| def lineInString(string, match) { |
| def lines = string.split("\n") |
| def result = lines.findAll { it.contains(match) } |
| return result[0] |
| } |
| |
| def getResult(string, match) { |
| line = lineInString(string, match) |
| a = line.split(match)[1].split(' ') |
| score = a[0] |
| if (a.size() > 1) |
| { |
| fail_text = a[1..-1].join(" ") |
| return [score, fail_text] |
| } |
| return [score, ""] |
| } |
| |
| def submitJobsToList(results) { |
| def all_jobs = [] |
| for (String result : results){ |
| jobs_s = result.split('JOBS: ') |
| if (jobs_s.size() > 1) { |
| all_jobs += jobs_s[1] |
| } |
| } |
| return(all_jobs) |
| } |
| |
| def configs = [] |
| def builds = [:] |
| def results = [:] |
| |
| node("docker-amd64-tf-m-bionic") { |
| stage("Init") { |
| cleanWs() |
| dir("tf-m-ci-scripts") { |
| checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]]) |
| } |
| } |
| stage("Configs") { |
| // Populate configs |
| listConfigs('tf-m-ci-scripts', configs, env.FILTER_GROUP) |
| results['builds'] = [:] |
| results['lava_jobs'] = [] |
| for (config in configs) { |
| builds[config] = buildConfig("tf-m-ci-scripts", config, env.FILTER_GROUP, results) |
| } |
| builds["docs"] = buildDocs(results) |
| } |
| } |
| |
| stage("Builds") { |
| def verify = 1 |
| def success = true |
| try { |
| parallel(builds) |
| } catch (Exception e) { |
| print(e) |
| manager.buildFailure() |
| verify = -1 |
| success = false |
| } finally { |
| print("Verifying status") |
| def failed_builds = filterFailedBuild(results['builds']) |
| emailNotification(success, 'build', failed_builds) |
| g = new Gerrit() |
| g.verifyStatus(verify, 'tf-m-build', 'build') |
| print("Building CSV") |
| generateBuildCsv(results['builds']) |
| writeSummary(results['builds']) |
| } |
| } |
| |
| node("docker-amd64-tf-m-bionic") { |
| stage("Copy Docs") { |
| if (env.JOB_NAME.equals("tf-m-build-and-test")) { |
| step([$class: 'CopyArtifact', projectName: 'tf-m-build-docs', |
| selector: specific("${results['docs'][0]}"), target: './docs/', |
| optional: true]) |
| archiveArtifacts artifacts: 'docs/**', allowEmptyArchive: true |
| } |
| else { |
| print("No doc copy for job: ${env.JOB_NAME}") |
| } |
| } |
| stage("Tests") { |
| dir("tf-m-ci-scripts") { |
| checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]]) |
| } |
| def all_jobs = [] |
| def success = true |
| print("Wait for LAVA results here...") |
| try { |
| all_jobs = submitJobsToList(results['lava_jobs']) |
| if (all_jobs.size() > 0) { |
| dir("tf-m-ci-scripts") { |
| withCredentials([usernamePassword(credentialsId: env.LAVA_CREDENTIALS, passwordVariable: 'LAVA_TOKEN', usernameVariable: 'LAVA_USER')]) { |
| output = sh(script: """./lava_helper/lava_wait_jobs.py --job-ids ${all_jobs.join(",")} \ |
| --lava-url ${env.LAVA_URL} --lava-user ${LAVA_USER} --lava-token ${LAVA_TOKEN} \ |
| --artifacts-path lava_artifacts --lava-timeout 7200 \ |
| """, returnStdout: true).trim() |
| archiveArtifacts artifacts: 'test_summary.*', allowEmptyArchive: true |
| archiveArtifacts artifacts: 'test_results.csv', allowEmptyArchive: true |
| g = new Gerrit() |
| def (boot_result, boot_output) = getResult(output, 'BOOT_RESULT: ') |
| if (boot_result) { |
| g.verifyStatus(boot_result, "lava_boot", "test") |
| } |
| def (test_result, test_output) = getResult(output, 'TEST_RESULT: ') |
| if (test_result) { |
| g.verifyStatus(test_result, "lava_test", "test") |
| } |
| if (boot_result.toInteger() < 1 || test_result.toInteger() < 1) { |
| error("Marking job as failed due to failed boots: ${boot_output} or tests: ${test_output}") |
| } |
| } |
| } |
| } |
| else { |
| print("There were no LAVA jobs to test.") |
| } |
| } |
| catch (Exception e) { |
| print("ERROR: ${e}") |
| success = false |
| } finally { |
| archiveArtifacts artifacts: 'tf-m-ci-scripts/lava_artifacts/**', allowEmptyArchive: true |
| emailNotification(success, 'test', filterFailedTest(output)) |
| cleanWs() |
| if (!success) { |
| error("There was an Error waiting for LAVA jobs") |
| } |
| } |
| } |
| } |