blob: 0d86f038339e128ba2b14c5518f2fa05c3ccc187 [file] [log] [blame]
Dean Birch62c4f082020-01-17 16:13:26 +00001#!/usr/bin/env groovy
2//-------------------------------------------------------------------------------
Xinyu Zhang433771e2022-04-01 16:49:17 +08003// Copyright (c) 2020-2022, Arm Limited and Contributors. All rights reserved.
Dean Birch62c4f082020-01-17 16:13:26 +00004//
5// SPDX-License-Identifier: BSD-3-Clause
6//
7//-------------------------------------------------------------------------------
8
Dean Birchd0f9f8c2020-03-26 11:10:33 +00009@Library('trustedfirmware') _
10import org.trustedfirmware.Gerrit
11import org.trustedfirmware.Summary
Dean Bircha6ede7e2020-03-13 14:00:33 +000012
Xinyu Zhang0aced4c2021-09-03 17:06:21 +080013failure_states = ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"]
14
Summer Qin3c2b5722021-05-26 10:43:45 +080015mapPlatform = ["cypress/psoc64": "psoc64",
16 "arm/mps2/an519": "AN519",
17 "arm/mps2/an521": "AN521",
Summer Qin3c2b5722021-05-26 10:43:45 +080018 "arm/mps3/an524": "AN524",
Bence Balogh176b78f2022-02-22 13:49:34 +010019 "arm/mps3/an547": "AN547",
20 "arm/mps3/an552": "AN552",
Bence Balogh8731a092022-05-24 17:24:54 +020021 "arm/mps3/corstone310_fvp": "corstone310",
Mark Horvathef57baa2022-09-12 13:36:36 +020022 "arm/musca_b1": "MUSCA_B1",
Arthur She19c0e1a2021-06-02 11:06:19 -070023 "arm/musca_s1": "MUSCA_S1",
Jamie Foxf3b8aa82022-09-08 11:52:01 +010024 "arm/rss": "RSS",
Xinyu Zhangfcb6aad2021-08-25 16:24:11 +080025 "stm/stm32l562e_dk": "stm32l562e_dk",
Xinyu Zhang6afdd612021-10-12 17:07:32 +080026 "arm/corstone1000": "corstone1000",
Arthur Shef3657742021-09-07 14:23:18 -070027 "nxp/lpcxpresso55s69": "lpcxpresso55s69"]
Xinyu Zhang5c4bbca2020-09-24 16:36:03 +080028
Xinyu Zhang5c4bbca2020-09-24 16:36:03 +080029mapBL2 = ["True": "--bl2",
30 "False": ""]
31
32mapTestPsaApi = ["OFF": "",
Xinyu Zhang39acb412021-07-09 20:35:19 +080033 "STORAGE": "STORAGE",
Xinyu Zhang5c4bbca2020-09-24 16:36:03 +080034 "CRYPTO": "Crypto",
35 "INITIAL_ATTESTATION": "Attest",
36 "IPC": "FF"]
37
Xinyu Zhang73ed2992021-09-15 11:38:23 +080038// LIB_MODEL, ISOLATION_LEVEL, TEST_REG, TEST_PSA_API, PROFILE, CONFIG_NAME
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +080039mapConfigs = [
Xinyu Zhang73ed2992021-09-15 11:38:23 +080040 ["True", "1", "False", "OFF", "N.A", "Default"],
41 ["False", "1", "False", "OFF", "N.A", "CoreIPC"],
42 ["False", "2", "False", "OFF", "N.A", "CoreIPCTfmLevel2"],
43 ["False", "3", "False", "OFF", "N.A", "CoreIPCTfmLevel3"],
44 ["True", "1", "False", "OFF", "profile_small", "DefaultProfileS"],
45 ["False", "2", "False", "OFF", "profile_medium", "DefaultProfileM"],
46 ["False", "3", "False", "OFF", "profile_large", "DefaultProfileL"],
47 ["True", "1", "True", "OFF", "N.A", "Regression"],
48 ["False", "1", "True", "OFF", "N.A", "RegressionIPC"],
49 ["False", "2", "True", "OFF", "N.A", "RegressionIPCTfmLevel2"],
50 ["False", "3", "True", "OFF", "N.A", "RegressionIPCTfmLevel3"],
51 ["True", "1", "True", "OFF", "profile_small", "RegressionProfileS"],
52 ["False", "2", "True", "OFF", "profile_medium", "RegressionProfileM"],
53 ["False", "3", "True", "OFF", "profile_large", "RegressionProfileL"],
54 ["True", "1", "False", "STORAGE", "N.A", "PsaApiTest (STORAGE)"],
55 ["True", "1", "False", "CRYPTO", "N.A", "PsaApiTest (Crypto)"],
56 ["True", "1", "False", "INITIAL_ATTESTATION", "N.A", "PsaApiTest (Attest)"],
57 ["True", "1", "False", "IPC", "N.A", "PsaApiTest (FF)"],
58 ["False", "1", "False", "STORAGE", "N.A", "PsaApiTestIPC (STORAGE)"],
59 ["False", "1", "False", "CRYPTO", "N.A", "PsaApiTestIPC (Crypto)"],
60 ["False", "1", "False", "INITIAL_ATTESTATION", "N.A", "PsaApiTestIPC (Attest)"],
61 ["False", "1", "False", "IPC", "N.A", "PsaApiTestIPC (FF)"],
62 ["False", "2", "False", "STORAGE", "N.A", "PsaApiTestIPCTfmLevel2 (STORAGE)"],
63 ["False", "2", "False", "CRYPTO", "N.A", "PsaApiTestIPCTfmLevel2 (Crypto)"],
64 ["False", "2", "False", "INITIAL_ATTESTATION", "N.A", "PsaApiTestIPCTfmLevel2 (Attest)"],
65 ["False", "2", "False", "IPC", "N.A", "PsaApiTestIPCTfmLevel2 (FF)"],
66 ["False", "3", "False", "STORAGE", "N.A", "PsaApiTestIPCTfmLevel3 (STORAGE)"],
67 ["False", "3", "False", "CRYPTO", "N.A", "PsaApiTestIPCTfmLevel3 (Crypto)"],
68 ["False", "3", "False", "INITIAL_ATTESTATION", "N.A", "PsaApiTestIPCTfmLevel3 (Attest)"],
69 ["False", "3", "False", "IPC", "N.A", "PsaApiTestIPCTfmLevel3 (FF)"],
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +080070]
71
72cfgs = ["Default", "CoreIPC", "CoreIPCTfmLevel2", "CoreIPCTfmLevel3",
73 "Regression", "RegressionIPC",
74 "RegressionIPCTfmLevel2", "RegressionIPCTfmLevel3",
75 "DefaultProfileS", "RegressionProfileS",
76 "DefaultProfileM", "RegressionProfileM", "RegressionProfileM PSOFF",
Xinyu Zhang9b1aef92021-03-12 15:36:44 +080077 "DefaultProfileL", "RegressionProfileL",
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +080078 "PsaApiTest (Attest)", "PsaApiTestIPC (Attest)",
79 "PsaApiTestIPCTfmLevel2 (Attest)",
80 "PsaApiTest (Crypto)", "PsaApiTestIPC (Crypto)",
81 "PsaApiTestIPCTfmLevel2 (Crypto)",
Xinyu Zhang39acb412021-07-09 20:35:19 +080082 "PsaApiTest (STORAGE)", "PsaApiTestIPC (STORAGE)",
83 "PsaApiTestIPCTfmLevel2 (STORAGE)",
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +080084 "PsaApiTestIPC (FF)",
85 "PsaApiTestIPCTfmLevel2 (FF)",
Xinyu Zhang39acb412021-07-09 20:35:19 +080086 "PsaApiTestIPCTfmLevel3 (STORAGE)", "PsaApiTestIPCTfmLevel3 (Crypto)",
87 "PsaApiTestIPCTfmLevel3 (Attest)", "PsaApiTestIPCTfmLevel3 (FF)"]
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +080088
Xinyu Zhangda895572022-07-18 17:52:46 +080089cfgSkipFVP = [
Xinyu Zhangfb80b5d2022-07-26 15:42:26 +080090 "AN519_GCC_IPC_2_REG_Debug_BL2",
91 "AN519_GCC_IPC_2_REG_Debug_BL2_MEDIUM",
92 "AN519_GCC_IPC_2_REG_Debug_BL2_MEDIUM_PSOFF",
93 "AN519_ARMCLANG_IPC_2_REG_Debug_BL2",
94 "AN519_ARMCLANG_IPC_2_REG_Debug_BL2_MEDIUM",
95 "AN521_ARMCLANG_IPC_2_REG_Debug_BL2",
96 "AN521_ARMCLANG_IPC_2_REG_Debug_BL2_NSCE",
97 "AN521_ARMCLANG_IPC_2_REG_Debug_BL2_MEDIUM",
Xinyu Zhangda895572022-07-18 17:52:46 +080098]
99
Xinyu Zhangaa3747f2020-12-24 16:27:06 +0800100@NonCPS
Xinyu Zhang5c4bbca2020-09-24 16:36:03 +0800101def generateLavaParam(build_params) {
102 def params = []
Xinyu Zhang589fd052022-04-19 17:54:16 +0800103 params += string(name: "TARGET_PLATFORM", \
104 value: mapPlatform[build_params["TFM_PLATFORM"]])
Xinyu Zhang5c4bbca2020-09-24 16:36:03 +0800105 params += string(name: "COMPILER", \
Xinyu Zhang433771e2022-04-01 16:49:17 +0800106 value: build_params["COMPILER"].split('_')[0])
Xinyu Zhang5c4bbca2020-09-24 16:36:03 +0800107 params += string(name: "PSA_API_SUITE", \
108 value: mapTestPsaApi[build_params["TEST_PSA_API"]])
109
Xinyu Zhangdbfadae2020-12-07 14:42:59 +0800110 configName = "Config"
Xinyu Zhang73ed2992021-09-15 11:38:23 +0800111 config_params = [build_params["LIB_MODEL"], build_params["ISOLATION_LEVEL"], \
Xinyu Zhangdbfadae2020-12-07 14:42:59 +0800112 build_params["TEST_REGRESSION"], build_params["TEST_PSA_API"], \
113 build_params["PROFILE"]]
Xinyu Zhang7c8d3372021-12-22 11:15:42 +0800114 // Regression Test is enabled if CRYPTO is ON
115 if (build_params["EXTRA_PARAMS"] == "CRYPTO_ON") {
116 config_params[2] = "True"
117 }
Xinyu Zhangdbfadae2020-12-07 14:42:59 +0800118 for (config in mapConfigs) {
Xinyu Zhang3f7e2f52021-09-02 13:43:57 +0800119 if (config_params == config[0..4]) {
120 configName += config[5].replace(' (', '_').replace(')', '')
Xinyu Zhangdbfadae2020-12-07 14:42:59 +0800121 break
122 }
Xinyu Zhang5c4bbca2020-09-24 16:36:03 +0800123 }
Xinyu Zhangdbfadae2020-12-07 14:42:59 +0800124 if (configName == "Config") {
125 configName = "ConfigDefault"
126 }
127 params += string(name: "PROJ_CONFIG", value: configName)
Xinyu Zhangaa3747f2020-12-24 16:27:06 +0800128 print("Params of ${configName} :")
129 print(config_params)
Xinyu Zhang5c4bbca2020-09-24 16:36:03 +0800130 return params
131}
132
Xinyu Zhang0aced4c2021-09-03 17:06:21 +0800133def submit_lava_tests(config, results, build_res, params, params_collection) {
134 print("Doing LAVA stuff for ${build_res.getAbsoluteUrl()}")
135 params += generateLavaParam(params_collection)
136 params += string(name: 'BUILD_NUMBER', value: "${build_res.number}")
137 params += string(name: 'BUILD_URL', value: build_res.getAbsoluteUrl())
138 params += string(name: 'LAVA_URL', value: env.LAVA_URL)
Paul Sokolovskydaa430a2022-01-06 09:59:16 +0300139 params += string(name: 'CI_SCRIPTS_REPO', value: env.CI_SCRIPTS_REPO)
Xinyu Zhang0aced4c2021-09-03 17:06:21 +0800140 params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
141 params += string(name: 'LAVA_CREDENTIALS', value: env.LAVA_CREDENTIALS)
142 params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN)
Xinyu Zhangda895572022-07-18 17:52:46 +0800143 // Workaround: Configs in cfgSkipFVP fail on FVP but pass on physical boards
144 if (params_collection['CONFIG_NAME'] in cfgSkipFVP) {
145 params += string(name: 'DEVICE_FILTER', value: "--physical-board-only")
146 } else {
147 params += string(name: 'DEVICE_FILTER', value: env.DEVICE_FILTER)
148 }
Xinyu Zhang0aced4c2021-09-03 17:06:21 +0800149 def lava_res = build(job: 'tf-m-lava-submit', parameters: params, propagate: false)
150 def lava_resubmitted = false
151 if (lava_res.result in failure_states) {
152 error("LAVA Create and Submit failed at ${lava_res.getAbsoluteUrl()}")
153 } else {
154 lava_des = lava_res.getDescription()
155 if (lava_des.contains(" Submitted twice!")) {
156 lava_resubmitted = true
157 lava_des = lava_des - " Submitted twice!"
158 }
159 results['lava_jobs'] += lava_des
160 }
161 links = "Build Config: ${config}\n"
162 links += "Build URL: ${build_res.getAbsoluteUrl()}\n"
163 links += "LAVA Submit: ${lava_res.getAbsoluteUrl()}"
164 if (lava_resubmitted) {
165 links += "\nLAVA Job Re-Submitted!"
166 }
167 print(links)
168}
169
Dean Birch62c4f082020-01-17 16:13:26 +0000170def listConfigs(ci_scripts_dir, config_list, filter_group) {
171 dir(ci_scripts_dir) {
172 echo "Obtaining list of configs."
Matthew Hartfb6fd362020-03-04 21:03:59 +0000173 echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}"
Dean Birch62c4f082020-01-17 16:13:26 +0000174 def build_config_list_raw = sh(script: """\
Matthew Hartfb6fd362020-03-04 21:03:59 +0000175python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}
Dean Birch62c4f082020-01-17 16:13:26 +0000176""", returnStdout: true).trim()
177 def build_config_list = build_config_list_raw.tokenize('\n')
178 config_list.addAll(build_config_list)
179 }
180}
181
Matthew Hartfb6fd362020-03-04 21:03:59 +0000182def buildConfig(ci_scripts_dir, config, filter_group, results) {
Dean Birch62c4f082020-01-17 16:13:26 +0000183 def params = []
Matthew Hartfb6fd362020-03-04 21:03:59 +0000184 def params_collection = [:]
Dean Birch62c4f082020-01-17 16:13:26 +0000185 def build_config_params
186 dir(ci_scripts_dir) {
187 echo "Obtaining build configuration for config ${config}"
Matthew Hartfb6fd362020-03-04 21:03:59 +0000188 echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}"
Dean Birch62c4f082020-01-17 16:13:26 +0000189 build_config_params = sh(script: """\
Matthew Hartfb6fd362020-03-04 21:03:59 +0000190python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}
Dean Birch62c4f082020-01-17 16:13:26 +0000191""", returnStdout: true).trim()
192 }
193 def lines = build_config_params.tokenize('\n')
194 for (String line : lines) {
195 def key, value
196 (key, value) = line.tokenize('=')
197 params += string(name: key, value: value)
Matthew Hartfb6fd362020-03-04 21:03:59 +0000198 params_collection[key] = value
Dean Birch62c4f082020-01-17 16:13:26 +0000199 }
200 params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000201 params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
202 params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER)
203 params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION)
Dean Birch62c4f082020-01-17 16:13:26 +0000204 params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
Karl Zhang02d30352020-08-20 13:48:52 +0800205 params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
Dean Birch62c4f082020-01-17 16:13:26 +0000206 params += string(name: 'CODE_REPO', value: env.CODE_REPO)
Karl Zhangf6f467e2020-07-10 16:24:45 +0800207 params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN)
Paul Sokolovskydaa430a2022-01-06 09:59:16 +0300208 params += string(name: 'CI_SCRIPTS_REPO', value: env.CI_SCRIPTS_REPO)
Colin Thorbinson58703db2020-11-24 12:02:19 +0000209 params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
Leonardo Sandoval7090b2c2021-09-17 13:20:44 -0500210 params += string(name: 'MCUBOOT_REFSPEC', value: env.MCUBOOT_REFSPEC)
211 params += string(name: 'MCUBOOT_URL', value: env.MCUBOOT_URL)
212 params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
Paul Sokolovsky8d288712022-01-13 00:53:03 +0300213 params += string(name: 'MBEDTLS_URL', value: env.MBEDTLS_URL)
Leonardo Sandoval7090b2c2021-09-17 13:20:44 -0500214 params += string(name: 'TFM_TESTS_REFSPEC', value: env.TFM_TESTS_REFSPEC)
215 params += string(name: 'TFM_TESTS_URL', value: env.TFM_TESTS_URL)
216 params += string(name: 'PSA_ARCH_TESTS_VERSION', value: env.PSA_ARCH_TESTS_VERSION)
217 params += string(name: 'PSA_ARCH_TESTS_URL', value: env.PSA_ARCH_TESTS_URL)
218 params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER)
Hugo L'Hostise55a2752021-01-27 11:09:08 +0000219 if (env.JOB_NAME.equals("tf-m-nightly")) { //Setting the Memory footprint gathering.
220 params += string(name: 'SQUAD_CONFIGURATIONS', value: env.SQUAD_CONFIGURATIONS)
221 }
Dean Bircha6ede7e2020-03-13 14:00:33 +0000222 return { -> results
223 def build_res = build(job: 'tf-m-build-config', parameters: params, propagate: false)
Dean Bircha6ede7e2020-03-13 14:00:33 +0000224 def build_url = build_res.getAbsoluteUrl()
Xinyu Zhang0aced4c2021-09-03 17:06:21 +0800225 results['builds'][build_res.number] = [build_res, config, params_collection]
226
Dean Bircha6ede7e2020-03-13 14:00:33 +0000227 print("${build_res.number}: ${config} ${build_res.result} ${build_url}")
Xinyu Zhangf2ba9112021-09-02 13:31:16 +0800228
229 // Filter out configs do not need LAVA tests
230
231 // Configs with build failure do not need LAVA tests
Xinyu Zhang6aa579a2022-04-20 11:36:27 +0800232 if (build_res.result in failure_states) {
Dean Bircha6ede7e2020-03-13 14:00:33 +0000233 error("Build failed at ${build_url}")
Arthur She3c0dadd2021-11-18 21:17:48 -0800234 } else {
235 // Build successful
Xinyu Zhang6aa579a2022-04-20 11:36:27 +0800236 // Job tf-m-extra-build does not need LAVA tests
237 if (env.JOB_NAME.equals("tf-m-extra-build")) {
238 print("LAVA is not needed in tf-m-extra-build job.")
239 }
Arthur She87602dc2022-02-06 14:42:18 -0800240 // Submit test job for NXP LPCXpresso55S69 & Cypress PSoC64
Xinyu Zhang6aa579a2022-04-20 11:36:27 +0800241 else if (params_collection["TFM_PLATFORM"].contains("lpcxpresso55s69") || params_collection["TFM_PLATFORM"].contains("psoc64")) {
Arthur She3c0dadd2021-11-18 21:17:48 -0800242 submit_lava_tests(config, results, build_res, params, params_collection)
243 } else {
244 // Configs without BL2 do not need LAVA tests
245 if (params_collection["BL2"] == "False") {
246 print("LAVA is not needed for ${build_url}")
247 }
Arthur She3c0dadd2021-11-18 21:17:48 -0800248 // LAVA tests on MUSCA_B1 are not needed in per-patch job
249 else if (params_collection["TFM_PLATFORM"].contains("musca_b1") && \
250 env.JOB_NAME.equals("tf-m-build-and-test")) {
251 print("LAVA is not needed for ${build_url}")
252 }
Satish Kumar1cfdd912022-08-01 09:24:07 +0100253 // LAVA tests on Corstone1000 FPGA config is not supported
254 else if (params_collection["TFM_PLATFORM"].contains("corstone1000") && \
255 params_collection["EXTRA_PARAMS"].contains("FPGA")) {
256 print("LAVA is not needed for ${build_url}")
257 }
Arthur She3c0dadd2021-11-18 21:17:48 -0800258 // Submit LAVA tests
259 else {
260 submit_lava_tests(config, results, build_res, params, params_collection)
261 }
262 }
Dean Birch62c4f082020-01-17 16:13:26 +0000263 }
264 }
265}
266
Matthew Hart06340d72020-06-15 16:08:20 +0100267def buildDocs(results) {
Dean Birch62c4f082020-01-17 16:13:26 +0000268 def params = []
269 params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000270 params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
271 params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER)
272 params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION)
Dean Birch62c4f082020-01-17 16:13:26 +0000273 params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
Dean Birch62c4f082020-01-17 16:13:26 +0000274 params += string(name: 'CODE_REPO', value: env.CODE_REPO)
Paul Sokolovskydaa430a2022-01-06 09:59:16 +0300275 params += string(name: 'CI_SCRIPTS_REPO', value: env.CI_SCRIPTS_REPO)
Colin Thorbinson58703db2020-11-24 12:02:19 +0000276 params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
Leonardo Sandoval7090b2c2021-09-17 13:20:44 -0500277 params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER)
Matthew Hart06340d72020-06-15 16:08:20 +0100278 return { -> results
Dean Birch62c4f082020-01-17 16:13:26 +0000279 def res = build(job: 'tf-m-build-docs', parameters: params, propagate:false)
280 print("${res.number}: Docs ${res.result} ${res.getAbsoluteUrl()}")
Matthew Hart06340d72020-06-15 16:08:20 +0100281 results['docs'] = [res.number, res.result, params]
Dean Bircha6ede7e2020-03-13 14:00:33 +0000282 if (res.result in ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"]) {
Dean Birch62c4f082020-01-17 16:13:26 +0000283 error("Build failed at ${res.getAbsoluteUrl()}")
284 }
285 }
286}
287
Xinyu Zhang38a18872020-11-23 16:45:28 +0800288def generateEmailBody(stage, failed_jobs) {
289 body = "Check console output at ${env.BUILD_URL} \n\n"
290
291 body += "Failed Jobs:\n"
292 failed_jobs.each { job ->
293 body += "${job.key} ${job.value}\n"
294 }
295
296 body += "\nFor detailed ${stage} results please refer to \
297 ${env.BUILD_URL}artifact/${stage}_results.csv \n"
298 return body
299}
300
301def emailNotification(results, stage, failed_jobs) {
Karl Zhang0413e972020-09-18 17:59:26 +0800302 script {
Xinyu Zhang953dc232022-09-07 18:27:35 +0800303 if (env.EMAIL_NOTIFICATION) {
Karl Zhang0413e972020-09-18 17:59:26 +0800304 def result = "Fail."
Karl Zhang182ecdf2020-10-10 09:52:12 +0800305 if (results == true) {
Karl Zhang0413e972020-09-18 17:59:26 +0800306 result = "Success."
Karl Zhang182ecdf2020-10-10 09:52:12 +0800307 print("Skip sending as ${result} for ${stage}")
308 }
309 else {
310 emailext (
311 subject: ("Job ${env.JOB_NAME} ${stage} ${env.BUILD_NUMBER} ${result}"),
Xinyu Zhang38a18872020-11-23 16:45:28 +0800312 body: generateEmailBody(stage, failed_jobs),
Karl Zhang182ecdf2020-10-10 09:52:12 +0800313 to: "${EMAIL_NOTIFICATION}"
314 )
315 }
Karl Zhang0413e972020-09-18 17:59:26 +0800316 }
317 } /* script */
318}
319
Xinyu Zhang38a18872020-11-23 16:45:28 +0800320def filterFailedBuild(results) {
321 def failed_builds = [:]
322 results.each { result ->
323 if (result.value[0].getResult() == "FAILURE") {
324 failed_builds[result.value[1]] = result.value[0].getAbsoluteUrl()
325 }
326 }
327 return failed_builds
328}
329
Xinyu Zhang82dab282022-10-09 16:33:19 +0800330def parseTestResults(output) {
331 // Verify test status
332 g = new Gerrit()
333 if (output.contains('FAILURE')) {
334 score = -1
335 } else {
336 score = 1
337 }
338 g.verifyStatus(score, "lava_test", "test")
339 if (score < 0) {
340 error("Marking job as failed due to failed boots: \"${boot_output}\" or tests: \"${test_output}\"")
341 }
342
343 // Generate test results summary
344 def test_results = [:]
345 records = output.split('\nLAVA Test Config:\n')
346 if (records.size() < 2) {
347 return test_results
348 }
349 records[1..-1].each { record ->
350 config_name = ""
351 metadata = [:]
352 record.split('\n').each { line ->
353 record_metadata = line.split(': ')
354 if (record_metadata[0] == 'Config Name') {
355 config_name = record_metadata[1]
356 } else {
357 metadata[record_metadata[0]] = record_metadata[1]
358 }
359 }
360 test_results[config_name] = metadata
361 }
362 return test_results
363}
364
Xinyu Zhang38a18872020-11-23 16:45:28 +0800365def filterFailedTest(string) {
366 def failed_tests = [:]
367 line = lineInString(string, "FAILURE_TESTS:")
Paul Sokolovskyc5cf9e62022-04-26 22:03:23 +0300368 if (line == null) {
369 return ["???"];
370 }
Xinyu Zhang38a18872020-11-23 16:45:28 +0800371 a = line.split(' ')
372 if (a.size() > 1) {
373 a = line.split(' ')[1..-1]
374 a.each { fail_test ->
375 config_link = fail_test.split(':')
376 failed_tests[config_link[0]] = config_link[1..-1].join(':')
377 }
378 }
379 return failed_tests
380}
381
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800382@NonCPS
383def generateCsvContent(results) {
384 def resultsParam = []
385 results.each { result ->
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800386 if (result.value[2]['BL2'] == "True") {
387 resultsParam.add([result.value[1], \
388 result.value[0].getResult(), \
389 result.value[2]['TFM_PLATFORM'], \
Xinyu Zhang433771e2022-04-01 16:49:17 +0800390 result.value[2]['COMPILER'].split('_')[0], \
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800391 result.value[2]['CMAKE_BUILD_TYPE'], \
392 result.value[2]['BL2'], \
Xinyu Zhang73ed2992021-09-15 11:38:23 +0800393 result.value[2]['LIB_MODEL'], \
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800394 result.value[2]['ISOLATION_LEVEL'], \
395 result.value[2]['TEST_REGRESSION'], \
396 result.value[2]['TEST_PSA_API'], \
Xinyu Zhang589fd052022-04-19 17:54:16 +0800397 result.value[2]['PROFILE']])
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800398 }
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800399 }
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800400 resultsParam.each { result ->
Xinyu Zhang433771e2022-04-01 16:49:17 +0800401 result[3] = result[3].split('_')[0]
Xinyu Zhangfb80b5d2022-07-26 15:42:26 +0800402 build_params = result[6..10]
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800403 configName = ""
404 for (map_cfg in mapConfigs) {
Xinyu Zhang3f7e2f52021-09-02 13:43:57 +0800405 if (build_params[0..4] == map_cfg[0..4]) {
406 configName = map_cfg[5]
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800407 break
408 }
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800409 }
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800410 if (configName == "") {
411 configName = "Default"
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800412 }
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800413 else if (configName == "RegressionProfileM") {
Xinyu Zhang3f7e2f52021-09-02 13:43:57 +0800414 if (build_params[5] == "OFF") {
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800415 configName = "RegressionProfileM PSOFF"
416 }
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800417 }
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800418 result.add(configName)
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800419 }
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800420 def csvContent = []
421 resultsParam.each { result ->
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800422 current_row = result[2..4]
423 cfgs.each {cfg ->
Xinyu Zhangfb80b5d2022-07-26 15:42:26 +0800424 if (cfg == result[11]) {
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800425 current_row.add(cfg)
426 current_row.add(result[1])
427 }
428 }
429 csvContent.add(current_row)
430 }
431 csvContent.sort{a,b -> a[0] <=> b[0] ?: a[1] <=> b[1] ?: a[2] <=> b[2]}
432 build_summary = []
433 current_platform = ""
434 current_compiler = ""
435 current_build_type = ""
436 csvContent.each { build_cfg ->
437 if (current_platform != build_cfg[0] || \
438 current_compiler != build_cfg[1] || \
439 current_build_type != build_cfg[2]) {
440 current_platform = build_cfg[0]
441 current_compiler = build_cfg[1]
442 current_build_type = build_cfg[2]
443 csv_line = [current_platform, current_compiler, current_build_type]
444 cfgs.each {
445 csv_line.add("N.A.")
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800446 }
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800447 build_summary.add(csv_line)
448 }
449 i = 0
450 cfgs.each { cfg ->
451 if (cfg == build_cfg[3]) {
452 build_summary[-1][3+i] = build_cfg[4]
453 }
454 i += 1
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800455 }
456 }
Xinyu Zhang4f2ef5a2020-11-09 18:11:43 +0800457 build_summary.add(0, ['Platform', 'Compiler', 'Cmake Build Type'])
458 build_summary[0] += cfgs
459 return build_summary
xinyu-tfmb4fc0412020-08-19 10:49:51 +0800460}
461
462def generateBuildCsv(results) {
463 def csvContent = generateCsvContent(results)
464 node("master") {
465 writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL
466 archiveArtifacts 'build_results.csv'
467 }
468}
Dean Bircha6ede7e2020-03-13 14:00:33 +0000469
470def buildCsv(results) {
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000471 def summary = new Summary();
Dean Bircha6ede7e2020-03-13 14:00:33 +0000472 def csvContent = summary.getBuildCsv(results)
473 node("master") {
474 writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL
475 archiveArtifacts 'build_results.csv'
476 }
477}
478
479def writeSummary(results) {
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000480 def summary = new Summary();
Dean Bircha6ede7e2020-03-13 14:00:33 +0000481 def buildLinks = summary.getLinks(results)
482 node("master") {
483 writeFile file: "build_links.html", text: buildLinks
484 archiveArtifacts 'build_links.html'
485 }
486}
487
Matthew Hartfb6fd362020-03-04 21:03:59 +0000488def lineInString(string, match) {
489 def lines = string.split("\n")
490 def result = lines.findAll { it.contains(match) }
491 return result[0]
492}
493
494def getResult(string, match) {
495 line = lineInString(string, match)
Dean Birch1d545c02020-05-29 14:09:21 +0100496 a = line.split(match)[1].split(' ')
497 score = a[0]
498 if (a.size() > 1)
499 {
500 fail_text = a[1..-1].join(" ")
501 return [score, fail_text]
502 }
503 return [score, ""]
Matthew Hartfb6fd362020-03-04 21:03:59 +0000504}
505
506def submitJobsToList(results) {
507 def all_jobs = []
508 for (String result : results){
509 jobs_s = result.split('JOBS: ')
510 if (jobs_s.size() > 1) {
511 all_jobs += jobs_s[1]
512 }
513 }
514 return(all_jobs)
515}
516
Dean Birch62c4f082020-01-17 16:13:26 +0000517def configs = []
518def builds = [:]
Matthew Hartfb6fd362020-03-04 21:03:59 +0000519def results = [:]
Dean Birch62c4f082020-01-17 16:13:26 +0000520
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800521timestamps {
522 node("docker-amd64-tf-m-bionic") {
523 stage("Init") {
524 cleanWs()
525 dir("tf-m-ci-scripts") {
526 checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
Xinyu Zhang8472b852021-09-04 00:13:34 +0800527 sh "git rev-parse --short HEAD"
Leonardo Sandoval7090b2c2021-09-17 13:20:44 -0500528 // Clone TF-M repositories so share folder can be reused by downstream jobs
529 sh "./clone.sh"
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800530 }
531 }
Xinyu Zhang6b456c22022-10-13 15:48:58 +0800532
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800533 stage("Configs") {
534 // Populate configs
535 listConfigs('tf-m-ci-scripts', configs, env.FILTER_GROUP)
536 results['builds'] = [:]
537 results['lava_jobs'] = []
538 for (config in configs) {
539 builds[config] = buildConfig("tf-m-ci-scripts", config, env.FILTER_GROUP, results)
540 }
Xinyu Zhang351f14c2021-11-15 14:30:09 +0800541 if (!env.JOB_NAME.equals("tf-m-extra-build")) {
542 builds["docs"] = buildDocs(results)
543 }
Dean Birch62c4f082020-01-17 16:13:26 +0000544 }
Karl Zhangfec84102020-06-24 09:56:36 +0800545
Xinyu Zhang6b456c22022-10-13 15:48:58 +0800546 stage("Builds") {
547 def verify = 1
548 def success = true
549 try {
550 parallel(builds)
551 } catch (Exception e) {
552 print(e)
553 manager.buildFailure()
554 verify = -1
555 success = false
556 } finally {
557 print("Verifying status")
558 def failed_builds = filterFailedBuild(results['builds'])
559 emailNotification(success, 'build', failed_builds)
560 g = new Gerrit()
561 g.verifyStatus(verify, 'tf-m-build', 'build')
562 print("Building CSV")
563 generateBuildCsv(results['builds'])
564 writeSummary(results['builds'])
565 }
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800566 }
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800567
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800568 stage("Tests") {
569 dir("tf-m-ci-scripts") {
570 checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
571 }
572 def all_jobs = []
573 def success = true
574 print("Wait for LAVA results here...")
575 try {
576 all_jobs = submitJobsToList(results['lava_jobs'])
Paul Sokolovsky437bc422022-02-01 17:57:13 +0300577 output = ""
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800578 if (all_jobs.size() > 0) {
Paul Sokolovsky3d522f72022-04-13 11:17:00 +0300579 dir(".") {
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800580 withCredentials([usernamePassword(credentialsId: env.LAVA_CREDENTIALS, passwordVariable: 'LAVA_TOKEN', usernameVariable: 'LAVA_USER')]) {
Paul Sokolovsky3d522f72022-04-13 11:17:00 +0300581 output = sh(script: """./tf-m-ci-scripts/lava_helper/lava_wait_jobs.py --job-ids ${all_jobs.join(",")} \
Xinyu Zhang6b456c22022-10-13 15:48:58 +0800582 --lava-url ${env.LAVA_URL} --lava-user ${LAVA_USER} --lava-token ${LAVA_TOKEN} \
583 --artifacts-path cfgs --lava-timeout 12000 \
584 """, returnStdout: true).trim()
Paul Sokolovsky1b024ac2022-02-01 22:26:23 +0300585 println("--- output from lava_wait_jobs.py ---")
Xinyu Zhange9033f62022-10-08 11:15:27 +0800586 println(output)
Paul Sokolovsky1b024ac2022-02-01 22:26:23 +0300587 println("--- end of output from lava_wait_jobs.py ---")
Xinyu Zhang82dab282022-10-09 16:33:19 +0800588 parseTestResults(output)
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800589 archiveArtifacts artifacts: 'test_summary.*', allowEmptyArchive: true
590 archiveArtifacts artifacts: 'test_results.csv', allowEmptyArchive: true
Paul Sokolovsky0136de02022-04-19 20:21:47 +0300591
592 if (env.CODE_COVERAGE_EN == "TRUE") {
593 println("Producing merged report")
594 sh(script: """./tf-m-ci-scripts/lava_helper/codecov_merge.sh""")
595 archiveArtifacts artifacts: 'merged_report/**', allowEmptyArchive: true
596 }
Matthew Hartfb6fd362020-03-04 21:03:59 +0000597 }
598 }
599 }
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800600 else {
601 print("There were no LAVA jobs to test.")
602 }
Matthew Hartfb6fd362020-03-04 21:03:59 +0000603 }
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800604 catch (Exception e) {
605 print("ERROR: ${e}")
606 success = false
607 } finally {
Paul Sokolovsky3d522f72022-04-13 11:17:00 +0300608 archiveArtifacts artifacts: 'cfgs/**', allowEmptyArchive: true
Xinyu Zhang4cdfd1b2021-05-21 15:10:49 +0800609 if (all_jobs.size() > 0) {
610 emailNotification(success, 'test', filterFailedTest(output))
611 }
612 cleanWs()
613 if (!success) {
614 error("There was an Error waiting for LAVA jobs")
615 }
Matthew Hartfb6fd362020-03-04 21:03:59 +0000616 }
Dean Bircha6ede7e2020-03-13 14:00:33 +0000617 }
618 }
Dean Bircha6ede7e2020-03-13 14:00:33 +0000619}