Refactor the LAVA job generation
Create a LAVA job per test
Use flasher support
Simplify templates
Include metadata
Add LAVA job generation for each parallel build
Implement the LAVA wait and parsing step for simple gerrit verification
Create FVP jobs as well
Change filenames for MPS2 fvp jobs
Fix FVP templates and add NOBL2 jobs
User docker-prefix and license-variable from Jenkins job environment
Change-Id: I4dca28a353bc908a570f578b539aeb9c4528f6fa
Signed-off-by: Dean Birch <dean.birch@arm.com>
diff --git a/jenkins/build-config.jpl b/jenkins/build-config.jpl
index b4263cd..5e0ff1f 100644
--- a/jenkins/build-config.jpl
+++ b/jenkins/build-config.jpl
@@ -31,7 +31,7 @@
])
}
dir("tf-m-ci-scripts") {
- git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ git url: '$CI_SCRIPTS_REPO', branch: '$CI_SCRIPTS_BRANCH', credentialsId: 'GIT_SSH_KEY'
}
dir("mbed-crypto") {
checkout(
@@ -39,7 +39,7 @@
poll: false,
scm: [
$class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
+ branches: [[name: 'FETCH_HEAD']],
userRemoteConfigs: [[
refspec: 'refs/tags/$MBEDCRYPTO_VERSION',
url: params.MBEDCRYPTO_URL
@@ -63,7 +63,7 @@
branches: [[name: 'FETCH_HEAD']],
userRemoteConfigs: [[
refspec: 'refs/tags/v20.03_API1.0',
- url: 'https://github.com/ARM-software/psa-arch-tests'
+ url: params.PSA_ARCH_TESTS_URL
]]
]
)
@@ -86,6 +86,8 @@
} finally {
g = new Gerrit()
g.verifyStatusInWorkspace(verify, env.CONFIG_NAME, 'build')
+ def buildStatus = (verify == 1) ? 'Successful' : 'Failed'
+ //g.commentInWorkspace("Build configuration ${env.CONFIG_NAME} ${buildStatus}: ${env.RUN_DISPLAY_URL}")
cleanWs()
}
}
diff --git a/jenkins/build-docs.jpl b/jenkins/build-docs.jpl
index 71993b7..983bf16 100644
--- a/jenkins/build-docs.jpl
+++ b/jenkins/build-docs.jpl
@@ -26,7 +26,7 @@
])
}
dir("tf-m-ci-scripts") {
- git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ git url: '$CI_SCRIPTS_REPO', branch: '$CI_SCRIPTS_BRANCH', credentialsId: 'GIT_SSH_KEY'
}
dir("mbed-crypto") {
checkout(
@@ -34,7 +34,7 @@
poll: false,
scm: [
$class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
+ branches: [[name: 'FETCH_HEAD']],
userRemoteConfigs: [[
refspec: 'refs/tags/$MBEDCRYPTO_VERSION',
url: params.MBEDCRYPTO_URL
@@ -63,6 +63,8 @@
} finally {
g = new Gerrit()
g.verifyStatusInWorkspace(verify, 'tf-m-build-docs', 'build')
+ def buildStatus = (verify == 1) ? 'Successful' : 'Failed'
+ //g.commentInWorkspace("Build docs ${buildStatus}: ${env.RUN_DISPLAY_URL}")
cleanWs()
}
}
diff --git a/jenkins/checkpatch.jpl b/jenkins/checkpatch.jpl
index 7855cb0..e9f242e 100644
--- a/jenkins/checkpatch.jpl
+++ b/jenkins/checkpatch.jpl
@@ -26,7 +26,7 @@
])
}
dir("tf-m-ci-scripts") {
- git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ git url: '$CI_SCRIPTS_REPO', branch: '$CI_SCRIPTS_BRANCH', credentialsId: 'GIT_SSH_KEY'
}
dir("mbed-crypto") {
checkout(
@@ -63,6 +63,8 @@
} finally {
g = new Gerrit()
g.verifyStatusInWorkspace(verify, 'checkpatch', 'static')
+ def buildStatus = (verify == 1) ? 'Successful' : 'Failed'
+ //g.commentInWorkspace("Build checkpatch ${buildStatus}: ${env.RUN_DISPLAY_URL}")
cleanWs()
}
}
diff --git a/jenkins/ci.jpl b/jenkins/ci.jpl
index e20b9a3..cedcdc1 100644
--- a/jenkins/ci.jpl
+++ b/jenkins/ci.jpl
@@ -13,23 +13,24 @@
def listConfigs(ci_scripts_dir, config_list, filter_group) {
dir(ci_scripts_dir) {
echo "Obtaining list of configs."
- echo "Running: ./configs.py -g ${filter_group}"
+ echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}"
def build_config_list_raw = sh(script: """\
-./configs.py -g ${filter_group}
+python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}
""", returnStdout: true).trim()
def build_config_list = build_config_list_raw.tokenize('\n')
config_list.addAll(build_config_list)
}
}
-def buildConfig(ci_scripts_dir, config, filter_group) {
+def buildConfig(ci_scripts_dir, config, filter_group, results) {
def params = []
+ def params_collection = [:]
def build_config_params
dir(ci_scripts_dir) {
echo "Obtaining build configuration for config ${config}"
- echo "Running: ./configs.py -g ${filter_group} ${config}"
+ echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}"
build_config_params = sh(script: """\
-./configs.py -g ${filter_group} ${config}
+python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}
""", returnStdout: true).trim()
}
def lines = build_config_params.tokenize('\n')
@@ -37,6 +38,7 @@
def key, value
(key, value) = line.tokenize('=')
params += string(name: key, value: value)
+ params_collection[key] = value
}
params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
@@ -60,8 +62,14 @@
print("Doing LAVA stuff for ${build_url}")
params += string(name: 'BUILD_NUMBER', value: "${build_res.number}")
params += string(name: 'BUILD_URL', value: build_url)
+ params += string(name: 'LAVA_URL', value: env.LAVA_URL)
def lava_res = build(job: 'tf-m-lava-submit', parameters: params, propagate: false)
- results['lava_jobs'] += lava_res.getDescription()
+ if (lava_res.result in failure_states) {
+ error("LAVA Create and Submit failed at ${lava_res.getAbsoluteUrl()}")
+ }
+ else {
+ results['lava_jobs'] += lava_res.getDescription()
+ }
}
}
}
@@ -104,14 +112,37 @@
}
}
+def lineInString(string, match) {
+ def lines = string.split("\n")
+ def result = lines.findAll { it.contains(match) }
+ return result[0]
+}
+
+def getResult(string, match) {
+ line = lineInString(string, match)
+ return(line.split(match)[1].split(' '))
+}
+
+def submitJobsToList(results) {
+ def all_jobs = []
+ for (String result : results){
+ jobs_s = result.split('JOBS: ')
+ if (jobs_s.size() > 1) {
+ all_jobs += jobs_s[1]
+ }
+ }
+ return(all_jobs)
+}
+
def configs = []
def builds = [:]
+def results = [:]
-node("master") {
+node("docker-amd64-xenial") {
stage("Init") {
cleanWs()
dir("tf-m-ci-scripts") {
- git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ git url: '$CI_SCRIPTS_REPO', branch: '$CI_SCRIPTS_BRANCH', credentialsId: 'GIT_SSH_KEY'
}
}
stage("Configs") {
@@ -120,7 +151,7 @@
results['builds'] = [:]
results['lava_jobs'] = []
for (config in configs) {
- builds[config] = buildConfig("tf-m-ci-scripts", config, env.FILTER_GROUP)
+ builds[config] = buildConfig("tf-m-ci-scripts", config, env.FILTER_GROUP, results)
}
builds["docs"] = buildDocs()
}
@@ -145,12 +176,50 @@
node("docker-amd64-xenial") {
stage("Tests") {
dir("tf-m-ci-scripts") {
- git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ git url: '$CI_SCRIPTS_REPO', branch: '$CI_SCRIPTS_BRANCH', credentialsId: 'GIT_SSH_KEY'
}
+ def all_jobs = []
+ def success = true
print("Wait for LAVA results here...")
- results['lava_jobs'].each { result ->
- print(result)
+ try {
+ all_jobs = submitJobsToList(results['lava_jobs'])
+ if (all_jobs.size() > 0) {
+ dir("tf-m-ci-scripts") {
+ withCredentials([usernamePassword(credentialsId: 'LAVA_CREDENTIALS', passwordVariable: 'LAVA_TOKEN', usernameVariable: 'LAVA_USER')]) {
+ output = sh(script: """./lava_helper/lava_wait_jobs.py --job-ids ${all_jobs.join(",")} \
+ --lava-url ${env.LAVA_URL} --lava-user ${LAVA_USER} --lava-token ${LAVA_TOKEN} \
+ --artifacts-path lava_artifacts \
+ """, returnStdout: true).trim()
+ archiveArtifacts artifacts: 'test_summary.*', allowEmptyArchive: true
+ print(output)
+ g = new Gerrit()
+ def boot_result = getResult(output, 'BOOT_RESULT: ')[0]
+ if (boot_result) {
+ g.verifyStatus(boot_result, "lava_boot", "test")
+ }
+ def test_result = getResult(output, 'TEST_RESULT: ')[0]
+ if (test_result) {
+ g.verifyStatus(test_result, "lava_test", "test")
+ }
+ if (boot_result.toInteger() < 1 || test_result.toInteger() < 1) {
+ error("Marking job as failed due to failed boots/tests")
+ }
+ }
+ }
+ }
+ else {
+ print("There were no LAVA jobs to test.")
+ }
+ }
+ catch (Exception e) {
+ print("ERROR: ${e}")
+ success = false
+ } finally {
+ archiveArtifacts artifacts: 'tf-m-ci-scripts/lava_artifacts/**', allowEmptyArchive: true
+ cleanWs()
+ if (!success) {
+ error("There was an Error waiting for LAVA jobs")
+ }
}
}
- cleanWs()
}
diff --git a/jenkins/cppcheck.jpl b/jenkins/cppcheck.jpl
index fb40a9f..9e2e756 100644
--- a/jenkins/cppcheck.jpl
+++ b/jenkins/cppcheck.jpl
@@ -26,7 +26,7 @@
])
}
dir("tf-m-ci-scripts") {
- git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ git url: '$CI_SCRIPTS_REPO', branch: '$CI_SCRIPTS_BRANCH', credentialsId: 'GIT_SSH_KEY'
}
dir("mbed-crypto") {
checkout(
@@ -66,6 +66,8 @@
} finally {
g = new Gerrit()
g.verifyStatusInWorkspace(verify, 'cppcheck', 'static')
+ def buildStatus = (verify == 1) ? 'Successful' : 'Failed'
+ //g.commentInWorkspace("Build cppcheck ${buildStatus}: ${env.RUN_DISPLAY_URL}")
cleanWs()
}
}
diff --git a/jenkins/lava-submit.jpl b/jenkins/lava-submit.jpl
index fbb8611..73d5b78 100644
--- a/jenkins/lava-submit.jpl
+++ b/jenkins/lava-submit.jpl
@@ -10,7 +10,7 @@
stage("Init") {
cleanWs()
dir("tf-m-ci-scripts") {
- git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ git url: '$CI_SCRIPTS_REPO', branch: '$CI_SCRIPTS_BRANCH', credentialsId: 'GIT_SSH_KEY'
}
}
stage("LAVA") {
@@ -25,12 +25,13 @@
--build-number ${env.BUILD_NUMBER} --output-dir lava_jobs \
--compiler ${env.COMPILER} --platform ${env.TARGET_PLATFORM} \
${bl2_string} --build-type ${env.CMAKE_BUILD_TYPE} \
---jenkins-build-url ${env.BUILD_URL} --proj-config ${env.PROJ_CONFIG}
+--jenkins-build-url ${env.BUILD_URL} --proj-config ${env.PROJ_CONFIG} \
+--docker-prefix ${env.DOCKER_PREFIX} --license-variable "${env.LICENSE_VARIABLE}"
""", returnStdout: true).trim()
print(res)
job_ids = sh(script: """./lava_helper/lava_submit_jobs.py \
- --lava-url https://tf.validation.linaro.org --job-dir lava_jobs \
- --lava-user ${LAVA_USER} --lava-token ${LAVA_TOKEN}
+ --lava-url ${env.LAVA_URL} --job-dir lava_jobs \
+ --lava-user ${LAVA_USER} --lava-token ${LAVA_TOKEN} | egrep "^JOBS"
""", returnStdout: true).trim()
currentBuild.setDescription(job_ids)
}
diff --git a/jenkins/static.jpl b/jenkins/static.jpl
index 3f75476..dccdd09 100644
--- a/jenkins/static.jpl
+++ b/jenkins/static.jpl
@@ -22,12 +22,23 @@
}
}
+def status = 1
+
stage("Static Checks") {
def checks = [:]
checks["cppcheck"] = trigger("tf-m-cppcheck")
checks["checkpatch"] = trigger("tf-m-checkpatch")
- parallel(checks)
+ try {
+ parallel(checks)
+ } catch (Exception e) {
+ status = -1
+ echo "Failed static checks, continuing with build."
+ }
}
stage("Trigger Build") {
parallel(["build":trigger("tf-m-build-and-test")])
+ // If previously failed at static checks, mark this as a failure
+ if (status < 0 ) {
+ error("Failing due to failed static checks.")
+ }
}