New simpler pipeline
Adding basic script to generate combinations of builds in python.
Probably can use some existing script for this, but this was simpler to
get working with pipeline.
Pipeline to query the python script for build configuration names,
obtain parameters for them, then trigger downstream jobs for them. Does
not yet run any LAVA tests.
Also putting build commands into this repository instead of just being
in the job. This makes it far easier to reproduce what would be run in
Jenkins.
Change-Id: Ie1a7431eb38191e7fcd23edebd284537878e2dde
Signed-off-by: Dean Birch <dean.birch@arm.com>
diff --git a/jenkins/build-config.jpl b/jenkins/build-config.jpl
new file mode 100644
index 0000000..d258dbd
--- /dev/null
+++ b/jenkins/build-config.jpl
@@ -0,0 +1,54 @@
+#!/usr/bin/env groovy
+//-------------------------------------------------------------------------------
+// Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//
+//-------------------------------------------------------------------------------
+
+node("docker-amd64-xenial") {
+ stage("Init") {
+ cleanWs()
+ dir("trusted-firmware-m") {
+ checkout(
+ poll: false,
+ scm: [
+ $class: 'GitSCM',
+ branches: [[name: '$GERRIT_BRANCH']],
+ extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
+ userRemoteConfigs: [[
+ credentialsId: 'GIT_SSH_KEY',
+ refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
+ ]]
+ ])
+ }
+ dir("tf-m-ci-scripts") {
+ git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ }
+ dir("mbed-crypto") {
+ checkout(
+ changelog: false,
+ poll: false,
+ scm: [
+ $class: 'GitSCM',
+ branches: [[name: 'FETCH_HEAD']],
+ userRemoteConfigs: [[
+ refspec: 'refs/tags/$MBEDCRYPTO_VERSION',
+ url: 'https://github.com/ARMmbed/mbed-crypto.git'
+ ]]
+ ]
+ )
+ }
+ sh """
+wget -O cmsis.pack -q https://github.com/ARM-software/CMSIS_5/releases/download/${CMSIS_VERSION}/ARM.CMSIS.${CMSIS_VERSION}.pack
+unzip -o -d CMSIS_5 cmsis.pack
+"""
+ }
+ stage("Build") {
+ sh "tf-m-ci-scripts/run-build.sh"
+ }
+ stage("Post") {
+ archiveArtifacts 'trusted-firmware-m/build/install/**'
+ cleanWs()
+ }
+}
diff --git a/jenkins/build-docs.jpl b/jenkins/build-docs.jpl
new file mode 100644
index 0000000..8c967fd
--- /dev/null
+++ b/jenkins/build-docs.jpl
@@ -0,0 +1,54 @@
+#!/usr/bin/env groovy
+//-------------------------------------------------------------------------------
+// Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//
+//-------------------------------------------------------------------------------
+
+node("docker-amd64-xenial") {
+ stage("Init") {
+ cleanWs()
+ dir("trusted-firmware-m") {
+ checkout(
+ poll: false,
+ scm: [
+ $class: 'GitSCM',
+ branches: [[name: '$GERRIT_BRANCH']],
+ extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
+ userRemoteConfigs: [[
+ credentialsId: 'GIT_SSH_KEY',
+ refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
+ ]]
+ ])
+ }
+ dir("tf-m-ci-scripts") {
+ git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ }
+ dir("mbed-crypto") {
+ checkout(
+ changelog: false,
+ poll: false,
+ scm: [
+ $class: 'GitSCM',
+ branches: [[name: 'FETCH_HEAD']],
+ userRemoteConfigs: [[
+ refspec: 'refs/tags/$MBEDCRYPTO_VERSION',
+ url: 'https://github.com/ARMmbed/mbed-crypto.git'
+ ]]
+ ]
+ )
+ }
+ sh """
+wget -O cmsis.pack -q https://github.com/ARM-software/CMSIS_5/releases/download/${CMSIS_VERSION}/ARM.CMSIS.${CMSIS_VERSION}.pack
+unzip -o -d CMSIS_5 cmsis.pack
+"""
+ }
+ stage("Build") {
+ sh "tf-m-ci-scripts/build-docs.sh"
+ }
+ stage("Post") {
+ archiveArtifacts 'trusted-firmware-m/build/install/**'
+ cleanWs()
+ }
+}
diff --git a/jenkins/checkpatch.jpl b/jenkins/checkpatch.jpl
new file mode 100644
index 0000000..f43ef58
--- /dev/null
+++ b/jenkins/checkpatch.jpl
@@ -0,0 +1,80 @@
+#!/usr/bin/env groovy
+//-------------------------------------------------------------------------------
+// Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//
+//-------------------------------------------------------------------------------
+
+def verifyStatus(value, stage_name) {
+ withCredentials([usernamePassword(credentialsId: 'VERIFY_STATUS', passwordVariable: 'VERIFY_PASSWORD', usernameVariable: 'VERIFY_USER')]) {
+ sh """
+if [ -z "\$GERRIT_HOST" ] ; then
+ echo Not running for a Gerrit change, skipping vote.
+ exit 0
+fi
+if [ ! -d venv ] ; then
+ virtualenv -p \$(which python3) venv
+fi
+. venv/bin/activate
+pip -q install requests
+./tf-m-ci-scripts/jenkins/verify.py --value ${value} --verify-name tf-m-${stage_name} --user \$VERIFY_USER
+"""
+ }
+}
+
+node("docker-amd64-xenial") {
+ stage("Init") {
+ cleanWs()
+ dir("trusted-firmware-m") {
+ checkout(
+ poll: false,
+ scm: [
+ $class: 'GitSCM',
+ branches: [[name: '$GERRIT_BRANCH']],
+ extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
+ userRemoteConfigs: [[
+ credentialsId: 'GIT_SSH_KEY',
+ refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
+ ]]
+ ])
+ }
+ dir("tf-m-ci-scripts") {
+ git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ }
+ dir("mbed-crypto") {
+ checkout(
+ changelog: false,
+ poll: false,
+ scm: [
+ $class: 'GitSCM',
+ branches: [[name: 'FETCH_HEAD']],
+ userRemoteConfigs: [[
+ refspec: 'refs/tags/$MBEDCRYPTO_VERSION',
+ url: 'https://github.com/ARMmbed/mbed-crypto.git'
+ ]]
+ ]
+ )
+ }
+ sh """
+wget -O cmsis.pack -q https://github.com/ARM-software/CMSIS_5/releases/download/${CMSIS_VERSION}/ARM.CMSIS.${CMSIS_VERSION}.pack
+unzip -o -d CMSIS_5 cmsis.pack
+"""
+ }
+ stage("Check") {
+ def verify = 1
+ try {
+ sh """
+cd trusted-firmware-m
+../tf-m-ci-scripts/run-checkpatch.sh -u
+../tf-m-ci-scripts/run-checkpatch.sh -r -l 1
+"""
+ } catch (Exception e) {
+ manager.buildFailure()
+ verify = -1
+ } finally {
+ verifyStatus(verify, 'checkpatch')
+ cleanWs()
+ }
+ }
+}
diff --git a/jenkins/ci.jpl b/jenkins/ci.jpl
new file mode 100644
index 0000000..9f52c62
--- /dev/null
+++ b/jenkins/ci.jpl
@@ -0,0 +1,120 @@
+#!/usr/bin/env groovy
+//-------------------------------------------------------------------------------
+// Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//
+//-------------------------------------------------------------------------------
+
+def listConfigs(ci_scripts_dir, config_list, filter_group) {
+ dir(ci_scripts_dir) {
+ echo "Obtaining list of configs."
+ echo "Running: ./configs.py -g ${filter_group}"
+ def build_config_list_raw = sh(script: """\
+./configs.py -g ${filter_group}
+""", returnStdout: true).trim()
+ def build_config_list = build_config_list_raw.tokenize('\n')
+ config_list.addAll(build_config_list)
+ }
+}
+
+def buildConfig(ci_scripts_dir, config, filter_group) {
+ def params = []
+ def build_config_params
+ dir(ci_scripts_dir) {
+ echo "Obtaining build configuration for config ${config}"
+ echo "Running: ./configs.py -g ${filter_group} ${config}"
+ build_config_params = sh(script: """\
+./configs.py -g ${filter_group} ${config}
+""", returnStdout: true).trim()
+ }
+ def lines = build_config_params.tokenize('\n')
+ for (String line : lines) {
+ def key, value
+ (key, value) = line.tokenize('=')
+ params += string(name: key, value: value)
+ }
+ params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
+ params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
+ params += string(name: 'CMSIS_VERSION', value: env.CMSIS_VERSION)
+ params += string(name: 'MBEDCRYPTO_VERSION', value: env.MBEDCRYPTO_VERSION)
+ params += string(name: 'CODE_REPO', value: env.CODE_REPO)
+ return {
+ def res = build(job: 'tf-m-build-config', parameters: params, propagate: false)
+ print("${res.number}: ${config} ${res.result} ${res.getAbsoluteUrl()}")
+ if (res.result == "FAILURE") {
+ error("Build failed at ${res.getAbsoluteUrl()}")
+ }
+ }
+}
+
+def buildDocs() {
+ def params = []
+ params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
+ params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
+ params += string(name: 'CMSIS_VERSION', value: env.CMSIS_VERSION)
+ params += string(name: 'MBEDCRYPTO_VERSION', value: env.MBEDCRYPTO_VERSION)
+ params += string(name: 'CODE_REPO', value: env.CODE_REPO)
+ return {
+ def res = build(job: 'tf-m-build-docs', parameters: params, propagate:false)
+ print("${res.number}: Docs ${res.result} ${res.getAbsoluteUrl()}")
+ if (res.result == "FAILURE") {
+ error("Build failed at ${res.getAbsoluteUrl()}")
+ }
+ }
+}
+
+def verifyStatus(value, stage_name) {
+ node("docker-amd64-xenial") {
+ cleanWs()
+ dir("tf-m-ci-scripts") {
+ git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ }
+ withCredentials([usernamePassword(credentialsId: 'VERIFY_STATUS', passwordVariable: 'VERIFY_PASSWORD', usernameVariable: 'VERIFY_USER')]) {
+ sh("""
+ if [ -z "\$GERRIT_HOST" ] ; then
+ echo Not running for a Gerrit change, skipping vote.
+ exit 0
+ fi
+ if [ ! -d venv ] ; then
+ virtualenv -p \$(which python3) venv
+ fi
+ . venv/bin/activate
+ pip -q install requests
+ ./tf-m-ci-scripts/jenkins/verify.py --value ${value} --verify-name tf-m-${stage_name} --user \$VERIFY_USER
+ """)
+ }
+ }
+}
+
+def configs = []
+def builds = [:]
+
+node("master") {
+ stage("Init") {
+ cleanWs()
+ dir("tf-m-ci-scripts") {
+ git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ }
+ }
+ stage("Configs") {
+ // Populate configs
+ listConfigs('tf-m-ci-scripts', configs, env.FILTER_GROUP)
+ for (config in configs) {
+ builds[config] = buildConfig("tf-m-ci-scripts", config, env.FILTER_GROUP)
+ }
+ builds["docs"] = buildDocs()
+ }
+}
+stage("Builds") {
+ def verify = 1
+ try {
+ parallel(builds)
+ } catch (Exception e) {
+ manager.buildFailure()
+ verify = -1
+ } finally {
+ verifyStatus(verify, 'build')
+ }
+}
+// TODO Test phase
diff --git a/jenkins/cppcheck.jpl b/jenkins/cppcheck.jpl
new file mode 100644
index 0000000..eacdc3a
--- /dev/null
+++ b/jenkins/cppcheck.jpl
@@ -0,0 +1,83 @@
+#!/usr/bin/env groovy
+//-------------------------------------------------------------------------------
+// Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//
+//-------------------------------------------------------------------------------
+
+def verifyStatus(value, stage_name) {
+ withCredentials([usernamePassword(credentialsId: 'VERIFY_STATUS', passwordVariable: 'VERIFY_PASSWORD', usernameVariable: 'VERIFY_USER')]) {
+ sh """
+if [ -z "\$GERRIT_HOST" ] ; then
+ echo Not running for a Gerrit change, skipping vote.
+ exit 0
+fi
+if [ ! -d venv ] ; then
+ virtualenv -p \$(which python3) venv
+fi
+. venv/bin/activate
+pip -q install requests
+./tf-m-ci-scripts/jenkins/verify.py --value ${value} --verify-name tf-m-${stage_name} --user \$VERIFY_USER
+"""
+ }
+}
+
+node("docker-amd64-xenial") {
+ stage("Init") {
+ cleanWs()
+ dir("trusted-firmware-m") {
+ checkout(
+ poll: false,
+ scm: [
+ $class: 'GitSCM',
+ branches: [[name: '$GERRIT_BRANCH']],
+ extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
+ userRemoteConfigs: [[
+ credentialsId: 'GIT_SSH_KEY',
+ refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
+ ]]
+ ])
+ }
+ dir("tf-m-ci-scripts") {
+ git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
+ }
+ dir("mbed-crypto") {
+ checkout(
+ changelog: false,
+ poll: false,
+ scm: [
+ $class: 'GitSCM',
+ branches: [[name: 'FETCH_HEAD']],
+ userRemoteConfigs: [[
+ refspec: 'refs/tags/$MBEDCRYPTO_VERSION',
+ url: 'https://github.com/ARMmbed/mbed-crypto.git'
+ ]]
+ ]
+ )
+ }
+ sh """
+wget -O cmsis.pack -q https://github.com/ARM-software/CMSIS_5/releases/download/${CMSIS_VERSION}/ARM.CMSIS.${CMSIS_VERSION}.pack
+unzip -o -d CMSIS_5 cmsis.pack
+"""
+ }
+ stage("Check") {
+ def verify = 1
+ try {
+ sh """
+cd trusted-firmware-m
+export GIT_REF_ARG=""
+if [ ! -z "\$GERRIT_PATCHSET_REVISION" ] ; then
+ export GIT_REF_ARG="HEAD"
+fi
+../tf-m-ci-scripts/run-cppcheck.sh -r \$GIT_REF_ARG
+"""
+ } catch (Exception e) {
+ manager.buildFailure()
+ verify = -1
+ } finally {
+ verifyStatus(verify, 'cppcheck')
+ cleanWs()
+ }
+ }
+}
diff --git a/jenkins/static.jpl b/jenkins/static.jpl
new file mode 100644
index 0000000..3f75476
--- /dev/null
+++ b/jenkins/static.jpl
@@ -0,0 +1,33 @@
+#!/usr/bin/env groovy
+//-------------------------------------------------------------------------------
+// Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//
+//-------------------------------------------------------------------------------
+
+
+def trigger(job_name) {
+ return {
+ def params = []
+ params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST ?: '')
+ params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH ?: '')
+ params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC ?: '')
+ params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER ?: '')
+ params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION ?: '')
+ params += string(name: 'CMSIS_VERSION', value: env.CMSIS_VERSION)
+ params += string(name: 'MBEDCRYPTO_VERSION', value: env.MBEDCRYPTO_VERSION)
+ params += string(name: 'CODE_REPO', value: env.CODE_REPO)
+ build(job: job_name, parameters: params)
+ }
+}
+
+stage("Static Checks") {
+ def checks = [:]
+ checks["cppcheck"] = trigger("tf-m-cppcheck")
+ checks["checkpatch"] = trigger("tf-m-checkpatch")
+ parallel(checks)
+}
+stage("Trigger Build") {
+ parallel(["build":trigger("tf-m-build-and-test")])
+}
diff --git a/jenkins/verify.py b/jenkins/verify.py
new file mode 100755
index 0000000..11c8eed
--- /dev/null
+++ b/jenkins/verify.py
@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""
+Posts a verification to Gerrit verify-status plugin.
+"""
+
+__copyright__ = """
+/*
+ * Copyright (c) 2020, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+
+import argparse
+import json
+import os
+import sys
+import requests
+
+
+def check_plugins(base_url, auth):
+ """
+ Checks if the verify-status plugin is installed
+ """
+ plugin_url = "{}/a/plugins/".format(base_url)
+ headers = {"Content-Type": "application/json; charset=UTF-8"}
+ try:
+ request = requests.get(plugin_url, auth=auth, headers=headers)
+ except requests.exceptions.RequestException as exception:
+ print("Error checking plugins: {}".format(str(exception)), file=sys.stderr)
+ sys.exit(1)
+ if request.status_code != 200:
+ print("Could not check if verify-status plugin is installed")
+ return
+ json_plugins = json.loads(request.text.replace(")]}'",""))
+ if "verify-status" not in json_plugins:
+ print("verify-status plugin not installed.")
+ sys.exit(0)
+
+
+def submit_verification(base_url, auth, changeset, patchset_revision, verify_details):
+ """
+ Handles the actual post request.
+ """
+ check_plugins(base_url, auth)
+ post_data = {
+ "verifications": {
+ verify_details["verify_name"]: {
+ "url": verify_details["job_url"],
+ "value": verify_details["value"],
+ "abstain": verify_details["abstain"],
+ "reporter": verify_details["reporter"],
+ "comment": verify_details["comment"],
+ "category": verify_details["category"],
+ "duration": verify_details["duration"],
+ }
+ }
+ }
+ submit_url = "{}/a/changes/{}/revisions/{}/verify-status~verifications".format(
+ base_url, changeset, patchset_revision
+ )
+ headers = {"Content-Type": "application/json; charset=UTF-8"}
+ post = None
+ try:
+ post = requests.post(
+ submit_url, data=json.dumps(post_data), auth=auth, headers=headers,
+ )
+ except requests.exceptions.RequestException as exception:
+ print("Error posting to verify-status:", file=sys.stderr)
+ print(str(exception), file=sys.stderr)
+ sys.exit(1)
+ if post.status_code == 204:
+ print("Gerrit verify-status posted successfully.")
+ else:
+ print(
+ "Error posting to verify-status: {}".format(post.status_code),
+ file=sys.stderr,
+ )
+ print(post.text, file=sys.stderr)
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ PARSER = argparse.ArgumentParser(
+ description="Submits a job verification to verify-status plugin of Gerrit"
+ )
+ PARSER.add_argument("--host", help="Gerrit Host", default=os.getenv("GERRIT_HOST"))
+ PARSER.add_argument("--job-url", help="Job URL.", default=os.getenv("BUILD_URL"))
+ PARSER.add_argument("--value", help="Verification value.")
+ PARSER.add_argument(
+ "--changeset",
+ help="Changeset in Gerrit to verify.",
+ default=os.getenv("GERRIT_CHANGE_NUMBER"),
+ )
+ PARSER.add_argument(
+ "--patchset-revision",
+ help="Commit SHA of revision in Gerrit to verify.",
+ default=os.getenv("GERRIT_PATCHSET_REVISION"),
+ )
+ PARSER.add_argument(
+ "--verify-name", help="Name to give the job verifcation message."
+ )
+ PARSER.add_argument(
+ "--user", help="Username to authenticate as.", default=os.getenv("VERIFY_USER")
+ )
+ PARSER.add_argument(
+ "--password",
+ help="Password or token to authenticate as. "
+ "Defaults to VERIFY_PASSWORD environment variable.",
+ default=os.getenv("VERIFY_PASSWORD"),
+ )
+ PARSER.add_argument(
+ "--abstain",
+ help="Whether this should affect the final voting value.",
+ action="store_true",
+ )
+ PARSER.add_argument("--reporter", help="Metadata for reporter.", default="")
+ PARSER.add_argument("--comment", help="Metadata for comment.", default="")
+ PARSER.add_argument("--category", help="Metadata for category.", default="")
+ PARSER.add_argument("--duration", help="Duration of the job.", default="")
+ PARSER.add_argument("--protocol", help="Protocol to use.", default="https")
+ PARSER.add_argument("--port", help="Port to use.", default=None)
+ ARGS = PARSER.parse_args()
+ submit_verification(
+ "{}://{}{}".format(ARGS.protocol, ARGS.host, ":{}".format(ARGS.port) if ARGS.port else ""),
+ (ARGS.user, ARGS.password),
+ ARGS.changeset,
+ ARGS.patchset_revision,
+ {
+ "verify_name": ARGS.verify_name,
+ "job_url": ARGS.job_url,
+ "value": int(ARGS.value),
+ "abstain": bool(ARGS.abstain),
+ "reporter": ARGS.reporter,
+ "comment": ARGS.comment,
+ "category": ARGS.category,
+ "duration": ARGS.duration,
+ },
+ )