Open CI Scripts: Initial Commit
* build_helper: Python script which builds sets
of configurations from a json file input
* checkpatch: Bash scripts helping with running checkpatch
* cppcheck: Bash script helping with running cppcheck
* lava_helper: Python script which generates a lava job
definition and parses the output of a lava dispatcher
* tfm_ci_pylib: Generic Python module for Open CI
* configs: Directory storing reference configurations
Change-Id: Ibda0cbfeb5b004b35fef3c2af4cb5c012f2672b4
Signed-off-by: Galanakis, Minos <minos.galanakis@linaro.org>
diff --git a/tfm_ci_pylib/tfm_builder.py b/tfm_ci_pylib/tfm_builder.py
new file mode 100644
index 0000000..07ed776
--- /dev/null
+++ b/tfm_ci_pylib/tfm_builder.py
@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+
+""" tfm_builder.py:
+
+ Build wrapping class that builds a specific tfm configuration """
+
+from __future__ import print_function
+
+__copyright__ = """
+/*
+ * Copyright (c) 2018-2019, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+__author__ = "Minos Galanakis"
+__email__ = "minos.galanakis@linaro.org"
+__project__ = "Trusted Firmware-M Open CI"
+__status__ = "stable"
+__version__ = "1.0"
+
+import os
+from .utils import *
+import shutil
+from .structured_task import structuredTask
+
+
+class TFM_Builder(structuredTask):
+ """ Wrap around tfm cmake system and spawn a thread to build the project.
+ """
+ _tfb_build_params = ["TARGET_PLATFORM",
+ "COMPILER",
+ "PROJ_CONFIG",
+ "CMAKE_BUILD_TYPE",
+ "WITH_MCUBOOT"
+ ]
+
+ _tfb_build_template = ("cmake -G \"Unix Makefiles\" -DPROJ_CONFIG=`"
+ "readlink -f %(PROJ_CONFIG)s.cmake` "
+ "-DTARGET_PLATFORM=%(TARGET_PLATFORM)s "
+ "-DCOMPILER=%(COMPILER)s "
+ "-DCMAKE_BUILD_TYPE=%(CMAKE_BUILD_TYPE)s "
+ "-DBL2=%(WITH_MCUBOOT)s "
+ "%(TFM_ROOT)s")
+
+ def __init__(self,
+ name, # Proccess name
+ tfm_dir, # TFM root directory
+ work_dir, # Current working directory(ie logs)
+ cfg_dict, # Input config dictionary of the following form
+ # input_dict = {"PROJ_CONFIG": "ConfigRegression",
+ # "TARGET_PLATFORM": "MUSCA_A",
+ # "COMPILER": "ARMCLANG",
+ # "CMAKE_BUILD_TYPE": "Debug"}
+ install=False, # Install library after build
+ build_threads=4, # Number of CPU thrads used in build
+ silent=False): # Silence stdout ouptut
+
+ self._tfb_cfg = cfg_dict
+ self._tfb_build_threads = build_threads
+ self._tfb_install = install
+ self._tfb_silent = silent
+ self._tfb_binaries = []
+
+ # Required by other methods, always set working directory first
+ self._tfb_work_dir = os.path.abspath(os.path.expanduser(work_dir))
+
+ self._tfb_tfm_dir = os.path.abspath(os.path.expanduser(tfm_dir))
+ # Entries will be filled after sanity test on cfg_dict dring pre_exec
+ self._tfb_build_dir = None
+ self._tfb_log_f = None
+ super(TFM_Builder, self).__init__(name=name)
+
+ def mute(self):
+ self._tfb_silent = True
+
+ def log(self):
+ """ Print and return the contents of log file """
+ with open(self._tfb_log_f, "r") as F:
+ log = F.read()
+ print(log)
+ return log
+
+ def report(self):
+ """Return the report on the job """
+ return self.unstash("Build Report")
+
+ def pre_eval(self):
+ """ Tests that need to be run in set-up state """
+
+ # Test that all required entries exist in config
+ diff = list(set(self._tfb_build_params) - set(self._tfb_cfg.keys()))
+ if diff:
+ print("Cound't find require build entry: %s in config" % diff)
+ return False
+ # TODO check validity of passed config values
+ # TODO test detection of srec
+ # self.srec_path = shutil.which("srec_cat")
+ return True
+
+ def pre_exec(self, eval_ret):
+ """ Create all required directories, files if they do not exist """
+
+ self._tfb_build_dir = os.path.join(self._tfb_work_dir,
+ self.get_name())
+ # Ensure we have a clean build directory
+ shutil.rmtree(self._tfb_build_dir, ignore_errors=True)
+
+ self._tfb_cfg["TFM_ROOT"] = self._tfb_tfm_dir
+
+ # Append the path for the config
+ self._tfb_cfg["PROJ_CONFIG"] = os.path.join(self._tfb_tfm_dir,
+ self._tfb_cfg[("PROJ_"
+ "CONFIG")])
+
+ # Log will be placed in work directory, named as the build dir
+ self._tfb_log_f = "%s.log" % self._tfb_build_dir
+
+ # Confirm that the work/build directory exists
+ for p in [self._tfb_work_dir, self._tfb_build_dir]:
+ if not os.path.exists(p):
+ os.makedirs(p)
+
+ # Calcuate a list of expected binaries
+ binaries = []
+
+ # If install is asserted pick the iems from the appropriate location
+ if self._tfb_install:
+
+ fvp_path = os.path.join(self._tfb_build_dir,
+ "install", "outputs", "fvp")
+ platform_path = os.path.join(self._tfb_build_dir,
+ "install",
+ "outputs",
+ self._tfb_cfg["TARGET_PLATFORM"])
+
+ # Generate a list of binaries included in both directories
+ common_bin_list = ["tfm_%s.%s" % (s, e) for s in ["s", "ns"]
+ for e in ["bin", "axf"]]
+ if self._tfb_cfg["WITH_MCUBOOT"]:
+ common_bin_list += ["mcuboot.%s" % e for e in ["bin", "axf"]]
+
+ # When building with bootloader extra binaries are expected
+ binaries += [os.path.join(platform_path, b) for b in
+ ["tfm_sign.bin",
+ "tfm_full.bin"]]
+ binaries += [os.path.join(fvp_path, b) for b in
+ ["tfm_s_ns_concatenated.bin",
+ "tfm_s_ns_signed.bin"]]
+
+ binaries += [os.path.join(p, b) for p in [fvp_path, platform_path]
+ for b in common_bin_list]
+
+ # Add Musca required binaries
+ if self._tfb_cfg["TARGET_PLATFORM"] == "MUSCA_A":
+ binaries += [os.path.join(platform_path,
+ "musca_firmware.hex")]
+
+ self._tfb_binaries = binaries
+
+ else:
+ binaries += [os.path.join(self._tfb_build_dir, "app", "tfm_ns")]
+ if "ConfigCoreTest" in self._tfb_build_dir:
+ binaries += [os.path.join(self._tfb_build_dir,
+ "unit_test", "tfm_s")]
+ else:
+ binaries += [os.path.join(self._tfb_build_dir, "app",
+ "secure_fw", "tfm_s")]
+ if self._tfb_cfg["WITH_MCUBOOT"]:
+ binaries += [os.path.join(self._tfb_build_dir,
+ "bl2", "ext", "mcuboot", "mcuboot")]
+
+ ext = ['.bin', '.axf']
+ self._tfb_binaries = ["%s%s" % (n, e) for n in binaries
+ for e in ext]
+
+ # Add Musca required binaries
+ if self._tfb_cfg["TARGET_PLATFORM"] == "MUSCA_A":
+ self._tfb_binaries += [os.path.join(self._tfb_build_dir,
+ "tfm_sign.bin")]
+ self._tfb_binaries += [os.path.join(self._tfb_build_dir,
+ "musca_firmware.hex")]
+
+ def get_binaries(self,
+ bootl=None,
+ bin_s=None,
+ bin_ns=None,
+ bin_sign=None,
+ filt=None):
+ """ Return the absolute location of binaries (from config)
+ if they exist. Can add a filter parameter which will only
+ consider entries with /filter/ in their path as a directory """
+ ret_boot = None
+ ret_bin_ns = None
+ ret_bin_s = None
+ ret_bin_sign = None
+
+ # Apply filter as a /filter/ string to the binary list
+ filt = "/" + filt + "/" if filter else None
+ binaries = list(filter(lambda x: filt in x, self._tfb_binaries)) \
+ if filt else self._tfb_binaries
+
+ for obj_file in binaries:
+ fname = os.path.split(obj_file)[-1]
+ if bootl:
+ if fname == bootl:
+ ret_boot = obj_file
+ continue
+ if bin_s:
+ if fname == bin_s:
+ ret_bin_s = obj_file
+ continue
+
+ if bin_ns:
+ if fname == bin_ns:
+ ret_bin_ns = obj_file
+ continue
+ if bin_sign:
+ if fname == bin_sign:
+ ret_bin_sign = obj_file
+ continue
+ return [ret_boot, ret_bin_s, ret_bin_ns, ret_bin_sign]
+
+ def task_exec(self):
+ """ Main tasks """
+
+ # Mark proccess running as status
+ self.set_status(-1)
+ # Go to build directory
+ os.chdir(self._tfb_build_dir)
+ # Compile the build commands
+ cmake_cmd = self._tfb_build_template % self._tfb_cfg
+ build_cmd = "cmake --build ./ -- -j %s" % self._tfb_build_threads
+
+ # Pass the report to later stages
+ rep = {"build_cmd": "%s" % build_cmd,
+ "cmake_cmd": "%s" % cmake_cmd}
+ self.stash("Build Report", rep)
+
+ # Calll camke to configure the project
+ if not subprocess_log(cmake_cmd,
+ self._tfb_log_f,
+ prefix=cmake_cmd,
+ silent=self._tfb_silent):
+ # Build it
+ if subprocess_log(build_cmd,
+ self._tfb_log_f,
+ append=True,
+ prefix=build_cmd,
+ silent=self._tfb_silent):
+ raise Exception("Build Failed please check log: %s" %
+ self._tfb_log_f)
+ else:
+ raise Exception("Cmake Failed please check log: %s" %
+ self._tfb_log_f)
+
+ if self._tfb_install:
+ install_cmd = "cmake --build ./ -- -j install"
+ if subprocess_log(install_cmd,
+ self._tfb_log_f,
+ append=True,
+ prefix=install_cmd,
+ silent=self._tfb_silent):
+ raise Exception(("Make install Failed."
+ " please check log: %s") % self._tfb_log_f)
+ if self._tfb_cfg["TARGET_PLATFORM"] == "MUSCA_A":
+ boot_f, s_bin, ns_bin, sns_signed_bin = self.get_binaries(
+ bootl="mcuboot.bin",
+ bin_s="tfm_s.bin",
+ bin_ns="tfm_ns.bin",
+ bin_sign="tfm_sign.bin",
+ filt="MUSCA_A")
+ self.convert_to_hex(boot_f, sns_signed_bin)
+ self._t_stop()
+
+ def sign_img(self, secure_bin, non_secure_bin):
+ """Join a secure and non secure image and sign them"""
+
+ imgtool_dir = os.path.join(self._tfb_tfm_dir,
+ "bl2/ext/mcuboot/scripts/")
+ flash_layout = os.path.join(self._tfb_tfm_dir,
+ "platform/ext/target/musca_a/"
+ "partition/flash_layout.h")
+ sign_cert = os.path.join(self._tfb_tfm_dir,
+ "bl2/ext/mcuboot/root-rsa-2048.pem")
+ sns_unsigned_bin = os.path.join(self._tfb_build_dir,
+ "sns_unsigned.bin")
+ sns_signed_bin = os.path.join(self._tfb_build_dir, "sns_signed.bin")
+
+ # Early versions of the tool hard relative imports, run from its dir
+ os.chdir(imgtool_dir)
+ assemble_cmd = ("python3 assemble.py -l %(layout)s -s %(s)s "
+ "-n %(ns)s -o %(sns)s") % {"layout": flash_layout,
+ "s": secure_bin,
+ "ns": non_secure_bin,
+ "sns": sns_unsigned_bin
+ }
+ sign_cmd = ("python3 imgtool.py sign -k %(cert)s --align 1 -v "
+ "1.0 -H 0x400 --pad 0x30000 "
+ "%(sns)s %(sns_signed)s") % {"cert": sign_cert,
+ "sns": sns_unsigned_bin,
+ "sns_signed": sns_signed_bin
+ }
+ run_proccess(assemble_cmd)
+ run_proccess(sign_cmd)
+ # Return to build directory
+ os.chdir(self._tfb_build_dir)
+ return sns_signed_bin
+
+ def convert_to_hex(self,
+ boot_bin,
+ sns_signed_bin,
+ qspi_base=0x200000,
+ boot_size=0x10000):
+ """Convert a signed image to an intel hex format with mcuboot """
+ if self._tfb_install:
+ platform_path = os.path.join(self._tfb_build_dir,
+ "install",
+ "outputs",
+ self._tfb_cfg["TARGET_PLATFORM"])
+ firmware_hex = os.path.join(platform_path, "musca_firmware.hex")
+ else:
+ firmware_hex = os.path.join(self._tfb_build_dir,
+ "musca_firmware.hex")
+
+ img_offset = qspi_base + boot_size
+ merge_cmd = ("srec_cat %(boot)s -Binary -offset 0x%(qspi_offset)x "
+ "%(sns_signed)s -Binary -offset 0x%(img_offset)x "
+ "-o %(hex)s -Intel") % {"boot": boot_bin,
+ "sns_signed": sns_signed_bin,
+ "hex": firmware_hex,
+ "qspi_offset": qspi_base,
+ "img_offset": img_offset
+ }
+ run_proccess(merge_cmd)
+ return
+
+ def post_eval(self):
+ """ Verify that the artefacts exist """
+ print("%s Post eval" % self.get_name())
+
+ ret_eval = False
+ rep = self.unstash("Build Report")
+ missing_binaries = list(filter(lambda x: not os.path.isfile(x),
+ self._tfb_binaries))
+
+ if len(missing_binaries):
+ print("ERROR: Could not locate the following binaries:")
+ print("\n".join(missing_binaries))
+
+ # Update the artifacts to not include missing ones
+ artf = [n for n in self._tfb_binaries if n not in missing_binaries]
+ # TODO update self._tfb_binaries
+ ret_eval = False
+ else:
+ print("SUCCESS: Produced binaries:")
+ print("\n".join(self._tfb_binaries))
+ ret_eval = True
+
+ artf = self._tfb_binaries
+
+ # Add artefact related information to report
+ rep["log"] = self._tfb_log_f
+ rep["missing_artefacts"] = missing_binaries
+ rep["artefacts"] = artf
+
+ rep["status"] = "Success" if ret_eval else "Failed"
+ self.stash("Build Report", rep)
+ return ret_eval
+
+ def post_exec(self, eval_ret):
+ """ """
+
+ if eval_ret:
+ print("TFM Builder %s was Successful" % self.get_name())
+ else:
+ print("TFM Builder %s was UnSuccessful" % self.get_name())