Open CI Scripts: Feature Update
* build_helper: Added --install argument to execute cmake install
* build_helper: Added the capability to parse axf files for
code/data/bss sizes and capture it to report
* build_helper: Added --relative-paths to calculate paths relative
to the root of the workspace
* build_helper_configs: Full restructure of config modules.
Extra build commands and expected artefacts can be defined per
platform basis
* Checkpatch: Added directive to ignore --ignore SPDX_LICENSE_TAG
and added the capability to run only on files changed in patch.
* CppCheck adjusted suppression directories for new external
libraries and code-base restructure
* Added fastmodel dispatcher. It will wrap around fastmodels
and test against a dynamically defined test_map. Fed with an
input of the build summary fastmodel dispatcher will detect
builds which have tests in the map and run them.
* Added Fastmodel configs for AN519 and AN521 platforms
* lava_helper. Added arguments for --override-jenkins-job/
--override-jenkins-url
* Adjusted JINJA2 template to include build number and
enable the overrides.
* Adjusted lava helper configs to support dual platform firmware
and added CoreIPC config
* Added report parser module to create/read/evaluate and
modify reports. Bash scripts for cppcheck checkpatch summaries
have been removed.
* Adjusted run_cppcheck/run_checkpatch for new project libraries,
new codebase structure and other tweaks.
* Restructured build manager, decoupling it from the tf-m
cmake requirements. Build manager can now dynamically build a
configuration from combination of parameters or can just execute
an array of build commands. Hardcoded tf-m assumptions have been
removed and moved into the configuration space.
* Build system can now produce MUSCA_A/ MUSCA_B1 binaries as well
as intel HEX files.
* Updated the utilities snippet collection in the tfm-ci-pylib.
Change-Id: Ifad7676e1cd47e3418e851b56dbb71963d85cd88
Signed-off-by: Minos Galanakis <minos.galanakis@linaro.org>
diff --git a/tfm_ci_pylib/tfm_builder.py b/tfm_ci_pylib/tfm_builder.py
index 37a1315..1908a8e 100644
--- a/tfm_ci_pylib/tfm_builder.py
+++ b/tfm_ci_pylib/tfm_builder.py
@@ -18,58 +18,48 @@
__email__ = "minos.galanakis@linaro.org"
__project__ = "Trusted Firmware-M Open CI"
__status__ = "stable"
-__version__ = "1.0"
+__version__ = "1.1"
import os
-from .utils import *
+import re
import shutil
+from .utils import *
from .structured_task import structuredTask
class TFM_Builder(structuredTask):
""" Wrap around tfm cmake system and spawn a thread to build the project.
"""
- _tfb_build_params = ["TARGET_PLATFORM",
- "COMPILER",
- "PROJ_CONFIG",
- "CMAKE_BUILD_TYPE",
- "WITH_MCUBOOT"
- ]
-
- _tfb_build_template = ("cmake -G \"Unix Makefiles\" -DPROJ_CONFIG=`"
- "readlink -f %(PROJ_CONFIG)s.cmake` "
- "-DTARGET_PLATFORM=%(TARGET_PLATFORM)s "
- "-DCOMPILER=%(COMPILER)s "
- "-DCMAKE_BUILD_TYPE=%(CMAKE_BUILD_TYPE)s "
- "-DBL2=%(WITH_MCUBOOT)s "
- "%(TFM_ROOT)s")
-
def __init__(self,
name, # Proccess name
- tfm_dir, # TFM root directory
work_dir, # Current working directory(ie logs)
cfg_dict, # Input config dictionary of the following form
# input_dict = {"PROJ_CONFIG": "ConfigRegression",
# "TARGET_PLATFORM": "MUSCA_A",
# "COMPILER": "ARMCLANG",
# "CMAKE_BUILD_TYPE": "Debug"}
- install=False, # Install library after build
- build_threads=4, # Number of CPU thrads used in build
- silent=False): # Silence stdout ouptut
+ build_threads=4, # Number of CPU thrads used in build
+ silent=False, # Silence stdout ouptut
+ img_sizes=False, # Use arm-none-eabi-size for size info
+ relative_paths=False): # Store relative paths in report
self._tfb_cfg = cfg_dict
self._tfb_build_threads = build_threads
- self._tfb_install = install
self._tfb_silent = silent
+ self._tfb_img_sizes = img_sizes
+ self._tfb_relative_paths = relative_paths
self._tfb_binaries = []
# Required by other methods, always set working directory first
self._tfb_work_dir = os.path.abspath(os.path.expanduser(work_dir))
- self._tfb_tfm_dir = os.path.abspath(os.path.expanduser(tfm_dir))
+ # Override code_base_dir with abspath
+ _code_dir = self._tfb_cfg["codebase_root_dir"]
+ self._tfb_code_dir = os.path.abspath(os.path.expanduser(_code_dir))
# Entries will be filled after sanity test on cfg_dict dring pre_exec
self._tfb_build_dir = None
self._tfb_log_f = None
+
super(TFM_Builder, self).__init__(name=name)
def mute(self):
@@ -77,10 +67,14 @@
def log(self):
""" Print and return the contents of log file """
- with open(self._tfb_log_f, "r") as F:
- log = F.read()
- print(log)
- return log
+ try:
+ with open(self._tfb_log_f, "r") as F:
+ log = F.read()
+ print(log)
+ return log
+ except FileNotFoundError:
+ print("Log %s not found" % self._tfb_log_f)
+ return ""
def report(self):
"""Return the report on the job """
@@ -89,14 +83,10 @@
def pre_eval(self):
""" Tests that need to be run in set-up state """
- # Test that all required entries exist in config
- diff = list(set(self._tfb_build_params) - set(self._tfb_cfg.keys()))
- if diff:
- print("Cound't find require build entry: %s in config" % diff)
+ if not os.path.isdir(self._tfb_code_dir):
+ print("Missing code-base directory:", self._tfb_code_dir)
return False
- # TODO check validity of passed config values
- # TODO test detection of srec
- # self.srec_path = shutil.which("srec_cat")
+
return True
def pre_exec(self, eval_ret):
@@ -107,14 +97,6 @@
# Ensure we have a clean build directory
shutil.rmtree(self._tfb_build_dir, ignore_errors=True)
- self._tfb_cfg["TFM_ROOT"] = self._tfb_tfm_dir
-
- # Append the path for the config
- self._tfb_cfg["PROJ_CONFIG"] = os.path.join(self._tfb_tfm_dir,
- "configs",
- self._tfb_cfg[("PROJ_"
- "CONFIG")])
-
# Log will be placed in work directory, named as the build dir
self._tfb_log_f = "%s.log" % self._tfb_build_dir
@@ -123,100 +105,6 @@
if not os.path.exists(p):
os.makedirs(p)
- # Calcuate a list of expected binaries
- binaries = []
-
- # If install is asserted pick the iems from the appropriate location
- if self._tfb_install:
-
- fvp_path = os.path.join(self._tfb_build_dir,
- "install", "outputs", "fvp")
- platform_path = os.path.join(self._tfb_build_dir,
- "install",
- "outputs",
- self._tfb_cfg["TARGET_PLATFORM"])
-
- # Generate a list of binaries included in both directories
- common_bin_list = ["tfm_%s.%s" % (s, e) for s in ["s", "ns"]
- for e in ["bin", "axf"]]
- if self._tfb_cfg["WITH_MCUBOOT"]:
- common_bin_list += ["mcuboot.%s" % e for e in ["bin", "axf"]]
-
- # When building with bootloader extra binaries are expected
- binaries += [os.path.join(platform_path, b) for b in
- ["tfm_sign.bin"]]
- binaries += [os.path.join(fvp_path, b) for b in
- ["tfm_s_ns_signed.bin"]]
-
- binaries += [os.path.join(p, b) for p in [fvp_path, platform_path]
- for b in common_bin_list]
-
- # Add Musca required binaries
- if self._tfb_cfg["TARGET_PLATFORM"] == "MUSCA_A":
- binaries += [os.path.join(platform_path,
- "musca_firmware.hex")]
-
- self._tfb_binaries = binaries
-
- else:
- binaries += [os.path.join(self._tfb_build_dir, "app", "tfm_ns")]
- binaries += [os.path.join(self._tfb_build_dir, "app",
- "secure_fw", "tfm_s")]
- if self._tfb_cfg["WITH_MCUBOOT"]:
- binaries += [os.path.join(self._tfb_build_dir,
- "bl2", "ext", "mcuboot", "mcuboot")]
-
- ext = ['.bin', '.axf']
- self._tfb_binaries = ["%s%s" % (n, e) for n in binaries
- for e in ext]
-
- # Add Musca required binaries
- if self._tfb_cfg["TARGET_PLATFORM"] == "MUSCA_A":
- self._tfb_binaries += [os.path.join(self._tfb_build_dir,
- "tfm_sign.bin")]
- self._tfb_binaries += [os.path.join(self._tfb_build_dir,
- "musca_firmware.hex")]
-
- def get_binaries(self,
- bootl=None,
- bin_s=None,
- bin_ns=None,
- bin_sign=None,
- filt=None):
- """ Return the absolute location of binaries (from config)
- if they exist. Can add a filter parameter which will only
- consider entries with /filter/ in their path as a directory """
- ret_boot = None
- ret_bin_ns = None
- ret_bin_s = None
- ret_bin_sign = None
-
- # Apply filter as a /filter/ string to the binary list
- filt = "/" + filt + "/" if filter else None
- binaries = list(filter(lambda x: filt in x, self._tfb_binaries)) \
- if filt else self._tfb_binaries
-
- for obj_file in binaries:
- fname = os.path.split(obj_file)[-1]
- if bootl:
- if fname == bootl:
- ret_boot = obj_file
- continue
- if bin_s:
- if fname == bin_s:
- ret_bin_s = obj_file
- continue
-
- if bin_ns:
- if fname == bin_ns:
- ret_bin_ns = obj_file
- continue
- if bin_sign:
- if fname == bin_sign:
- ret_bin_sign = obj_file
- continue
- return [ret_boot, ret_bin_s, ret_bin_ns, ret_bin_sign]
-
def task_exec(self):
""" Main tasks """
@@ -224,141 +112,100 @@
self.set_status(-1)
# Go to build directory
os.chdir(self._tfb_build_dir)
- # Compile the build commands
- cmake_cmd = self._tfb_build_template % self._tfb_cfg
- build_cmd = "cmake --build ./ -- -j %s" % self._tfb_build_threads
+
+ build_cmds = self._tfb_cfg["build_cmds"]
+
+ threads_no_rex = re.compile(r'.*(-j\s?(\d+))')
# Pass the report to later stages
- rep = {"build_cmd": "%s" % build_cmd,
- "cmake_cmd": "%s" % cmake_cmd}
+ rep = {"build_cmd": "%s" % ",".join(build_cmds)}
self.stash("Build Report", rep)
- # Calll camke to configure the project
- if not subprocess_log(cmake_cmd,
- self._tfb_log_f,
- prefix=cmake_cmd,
- silent=self._tfb_silent):
+ # Calll cmake to configure the project
+ for build_cmd in build_cmds:
+ # if a -j parameter is passed as user argument
+ user_set_threads_match = threads_no_rex.findall(build_cmd)
+
+ if user_set_threads_match:
+ # Unpack the regex groups (fullmatch, decimal match)
+ user_jtxt, user_set_threads = user_set_threads_match[0]
+ if int(user_set_threads) > self._tfb_build_threads:
+ print("Ignoring user requested n=%s threads because it"
+ " exceeds the maximum thread set ( %d )" %
+ (user_set_threads, self._tfb_build_threads))
+ thread_no = self._tfb_build_threads
+ else:
+ print("Using %s build threads" % user_set_threads)
+ thread_no = user_set_threads
+ build_cmd = build_cmd.replace(user_jtxt,
+ "-j %s " % thread_no)
+
# Build it
if subprocess_log(build_cmd,
self._tfb_log_f,
append=True,
prefix=build_cmd,
silent=self._tfb_silent):
+
raise Exception("Build Failed please check log: %s" %
self._tfb_log_f)
- else:
- raise Exception("Cmake Failed please check log: %s" %
- self._tfb_log_f)
- if self._tfb_install:
- install_cmd = "cmake --build ./ -- -j install"
- if subprocess_log(install_cmd,
- self._tfb_log_f,
- append=True,
- prefix=install_cmd,
- silent=self._tfb_silent):
- raise Exception(("Make install Failed."
- " please check log: %s") % self._tfb_log_f)
- if self._tfb_cfg["TARGET_PLATFORM"] == "MUSCA_A":
- boot_f, s_bin, ns_bin, sns_signed_bin = self.get_binaries(
- bootl="mcuboot.bin",
- bin_s="tfm_s.bin",
- bin_ns="tfm_ns.bin",
- bin_sign="tfm_sign.bin",
- filt="MUSCA_A")
- self.convert_to_hex(boot_f, sns_signed_bin)
self._t_stop()
- def sign_img(self, secure_bin, non_secure_bin):
- """Join a secure and non secure image and sign them"""
-
- imgtool_dir = os.path.join(self._tfb_tfm_dir,
- "bl2/ext/mcuboot/scripts/")
- flash_layout = os.path.join(self._tfb_tfm_dir,
- "platform/ext/target/musca_a/"
- "partition/flash_layout.h")
- sign_cert = os.path.join(self._tfb_tfm_dir,
- "bl2/ext/mcuboot/root-rsa-2048.pem")
- sns_unsigned_bin = os.path.join(self._tfb_build_dir,
- "sns_unsigned.bin")
- sns_signed_bin = os.path.join(self._tfb_build_dir, "sns_signed.bin")
-
- # Early versions of the tool hard relative imports, run from its dir
- os.chdir(imgtool_dir)
- assemble_cmd = ("python3 assemble.py -l %(layout)s -s %(s)s "
- "-n %(ns)s -o %(sns)s") % {"layout": flash_layout,
- "s": secure_bin,
- "ns": non_secure_bin,
- "sns": sns_unsigned_bin
- }
- sign_cmd = ("python3 imgtool.py sign -k %(cert)s --align 1 -v "
- "1.0 -H 0x400 --pad 0x30000 "
- "%(sns)s %(sns_signed)s") % {"cert": sign_cert,
- "sns": sns_unsigned_bin,
- "sns_signed": sns_signed_bin
- }
- run_proccess(assemble_cmd)
- run_proccess(sign_cmd)
- # Return to build directory
- os.chdir(self._tfb_build_dir)
- return sns_signed_bin
-
- def convert_to_hex(self,
- boot_bin,
- sns_signed_bin,
- qspi_base=0x200000,
- boot_size=0x10000):
- """Convert a signed image to an intel hex format with mcuboot """
- if self._tfb_install:
- platform_path = os.path.join(self._tfb_build_dir,
- "install",
- "outputs",
- self._tfb_cfg["TARGET_PLATFORM"])
- firmware_hex = os.path.join(platform_path, "musca_firmware.hex")
- else:
- firmware_hex = os.path.join(self._tfb_build_dir,
- "musca_firmware.hex")
-
- img_offset = qspi_base + boot_size
- merge_cmd = ("srec_cat %(boot)s -Binary -offset 0x%(qspi_offset)x "
- "%(sns_signed)s -Binary -offset 0x%(img_offset)x "
- "-o %(hex)s -Intel") % {"boot": boot_bin,
- "sns_signed": sns_signed_bin,
- "hex": firmware_hex,
- "qspi_offset": qspi_base,
- "img_offset": img_offset
- }
- run_proccess(merge_cmd)
- return
-
def post_eval(self):
""" Verify that the artefacts exist """
print("%s Post eval" % self.get_name())
ret_eval = False
rep = self.unstash("Build Report")
- missing_binaries = list(filter(lambda x: not os.path.isfile(x),
- self._tfb_binaries))
- if len(missing_binaries):
- print("ERROR: Could not locate the following binaries:")
- print("\n".join(missing_binaries))
-
- # Update the artifacts to not include missing ones
- artf = [n for n in self._tfb_binaries if n not in missing_binaries]
- # TODO update self._tfb_binaries
- ret_eval = False
- else:
- print("SUCCESS: Produced binaries:")
- print("\n".join(self._tfb_binaries))
- ret_eval = True
-
- artf = self._tfb_binaries
+ artefacts = list_filtered_tree(self._tfb_work_dir, r'%s' %
+ self._tfb_cfg["artifact_capture_rex"])
# Add artefact related information to report
rep["log"] = self._tfb_log_f
- rep["missing_artefacts"] = missing_binaries
- rep["artefacts"] = artf
+
+ if not len(artefacts):
+ print("ERROR: Could not capture any binaries:")
+
+ # TODO update self._tfb_binaries
+ ret_eval = False
+ else:
+ print("SUCCESS: Produced the following binaries:")
+ print("\n\t".join(artefacts))
+ ret_eval = True
+
+ rep["artefacts"] = artefacts
+
+ # Proccess the artifacts into file structures
+ art_files = {}
+ for art_item in artefacts:
+ art_f = {"pl_source": 1,
+ "resource": art_item if not self._tfb_relative_paths
+ else resolve_rel_path(art_item),
+ "size": {"bytes": str(os.path.getsize(art_item))}
+ }
+ if self._tfb_img_sizes and ".axf" in art_item:
+ eabi_size, _ = arm_non_eabi_size(art_item)
+ art_f["size"]["text"] = eabi_size["text"]
+ art_f["size"]["data"] = eabi_size["data"]
+ art_f["size"]["bss"] = eabi_size["bss"]
+ # filename is used as key for artfacts
+ art_files[os.path.split(art_item)[-1]] = art_f
+ rep["artefacts"] = art_files
+
+ if "required_artefacts" in self._tfb_cfg.keys():
+ if len(self._tfb_cfg["required_artefacts"]):
+ print("Searching for required binaries")
+ missing_binaries = list(filter(lambda x: not os.path.isfile(x),
+ self._tfb_cfg["required_artefacts"]))
+ if len(missing_binaries):
+ rep["missing_artefacts"] = missing_binaries
+ print("ERROR: Missing required artefacts:")
+ print("\n".join(missing_binaries))
+ ret_eval = False
+ else:
+ ret_eval = True
rep["status"] = "Success" if ret_eval else "Failed"
self.stash("Build Report", rep)
@@ -371,3 +218,7 @@
print("TFM Builder %s was Successful" % self.get_name())
else:
print("TFM Builder %s was UnSuccessful" % self.get_name())
+
+
+if __name__ == "__main__":
+ pass