Open CI Scripts: Feature Update
* build_helper: Added --install argument to execute cmake install
* build_helper: Added the capability to parse axf files for
code/data/bss sizes and capture it to report
* build_helper: Added --relative-paths to calculate paths relative
to the root of the workspace
* build_helper_configs: Full restructure of config modules.
Extra build commands and expected artefacts can be defined per
platform basis
* Checkpatch: Added directive to ignore --ignore SPDX_LICENSE_TAG
and added the capability to run only on files changed in patch.
* CppCheck adjusted suppression directories for new external
libraries and code-base restructure
* Added fastmodel dispatcher. It will wrap around fastmodels
and test against a dynamically defined test_map. Fed with an
input of the build summary fastmodel dispatcher will detect
builds which have tests in the map and run them.
* Added Fastmodel configs for AN519 and AN521 platforms
* lava_helper. Added arguments for --override-jenkins-job/
--override-jenkins-url
* Adjusted JINJA2 template to include build number and
enable the overrides.
* Adjusted lava helper configs to support dual platform firmware
and added CoreIPC config
* Added report parser module to create/read/evaluate and
modify reports. Bash scripts for cppcheck checkpatch summaries
have been removed.
* Adjusted run_cppcheck/run_checkpatch for new project libraries,
new codebase structure and other tweaks.
* Restructured build manager, decoupling it from the tf-m
cmake requirements. Build manager can now dynamically build a
configuration from combination of parameters or can just execute
an array of build commands. Hardcoded tf-m assumptions have been
removed and moved into the configuration space.
* Build system can now produce MUSCA_A/ MUSCA_B1 binaries as well
as intel HEX files.
* Updated the utilities snippet collection in the tfm-ci-pylib.
Change-Id: Ifad7676e1cd47e3418e851b56dbb71963d85cd88
Signed-off-by: Minos Galanakis <minos.galanakis@linaro.org>
diff --git a/build_helper/build_helper.py b/build_helper/build_helper.py
index ea8e8f3..58957d7 100755
--- a/build_helper/build_helper.py
+++ b/build_helper/build_helper.py
@@ -20,17 +20,17 @@
__email__ = "minos.galanakis@linaro.org"
__project__ = "Trusted Firmware-M Open CI"
__status__ = "stable"
-__version__ = "1.0"
+__version__ = "1.1"
import os
import sys
import time
import argparse
import datetime
-from build_helper_configs import config_AN521
+from build_helper_configs import _builtin_configs
try:
- from tfm_ci_pylib.utils import get_cmd_args, load_json
+ from tfm_ci_pylib.utils import get_cmd_args
from tfm_ci_pylib.tfm_build_manager import TFM_Build_Manager
except ImportError:
dir_path = os.path.dirname(os.path.realpath(__file__))
@@ -39,7 +39,15 @@
from tfm_ci_pylib.tfm_build_manager import TFM_Build_Manager
-def build(tfm_dir, build_dir, buid_report_f, build_config):
+def build(tfm_dir,
+ build_dir,
+ buid_report_f,
+ build_config,
+ parallel_builds=3,
+ build_threads=3,
+ build_install=True,
+ image_sizes=False,
+ relative_paths=False):
""" Instantiate a build manager class and build all configurations """
start_time = time.time()
@@ -48,7 +56,11 @@
work_dir=build_dir,
cfg_dict=build_config,
report=buid_report_f,
- install=True)
+ parallel_builds=parallel_builds,
+ build_threads=build_threads,
+ install=build_install,
+ img_sizes=image_sizes,
+ relative_paths=relative_paths)
bm.start()
bm.join()
build_report = bm.get_report()
@@ -68,13 +80,27 @@
print("Failed to load config %s. Exception: %s" % (build_config,
e.msg))
sys.exit(1)
+ elif user_args.config:
+ if user_args.config in _builtin_configs.keys():
+ build_config = _builtin_configs[user_args.config.lower()]
+ else:
+ print("Configuration %s is not defined in built-in configs" %
+ user_args.config)
+ sys.exit(1)
else:
- build_config = config_AN521
+ print("Error: Configuration not specificed")
+ sys.exit(1)
+
# Build everything
build_status, build_report = build(user_args.tfm_dir,
user_args.build_dir,
user_args.report,
- build_config)
+ build_config,
+ user_args.parallel_builds,
+ user_args.thread_no,
+ user_args.install,
+ user_args.image_sizes,
+ user_args.relative_paths)
if not build_report:
print("Build Report Empty, check build status")
@@ -82,15 +108,16 @@
if build_status:
print("Build Failed")
- sys.exit(1)
+ if user_args.eif:
+ sys.exit(1)
# pprint(build_report)
- print("Build Complete!")
+ print("Build Helper Quitting!")
sys.exit(0)
if __name__ == "__main__":
- # Calcuate the workspace root directory relative to the script location
+ # Calculate the workspace root directory relative to the script location
# Equivalent to realpath $(dirname ./build_helper/build_helper.py)/../../
root_path = os.path.dirname(os.path.realpath(__file__))
for i in range(2):
@@ -102,7 +129,17 @@
action="store",
default="./builds",
help="Where to generate the artifacts")
- parser.add_argument("-c", "--config_file",
+ parser.add_argument("-c", "--config",
+ dest="config",
+ action="store",
+ help="Which of the built-in configs to run."
+ "(%s)" % "/ ".join(_builtin_configs.keys()))
+ parser.add_argument("-e", "--error_if_failed",
+ dest="eif",
+ action="store_true",
+ help="If set will change the script exit code if one "
+ "or more builds fail")
+ parser.add_argument("-f", "--config_file",
dest="config_f",
action="store",
help="Manual configuration override file (JSON)")
@@ -110,10 +147,36 @@
dest="report",
action="store",
help="JSON file containing build report")
+ parser.add_argument("-i", "--install",
+ dest="install",
+ action="store_true",
+ help="Run make install after building config")
parser.add_argument("-t", "--tfm_dir",
dest="tfm_dir",
action="store",
default=os.path.join(root_path, "tf-m"),
help="TFM directory")
-
+ parser.add_argument("-s", "--image-sizes",
+ dest="image_sizes",
+ action="store_true",
+ help="Run arm-none-eabi-size to axf files "
+ "generated by build")
+ parser.add_argument("-l", "--relative-paths",
+ dest="relative_paths",
+ action="store_true",
+ help="When set paths stored in report will be stored"
+ "in a relative path to the execution directory."
+ "Recommended for Jenkins Builds.")
+ parser.add_argument("-p", "--parallel-builds",
+ type=int,
+ dest="parallel_builds",
+ action="store",
+ default=3,
+ help="Number of builds jobs to run in parallel.")
+ parser.add_argument("-n", "--number-of-threads",
+ type=int,
+ dest="thread_no",
+ action="store",
+ default=3,
+ help="Number of threads to use per build job.")
main(get_cmd_args(parser=parser))
diff --git a/build_helper/build_helper_configs.py b/build_helper/build_helper_configs.py
index 39436c8..e48abeb 100644
--- a/build_helper/build_helper_configs.py
+++ b/build_helper/build_helper_configs.py
@@ -18,37 +18,255 @@
__email__ = "minos.galanakis@linaro.org"
__project__ = "Trusted Firmware-M Open CI"
__status__ = "stable"
-__version__ = "1.0"
+__version__ = "1.1"
+# common parameters for tf-m build system
+# This configuration template will be passed into the tfm-builder module after
+# the template evaluation is converted to a command
+
+_common_tfm_builder_cfg = {
+ "config_type": "tf-m",
+ "codebase_root_dir": "tf-m",
+ # Order to which the variants are evaluated. This affects the name of
+ # variant configuration and the wildcard replacement logic in invalid
+ # configuration tuples
+ "sort_order": ["target_platform",
+ "compiler",
+ "proj_config",
+ "cmake_build_type",
+ "with_mcuboot"],
+
+ # Keys for the templace will come from the combinations of parameters
+ # provided in the seed dictionary.
+
+ "config_template": (
+ "cmake -G \"Unix Makefiles\" "
+ "-DPROJ_CONFIG=`"
+ "readlink -f %(codebase_root_dir)s/configs/%(proj_config)s.cmake` "
+ "-DTARGET_PLATFORM=%(target_platform)s "
+ "-DCOMPILER=%(compiler)s "
+ "-DCMAKE_BUILD_TYPE=%(cmake_build_type)s "
+ "-DBL2=%(with_mcuboot)s "
+ "%(codebase_root_dir)s"),
+
+ # A small subset of string substitution params is allowed in commands.
+ # tfm_build_manager will replace %(_tbm_build_dir_)s, %(_tbm_code_dir_)s,
+ # _tbm_target_platform_ with the paths set when building
+
+ "artifact_capture_rex": (r'%(_tbm_build_dir_)s/install/outputs/'
+ r'(?:fvp|AN521|AN519|MUSCA_A|MUSCA_B1)'
+ r'/(\w+\.(?:axf|bin|hex))$'),
+
+ # ALL commands will be executed for every build.
+ # Other keys will append extra commands when matching target_platform
+ "build_cmds": {"all": ["cmake --build ./ -- -j 2 install"],
+ "MUSCA_A": [("srec_cat "
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/mcuboot.bin "
+ "-Binary -offset 0x200000 "
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/tfm_sign.bin "
+ "-Binary -offset 0x220000 -o "
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s"
+ "/tfm.hex -Intel")],
+ "MUSCA_B1": [("srec_cat "
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/mcuboot.bin "
+ "-Binary -offset 0x200000 "
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/tfm_sign.bin "
+ "-Binary -offset 0x220000 -o "
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s"
+ "/tfm.hex -Intel")]
+ },
+
+ # (Optional) If set will fail if those artefacts are missing post build
+ "required_artefacts": {"all": [
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/tfm_s.bin",
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/tfm_ns.bin"],
+ "MUSCA_A": [
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/tfm.hex",
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/mcuboot.bin",
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/tfm_sign.bin"],
+ "MUSCA_B1": [
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/tfm.hex",
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/mcuboot.bin",
+ "%(_tbm_build_dir_)s/install/outputs/"
+ "%(_tbm_target_platform_)s/tfm_sign.bin"]
+ }
+}
# Configure build manager to build several combinations
-config_AN521 = {"platform": ["AN521"],
+config_AN521 = {"seed_params": {
+ "target_platform": ["AN521"],
"compiler": ["GNUARM"],
- "config": ["ConfigRegression",
- "ConfigDefault"],
- "build": ["Debug"],
- "with_mcuboot": [True],
+ "proj_config": ["ConfigRegression",
+ "ConfigCoreIPC",
+ "ConfigCoreIPCTfmLevel2",
+ "ConfigDefault"],
+ "cmake_build_type": ["Debug", "Release"],
+ "with_mcuboot": [True, False],
+ },
+ "common_params": _common_tfm_builder_cfg,
# invalid configuations can be added as tuples of adjustable
# resolution "AN521" will reject all combinations for that
# platform while ("AN521", "GNUARM") will only reject GCC ones
"invalid": []
}
-_builtin_configs = {"AN521_gnuarm_Config_DRC": config_AN521}
+
+# Configure build manager to build several combinations
+config_AN519 = {"seed_params": {
+ "target_platform": ["AN519"],
+ "compiler": ["GNUARM"],
+ "proj_config": ["ConfigRegression",
+ "ConfigCoreIPC",
+ "ConfigCoreIPCTfmLevel2",
+ "ConfigDefault"],
+ "cmake_build_type": ["Debug", "Release"],
+ "with_mcuboot": [True, False],
+ },
+ "common_params": _common_tfm_builder_cfg,
+ # invalid configuations can be added as tuples of adjustable
+ # resolution "AN521" will reject all combinations for that
+ # platform while ("AN521", "GNUARM") will only reject GCC ones
+ "invalid": []
+ }
+
+config_IPC = {"seed_params": {
+ "target_platform": ["AN521", "AN519", "MUSCA_A", "MUSCA_B1"],
+ "compiler": ["ARMCLANG", "GNUARM"],
+ "proj_config": ["ConfigCoreIPC",
+ "ConfigCoreIPCTfmLevel2"],
+ "cmake_build_type": ["Debug", "Release"],
+ "with_mcuboot": [True, False],
+ },
+ "common_params": _common_tfm_builder_cfg,
+ # invalid configuations can be added as tuples of adjustable
+ # resolution "AN521" will reject all combinations for that
+ # platform while ("AN521", "GNUARM") will only reject GCC
+ "invalid": [("MUSCA_B1", "*", "*", "*", False)]
+ }
+
+# Configure build manager to build the maximum number of configurations
+config_full = {"seed_params": {
+ "target_platform": ["AN521", "AN519", "MUSCA_A", "MUSCA_B1"],
+ "compiler": ["ARMCLANG", "GNUARM"],
+ "proj_config": ["ConfigRegression",
+ "ConfigCoreIPC",
+ "ConfigCoreIPCTfmLevel2",
+ "ConfigDefault"],
+ "cmake_build_type": ["Debug", "Release"],
+ "with_mcuboot": [True, False],
+ },
+ "common_params": _common_tfm_builder_cfg,
+ # invalid configuations can be added as tuples of adjustable
+ # resolution "AN521" will reject all combinations for that
+ # platform while ("AN521", "GNUARM") will only reject GCC ones
+ "invalid": [("MUSCA_A", "*", "*", "*", False),
+ ("MUSCA_B1", "*", "*", "*", False)]
+ }
+
+config_MUSCA_A = {"seed_params": {
+ "target_platform": ["MUSCA_A"],
+ "compiler": ["GNUARM"],
+ "proj_config": ["ConfigRegression",
+ "ConfigCoreIPC",
+ "ConfigCoreIPCTfmLevel2",
+ "ConfigDefault"],
+ "cmake_build_type": ["Debug", "Release"],
+ "with_mcuboot": [True],
+ },
+ "common_params": _common_tfm_builder_cfg,
+ # invalid configuations can be added as tuples of adjustable
+ # resolution "AN521" will reject all combinations for that
+ # platform while ("AN521", "GNUARM") will only reject GCC
+ "invalid": [("MUSCA_A", "*", "*", "*", False)]
+ }
+
+config_MUSCA_B1 = {"seed_params": {
+ "target_platform": ["MUSCA_B1"],
+ "compiler": ["GNUARM"],
+ "proj_config": ["ConfigRegression",
+ "ConfigCoreIPC",
+ "ConfigCoreIPCTfmLevel2",
+ "ConfigDefault"],
+ "cmake_build_type": ["Debug", "Release"],
+ "with_mcuboot": [True],
+ },
+ "common_params": _common_tfm_builder_cfg,
+ # invalid configuations can be added as tuples of adjustable
+ # resolution "AN521" will reject all combinations for that
+ # platform while ("AN521", "GNUARM") will only reject GCC
+ "invalid": [("MUSCA_B1", "*", "*", "*", False)]
+ }
+
+# Configruation used for document building
+config_doxygen = {"common_params": {
+ "config_type": "tf-m_documents",
+ "codebase_root_dir": "tf-m",
+ "build_cmds": {"all": ["cmake -G \"Unix Makefiles\" "
+ "-DPROJ_CONFIG=`readlink -f "
+ "%(_tbm_code_dir_)s/"
+ "configs/ConfigDefault.cmake` "
+ "-DTARGET_PLATFORM=AN521 "
+ "-DCOMPILER=GNUARM "
+ "-DCMAKE_BUILD_TYPE=Debug "
+ "-DBL2=True "
+ "%(_tbm_code_dir_)s/",
+ "cmake --build ./ -- install_doc",
+ "cmake --build ./ "
+ "-- install_userguide"]},
+ "artifact_capture_rex": r'%(_tbm_build_dir_)s/install/'
+ r'doc/reference_manual/(?:pdf|html)'
+ r'/(\w+\.(?:html|md|pdf))$',
+ },
+ "invalid": []
+ }
+
+# Configruation used in testing
+config_debug = {"seed_params": {
+ "target_platform": ["AN521"],
+ "compiler": ["ARMCLANG"],
+ "proj_config": ["ConfigDefault"],
+ "cmake_build_type": ["Debug"],
+ "with_mcuboot": [True],
+ },
+ "common_params": _common_tfm_builder_cfg,
+ # invalid configuations can be added as tuples of adjustable
+ # resolution "AN521" will reject all combinations for that
+ # platform while ("AN521", "GNUARM") will only reject GCC ones
+ "invalid": [("*", "GNUARM", "*", "*", False),
+ ("AN521", "ARMCLANG", "ConfigRegression",
+ "Release", False),
+ ]
+ }
+
+_builtin_configs = {"full": config_full,
+ "an521": config_AN521,
+ "an519": config_AN519,
+ "musca_a": config_MUSCA_A,
+ "musca_b1": config_MUSCA_B1,
+ "ipc": config_IPC,
+ "doxygen": config_doxygen,
+ "debug": config_debug}
if __name__ == '__main__':
import os
- import sys
- try:
- from tfm_ci_pylib.utils import export_config_map
- except ImportError:
- dir_path = os.path.dirname(os.path.realpath(__file__))
- sys.path.append(os.path.join(dir_path, "../"))
- from tfm_ci_pylib.utils import export_config_map
- if len(sys.argv) == 2:
- if sys.argv[1] == "--export":
- export_config_map(_builtin_configs)
- if len(sys.argv) == 3:
- if sys.argv[1] == "--export":
- export_config_map(_builtin_configs, sys.argv[2])
+ # Default behavior is to export refference config when called
+ _dir = os.getcwd()
+ from utils import save_json
+ for _cname, _cfg in _builtin_configs.items():
+ _fname = os.path.join(_dir, _cname + ".json")
+ print("Exporting config %s" % _fname)
+ save_json(_fname, _cfg)