Add local CI scripts for TFM 1.0 release

These scripts are based on current CI scripts
    iot-sw/tfm-ci-scripts with commit 5acf417

Major changes:
    Add configs to build all combinations for release required
    Codebase dynamically change when build PSA suite and OTP for parallel
    Build rules for PSA DEV API/PSA FF/ARCH TEST/OTP
    Build with parallel as much at it can be on the host
    Add test PSA ARCH on AN521 FVP for PSA DEV API
    Support PSA FF/DEV API tests on AN521 FVP
    Add “Minsizerel” type support for FVP tests
    Some hard-code or temporary changes that need to refine

Change-Id: I497b97e7cc57924295321259068b333a81a8f1a6
Signed-off-by: Karl Zhang <karl.zhang@arm.com>
diff --git a/tfm_ci_pylib/fastmodel_wrapper/__init__.py b/tfm_ci_pylib/fastmodel_wrapper/__init__.py
index d59ebcc..a1fca97 100644
--- a/tfm_ci_pylib/fastmodel_wrapper/__init__.py
+++ b/tfm_ci_pylib/fastmodel_wrapper/__init__.py
@@ -18,4 +18,6 @@
 
 from .config_templates import template_default_config, \
     template_regression_config, template_coreipc_config, \
-    template_coreipctfmlevel2_config
+    template_coreipctfmlevel2_config, template_regressionipc_config, \
+    template_regressionipctfmlevel2_config, template_psaapitestipctfmlevel2_config, \
+    template_psaapitestipc_config, template_psaapitest_config
diff --git a/tfm_ci_pylib/fastmodel_wrapper/config_templates.py b/tfm_ci_pylib/fastmodel_wrapper/config_templates.py
index 2f97652..fea5b7c 100644
--- a/tfm_ci_pylib/fastmodel_wrapper/config_templates.py
+++ b/tfm_ci_pylib/fastmodel_wrapper/config_templates.py
@@ -44,6 +44,7 @@
         "config": "%(config)s",
         "platform": "%(platform)s",
         "compiler": "%(compiler)s",
+        "psa_suite": "%(psa_suite)s",
         "build_type": "%(build_type)s",
         "bootloader": "%(bootloader)s"
     }
@@ -228,3 +229,108 @@
     # Testing will stop if string is reached
     _cfg["test_end_string"] = "Secure image initializing"
     _cfg["simlimit"] = "1200"
+
+class template_psaapitestipctfmlevel2_config(template_cfg):
+    """ Will automatically populate the required information for tfm
+    coreipc tfmlevel2 configuration testing. User still needs to set the
+    buildpath, platform, compiler variants """
+
+    _cfg = deepcopy(template_cfg._cfg)
+
+    _vdict = deepcopy(template_cfg._vdict)
+
+    # Set defaults across all variants
+    _vdict["build_path"] = "build-ci-all"
+
+    _vdict["app_bin_path"] = "install/outputs/fvp"
+    _vdict["data_bin_path"] = "install/outputs/fvp"
+
+    _vdict["variant_name_tpl"] = "%(platform)s_%(compiler)s_%(config)s_" + \
+        "%(psa_suite)s_%(build_type)s_%(bootloader)s"
+
+    # Mofify the %(config)s parameter of the template
+    _vdict["config"] = "ConfigPsaApiTestIPCTfmLevel2"
+    _cfg["terminal_log"] = _cfg["terminal_log"] % _vdict
+
+    # System supports two types of matching with
+    # test_case_id and result match group and only test_case_id
+    _cfg["test_rex"] = (r'Entering standby'
+                        )
+
+    # test_case_id capture group Should match test_cases entries
+    _cfg["test_cases"] = [
+        'Secure image initializing!',
+    ]
+    # Testing will stop if string is reached
+    _cfg["test_end_string"] = "Entering standby.."
+    _cfg["simlimit"] = "1200"
+
+class template_psaapitestipc_config(template_cfg):
+    """ Will automatically populate the required information for tfm
+    coreipc tfmlevel2 configuration testing. User still needs to set the
+    buildpath, platform, compiler variants """
+
+    _cfg = deepcopy(template_cfg._cfg)
+
+    _vdict = deepcopy(template_cfg._vdict)
+
+    # Set defaults across all variants
+    _vdict["build_path"] = "build-ci-all"
+
+    _vdict["app_bin_path"] = "install/outputs/fvp"
+    _vdict["data_bin_path"] = "install/outputs/fvp"
+
+    _vdict["variant_name_tpl"] = "%(platform)s_%(compiler)s_%(config)s_" + \
+        "%(psa_suite)s_%(build_type)s_%(bootloader)s"
+
+    # Mofify the %(config)s parameter of the template
+    _vdict["config"] = "ConfigPsaApiTestIPC"
+    _cfg["terminal_log"] = _cfg["terminal_log"] % _vdict
+
+    # System supports two types of matching with
+    # test_case_id and result match group and only test_case_id
+    _cfg["test_rex"] = (r'Entering standby'
+                        )
+
+    # test_case_id capture group Should match test_cases entries
+    _cfg["test_cases"] = [
+        'Secure image initializing!',
+    ]
+    # Testing will stop if string is reached
+    _cfg["test_end_string"] = "Entering standby.."
+    _cfg["simlimit"] = "1200"
+
+class template_psaapitest_config(template_cfg):
+    """ Will automatically populate the required information for tfm
+    coreipc tfmlevel2 configuration testing. User still needs to set the
+    buildpath, platform, compiler variants """
+
+    _cfg = deepcopy(template_cfg._cfg)
+
+    _vdict = deepcopy(template_cfg._vdict)
+
+    # Set defaults across all variants
+    _vdict["build_path"] = "build-ci-all"
+
+    _vdict["app_bin_path"] = "install/outputs/fvp"
+    _vdict["data_bin_path"] = "install/outputs/fvp"
+
+    _vdict["variant_name_tpl"] = "%(platform)s_%(compiler)s_%(config)s_" + \
+        "%(psa_suite)s_%(build_type)s_%(bootloader)s"
+
+    # Mofify the %(config)s parameter of the template
+    _vdict["config"] = "ConfigPsaApiTest"
+    _cfg["terminal_log"] = _cfg["terminal_log"] % _vdict
+
+    # System supports two types of matching with
+    # test_case_id and result match group and only test_case_id
+    _cfg["test_rex"] = (r'Entering standby'
+                        )
+
+    # test_case_id capture group Should match test_cases entries
+    _cfg["test_cases"] = [
+        'Secure image initializing!',
+    ]
+    # Testing will stop if string is reached
+    _cfg["test_end_string"] = "Entering standby.."
+    _cfg["simlimit"] = "1200"
diff --git a/tfm_ci_pylib/fastmodel_wrapper/fastmodel_wrapper.py b/tfm_ci_pylib/fastmodel_wrapper/fastmodel_wrapper.py
index 7566c2e..d49cd6c 100755
--- a/tfm_ci_pylib/fastmodel_wrapper/fastmodel_wrapper.py
+++ b/tfm_ci_pylib/fastmodel_wrapper/fastmodel_wrapper.py
@@ -257,6 +257,7 @@
 
         # Convert to list
         cmd = self.compile_cmd().split(" ")
+        print("fvp cmd ", self.compile_cmd())
 
         # Run it as subproccess
         self.fvp_proc = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=False)
@@ -286,9 +287,10 @@
                     queue.put(line)
 
                 # If the text end string is found terminate
-                if self.test_end_string in str(line):
+                if str(line).find(self.test_end_string) > 0:
 
                     queue.put("Found End String \"%s\"" % self.test_end_string)
+                    print("Found End String \"%s\"" % self.test_end_string)
                     self.test_complete = True
                     self.stop()
                     break
@@ -344,7 +346,7 @@
             print("Could not find all binaries from %s" % ", ".join(bin_list))
             print("Missing Files:", ", ".join(find_missing_files(bin_list)))
             sys.exit(1)
-
+        self.show_cmd()
         self.pids.append(self.run_fpv())
         self.pids.append(self.run_monitor())
         print("Spawned Proccesses with PID %s" % repr(self.pids)[1:-1])
diff --git a/tfm_ci_pylib/fastmodel_wrapper/fastmodel_wrapper_config.py b/tfm_ci_pylib/fastmodel_wrapper/fastmodel_wrapper_config.py
index 0c2f60a..2180620 100644
--- a/tfm_ci_pylib/fastmodel_wrapper/fastmodel_wrapper_config.py
+++ b/tfm_ci_pylib/fastmodel_wrapper/fastmodel_wrapper_config.py
@@ -200,6 +200,25 @@
                 _cfg["application"] = _cfg["application"] % _vdict
                 _cfg["data"] = _cfg["data"] % _vdict
 
+                if _vdict["psa_suite"] == "FF":
+                    print("TfmFastModelConfig override:")
+                    _cfg["parameters"] = [
+                        "fvp_mps2.platform_type=2",
+                        "cpu0.baseline=0",
+                        "cpu0.INITVTOR_S=0x10080400",
+                        "cpu0.semihosting-enable=0",
+                        "fvp_mps2.DISABLE_GATING=0",
+                        "fvp_mps2.telnetterminal0.start_telnet=0",
+                        "fvp_mps2.telnetterminal1.start_telnet=0",
+                        "fvp_mps2.telnetterminal2.start_telnet=0",
+                        "fvp_mps2.telnetterminal0.quiet=1",
+                        "fvp_mps2.telnetterminal1.quiet=1",
+                        "fvp_mps2.telnetterminal2.quiet=1",
+                        "fvp_mps2.UART2.out_file=$TERM_FILE",
+                        "fvp_mps2.UART2.unbuffered_output=1",
+                        "fvp_mps2.UART0.shutdown_on_eot=1",
+                        "fvp_mps2.mps2_visualisation.disable-visualisation=1"]
+
                 _name = cls._name % _vdict
 
         return TfmFastModelConfig
diff --git a/tfm_ci_pylib/tfm_build_manager.py b/tfm_ci_pylib/tfm_build_manager.py
index 4ce848a..56bf469 100644
--- a/tfm_ci_pylib/tfm_build_manager.py
+++ b/tfm_ci_pylib/tfm_build_manager.py
@@ -22,6 +22,7 @@
 
 import os
 import sys
+from .utils import *
 from time import time
 from copy import deepcopy
 from .utils import gen_cfg_combinations, list_chunks, load_json,\
@@ -60,6 +61,8 @@
 
         self._tbm_tfm_dir = os.path.abspath(os.path.expanduser(tfm_dir))
 
+        print("bm param tfm_dir %s" % tfm_dir)
+        print("bm %s %s %s" % (work_dir, cfg_dict, self._tbm_work_dir))
         # Internal flag to tag simple (non combination formatted configs)
         self.simple_config = False
         self._tbm_report = report
@@ -67,6 +70,8 @@
         self._tbm_cfg = self.load_config(cfg_dict, self._tbm_work_dir)
         self._tbm_build_cfg, \
             self.tbm_common_cfg = self.parse_config(self._tbm_cfg)
+        self._tfb_code_base_updated = False
+        self._tfb_log_f = "CodeBasePrepare.log"
 
         super(TFM_Build_Manager, self).__init__(name="TFM_Build_Manager")
 
@@ -124,6 +129,66 @@
                                 "of type %s" % (key, config[key]))
         return config
 
+    def pre_build(self, build_cfg):
+        print("pre_build start %s \r\nself._tfb_cfg %s\r\n" %
+                (self, build_cfg))
+
+        try:
+            if self._tfb_code_base_updated:
+                print("Code base has been updated")
+                return True
+
+            self._tfb_code_base_updated = True
+
+            if "build_psa_api" in build_cfg:
+                # FF IPC build needs repo manifest update for TFM and PSA arch test
+                if "build_ff_ipc" in build_cfg:
+                    print("Checkout to FF IPC code base")
+                    os.chdir(build_cfg["codebase_root_dir"] + "/../psa-arch-tests/api-tests")
+                    _api_test_manifest = "git checkout . ; python3 tools/scripts/manifest_update.py"
+                    if subprocess_log(_api_test_manifest,
+                                      self._tfb_log_f,
+                                      append=True,
+                                      prefix=_api_test_manifest):
+
+                        raise Exception("Python Failed please check log: %s" %
+                                        self._tfb_log_f)
+
+                    _api_test_manifest_tfm = "python3 tools/tfm_parse_manifest_list.py -m tools/tfm_psa_ff_test_manifest_list.yaml append"
+                    os.chdir(build_cfg["codebase_root_dir"])
+                    if subprocess_log(_api_test_manifest_tfm,
+                                      self._tfb_log_f,
+                                      append=True,
+                                      prefix=_api_test_manifest_tfm):
+
+                        raise Exception("Python TFM Failed please check log: %s" %
+                                        self._tfb_log_f)
+                    return True
+
+            print("Checkout to default code base")
+            os.chdir(build_cfg["codebase_root_dir"] + "/../psa-arch-tests/api-tests")
+            _api_test_manifest = "git checkout ."
+            if subprocess_log(_api_test_manifest,
+                              self._tfb_log_f,
+                              append=True,
+                              prefix=_api_test_manifest):
+
+                raise Exception("Python Failed please check log: %s" %
+                                self._tfb_log_f)
+
+            _api_test_manifest_tfm = "python3 tools/tfm_parse_manifest_list.py"
+            os.chdir(build_cfg["codebase_root_dir"])
+            if subprocess_log(_api_test_manifest_tfm,
+                              self._tfb_log_f,
+                              append=True,
+                              prefix=_api_test_manifest_tfm):
+
+                raise Exception("Python TFM Failed please check log: %s" %
+                                self._tfb_log_f)
+        finally:
+            print("python pass after builder prepare")
+            os.chdir(build_cfg["codebase_root_dir"] + "/../")
+
     def task_exec(self):
         """ Create a build pool and execute them in parallel """
 
@@ -169,7 +234,8 @@
         # When a seed pool is provided iterate through the entries
         # and update platform spefific parameters
         elif len(self._tbm_build_cfg):
-
+            print("\r\n_tbm_build_cfg %s\r\n tbm_common_cfg %s\r\n" \
+             % (self._tbm_build_cfg, self.tbm_common_cfg))
             for name, i in self._tbm_build_cfg.items():
                 # Do not modify the original config
                 build_cfg = deepcopy(self.tbm_common_cfg)
@@ -193,11 +259,62 @@
 
                 # Merge the two dictionaries since the template may contain
                 # fixed and combinations seed parameters
-                cmd0 = build_cfg["config_template"] % \
-                    dict(dict(i._asdict()), **build_cfg)
+                if i.proj_config.startswith("ConfigPsaApiTest"):
+                    #PSA API tests only
+                    #TODO i._asdict()["tfm_build_dir"] = self._tbm_work_dir
+                    cmd0 = build_cfg["config_template_psa_api"] % \
+                        {**dict(i._asdict()), **build_cfg}
+                    cmd0 += " -DPSA_API_TEST_BUILD_PATH=" + self._tbm_work_dir + \
+                            "/" + name + "/BUILD"
 
-                # Prepend configuration commoand as the first cmd
+                    if i.psa_api_suit == "FF":
+                        cmd0 += " -DPSA_API_TEST_IPC=ON"
+                        cmd2 = "cmake " + build_cfg["codebase_root_dir"] + "/../psa-arch-tests/api-tests/ " + \
+                            "-G\"Unix Makefiles\" -DTARGET=tgt_ff_tfm_" + \
+                            i.target_platform.lower() +" -DCPU_ARCH=armv8m_ml -DTOOLCHAIN=" + \
+                            i.compiler + " -DSUITE=IPC -DPSA_INCLUDE_PATHS=\"" + \
+                            build_cfg["codebase_root_dir"] + "/interface/include/"
+
+                        cmd2 += ";" + build_cfg["codebase_root_dir"] + \
+                            "/../psa-arch-tests/api-tests/platform/manifests\"" + \
+                            " -DINCLUDE_PANIC_TESTS=1 -DPLATFORM_PSA_ISOLATION_LEVEL=" + \
+                            (("2") if i.proj_config.find("TfmLevel2") > 0 else "1") + \
+                            " -DSP_HEAP_MEM_SUPP=0"
+                        if i.target_platform == "MUSCA_B1":
+                            cmd0 += " -DSST_RAM_FS=ON"
+                        build_cfg["build_ff_ipc"] = "IPC"
+                    else:
+                        cmd0 += " -DPSA_API_TEST_" + i.psa_api_suit + "=ON"
+                        cmd2 = "cmake " + build_cfg["codebase_root_dir"] + "/../psa-arch-tests/api-tests/ " + \
+                            "-G\"Unix Makefiles\" -DTARGET=tgt_dev_apis_tfm_" + \
+                            i.target_platform.lower() +" -DCPU_ARCH=armv8m_ml -DTOOLCHAIN=" + \
+                            i.compiler + " -DSUITE=" + i.psa_api_suit +" -DPSA_INCLUDE_PATHS=\"" + \
+                            build_cfg["codebase_root_dir"] + "/interface/include/\""
+
+                    cmd2 += " -DCMAKE_BUILD_TYPE=" + i.cmake_build_type
+
+                    cmd3 = "cmake --build ."
+                    build_cfg["build_psa_api"] = cmd2 + " ; " + cmd3
+
+                else:
+                    cmd0 = build_cfg["config_template"] % \
+                        {**dict(i._asdict()), **build_cfg}
+
+                try:
+                    if i.__str__().find("with_OTP") > 0:
+                        cmd0 += " -DCRYPTO_HW_ACCELERATOR_OTP_STATE=ENABLED"
+                    else:
+                        build_cfg["build_cmds"][0] += " -j 2"
+                    if cmd0.find("SST_RAM_FS=ON") < 0 and i.target_platform == "MUSCA_B1":
+                        cmd0 += " -DSST_RAM_FS=OFF -DITS_RAM_FS=OFF"
+                except Exception as E:
+                    pass
+
+                # Prepend configuration commoand as the first cmd [cmd1] + [cmd2] + [cmd3] +
                 build_cfg["build_cmds"] = [cmd0] + build_cfg["build_cmds"]
+                print("cmd0 %s\r\n" % (build_cfg["build_cmds"]))
+                if "build_psa_api" in build_cfg:
+                    print("cmd build_psa_api %s\r\n" % build_cfg["build_psa_api"])
 
                 # Set the overrid params
                 over_dict = {"_tbm_build_dir_": os.path.join(
@@ -211,7 +328,7 @@
                 build_cfg = self.override_tbm_cfg_params(build_cfg,
                                                          over_params,
                                                          **over_dict)
-
+                self.pre_build(build_cfg)
                 # Overrides path in expected artefacts
                 print("Loading config %s" % name)
 
@@ -381,7 +498,7 @@
             fl = ([k for k, v in full_rep.items() if v['status'] == 'Failed'])
             ps = ([k for k, v in full_rep.items() if v['status'] == 'Success'])
         except Exception as E:
-            print("No report generated")
+            print("No report generated", E)
             return
         if fl:
             print_test(t_list=fl, status="failed", tname="Builds")
@@ -401,6 +518,7 @@
             # optional parameters.
             tags = [n for n in static_config["sort_order"]
                     if n in seed_config.keys()]
+            print("!!!!!!!!!!!gen list %s\r\n" % tags)
 
             data = []
             for key in tags:
@@ -421,7 +539,12 @@
 
             # Replace bollean vaiables with more BL2/NOBL2 and use it as"
             # configuration name.
-            ret_cfg[i_str.replace("True", "BL2").replace("False", "NOBL2")] = i
+            i_str = i_str.replace("True", "BL2").replace("False", "NOBL2")
+            i_str = i_str.replace("CRYPTO", "Crypto")
+            i_str = i_str.replace("PROTECTED_STORAGE", "PS")
+            i_str = i_str.replace("INITIAL_ATTESTATION", "Attest")
+            i_str = i_str.replace("INTERNAL_TRUSTED_STORAGE", "ITS")
+            ret_cfg[i_str] = i
 
         return ret_cfg
 
diff --git a/tfm_ci_pylib/tfm_builder.py b/tfm_ci_pylib/tfm_builder.py
index 1908a8e..2125727 100644
--- a/tfm_ci_pylib/tfm_builder.py
+++ b/tfm_ci_pylib/tfm_builder.py
@@ -22,6 +22,7 @@
 
 import os
 import re
+import time
 import shutil
 from .utils import *
 from .structured_task import structuredTask
@@ -105,11 +106,165 @@
             if not os.path.exists(p):
                 os.makedirs(p)
 
+    def pre_build(self):
+        print("builder start %s \r\nself._tfb_cfg %s\r\n" %
+                (self, self._tfb_cfg))
+
+        try:
+            self._lock.acquire()
+            if self._tfb_code_base_updated:
+                print("Code base has been updated")
+                return True
+
+            self._tfb_code_base_updated = True
+            self._lock.release()
+
+            if "build_psa_api" in self._tfb_cfg:
+                # FF IPC build needs repo manifest update for TFM and PSA arch test
+                if "build_ff_ipc" in self._tfb_cfg:
+                    print("Checkout to FF IPC code base")
+                    os.chdir(self._tfb_cfg["codebase_root_dir"] + "/../psa-arch-tests/api-tests")
+                    _api_test_manifest = "git checkout . ; python3 tools/scripts/manifest_update.py"
+                    if subprocess_log(_api_test_manifest,
+                                      self._tfb_log_f,
+                                      append=True,
+                                      prefix=_api_test_manifest,
+                                      silent=self._tfb_silent):
+
+                        raise Exception("Python Failed please check log: %s" %
+                                        self._tfb_log_f)
+
+                    _api_test_manifest_tfm = "python3 tools/tfm_parse_manifest_list.py -m tools/tfm_psa_ff_test_manifest_list.yaml append"
+                    os.chdir(self._tfb_cfg["codebase_root_dir"])
+                    if subprocess_log(_api_test_manifest_tfm,
+                                      self._tfb_log_f,
+                                      append=True,
+                                      prefix=_api_test_manifest_tfm,
+                                      silent=self._tfb_silent):
+
+                        raise Exception("Python TFM Failed please check log: %s" %
+                                        self._tfb_log_f)
+                else:
+                    print("Checkout to default code base")
+                    os.chdir(self._tfb_cfg["codebase_root_dir"] + "/../psa-arch-tests/api-tests")
+                    _api_test_manifest = "git checkout ."
+                    if subprocess_log(_api_test_manifest,
+                                      self._tfb_log_f,
+                                      append=True,
+                                      prefix=_api_test_manifest,
+                                      silent=self._tfb_silent):
+
+                        raise Exception("Python Failed please check log: %s" %
+                                        self._tfb_log_f)
+
+                    _api_test_manifest_tfm = "python3 tools/tfm_parse_manifest_list.py"
+                    os.chdir(self._tfb_cfg["codebase_root_dir"])
+                    if subprocess_log(_api_test_manifest_tfm,
+                                      self._tfb_log_f,
+                                      append=True,
+                                      prefix=_api_test_manifest_tfm,
+                                      silent=self._tfb_silent):
+
+                        raise Exception("Python TFM Failed please check log: %s" %
+                                        self._tfb_log_f)
+        finally:
+            print("python pass after builder prepare")
+
+            p = self._tfb_build_dir + "/BUILD"
+            if not os.path.exists(p):
+                os.makedirs(p)
+
+            os.chdir(p)
+            if subprocess_log(self._tfb_cfg["build_psa_api"],
+                              self._tfb_log_f,
+                              append=True,
+                              prefix=self._tfb_cfg["build_psa_api"],
+                              silent=self._tfb_silent):
+
+                raise Exception("Build Failed please check log: %s" %
+                                self._tfb_log_f)
+    def copy_tfm(self):
+        """ Copy a new TFM for crypto compile """
+
+        cp_cmd = "cp -r " + self._tfb_cfg["codebase_root_dir"] + " " + \
+            self._tfb_build_dir
+        if subprocess_log(cp_cmd,
+                          self._tfb_log_f,
+                          append=True,
+                          prefix=cp_cmd,
+                          silent=self._tfb_silent):
+
+            raise Exception("Build Failed please check log: %s" %
+                            self._tfb_log_f)
+
+        cp_cmd = "cp -r " + self._tfb_cfg["codebase_root_dir"] + "/../mbed-crypto " + \
+            self._tfb_build_dir
+        if subprocess_log(cp_cmd,
+                          self._tfb_log_f,
+                          append=True,
+                          prefix=cp_cmd,
+                          silent=self._tfb_silent):
+
+            raise Exception("Build Failed please check log: %s" %
+                            self._tfb_log_f)
+
+        cp_cmd = "cp -r " + self._tfb_cfg["codebase_root_dir"] + "/../psa-arch-tests " + \
+            self._tfb_build_dir
+        if subprocess_log(cp_cmd,
+                          self._tfb_log_f,
+                          append=True,
+                          prefix=cp_cmd,
+                          silent=self._tfb_silent):
+
+            raise Exception("Build Failed please check log: %s" %
+                            self._tfb_log_f)
+
+
+        cp_cmd = "mkdir -p " + self._tfb_build_dir + "/CMSIS_5/CMSIS ; "
+        cp_cmd += "cp -r " + self._tfb_cfg["codebase_root_dir"] + "/../CMSIS_5/CMSIS/RTOS2 " + \
+                self._tfb_build_dir + "/CMSIS_5/CMSIS"
+
+        if subprocess_log(cp_cmd,
+                          self._tfb_log_f,
+                          append=True,
+                          prefix=cp_cmd,
+                          silent=self._tfb_silent):
+
+            raise Exception("Build Failed please check log: %s" %
+                            self._tfb_log_f)
+
+        self._tfb_cfg["build_cmds"][0] = \
+            self._tfb_cfg["build_cmds"][0].replace(self._tfb_cfg["codebase_root_dir"],
+            self._tfb_build_dir + "/tf-m")
+
     def task_exec(self):
         """ Main tasks """
 
         # Mark proccess running as status
         self.set_status(-1)
+        print("builder _tfb_cfg %s" % self._tfb_cfg)
+
+        if "build_psa_api" in self._tfb_cfg:
+            p = self._tfb_build_dir + "/BUILD"
+            if not os.path.exists(p):
+                os.makedirs(p)
+            os.chdir(p)
+            if subprocess_log(self._tfb_cfg["build_psa_api"],
+                              self._tfb_log_f,
+                              append=True,
+                              prefix=self._tfb_cfg["build_psa_api"],
+                              silent=self._tfb_silent):
+                raise Exception("Build Failed please check log: %s" %
+                                self._tfb_log_f)
+
+        if self._tfb_cfg["build_cmds"].__str__().find( \
+                "CRYPTO_HW_ACCELERATOR_OTP_STATE=ENABLED") > 0:
+            self.copy_tfm()
+            time.sleep(15)
+            os.sync()
+            print("new build cmd ", self._tfb_cfg["build_cmds"][0])
+
+        print("Go to build directory")
         # Go to build directory
         os.chdir(self._tfb_build_dir)
 
@@ -141,6 +296,7 @@
                                               "-j %s " % thread_no)
 
             # Build it
+            print("~builder build_cmd %s\r\n" % build_cmd)
             if subprocess_log(build_cmd,
                               self._tfb_log_f,
                               append=True,
diff --git a/tfm_ci_pylib/utils.py b/tfm_ci_pylib/utils.py
index 5df108a..c2312e5 100755
--- a/tfm_ci_pylib/utils.py
+++ b/tfm_ci_pylib/utils.py
@@ -357,7 +357,7 @@
 
     eabi_size = check_output(["arm-none-eabi-size",
                               filename],
-                             timeout=2).decode('UTF-8').rstrip()
+                             timeout=18).decode('UTF-8').rstrip()
 
     size_data = re.search(size_info_rex, eabi_size)