jenkins/ci.jpl

#!/usr/bin/env groovy
//-------------------------------------------------------------------------------
// Copyright (c) 2020-2022, Arm Limited and Contributors. All rights reserved.
//
// SPDX-License-Identifier: BSD-3-Clause
//
//-------------------------------------------------------------------------------

@Library('trustedfirmware') _
import org.trustedfirmware.Gerrit
import org.trustedfirmware.Summary

failure_states = ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"]

mapPlatform = ["cypress/psoc64":                "psoc64",
               "arm/mps2/an519":                "AN519",
               "arm/mps2/an521":                "AN521",
               "arm/mps2/an539":                "AN539",
               "arm/mps2/sse-200_aws":          "SSE-200_AWS",
               "arm/mps3/an524":                "AN524",
               "arm/mps3/an547":                "AN547",
               "arm/mps3/an552":                "AN552",
               "arm/mps3/corstone310_fvp":      "corstone310",
               "arm/musca_b1/sse_200":          "MUSCA_B1",
               "arm/musca_b1/secure_enclave":   "MUSCA_B1_SE",
               "arm/musca_s1":                  "MUSCA_S1",
               "stm/stm32l562e_dk":             "stm32l562e_dk",
               "arm/corstone1000":              "corstone1000",
               "nxp/lpcxpresso55s69":           "lpcxpresso55s69"]

mapBL2 = ["True":  "--bl2",
          "False": ""]

mapTestPsaApi = ["OFF":                      "",
                 "STORAGE":                  "STORAGE",
                 "CRYPTO":                   "Crypto",
                 "INITIAL_ATTESTATION":      "Attest",
                 "IPC":                      "FF"]

// LIB_MODEL, ISOLATION_LEVEL, TEST_REG, TEST_PSA_API, PROFILE, CONFIG_NAME
mapConfigs = [
  ["True",  "1", "False", "OFF",                      "N.A",            "Default"],
  ["False", "1", "False", "OFF",                      "N.A",            "CoreIPC"],
  ["False", "2", "False", "OFF",                      "N.A",            "CoreIPCTfmLevel2"],
  ["False", "3", "False", "OFF",                      "N.A",            "CoreIPCTfmLevel3"],
  ["True",  "1", "False", "OFF",                      "profile_small",  "DefaultProfileS"],
  ["False", "2", "False", "OFF",                      "profile_medium", "DefaultProfileM"],
  ["False", "3", "False", "OFF",                      "profile_large",  "DefaultProfileL"],
  ["True",  "1", "True",  "OFF",                      "N.A",            "Regression"],
  ["False", "1", "True",  "OFF",                      "N.A",            "RegressionIPC"],
  ["False", "2", "True",  "OFF",                      "N.A",            "RegressionIPCTfmLevel2"],
  ["False", "3", "True",  "OFF",                      "N.A",            "RegressionIPCTfmLevel3"],
  ["True",  "1", "True",  "OFF",                      "profile_small",  "RegressionProfileS"],
  ["False", "2", "True",  "OFF",                      "profile_medium", "RegressionProfileM"],
  ["False", "3", "True",  "OFF",                      "profile_large",  "RegressionProfileL"],
  ["True",  "1", "False", "STORAGE",                  "N.A",            "PsaApiTest (STORAGE)"],
  ["True",  "1", "False", "CRYPTO",                   "N.A",            "PsaApiTest (Crypto)"],
  ["True",  "1", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTest (Attest)"],
  ["True",  "1", "False", "IPC",                      "N.A",            "PsaApiTest (FF)"],
  ["False", "1", "False", "STORAGE",                  "N.A",            "PsaApiTestIPC (STORAGE)"],
  ["False", "1", "False", "CRYPTO",                   "N.A",            "PsaApiTestIPC (Crypto)"],
  ["False", "1", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTestIPC (Attest)"],
  ["False", "1", "False", "IPC",                      "N.A",            "PsaApiTestIPC (FF)"],
  ["False", "2", "False", "STORAGE",                  "N.A",            "PsaApiTestIPCTfmLevel2 (STORAGE)"],
  ["False", "2", "False", "CRYPTO",                   "N.A",            "PsaApiTestIPCTfmLevel2 (Crypto)"],
  ["False", "2", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTestIPCTfmLevel2 (Attest)"],
  ["False", "2", "False", "IPC",                      "N.A",            "PsaApiTestIPCTfmLevel2 (FF)"],
  ["False", "3", "False", "STORAGE",                  "N.A",            "PsaApiTestIPCTfmLevel3 (STORAGE)"],
  ["False", "3", "False", "CRYPTO",                   "N.A",            "PsaApiTestIPCTfmLevel3 (Crypto)"],
  ["False", "3", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTestIPCTfmLevel3 (Attest)"],
  ["False", "3", "False", "IPC",                      "N.A",            "PsaApiTestIPCTfmLevel3 (FF)"],
]

cfgs = ["Default", "CoreIPC", "CoreIPCTfmLevel2", "CoreIPCTfmLevel3",
        "Regression", "RegressionIPC",
        "RegressionIPCTfmLevel2", "RegressionIPCTfmLevel3",
        "DefaultProfileS", "RegressionProfileS",
        "DefaultProfileM", "RegressionProfileM", "RegressionProfileM PSOFF",
        "DefaultProfileL", "RegressionProfileL",
        "PsaApiTest (Attest)", "PsaApiTestIPC (Attest)",
        "PsaApiTestIPCTfmLevel2 (Attest)",
        "PsaApiTest (Crypto)", "PsaApiTestIPC (Crypto)",
        "PsaApiTestIPCTfmLevel2 (Crypto)",
        "PsaApiTest (STORAGE)", "PsaApiTestIPC (STORAGE)",
        "PsaApiTestIPCTfmLevel2 (STORAGE)",
        "PsaApiTestIPC (FF)",
        "PsaApiTestIPCTfmLevel2 (FF)",
        "PsaApiTestIPCTfmLevel3 (STORAGE)", "PsaApiTestIPCTfmLevel3 (Crypto)",
        "PsaApiTestIPCTfmLevel3 (Attest)", "PsaApiTestIPCTfmLevel3 (FF)"]

cfgSkipFVP = [
  "AN519_GCC_IPC_2_REG_Debug_BL2",
  "AN519_GCC_IPC_2_REG_Debug_BL2_MEDIUM",
  "AN519_GCC_IPC_2_REG_Debug_BL2_MEDIUM_PSOFF",
  "AN519_ARMCLANG_IPC_2_REG_Debug_BL2",
  "AN519_ARMCLANG_IPC_2_REG_Debug_BL2_MEDIUM",
  "AN521_ARMCLANG_IPC_2_REG_Debug_BL2",
  "AN521_ARMCLANG_IPC_2_REG_Debug_BL2_NSCE",
  "AN521_ARMCLANG_IPC_2_REG_Debug_BL2_MEDIUM",
]

@NonCPS
def generateLavaParam(build_params) {
  def params = []
  params += string(name: "TARGET_PLATFORM", \
                   value: mapPlatform[build_params["TFM_PLATFORM"]])
  params += string(name: "COMPILER", \
                   value: build_params["COMPILER"].split('_')[0])
  params += string(name: "PSA_API_SUITE", \
                   value: mapTestPsaApi[build_params["TEST_PSA_API"]])

  configName = "Config"
  config_params = [build_params["LIB_MODEL"], build_params["ISOLATION_LEVEL"], \
                   build_params["TEST_REGRESSION"], build_params["TEST_PSA_API"], \
                   build_params["PROFILE"]]
  // Regression Test is enabled if CRYPTO is ON
  if (build_params["EXTRA_PARAMS"] == "CRYPTO_ON") {
    config_params[2] = "True"
  }
  for (config in mapConfigs) {
    if (config_params == config[0..4]) {
      configName += config[5].replace(' (', '_').replace(')', '')
      break
    }
  }
  if (configName == "Config") {
    configName = "ConfigDefault"
  }
  params += string(name: "PROJ_CONFIG", value: configName)
  print("Params of ${configName} :")
  print(config_params)
  return params
}

def submit_lava_tests(config, results, build_res, params, params_collection) {
  print("Doing LAVA stuff for ${build_res.getAbsoluteUrl()}")
  params += generateLavaParam(params_collection)
  params += string(name: 'BUILD_NUMBER', value: "${build_res.number}")
  params += string(name: 'BUILD_URL', value: build_res.getAbsoluteUrl())
  params += string(name: 'LAVA_URL', value: env.LAVA_URL)
  params += string(name: 'CI_SCRIPTS_REPO', value: env.CI_SCRIPTS_REPO)
  params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
  params += string(name: 'LAVA_CREDENTIALS', value: env.LAVA_CREDENTIALS)
  params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN)
  // Workaround: Configs in cfgSkipFVP fail on FVP but pass on physical boards
  if (params_collection['CONFIG_NAME'] in cfgSkipFVP) {
    params += string(name: 'DEVICE_FILTER', value: "--physical-board-only")
  } else {
    params += string(name: 'DEVICE_FILTER', value: env.DEVICE_FILTER)
  }
  def lava_res = build(job: 'tf-m-lava-submit', parameters: params, propagate: false)
  def lava_resubmitted = false
  if (lava_res.result in failure_states) {
    error("LAVA Create and Submit failed at ${lava_res.getAbsoluteUrl()}")
  } else {
    lava_des = lava_res.getDescription()
    if (lava_des.contains(" Submitted twice!")) {
      lava_resubmitted = true
      lava_des = lava_des - " Submitted twice!"
    }
    results['lava_jobs'] += lava_des
  }
  links = "Build Config: ${config}\n"
  links += "Build URL: ${build_res.getAbsoluteUrl()}\n"
  links += "LAVA Submit: ${lava_res.getAbsoluteUrl()}"
  if (lava_resubmitted) {
    links += "\nLAVA Job Re-Submitted!"
  }
  print(links)
}

def listConfigs(ci_scripts_dir, config_list, filter_group) {
  dir(ci_scripts_dir) {
    echo "Obtaining list of configs."
    echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}"
    def build_config_list_raw = sh(script: """\
python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}
""", returnStdout: true).trim()
    def build_config_list = build_config_list_raw.tokenize('\n')
    config_list.addAll(build_config_list)
  }
}

def buildConfig(ci_scripts_dir, config, filter_group, results) {
  def params = []
  def params_collection = [:]
  def build_config_params
  dir(ci_scripts_dir) {
    echo "Obtaining build configuration for config ${config}"
    echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}"
    build_config_params = sh(script: """\
python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}
""", returnStdout: true).trim()
  }
  def lines = build_config_params.tokenize('\n')
  for (String line : lines) {
    def key, value
    (key, value) = line.tokenize('=')
    params += string(name: key, value: value)
    params_collection[key] = value
  }
  params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
  params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
  params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER)
  params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION)
  params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
  params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
  params += string(name: 'CODE_REPO', value: env.CODE_REPO)
  params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN)
  params += string(name: 'CI_SCRIPTS_REPO', value: env.CI_SCRIPTS_REPO)
  params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
  params += string(name: 'MCUBOOT_REFSPEC', value: env.MCUBOOT_REFSPEC)
  params += string(name: 'MCUBOOT_URL', value: env.MCUBOOT_URL)
  params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
  params += string(name: 'MBEDTLS_URL', value: env.MBEDTLS_URL)
  params += string(name: 'TFM_TESTS_REFSPEC', value: env.TFM_TESTS_REFSPEC)
  params += string(name: 'TFM_TESTS_URL', value: env.TFM_TESTS_URL)
  params += string(name: 'PSA_ARCH_TESTS_VERSION', value: env.PSA_ARCH_TESTS_VERSION)
  params += string(name: 'PSA_ARCH_TESTS_URL', value: env.PSA_ARCH_TESTS_URL)
  params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER)
  if (env.JOB_NAME.equals("tf-m-nightly")) { //Setting the Memory footprint gathering.
     params += string(name: 'SQUAD_CONFIGURATIONS', value: env.SQUAD_CONFIGURATIONS)
  }
  return { -> results
    def build_res = build(job: 'tf-m-build-config', parameters: params, propagate: false)
    def build_url = build_res.getAbsoluteUrl()
    results['builds'][build_res.number] = [build_res, config, params_collection]

    print("${build_res.number}: ${config} ${build_res.result} ${build_url}")

    // Filter out configs do not need LAVA tests

    // Configs with build failure do not need LAVA tests
    if (build_res.result in failure_states) {
      error("Build failed at ${build_url}")
    } else {
        // Build successful
        // Job tf-m-extra-build does not need LAVA tests
        if (env.JOB_NAME.equals("tf-m-extra-build")) {
          print("LAVA is not needed in tf-m-extra-build job.")
        }
        // Submit test job for NXP LPCXpresso55S69 & Cypress PSoC64
        else if (params_collection["TFM_PLATFORM"].contains("lpcxpresso55s69") || params_collection["TFM_PLATFORM"].contains("psoc64")) {
          submit_lava_tests(config, results, build_res, params, params_collection)
        } else {
            // Configs without BL2 do not need LAVA tests
            if (params_collection["BL2"] == "False") {
              print("LAVA is not needed for ${build_url}")
            }
            // LAVA tests on MUSCA_B1 are not needed in per-patch job
            else if (params_collection["TFM_PLATFORM"].contains("musca_b1") && \
                     env.JOB_NAME.equals("tf-m-build-and-test")) {
              print("LAVA is not needed for ${build_url}")
            }
            // LAVA tests on Corstone1000 FPGA config is not supported
            else if (params_collection["TFM_PLATFORM"].contains("corstone1000") && \
                     params_collection["EXTRA_PARAMS"].contains("FPGA")) {
              print("LAVA is not needed for ${build_url}")
            }
            // Submit LAVA tests
            else {
              submit_lava_tests(config, results, build_res, params, params_collection)
            }
        }
    }
  }
}

def buildDocs(results) {
  def params = []
  params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
  params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
  params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER)
  params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION)
  params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
  params += string(name: 'CODE_REPO', value: env.CODE_REPO)
  params += string(name: 'CI_SCRIPTS_REPO', value: env.CI_SCRIPTS_REPO)
  params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
  params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER)
  return { -> results
    def res = build(job: 'tf-m-build-docs', parameters: params, propagate:false)
    print("${res.number}: Docs ${res.result} ${res.getAbsoluteUrl()}")
    results['docs'] = [res.number, res.result, params]
    if (res.result in ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"]) {
      error("Build failed at ${res.getAbsoluteUrl()}")
    }
  }
}

def generateEmailBody(stage, failed_jobs) {
  body = "Check console output at ${env.BUILD_URL} \n\n"

  body += "Failed Jobs:\n"
  failed_jobs.each { job ->
    body += "${job.key}  ${job.value}\n"
  }

  body += "\nFor detailed ${stage} results please refer to \
           ${env.BUILD_URL}artifact/${stage}_results.csv \n"
  return body
}

def emailNotification(results, stage, failed_jobs) {
  script {
   if (env.JOB_NAME.equals("tf-m-nightly") && !env.EMAIL_NOTIFICATION.equals('')) {
      def result = "Fail."
      if (results == true) {
        result = "Success."
        print("Skip sending as ${result} for ${stage}")
      }
      else {
        emailext (
            subject: ("Job ${env.JOB_NAME} ${stage} ${env.BUILD_NUMBER} ${result}"),
            body: generateEmailBody(stage, failed_jobs),
            to: "${EMAIL_NOTIFICATION}"
        )
      }
    }
  } /* script */
}

def filterFailedBuild(results) {
  def failed_builds = [:]
  results.each { result ->
    if (result.value[0].getResult() == "FAILURE") {
      failed_builds[result.value[1]] = result.value[0].getAbsoluteUrl()
    }
  }
  return failed_builds
}

def filterFailedTest(string) {
  def failed_tests = [:]
  line = lineInString(string, "FAILURE_TESTS:")
  if (line == null) {
    return ["???"];
  }
  a = line.split(' ')
  if (a.size() > 1) {
    a = line.split(' ')[1..-1]
    a.each { fail_test ->
      config_link = fail_test.split(':')
      failed_tests[config_link[0]] = config_link[1..-1].join(':')
  }
  }
  return failed_tests
}

@NonCPS
def generateCsvContent(results) {
  def resultsParam = []
  results.each { result ->
    if (result.value[2]['BL2'] == "True") {
      resultsParam.add([result.value[1], \
                        result.value[0].getResult(), \
                        result.value[2]['TFM_PLATFORM'], \
                        result.value[2]['COMPILER'].split('_')[0], \
                        result.value[2]['CMAKE_BUILD_TYPE'], \
                        result.value[2]['BL2'], \
                        result.value[2]['LIB_MODEL'], \
                        result.value[2]['ISOLATION_LEVEL'], \
                        result.value[2]['TEST_REGRESSION'], \
                        result.value[2]['TEST_PSA_API'], \
                        result.value[2]['PROFILE']])
    }
  }
  resultsParam.each { result ->
    result[3] = result[3].split('_')[0]
    build_params = result[6..10]
    configName = ""
    for (map_cfg in mapConfigs) {
      if (build_params[0..4] == map_cfg[0..4]) {
        configName = map_cfg[5]
        break
      }
    }
    if (configName == "") {
      configName = "Default"
    }
    else if (configName == "RegressionProfileM") {
      if (build_params[5] == "OFF") {
        configName = "RegressionProfileM PSOFF"
      }
    }
    result.add(configName)
  }
  def csvContent = []
  resultsParam.each { result ->
    current_row = result[2..4]
    cfgs.each {cfg ->
      if (cfg == result[11]) {
        current_row.add(cfg)
        current_row.add(result[1])
      }
    }
    csvContent.add(current_row)
  }
  csvContent.sort{a,b -> a[0] <=> b[0] ?: a[1] <=> b[1] ?: a[2] <=> b[2]}
  build_summary = []
  current_platform = ""
  current_compiler = ""
  current_build_type = ""
  csvContent.each { build_cfg ->
    if (current_platform != build_cfg[0] || \
        current_compiler != build_cfg[1] || \
        current_build_type != build_cfg[2]) {
          current_platform = build_cfg[0]
          current_compiler = build_cfg[1]
          current_build_type = build_cfg[2]
          csv_line = [current_platform, current_compiler, current_build_type]
          cfgs.each {
            csv_line.add("N.A.")
          }
          build_summary.add(csv_line)
        }
    i = 0
    cfgs.each { cfg ->
      if (cfg == build_cfg[3]) {
        build_summary[-1][3+i] = build_cfg[4]
      }
      i += 1
    }
  }
  build_summary.add(0, ['Platform', 'Compiler', 'Cmake Build Type'])
  build_summary[0] += cfgs
  return build_summary
}

def generateBuildCsv(results) {
  def csvContent = generateCsvContent(results)
  node("master") {
    writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL
    archiveArtifacts 'build_results.csv'
  }
}

def buildCsv(results) {
  def summary = new Summary();
  def csvContent = summary.getBuildCsv(results)
  node("master") {
    writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL
    archiveArtifacts 'build_results.csv'
  }
}

def writeSummary(results) {
  def summary = new Summary();
  def buildLinks = summary.getLinks(results)
  node("master") {
    writeFile file: "build_links.html", text: buildLinks
    archiveArtifacts 'build_links.html'
  }
}

def lineInString(string, match) {
  def lines = string.split("\n")
  def result = lines.findAll { it.contains(match) }
  return result[0]
}

def showLinks(string) {
  def lines = string.split("\n")
  def result = lines.findAll { it.contains("Build Config: ")}
  links = result.join("\n")
  print(links)
}

def getResult(string, match) {
  line = lineInString(string, match)
  a = line.split(match)[1].split(' ')
  score = a[0]
  if (a.size() > 1)
  {
    fail_text = a[1..-1].join(" ")
    return [score, fail_text]
  }
  return [score, ""]
}

def submitJobsToList(results) {
  def all_jobs = []
  for (String result : results){
    jobs_s = result.split('JOBS: ')
    if (jobs_s.size() > 1) {
      all_jobs += jobs_s[1]
    }
  }
  return(all_jobs)
}

def configs = []
def builds = [:]
def results = [:]

timestamps {
  node("docker-amd64-tf-m-bionic") {
    stage("Init") {
      cleanWs()
      dir("tf-m-ci-scripts") {
        checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
        sh "git rev-parse --short HEAD"
        // Clone TF-M repositories so share folder can be reused by downstream jobs
        sh "./clone.sh"
      }
    }
    stage("Configs") {
      // Populate configs
      listConfigs('tf-m-ci-scripts', configs, env.FILTER_GROUP)
      results['builds'] = [:]
      results['lava_jobs'] = []
      for (config in configs) {
        builds[config] = buildConfig("tf-m-ci-scripts", config, env.FILTER_GROUP, results)
      }
      if (!env.JOB_NAME.equals("tf-m-extra-build")) {
        builds["docs"] = buildDocs(results)
      }
    }
  }

  stage("Builds") {
    def verify = 1
    def success = true
    try {
      parallel(builds)
    } catch (Exception e) {
      print(e)
      manager.buildFailure()
      verify = -1
      success = false
    } finally {
      print("Verifying status")
      def failed_builds = filterFailedBuild(results['builds'])
      emailNotification(success, 'build', failed_builds)
      g = new Gerrit()
      g.verifyStatus(verify, 'tf-m-build', 'build')
      print("Building CSV")
      generateBuildCsv(results['builds'])
      writeSummary(results['builds'])
    }
  }

  node("docker-amd64-tf-m-bionic") {
    stage("Tests") {
      dir("tf-m-ci-scripts") {
        checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
      }
      def all_jobs = []
      def success = true
      print("Wait for LAVA results here...")
      try {
        all_jobs = submitJobsToList(results['lava_jobs'])
        output = ""
        if (all_jobs.size() > 0) {
          dir(".") {
            withCredentials([usernamePassword(credentialsId: env.LAVA_CREDENTIALS, passwordVariable: 'LAVA_TOKEN', usernameVariable: 'LAVA_USER')]) {
              output = sh(script: """./tf-m-ci-scripts/lava_helper/lava_wait_jobs.py --job-ids ${all_jobs.join(",")} \
    --lava-url ${env.LAVA_URL} --lava-user ${LAVA_USER} --lava-token ${LAVA_TOKEN} \
    --artifacts-path cfgs --lava-timeout 12000 \
    """, returnStdout: true).trim()
              println("--- output from lava_wait_jobs.py ---")
              println(output)
              println("--- end of output from lava_wait_jobs.py ---")
              showLinks(output)
              archiveArtifacts artifacts: 'test_summary.*', allowEmptyArchive: true
              archiveArtifacts artifacts: 'test_results.csv', allowEmptyArchive: true
              g = new Gerrit()
              def (boot_result, boot_output) = getResult(output, 'BOOT_RESULT: ')
              if (boot_result) {
                g.verifyStatus(boot_result, "lava_boot", "test")
              }
              def (test_result, test_output) = getResult(output, 'TEST_RESULT: ')
              if (test_result) {
                g.verifyStatus(test_result, "lava_test", "test")
              }
              if (boot_result.toInteger() < 1 || test_result.toInteger() < 1) {
                error("Marking job as failed due to failed boots: \"${boot_output}\" or tests: \"${test_output}\"")
              }

              if (env.CODE_COVERAGE_EN == "TRUE") {
                println("Producing merged report")
                sh(script: """./tf-m-ci-scripts/lava_helper/codecov_merge.sh""")
                archiveArtifacts artifacts: 'merged_report/**', allowEmptyArchive: true
              }
            }
          }
        }
        else {
          print("There were no LAVA jobs to test.")
        }
      }
      catch (Exception e) {
        print("ERROR: ${e}")
        success = false
      } finally {
        archiveArtifacts artifacts: 'cfgs/**', allowEmptyArchive: true
        if (all_jobs.size() > 0) {
          emailNotification(success, 'test', filterFailedTest(output))
        }
        cleanWs()
        if (!success) {
          error("There was an Error waiting for LAVA jobs")
        }
      }
    }
  }
}