LAVA: Update LAVA test log and job name format
To make CI logs more user-friendly, show test device type of each test
job and align the test job name with build configs.
Remove the redundant job result logs. It is good enough to print the
test result once for each jobs.
Signed-off-by: Xinyu Zhang <xinyu.zhang@arm.com>
Change-Id: I33dad21e5642f7d2b1a735a177ef4489ba80eed6
diff --git a/jenkins/ci.jpl b/jenkins/ci.jpl
index f4c40e5..0d86f03 100644
--- a/jenkins/ci.jpl
+++ b/jenkins/ci.jpl
@@ -327,6 +327,41 @@
return failed_builds
}
+def parseTestResults(output) {
+ // Verify test status
+ g = new Gerrit()
+ if (output.contains('FAILURE')) {
+ score = -1
+ } else {
+ score = 1
+ }
+ g.verifyStatus(score, "lava_test", "test")
+ if (score < 0) {
+ error("Marking job as failed due to failed boots: \"${boot_output}\" or tests: \"${test_output}\"")
+ }
+
+ // Generate test results summary
+ def test_results = [:]
+ records = output.split('\nLAVA Test Config:\n')
+ if (records.size() < 2) {
+ return test_results
+ }
+ records[1..-1].each { record ->
+ config_name = ""
+ metadata = [:]
+ record.split('\n').each { line ->
+ record_metadata = line.split(': ')
+ if (record_metadata[0] == 'Config Name') {
+ config_name = record_metadata[1]
+ } else {
+ metadata[record_metadata[0]] = record_metadata[1]
+ }
+ }
+ test_results[config_name] = metadata
+ }
+ return test_results
+}
+
def filterFailedTest(string) {
def failed_tests = [:]
line = lineInString(string, "FAILURE_TESTS:")
@@ -550,20 +585,9 @@
println("--- output from lava_wait_jobs.py ---")
println(output)
println("--- end of output from lava_wait_jobs.py ---")
+ parseTestResults(output)
archiveArtifacts artifacts: 'test_summary.*', allowEmptyArchive: true
archiveArtifacts artifacts: 'test_results.csv', allowEmptyArchive: true
- g = new Gerrit()
- def (boot_result, boot_output) = getResult(output, 'BOOT_RESULT: ')
- if (boot_result) {
- g.verifyStatus(boot_result, "lava_boot", "test")
- }
- def (test_result, test_output) = getResult(output, 'TEST_RESULT: ')
- if (test_result) {
- g.verifyStatus(test_result, "lava_test", "test")
- }
- if (boot_result.toInteger() < 1 || test_result.toInteger() < 1) {
- error("Marking job as failed due to failed boots: \"${boot_output}\" or tests: \"${test_output}\"")
- }
if (env.CODE_COVERAGE_EN == "TRUE") {
println("Producing merged report")
diff --git a/lava_helper/jinja2_templates/base.jinja2 b/lava_helper/jinja2_templates/base.jinja2
index d3159fa..8e88e35 100644
--- a/lava_helper/jinja2_templates/base.jinja2
+++ b/lava_helper/jinja2_templates/base.jinja2
@@ -14,6 +14,7 @@
platform: {{ platform }}
build_name: {{ build_name }}
recovery_image_url: {{ recovery_image_url }}
+ device_type: {{ device_type }}
build_job_url: {{ build_job_url }}
{%- endblock %}
diff --git a/lava_helper/jinja2_templates/lpcxpresso55s69.jinja2 b/lava_helper/jinja2_templates/lpcxpresso55s69.jinja2
index 1a3209d..5de0d9b 100644
--- a/lava_helper/jinja2_templates/lpcxpresso55s69.jinja2
+++ b/lava_helper/jinja2_templates/lpcxpresso55s69.jinja2
@@ -1,5 +1,5 @@
{#------------------------------------------------------------------------------
-# Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+# Copyright (c) 2020-2022, Arm Limited and Contributors. All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
#
@@ -43,5 +43,6 @@
name: {{ name }}
platform: {{ platform }}
build_name: {{ build_name }}
+ device_type: {{ device_type }}
build_job_url: {{ build_job_url }}
{% endblock %}
diff --git a/lava_helper/jinja2_templates/musca_b1.jinja2 b/lava_helper/jinja2_templates/musca_b1.jinja2
index 0b69e95..5233e95 100644
--- a/lava_helper/jinja2_templates/musca_b1.jinja2
+++ b/lava_helper/jinja2_templates/musca_b1.jinja2
@@ -1,5 +1,5 @@
{#------------------------------------------------------------------------------
-# Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+# Copyright (c) 2020-2022, Arm Limited and Contributors. All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
#
@@ -43,5 +43,6 @@
name: {{ name }}
platform: {{ platform }}
build_name: {{ build_name }}
+ device_type: {{ device_type }}
build_job_url: {{ build_job_url }}
{% endblock %}
diff --git a/lava_helper/jinja2_templates/psoc64.jinja2 b/lava_helper/jinja2_templates/psoc64.jinja2
index 651efee..e89c9f0 100644
--- a/lava_helper/jinja2_templates/psoc64.jinja2
+++ b/lava_helper/jinja2_templates/psoc64.jinja2
@@ -1,5 +1,5 @@
{#------------------------------------------------------------------------------
-# Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+# Copyright (c) 2020-2022, Arm Limited and Contributors. All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
#
@@ -42,5 +42,6 @@
name: {{ name }}
platform: {{ platform }}
build_name: {{ build_name }}
+ device_type: {{ device_type }}
build_job_url: {{ build_job_url }}
{% endblock %}
diff --git a/lava_helper/jinja2_templates/qemu_mps2_bl2.jinja2 b/lava_helper/jinja2_templates/qemu_mps2_bl2.jinja2
index e7c7a54..644b6c4 100644
--- a/lava_helper/jinja2_templates/qemu_mps2_bl2.jinja2
+++ b/lava_helper/jinja2_templates/qemu_mps2_bl2.jinja2
@@ -65,6 +65,7 @@
name: {{ name }}
platform: {{ platform }}
build_name: {{ build_name }}
+ device_type: {{ device_type }}
build_job_url: {{ build_job_url }}
firmware_url: {{ firmware_url }}
bootloader_url: {{ bootloader_url }}
diff --git a/lava_helper/jinja2_templates/stm32l562e_dk.jinja2 b/lava_helper/jinja2_templates/stm32l562e_dk.jinja2
index af9f19f..dbe9bcb 100644
--- a/lava_helper/jinja2_templates/stm32l562e_dk.jinja2
+++ b/lava_helper/jinja2_templates/stm32l562e_dk.jinja2
@@ -1,5 +1,5 @@
{#------------------------------------------------------------------------------
-# Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+# Copyright (c) 2020-2022, Arm Limited and Contributors. All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
#
@@ -43,5 +43,6 @@
name: {{ name }}
platform: {{ platform }}
build_name: {{ build_name }}
+ device_type: {{ device_type }}
build_job_url: {{ build_job_url }}
{% endblock %}
diff --git a/lava_helper/lava_create_jobs.py b/lava_helper/lava_create_jobs.py
index 207a9a5..ce14d24 100755
--- a/lava_helper/lava_create_jobs.py
+++ b/lava_helper/lava_create_jobs.py
@@ -35,19 +35,6 @@
return "{}/{}".format(recovery_store_url.rstrip('/'), recovery)
-def get_job_name(name, params, job):
- return "{}_{}_{}_{}_{}_{}_{}_{}".format(
- name,
- job,
- params["platform"],
- params["build_no"],
- params["compiler"],
- params["build_type"],
- params["boot_type"],
- params["name"],
- )
-
-
def load_config_overrides(user_args, config_key):
"""Load a configuration from multiple locations and override it with user provided
arguments"""
@@ -133,9 +120,7 @@
)
params.update(
{
- "job_name": get_job_name(
- config["job_name"], params, user_args.jenkins_job,
- ),
+ "job_name": "{}_{}_{}".format(os.getenv('CONFIG_NAME'), params['build_no'], params["device_type"]),
"build_name": os.getenv('CONFIG_NAME')
}
)
diff --git a/lava_helper/lava_wait_jobs.py b/lava_helper/lava_wait_jobs.py
index 789fae2..80bd3fd 100755
--- a/lava_helper/lava_wait_jobs.py
+++ b/lava_helper/lava_wait_jobs.py
@@ -71,10 +71,9 @@
def process_finished_jobs(finished_jobs, user_args):
print_lava_urls(finished_jobs, user_args)
- job_links(finished_jobs, user_args)
- boot_report(finished_jobs, user_args)
test_report(finished_jobs, user_args)
failure_report(finished_jobs, user_args)
+ job_links(finished_jobs, user_args)
csv_report(finished_jobs)
codecov_helper.coverage_reports(finished_jobs, user_args)
@@ -123,8 +122,8 @@
target_log = os.path.join(job_dir, 'target_log.txt')
config = os.path.join(job_dir, 'config.tar.bz2')
results_file = os.path.join(job_dir, 'results.yaml')
- definition, metadata = lava.get_job_definition(job_id, def_path)
- jobs[job_id]['metadata'] = metadata
+ definition = lava.get_job_definition(job_id, def_path)
+ jobs[job_id]['metadata'] = definition.get('metadata', [])
time.sleep(0.2) # be friendly to LAVA
lava.get_job_log(job_id, target_log)
time.sleep(0.2)
@@ -147,10 +146,13 @@
def job_links(jobs, user_args):
job_links = ""
for job, info in jobs.items():
- job_links += "Build Config: {}\n".format(info['metadata']['build_name'])
+ job_links += "\nLAVA Test Config:\n"
+ job_links += "Config Name: {}\n".format(info['metadata']['build_name'])
+ job_links += "Test Result: {}\n".format(info['result'])
+ job_links += "Device Type: {}\n".format(info['metadata']['device_type'])
job_links += "Build link: {}\n".format(info['metadata']['build_job_url'])
job_links += "LAVA link: {}\n".format(lava_id_to_url(job, user_args))
- job_links += "TFM LOG: {}artifact/{}/target_log.txt\n\n".format(os.getenv("BUILD_URL"), info['job_dir'])
+ job_links += "TFM LOG: {}artifact/{}/target_log.txt\n".format(os.getenv("BUILD_URL"), info['job_dir'])
print(job_links)
def csv_report(jobs):
@@ -183,18 +185,6 @@
writer.writeheader()
writer.writerows(lava_jobs)
-def boot_report(jobs, user_args):
- incomplete_jobs = []
- for job, info in jobs.items():
- if info['health'] != 'Complete':
- if info['error_reason'] == 'Infrastructure':
- info_print("Job {} failed with Infrastructure error".format(job))
- incomplete_jobs.append(job)
- incomplete_output = [lava_id_to_url(x, user_args) for x in incomplete_jobs];
- if len(incomplete_jobs) > 0:
- print("BOOT_RESULT: -1 Failed: {}".format(incomplete_output))
- else:
- print("BOOT_RESULT: +1")
def failure_report(jobs, user_args):
failed_report = "FAILURE_TESTS:"
@@ -203,6 +193,9 @@
failed_report += " {}:{}artifact/{}/target_log.txt\n".format(info['metadata']['build_name'],
os.getenv("BUILD_URL"),
info['job_dir'])
+ info['result'] = 'FAILURE'
+ else:
+ info['result'] = 'SUCCESS'
print(failed_report)
def remove_lava_dupes(results):
@@ -220,6 +213,9 @@
fail_j = []
jinja_data = []
for job, info in jobs.items():
+ if info['health'] != 'Complete':
+ fail_j.append(job)
+ continue
results_file = os.path.join(info['job_dir'], 'results.yaml')
if not os.path.exists(results_file) or (os.path.getsize(results_file) == 0):
fail_j.append(job)
@@ -235,11 +231,6 @@
if result['result'] == 'fail':
fail_j.append(job) if job not in fail_j else fail_j
time.sleep(0.5) # be friendly to LAVA
- fail_output = [lava_id_to_url(x, user_args) for x in fail_j]
- if len(fail_j) > 0:
- print("TEST_RESULT: -1 Failed: {}".format(fail_output))
- else:
- print("TEST_RESULT: +1")
data = {}
data['jobs'] = jinja_data
render_jinja(data)
diff --git a/tfm_ci_pylib/lava_rpc_connector.py b/tfm_ci_pylib/lava_rpc_connector.py
index de940bd..81c5af9 100644
--- a/tfm_ci_pylib/lava_rpc_connector.py
+++ b/tfm_ci_pylib/lava_rpc_connector.py
@@ -9,7 +9,7 @@
__copyright__ = """
/*
- * Copyright (c) 2018-2021, Arm Limited. All rights reserved.
+ * Copyright (c) 2018-2022, Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*
@@ -95,7 +95,7 @@
with open(yaml_out_file, "w") as F:
F.write(str(job_def))
def_o = yaml.safe_load(job_def)
- return job_def, def_o.get('metadata', [])
+ return def_o
def get_job_log(self, job_id, target_out_file):
auth_headers = {"Authorization": "Token %s" % self.token}