lava_rpc_connector.py: refactor get_job_log
Fix https://projects.linaro.org/browse/LSS-2002
target_log can be truncated when locales aren't installed.
To fix the issue, we install locales and set them to UTF-8.
In addition, we handle unicode (non-ascii) characters.
Refactor the code for this purpose and simplify the code.
The job log returned by LAVA is a YAML document, instead of looping and
loading each line of the document, load the YAML only once.
Then we can process and handle the message.
Signed-off-by: Fathi Boudra <fathi.boudra@linaro.org>
Change-Id: I1edc4de7e340f0752817d170aef0a74afe461f65
diff --git a/tfm_ci_pylib/lava_rpc_connector.py b/tfm_ci_pylib/lava_rpc_connector.py
index 80e0ff6..c5b3a48 100644
--- a/tfm_ci_pylib/lava_rpc_connector.py
+++ b/tfm_ci_pylib/lava_rpc_connector.py
@@ -9,7 +9,7 @@
__copyright__ = """
/*
- * Copyright (c) 2018-2020, Arm Limited. All rights reserved.
+ * Copyright (c) 2018-2021, Arm Limited. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*
@@ -102,25 +102,24 @@
log_url = "{server_url}/jobs/{job_id}/logs/".format(
server_url=self.server_api, job_id=job_id
)
- r = requests.get(log_url, stream=True, headers=auth_headers)
- if r.status_code != 200:
- print("{} - {}".format(log_url, r.status_code))
- return
- with open(target_out_file, "w") as target_out:
- try:
- for line in r.iter_lines():
- line = line.decode('utf-8')
- try:
- if ('target' in line) or ('feedback' in line):
- line_yaml = yaml.load(line)[0]
- if line_yaml['lvl'] in ['target', 'feedback']:
- target_out.write("{}\n".format(line_yaml['msg']))
- except yaml.parser.ParserError as e:
- continue
- except yaml.scanner.ScannerError as e:
- continue
- except Exception as e:
- pass
+ with requests.get(log_url, stream=True, headers=auth_headers) as r:
+ if r.status_code != 200:
+ print("{} - {}".format(log_url, r.status_code))
+ return
+ log_list = yaml.load(r.content, Loader=yaml.SafeLoader)
+ with open(target_out_file, "w") as target_out:
+ for line in log_list:
+ level = line["lvl"]
+ if (level == "target") or (level == "feedback"):
+ try:
+ target_out.write("{}\n".format(line["msg"]))
+ except UnicodeEncodeError:
+ msg = (
+ line["msg"]
+ .encode("ascii", errors="replace")
+ .decode("ascii")
+ )
+ target_out.write("{}\n".format(msg))
def get_job_config(self, job_id, config_out_file):
config_url = "{}/configuration".format(self.server_job_prefix % job_id)