Add code coverage tool
Change-Id: I9a90ba54af84b03165f395d97c9067acb10db0ec
Signed-off-by: Raef Coles <raef.coles@arm.com>
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..95c151a
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "code_coverage/qa-tools"]
+ path = code_coverage/qa-tools
+ url = https://git.gitlab.arm.com/tooling/qa-tools.git
diff --git a/code_coverage/generate_report_config_json.py b/code_coverage/generate_report_config_json.py
new file mode 100755
index 0000000..08c399a
--- /dev/null
+++ b/code_coverage/generate_report_config_json.py
@@ -0,0 +1,104 @@
+#!/usr/bin/python3
+# -----------------------------------------------------------------------------
+# Copyright (c) 2024, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# -----------------------------------------------------------------------------
+
+import argparse
+import logging
+import json
+from os import listdir
+from os.path import join, isfile, relpath
+import subprocess
+
+class Source:
+ def __init__(self, location : str):
+ self.type = "git"
+ self.location = location
+ self.refspec = ""
+
+ with open(join(location, ".git", "config"), "rt") as git_config_file:
+ url_line = [x for x in git_config_file.readlines() if "url" in x][0]
+ self.url = url_line.rstrip().replace("\turl = ", "").rstrip()
+
+ with open(join(location, ".git", "HEAD"), "rt") as git_HEAD_file:
+ self.commit = git_HEAD_file.read().rstrip()
+
+ self.location = relpath(location, args.source_dir)
+
+def get_tfm_dependencies(build_dir : str) -> [Source]:
+ dependencies = []
+
+ with open(join(build_dir, "CMakeCache.txt"), "rt") as cmakecache_file:
+ cmakecache = cmakecache_file.readlines()
+ variables = [x.rstrip().split("=") for x in cmakecache if "=" in x]
+ path_variables = [x for x in variables if "PATH" in x[0]]
+
+ for _,p in path_variables:
+ try:
+ dependencies.append(Source(p))
+ except (FileNotFoundError, NotADirectoryError):
+ continue
+
+ return dependencies
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--build_dir", help="TF-M build directory", required=True)
+parser.add_argument("--source_dir", help="TF-M source directory", required=True)
+parser.add_argument("--tools_binary_dir", help="Binary dir in which objdump etc reside", required=False)
+parser.add_argument("--log_level", help="Log level", choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], default="ERROR")
+parser.add_argument("--output_config_file", help="output JSON file", required=True)
+parser.add_argument("--output_intermediate_file", help="output intermediate file", required=True)
+parser.add_argument("trace_file", nargs="+", help="input trace log files")
+args = parser.parse_args()
+
+# logging setup
+logging.basicConfig(level=args.log_level)
+
+configuration = {
+ "remove_workspace": True,
+ "include_assembly": True,
+}
+
+if (args.tools_binary_dir):
+ tools_prefix = args.tools_binary_dir
+else:
+ tools_prefix = ""
+
+tfm_source = Source(args.source_dir)
+dependencies = get_tfm_dependencies(args.build_dir)
+
+parameters = {
+ "objdump" : join(tools_prefix, "arm-none-eabi-objdump"),
+ "readelf" : join(tools_prefix, "arm-none-eabi-readelf"),
+ "sources" : [
+ {
+ "type" : x.type,
+ "URL": x.url,
+ "COMMIT" : x.commit,
+ "REFSPEC" : x.refspec,
+ "LOCATION" : x.location,
+ } for x in [tfm_source] + dependencies],
+ "workspace": args.source_dir,
+ "output_file": args.output_intermediate_file,
+}
+
+bin_dir = join(args.build_dir, "bin")
+elf_files = [join(bin_dir, x) for x in listdir(bin_dir) if isfile(join(bin_dir, x)) and "elf" in x]
+
+elfs = [
+ {
+ "name": x,
+ "traces": args.trace_file,
+ } for x in elf_files]
+
+output = {
+ "configuration": configuration,
+ "parameters": parameters,
+ "elfs": elfs,
+}
+
+with open(args.output_config_file, "w+") as output_file:
+ json.dump(output, output_file)
diff --git a/code_coverage/ingest_tarmac.py b/code_coverage/ingest_tarmac.py
new file mode 100755
index 0000000..64d9251
--- /dev/null
+++ b/code_coverage/ingest_tarmac.py
@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+# -----------------------------------------------------------------------------
+# Copyright (c) 2024, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# -----------------------------------------------------------------------------
+
+import argparse
+import logging
+import re
+import elftools
+
+def parse_line(line: str) -> str:
+ split = line.split(" ")
+ addr = split[5]
+ size = len(split[6]) // 2
+ logging.debug("Instruction at {} of size {}".format(addr, size))
+ return (addr, size)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--input_file", help="tarmac file to input", required=True)
+parser.add_argument("--log_level", help="Log level", choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], default="ERROR")
+parser.add_argument("--output_file", help="output file, in qa-tools format", required=True)
+args = parser.parse_args()
+
+# logging setup
+logging.basicConfig(level=args.log_level)
+
+with open(args.input_file, "rt") as input_file:
+ trace = input_file.read()
+
+instructions = re.findall("[0-9]* [a-z]{2} [a-z\.]* IT .*", trace)
+
+hit_counts = {}
+
+for i in instructions:
+ addr = parse_line(i)
+ if addr in hit_counts.keys():
+ hit_counts[addr] += 1
+ else:
+ hit_counts[addr] = 1
+
+with open(args.output_file, "w+") as output_file:
+ output_file.writelines(["{} {} {}\n".format(x[0], hit_counts[x], x[1]) for x in hit_counts.keys()])
diff --git a/code_coverage/qa-tools b/code_coverage/qa-tools
new file mode 160000
index 0000000..d4db9a7
--- /dev/null
+++ b/code_coverage/qa-tools
@@ -0,0 +1 @@
+Subproject commit d4db9a70c69e8ae60167962297f8a43d893e43cf
diff --git a/code_coverage/tfm_generate_coverage_report.sh b/code_coverage/tfm_generate_coverage_report.sh
new file mode 100755
index 0000000..fe0853a
--- /dev/null
+++ b/code_coverage/tfm_generate_coverage_report.sh
@@ -0,0 +1,122 @@
+#!/bin/bash
+
+# Copyright (c) 2024, Arm Limited. All rights reserved.
+# SPDX-License-Identifier: BSD-3-Clause
+
+error()
+{
+ printf "\e[31m[ERR] $1\e[0m\n" 1>&2
+ if test -n "$2"
+ then
+ exit $2
+ else
+ exit 1
+ fi
+}
+
+usage() {
+ echo "$0 --source_dir <source_dir> --build_dir <build_dir> --output_dir <output_dir> data_file [data_file ...]"
+}
+
+set -ex
+
+SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}")"
+
+# Parse arguments
+while test $# -gt 0; do
+ case $1 in
+ -s|--source_dir)
+ SOURCE_DIR="$2"
+ shift
+ shift
+ ;;
+ -b|--build_dir)
+ BUILD_DIR="$2"
+ shift
+ shift
+ ;;
+ -b|--output_dir)
+ OUTPUT_DIR="$2"
+ shift
+ shift
+ ;;
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
+if test -z "$SOURCE_DIR"
+then
+ usage
+ error "No source dir specified"
+fi
+
+if test -z "$BUILD_DIR"
+then
+ usage
+ error "No build dir specified"
+fi
+
+if test -z "$OUTPUT_DIR"
+then
+ usage
+ error "No output dir specified"
+fi
+
+if ! test $# -gt 0
+then
+ usage
+ error "At least one data file must be input"
+fi
+
+info_dir=$(mktemp -d)
+
+for x in "$@"
+do
+ tmpdir=$(mktemp -d)
+
+ if ${SCRIPT_DIR}/ingest_tarmac.py \
+ --input_file "$x" \
+ --output_file "${tmpdir}/$(basename "$x").data"
+ then
+ input_file="${tmpdir}/$(basename "$x").data"
+ else
+ input_file="$x"
+ fi
+
+
+ ${SCRIPT_DIR}/generate_report_config_json.py \
+ --source_dir "${SOURCE_DIR}" \
+ --build_dir "${BUILD_DIR}" \
+ --output_config_file "${tmpdir}/$(basename "$x")_config.json" \
+ --output_intermediate_file "${tmpdir}/$(basename "$x")_intermediate.json" \
+ "$input_file"
+
+ python3 ${SCRIPT_DIR}/qa-tools/coverage-tool/coverage-reporting/intermediate_layer.py \
+ --config-json "${tmpdir}/$(basename "$x")_config.json"
+
+ python3 ${SCRIPT_DIR}/qa-tools/coverage-tool/coverage-reporting/generate_info_file.py \
+ --workspace ${SOURCE_DIR} \
+ --json "${tmpdir}/$(basename "$x")_intermediate.json" \
+ --info ${info_dir}/${RANDOM}${RANDOM}.info
+done
+
+info_file="$(mktemp).info"
+
+if test $(find "$info_dir" -type f | wc -l) -gt 1
+then
+ arguments=$(find "$info_dir" -type f | xargs -I{} echo "-a {}")
+
+ python3 ${SCRIPT_DIR}/qa-tools/coverage-tool/coverage-reporting/merge.py \
+ $arguments \
+ -o ${info_file}
+else
+ info_file=$(find "$info_dir" -type f)
+fi
+
+genhtml --branch-coverage "${info_file}" --output-directory "$OUTPUT_DIR"