script/graphs/tf-main-results.bash - next/ci/tf-a-ci-scripts - TrustedFirmware Git Browser

 #!/usr/bin/env bash
 #
 # Copyright (c) 2022 Arm Limited. All rights reserved.
 #
 # SPDX-License-Identifier: BSD-3-Clause
 #

 set -euo pipefail

 # Build and Jenkins URL.
 sub_build_url=${1}
 job_name="${2:-"tf-a-ci-gateway"}"
 filter=${3:-".*"}

 jenkins="${sub_build_url%%/job*}"

 # Utilise default paths to output files if none provided.
 job_target="$(dirname ${sub_build_url#*/job/})"
 PNGFILE=${PNGFILE:=${job_target}-result.png}
 CSVFILE=${CSVFILE:=${job_target}-result.csv}

 # Remove csv output file if it exists to append to empty file
 : > "${CSVFILE}"

 readarray -t sub_builds < <(curl -sSL "${sub_build_url}/api/json" | jq -Rr '
 	fromjson? | [
 		.subBuilds[]? | select(.jobName == "'${job_name}'") | .url
 	] | .[]')

 # Get a csv list of relative paths to report.json, or "-" if no report.json
 report_rel_paths_url="${jenkins}/{$(echo $(IFS=,; echo "${sub_builds[*]}"))}/api/json"

 readarray -t report_paths < <(curl -fsSL --fail-early "${report_rel_paths_url}" \
 					| sed 's/--_curl_--.*$//' \
 					| sed -e 's/^{/'$(printf "\x1e")'{/' \
 					| jq -sr --seq '
 				 	[ .[]
 						| [ .artifacts[]?
 						| select (.fileName == "report.json")
 						| .relativePath ]
 						| if length > 0 then .[] else "-" end ]
 					| .[]')

 # Combine sub build urls with relative path to "report.json"
 # the empty entries are intentionally kept as -, so the output array can
 # be mapped onto ${sub_build_list}
 report_urls="$jenkins/{"
 for i in "${!sub_builds[@]}"
 do
 	if [[ ${report_paths[i]} != "-" ]]
 	then
 		report_urls="${report_urls}${sub_builds[i]}/artifact/${report_paths[i]},"
 	fi
 done

 # Strip last comma and add closing brace
 report_urls="${report_urls%?}}"

 # Get Child build information from each report.json.
 readarray -t child_file_list_array < <(curl -sL "${report_urls}" -o -\
                         | sed 's/--_curl_--.*$//' \
                         | jq -sr --arg FILTER "${filter}" \
                             '[.[]
                              | .job as $job
                              | [ .child_build_numbers?, [(.test_files[]
                                | sub("\\.test";"")
                                | split("%") as $config
                                | { group: $config[1], suite: $config[2]})]]
                              | transpose
                              | map( {($job + "/" + .[0]) : .[1]} | to_entries )
                              | add
                              | map(select(.value.suite | test(".*"; "il")?
 							               and (endswith("nil")
 										     or endswith("norun-fip.dummy") | not)))
                              | if ( length > 0 )
                                 then .[] else empty end
                              | .value.group, (.value.suite | gsub("\\,nil";"")), .key]
 							 | .[]')

 # These three arrays should be the same length, and values at the same index
 # correspond to the same child build
 declare -a tftf_keys tftf_suite tftf_group

 for i in $(seq 0 3 $((${#child_file_list_array[@]}-1))) ; do
 	tftf_group+=("${child_file_list_array[$i]}")
 	tftf_suite+=("${child_file_list_array[$i+1]}")
 	tftf_keys+=("${child_file_list_array[$i+2]}")
 done


 child_output_results_url="${jenkins}/job/{$(echo $(IFS=,; echo "${tftf_keys[*]}"))}/api/json"

 # Retrieve relative path to either "uart0_full.txt" (FVP) or
 # "job_output.log" (LAVA) for each child job. Once again values where no match
 # is found are intentionally kept as "-" so the array can be correlated with
 # ${tftf_suite}.
 readarray -t child_output_results < <(curl -fsSL --fail-early "$child_output_results_url" \
                  | sed 's/}{/}\n{/g' \
                  | jq -sr '[ .[]
                              | ([ .artifacts[]?
                                   | select(.fileName == "uart0_full.txt"
                                   or .fileName == "job_output.log"
                                   or .fileName == "lava-uart0.log") ]
                              |  if length > 0
                                 then .[0].relativePath else "-" end), .result ]
                            | .[]')

 # Combine job and child_build number with relative path to output file
 testlog_urls="${jenkins}/job/{"
 tftf_child_results=()

 for i in $(seq 0 2 $((${#child_output_results[@]}-1))) ; do
 	testlog_urls+="${tftf_keys[$((i/2))]}/artifact/${child_output_results[$i]},"
 	tftf_child_results+=(${child_output_results[$((i+1))]})
 done

 # Remove final comma and append a closing brace
 testlog_urls="${testlog_urls%?}}"

 # Retrieve the log for each child with --include to also retrieve the HTTP
 # header and grep for a block like:
 # Tests Skipped : 125
 # Tests Passed  : 45
 # Tests Failed  : 0
 # Tests Crashed : 0
 #
 # If none is found the line HTTP is used to delemit each entry
 #
 # Logs from Lava each message is wrapped with braces and has some pre-amble,
 # which is removed with sed.

 tftf_result_keys=(
         "TestGroup" "TestSuite" "URL" "Result" "Passed" "Failed" "Crashed" "Skipped"
 )
 declare -A results_split="( $(for ord_ in ${tftf_result_keys[@]} ; do echo -n "[$ord_]=\"\" "; done))"
 declare output_csv_str="" csv_row=""

 read -ra tftf_urls <<< "$(eval "echo ${testlog_urls}")"

 # FIXME adjust this so we can handle both LAVA logs
 # for each test suite
 # curl the result log if its not '-'
 # remove debug information
 # if results is none:
 	# use "Result"
 # else
 	# read each key
 # write row to csv

 # Sort results into rows:
 for i in ${!tftf_suite[*]}; do
 	results_split["TestGroup"]="${tftf_group[$i]:-}"
 	results_split["TestSuite"]="\"${tftf_suite[$i]:-}\""
 	results_split["URL"]="${tftf_urls[$i]:-}"
 	results_split["Result"]="${tftf_child_results[$i]:-}"

 	# Skipped/Crashed are always zero if no test block is found
 	results_split["Skipped"]="0"
 	results_split["Crashed"]="0"
 	if [[ "${results_split["Result"]}" == "SUCCESS" ]];
 	then
 		results_split["Passed"]="1"
 		results_split["Failed"]="0"
 	else
 		results_split["Passed"]="0"
 		results_split["Failed"]="1"
 	fi

 	readarray -t raw_result < <(curl -sL --include "${results_split["URL"]}" \
 			| sed 's/.*msg": "//g' \
 			| grep --text -E "^Tests|HTTP\/")

 	for line in "${raw_result[@]}"; do
 		if [[ "${line}" == Test* ]]
 		then
 			k=$(echo "${line}" | awk -F ' ' '{print $2}')
 			count="${line//[!0-9]/}"
 			results_split[$k]=$count
 		fi
 	done

 	# Generate CSV row using array of ordinals to align with headers.
 	readarray -t row < <(for k in ${tftf_result_keys[@]} ; do echo "${results_split[$k]}"; done )
 	output_csv_str="${output_csv_str} $(echo $(IFS=,; echo "${row[*]}"))"
 	unset results_split[{..}] row
 done

 # Replace spaces in header with commas and print to the output file.
 echo $(IFS=,; echo "${tftf_result_keys[*]}") > ${CSVFILE}

 # Sort Filenames alphabetically and store in csv for gnuplot
 sorted=($(IFS=$' '; sort <<<$output_csv_str))
 printf "%b\n" "${sorted[@]}" >> ${CSVFILE}

 # Produce PNG image of graph using gnuplot and .plot description file
 gnuplot -e "jenkins_id='$sub_build_url'" -c ${0%bash}plot \
         "$CSVFILE" > "$PNGFILE"
	#!/usr/bin/env bash
	#
	# Copyright (c) 2022 Arm Limited. All rights reserved.
	#
	# SPDX-License-Identifier: BSD-3-Clause
	#

	set -euo pipefail

	# Build and Jenkins URL.
	sub_build_url=${1}
	job_name="${2:-"tf-a-ci-gateway"}"
	filter=${3:-".*"}

	jenkins="${sub_build_url%%/job*}"

	# Utilise default paths to output files if none provided.
	job_target="$(dirname ${sub_build_url#*/job/})"
	PNGFILE=${PNGFILE:=${job_target}-result.png}
	CSVFILE=${CSVFILE:=${job_target}-result.csv}

	# Remove csv output file if it exists to append to empty file
	: > "${CSVFILE}"

	readarray -t sub_builds < <(curl -sSL "${sub_build_url}/api/json" \| jq -Rr '
	fromjson? \| [
	.subBuilds[]? \| select(.jobName == "'${job_name}'") \| .url
	] \| .[]')

	# Get a csv list of relative paths to report.json, or "-" if no report.json
	report_rel_paths_url="${jenkins}/{$(echo $(IFS=,; echo "${sub_builds[*]}"))}/api/json"

	readarray -t report_paths < <(curl -fsSL --fail-early "${report_rel_paths_url}" \
	\| sed 's/--_curl_--.*$//' \
	\| sed -e 's/^{/'$(printf "\x1e")'{/' \
	\| jq -sr --seq '
	[ .[]
	\| [ .artifacts[]?
	\| select (.fileName == "report.json")
	\| .relativePath ]
	\| if length > 0 then .[] else "-" end ]
	\| .[]')

	# Combine sub build urls with relative path to "report.json"
	# the empty entries are intentionally kept as -, so the output array can
	# be mapped onto ${sub_build_list}
	report_urls="$jenkins/{"
	for i in "${!sub_builds[@]}"
	do
	if [[ ${report_paths[i]} != "-" ]]
	then
	report_urls="${report_urls}${sub_builds[i]}/artifact/${report_paths[i]},"
	fi
	done

	# Strip last comma and add closing brace
	report_urls="${report_urls%?}}"

	# Get Child build information from each report.json.
	readarray -t child_file_list_array < <(curl -sL "${report_urls}" -o -\
	\| sed 's/--_curl_--.*$//' \
	\| jq -sr --arg FILTER "${filter}" \
	'[.[]
	\| .job as $job
	\| [ .child_build_numbers?, [(.test_files[]
	\| sub("\\.test";"")
	\| split("%") as $config
	\| { group: $config[1], suite: $config[2]})]]
	\| transpose
	\| map( {($job + "/" + .[0]) : .[1]} \| to_entries )
	\| add
	\| map(select(.value.suite \| test(".*"; "il")?
	and (endswith("nil")
	or endswith("norun-fip.dummy") \| not)))
	\| if ( length > 0 )
	then .[] else empty end
	\| .value.group, (.value.suite \| gsub("\\,nil";"")), .key]
	\| .[]')

	# These three arrays should be the same length, and values at the same index
	# correspond to the same child build
	declare -a tftf_keys tftf_suite tftf_group

	for i in $(seq 0 3 $((${#child_file_list_array[@]}-1))) ; do
	tftf_group+=("${child_file_list_array[$i]}")
	tftf_suite+=("${child_file_list_array[$i+1]}")
	tftf_keys+=("${child_file_list_array[$i+2]}")
	done


	child_output_results_url="${jenkins}/job/{$(echo $(IFS=,; echo "${tftf_keys[*]}"))}/api/json"

	# Retrieve relative path to either "uart0_full.txt" (FVP) or
	# "job_output.log" (LAVA) for each child job. Once again values where no match
	# is found are intentionally kept as "-" so the array can be correlated with
	# ${tftf_suite}.
	readarray -t child_output_results < <(curl -fsSL --fail-early "$child_output_results_url" \
	\| sed 's/}{/}\n{/g' \
	\| jq -sr '[ .[]
	\| ([ .artifacts[]?
	\| select(.fileName == "uart0_full.txt"
	or .fileName == "job_output.log"
	or .fileName == "lava-uart0.log") ]
	\| if length > 0
	then .[0].relativePath else "-" end), .result ]
	\| .[]')

	# Combine job and child_build number with relative path to output file
	testlog_urls="${jenkins}/job/{"
	tftf_child_results=()

	for i in $(seq 0 2 $((${#child_output_results[@]}-1))) ; do
	testlog_urls+="${tftf_keys[$((i/2))]}/artifact/${child_output_results[$i]},"
	tftf_child_results+=(${child_output_results[$((i+1))]})
	done

	# Remove final comma and append a closing brace
	testlog_urls="${testlog_urls%?}}"

	# Retrieve the log for each child with --include to also retrieve the HTTP
	# header and grep for a block like:
	# Tests Skipped : 125
	# Tests Passed : 45
	# Tests Failed : 0
	# Tests Crashed : 0
	#
	# If none is found the line HTTP is used to delemit each entry
	#
	# Logs from Lava each message is wrapped with braces and has some pre-amble,
	# which is removed with sed.

	tftf_result_keys=(
	"TestGroup" "TestSuite" "URL" "Result" "Passed" "Failed" "Crashed" "Skipped"
	)
	declare -A results_split="( $(for ord_ in ${tftf_result_keys[@]} ; do echo -n "[$ord_]=\"\" "; done))"
	declare output_csv_str="" csv_row=""

	read -ra tftf_urls <<< "$(eval "echo ${testlog_urls}")"

	# FIXME adjust this so we can handle both LAVA logs
	# for each test suite
	# curl the result log if its not '-'
	# remove debug information
	# if results is none:
	# use "Result"
	# else
	# read each key
	# write row to csv

	# Sort results into rows:
	for i in ${!tftf_suite[*]}; do
	results_split["TestGroup"]="${tftf_group[$i]:-}"
	results_split["TestSuite"]="\"${tftf_suite[$i]:-}\""
	results_split["URL"]="${tftf_urls[$i]:-}"
	results_split["Result"]="${tftf_child_results[$i]:-}"

	# Skipped/Crashed are always zero if no test block is found
	results_split["Skipped"]="0"
	results_split["Crashed"]="0"
	if [[ "${results_split["Result"]}" == "SUCCESS" ]];
	then
	results_split["Passed"]="1"
	results_split["Failed"]="0"
	else
	results_split["Passed"]="0"
	results_split["Failed"]="1"
	fi

	readarray -t raw_result < <(curl -sL --include "${results_split["URL"]}" \
	\| sed 's/.*msg": "//g' \
	\| grep --text -E "^Tests\|HTTP\/")

	for line in "${raw_result[@]}"; do
	if [[ "${line}" == Test* ]]
	then
	k=$(echo "${line}" \| awk -F ' ' '{print $2}')
	count="${line//[!0-9]/}"
	results_split[$k]=$count
	fi
	done

	# Generate CSV row using array of ordinals to align with headers.
	readarray -t row < <(for k in ${tftf_result_keys[@]} ; do echo "${results_split[$k]}"; done )
	output_csv_str="${output_csv_str} $(echo $(IFS=,; echo "${row[*]}"))"
	unset results_split[{..}] row
	done

	# Replace spaces in header with commas and print to the output file.
	echo $(IFS=,; echo "${tftf_result_keys[*]}") > ${CSVFILE}

	# Sort Filenames alphabetically and store in csv for gnuplot
	sorted=($(IFS=$' '; sort <<<$output_csv_str))
	printf "%b\n" "${sorted[@]}" >> ${CSVFILE}

	# Produce PNG image of graph using gnuplot and .plot description file
	gnuplot -e "jenkins_id='$sub_build_url'" -c ${0%bash}plot \
	"$CSVFILE" > "$PNGFILE"