blob: 224de12b749ad3dc60dd410cc7f48506fffb7d33 [file] [log] [blame]
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +02001# Process the test results
2# Test status (like passed, or failed with error code)
3
4import argparse
5import re
6import TestScripts.NewParser as parse
7import TestScripts.CodeGen
8from collections import deque
9import os.path
10import csv
Christophe Favergeonf76a8032019-08-09 09:15:50 +010011import TestScripts.ParseTrace
Christophe Favergeon30c03792019-10-03 12:47:41 +010012import colorama
13from colorama import init,Fore, Back, Style
Christophe Favergeonf76a8032019-08-09 09:15:50 +010014
Christophe Favergeon30c03792019-10-03 12:47:41 +010015init()
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +020016
Christophe Favergeon6f8eee92019-10-09 12:21:27 +010017def errorStr(id):
18 if id == 1:
19 return("UNKNOWN_ERROR")
20 if id == 2:
21 return("Equality error")
22 if id == 3:
23 return("Absolute difference error")
24 if id == 4:
25 return("Relative difference error")
26 if id == 5:
27 return("SNR error")
28 if id == 6:
29 return("Different length error")
30 if id == 7:
31 return("Assertion error")
32 if id == 8:
33 return("Memory allocation error")
34 if id == 9:
35 return("Empty pattern error")
36 if id == 10:
37 return("Buffer tail corrupted")
Christophe Favergeonf055bd32019-10-15 12:30:30 +010038 if id == 11:
39 return("Close float error")
Christophe Favergeon6f8eee92019-10-09 12:21:27 +010040
41 return("Unknown error %d" % id)
42
43
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +020044def findItem(root,path):
45 """ Find a node in a tree
46
47 Args:
48 path (list) : A list of node ID
49 This list is describing a path in the tree.
50 By starting from the root and following this path,
51 we can find the node in the tree.
52 Raises:
53 Nothing
54 Returns:
55 TreeItem : A node
56 """
57 # The list is converted into a queue.
58 q = deque(path)
59 q.popleft()
60 c = root
61 while q:
62 n = q.popleft()
63 # We get the children based on its ID and continue
64 c = c[n-1]
65 return(c)
66
67def joinit(iterable, delimiter):
68 # Intersperse a delimiter between element of a list
69 it = iter(iterable)
70 yield next(it)
71 for x in it:
72 yield delimiter
73 yield x
74
75# Return test result as a text tree
76class TextFormatter:
77 def start(self):
78 None
79
80 def printGroup(self,elem,theId):
81 if elem is None:
82 elem = root
83 message=elem.data["message"]
84 if not elem.data["deprecated"]:
85 kind = "Suite"
86 ident = " " * elem.ident
87 if elem.kind == TestScripts.Parser.TreeElem.GROUP:
88 kind = "Group"
89 #print(elem.path)
Christophe Favergeon30c03792019-10-03 12:47:41 +010090 print(Style.BRIGHT + ("%s%s : %s (%d)" % (ident,kind,message,theId)) + Style.RESET_ALL)
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +020091
Christophe Favergeon4f462732019-11-13 14:11:14 +010092 def printTest(self,elem, theId, theError,errorDetail,theLine,passed,cycles,params):
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +020093 message=elem.data["message"]
94 if not elem.data["deprecated"]:
95 kind = "Test"
96 ident = " " * elem.ident
Christophe Favergeon30c03792019-10-03 12:47:41 +010097 p=Fore.RED + "FAILED" + Style.RESET_ALL
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +020098 if passed == 1:
Christophe Favergeon30c03792019-10-03 12:47:41 +010099 p= Fore.GREEN + "PASSED" + Style.RESET_ALL
100 print("%s%s %s(%d)%s : %s (cycles = %d)" % (ident,message,Style.BRIGHT,theId,Style.RESET_ALL,p,cycles))
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200101 if params:
102 print("%s %s" % (ident,params))
103 if passed != 1:
Christophe Favergeon6f8eee92019-10-09 12:21:27 +0100104 print(Fore.RED + ("%s %s at line %d" % (ident, errorStr(theError), theLine)) + Style.RESET_ALL)
Christophe Favergeon4f462732019-11-13 14:11:14 +0100105 if (len(errorDetail)>0):
106 print(Fore.RED + ident + " " + errorDetail + Style.RESET_ALL)
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200107
108 def pop(self):
109 None
110
111 def end(self):
112 None
113
Christophe Favergeone972cbd2019-11-19 15:54:13 +0100114# Return test result as a text tree
115class HTMLFormatter:
116 def __init__(self):
117 self.nb=1
118 self.suite=False
119
120 def start(self):
121 print("<html><head><title>Test Results</title></head><body>")
122
123 def printGroup(self,elem,theId):
124 if elem is None:
125 elem = root
126 message=elem.data["message"]
127 if not elem.data["deprecated"]:
128 kind = "Suite"
129 ident = " " * elem.ident
130 if elem.kind == TestScripts.Parser.TreeElem.GROUP:
131 kind = "Group"
132 if kind == "Group":
133 print("<h%d> %s (%d) </h%d>" % (self.nb,message,theId,self.nb))
134 else:
135 print("<h%d> %s (%d) </h%d>" % (self.nb,message,theId,self.nb))
136 self.suite=True
137 print("<table style=\"width:100%\">")
138 print("<tr>")
139 print("<td>Name</td>")
140 print("<td>ID</td>")
141 print("<td>Status</td>")
142 print("<td>Cycles</td>")
143 print("</tr>")
144 self.nb = self.nb + 1
145
146 def printTest(self,elem, theId, theError,errorDetail,theLine,passed,cycles,params):
147 message=elem.data["message"]
148 if not elem.data["deprecated"]:
149 kind = "Test"
150 ident = " " * elem.ident
151 p="<font color=\"red\">FAILED</font>"
152 if passed == 1:
153 p= "<font color=\"green\">PASSED</font>"
154 print("<tr>")
155 print("<td><pre>%s</pre></td>" % message)
156 print("<td>%d</td>" % theId)
157 print("<td>%s</td>" % p)
158 print("<td>%d</td>" % cycles)
159 print("</tr>")
160 #if params:
161 # print("%s %s" % (ident,params))
162 if passed != 1:
163
164 print("<tr><td colspan=4><font color=\"red\">%s at line %d</font></td></tr>" % (errorStr(theError), theLine))
165 if (len(errorDetail)>0):
166 print("<tr><td colspan=4><font color=\"red\">" + errorDetail + "</font></td></tr>")
167
168 def pop(self):
169 if self.suite:
170 print("</table>")
171 self.nb = self.nb - 1
172 self.suite=False
173
174 def end(self):
175 print("</body></html>")
176
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200177# Return test result as a CSV
178class CSVFormatter:
179
180 def __init__(self):
181 self.name=[]
182 self._start=True
183
184 def start(self):
185 print("CATEGORY,NAME,ID,STATUS,CYCLES,PARAMS")
186
187 def printGroup(self,elem,theId):
188 if elem is None:
189 elem = root
190 # Remove Root from category name in CSV file.
191 if not self._start:
192 self.name.append(elem.data["class"])
193 else:
194 self._start=False
195 message=elem.data["message"]
196 if not elem.data["deprecated"]:
197 kind = "Suite"
198 ident = " " * elem.ident
199 if elem.kind == TestScripts.Parser.TreeElem.GROUP:
200 kind = "Group"
201
Christophe Favergeon4f462732019-11-13 14:11:14 +0100202 def printTest(self,elem, theId, theError, errorDetail,theLine,passed,cycles,params):
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200203 message=elem.data["message"]
204 if not elem.data["deprecated"]:
205 kind = "Test"
206 name=elem.data["class"]
207 category= "".join(list(joinit(self.name,":")))
208 print("%s,%s,%d,%d,%d,\"%s\"" % (category,name,theId,passed,cycles,params))
209
210 def pop(self):
211 if self.name:
212 self.name.pop()
213
214 def end(self):
215 None
216
217class MathematicaFormatter:
218
219 def __init__(self):
220 self._hasContent=[False]
221 self._toPop=[]
222
223 def start(self):
224 None
225
226 def printGroup(self,elem,theId):
227 if self._hasContent[len(self._hasContent)-1]:
228 print(",",end="")
229
230 print("<|")
231 self._hasContent[len(self._hasContent)-1] = True
232 self._hasContent.append(False)
233 if elem is None:
234 elem = root
235 message=elem.data["message"]
236 if not elem.data["deprecated"]:
237
238 kind = "Suite"
239 ident = " " * elem.ident
240 if elem.kind == TestScripts.Parser.TreeElem.GROUP:
241 kind = "Group"
242 print("\"%s\" ->" % (message))
243 #if kind == "Suite":
244 print("{",end="")
245 self._toPop.append("}")
246 #else:
247 # self._toPop.append("")
248
Christophe Favergeon4f462732019-11-13 14:11:14 +0100249 def printTest(self,elem, theId, theError,errorDetail,theLine,passed,cycles,params):
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200250 message=elem.data["message"]
251 if not elem.data["deprecated"]:
252 kind = "Test"
253 ident = " " * elem.ident
254 p="FAILED"
255 if passed == 1:
256 p="PASSED"
257 parameters=""
258 if params:
259 parameters = "%s" % params
260 if self._hasContent[len(self._hasContent)-1]:
261 print(",",end="")
262 print("<|\"NAME\" -> \"%s\",\"ID\" -> %d,\"STATUS\" -> \"%s\",\"CYCLES\" -> %d,\"PARAMS\" -> \"%s\"|>" % (message,theId,p,cycles,parameters))
263 self._hasContent[len(self._hasContent)-1] = True
264 #if passed != 1:
265 # print("%s Error = %d at line %d" % (ident, theError, theLine))
266
267 def pop(self):
268 print(self._toPop.pop(),end="")
269 print("|>")
270 self._hasContent.pop()
271
272 def end(self):
273 None
274
275NORMAL = 1
276INTEST = 2
277TESTPARAM = 3
Christophe Favergeon4f462732019-11-13 14:11:14 +0100278ERRORDESC = 4
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200279
280def createMissingDir(destPath):
281 theDir=os.path.normpath(os.path.dirname(destPath))
282 if not os.path.exists(theDir):
283 os.makedirs(theDir)
284
285def correctPath(path):
286 while (path[0]=="/") or (path[0] == "\\"):
287 path = path[1:]
288 return(path)
289
290def extractDataFiles(results,outputDir):
291 infile = False
292 f = None
293 for l in results:
294 if re.match(r'^.*D:[ ].*$',l):
295 if infile:
296 if re.match(r'^.*D:[ ]END$',l):
297 infile = False
298 if f:
299 f.close()
300 else:
301 if f:
302 m = re.match(r'^.*D:[ ](.*)$',l)
303 data = m.group(1)
304 f.write(data)
305 f.write("\n")
306
307 else:
308 m = re.match(r'^.*D:[ ](.*)$',l)
309 path = str(m.group(1))
310 infile = True
311 destPath = os.path.join(outputDir,correctPath(path))
312 createMissingDir(destPath)
313 f = open(destPath,"w")
314
315
316
317def writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config):
318 if benchFile:
319 name=elem.data["class"]
320 category= elem.categoryDesc()
Christophe Favergeon37b86222019-07-17 11:49:00 +0200321 old=""
322 if "testData" in elem.data:
323 if "oldID" in elem.data["testData"]:
324 old=elem.data["testData"]["oldID"]
325 benchFile.write("\"%s\",\"%s\",%d,\"%s\",%s,%d,%s\n" % (category,name,theId,old,params,cycles,config))
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200326
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100327def getCyclesFromTrace(trace):
328 if not trace:
329 return(0)
330 else:
331 return(TestScripts.ParseTrace.getCycles(trace))
332
Christophe Favergeon5cacf9d2019-08-14 10:41:17 +0200333def analyseResult(resultPath,root,results,embedded,benchmark,trace,formatter):
Christophe Favergeonbe7efb42019-08-09 10:17:03 +0100334 calibration = 0
335 if trace:
336 # First cycle in the trace is the calibration data
337 # The noramlisation factor must be coherent with the C code one.
338 calibration = int(getCyclesFromTrace(trace) / 20)
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200339 formatter.start()
340 path = []
341 state = NORMAL
342 prefix=""
343 elem=None
344 theId=None
345 theError=None
Christophe Favergeon4f462732019-11-13 14:11:14 +0100346 errorDetail=""
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200347 theLine=None
348 passed=0
349 cycles=None
350 benchFile = None
351 config=""
352 if embedded:
353 prefix = ".*S:[ ]"
354
355 # Parse the result file.
356 # NORMAL mode is when we are parsing suite or group.
357 # Otherwise we are parsing a test and we need to analyse the
358 # test result.
359 # TESTPARAM is used to read parameters of the test.
360 # Format of output is:
361 #node ident : s id or g id or t or u
362 #test status : id error linenb status Y or N (Y when passing)
363 #param for this test b x,x,x,x or b alone if not param
364 #node end : p
365 # In FPGA mode:
366 #Prefix S:[ ] before driver dump
367 # D:[ ] before data dump (output patterns)
368
369 for l in results:
370 l = l.strip()
371 if not re.match(r'^.*D:[ ].*$',l):
372 if state == NORMAL:
373 if len(l) > 0:
374 # Line starting with g or s is a suite or group.
375 # In FPGA mode, those line are prefixed with 'S: '
376 # and data file with 'D: '
377 if re.match(r'^%s[gs][ ]+[0-9]+.*$' % prefix,l):
378 # Extract the test id
379 theId=re.sub(r'^%s[gs][ ]+([0-9]+).*$' % prefix,r'\1',l)
380 theId=int(theId)
381 path.append(theId)
382 # From a list of id, find the TreeElem in the Parsed tree
383 # to know what is the node.
384 elem = findItem(root,path)
385 # Display formatted output for this node
386 if elem.params:
387 #print(elem.params.full)
388 benchPath = os.path.join(benchmark,elem.fullPath(),"fullBenchmark.csv")
389 createMissingDir(benchPath)
390 if benchFile:
391 printf("ERROR BENCH FILE %s ALREADY OPEN" % benchPath)
392 benchFile.close()
393 benchFile=None
394 benchFile=open(benchPath,"w")
395 header = "".join(list(joinit(elem.params.full,",")))
396 # A test and a benchmark are different
397 # so we don't dump a status and error
398 # A status and error in a benchmark would
399 # impact the cycles since the test
400 # would be taken into account in the measurement
401 # So benchmark are always passing and contain no test
402 #benchFile.write("ID,%s,PASSED,ERROR,CYCLES\n" % header)
403 csvheaders = ""
404
Christophe Favergeon5cacf9d2019-08-14 10:41:17 +0200405 with open(os.path.join(resultPath,'currentConfig.csv'), 'r') as f:
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200406 reader = csv.reader(f)
407 csvheaders = next(reader, None)
408 configList = list(reader)
409 #print(configList)
410 config = "".join(list(joinit(configList[0],",")))
411 configHeaders = "".join(list(joinit(csvheaders,",")))
Christophe Favergeon37b86222019-07-17 11:49:00 +0200412 benchFile.write("CATEGORY,NAME,ID,OLDID,%s,CYCLES,%s\n" % (header,configHeaders))
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200413
414 formatter.printGroup(elem,theId)
415
416 # If we have detected a test, we switch to test mode
417 if re.match(r'^%s[t][ ]*$' % prefix,l):
418 state = INTEST
419
420
421 # Pop
422 # End of suite or group
423 if re.match(r'^%sp.*$' % prefix,l):
424 if benchFile:
425 benchFile.close()
426 benchFile=None
427 path.pop()
428 formatter.pop()
429 elif state == INTEST:
430 if len(l) > 0:
431 # In test mode, we are looking for test status.
432 # A line starting with S
433 # (There may be empty lines or line for data files)
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100434 passRe = r'^%s([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([t0-9]+)[ ]+([YN]).*$' % prefix
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200435 if re.match(passRe,l):
436 # If we have found a test status then we will start again
437 # in normal mode after this.
438
439 m = re.match(passRe,l)
440
441 # Extract test ID, test error code, line number and status
442 theId=m.group(1)
443 theId=int(theId)
444
445 theError=m.group(2)
446 theError=int(theError)
447
448 theLine=m.group(3)
449 theLine=int(theLine)
450
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100451 maybeCycles = m.group(4)
452 if maybeCycles == "t":
Christophe Favergeonbe7efb42019-08-09 10:17:03 +0100453 cycles = getCyclesFromTrace(trace) - calibration
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100454 else:
455 cycles = int(maybeCycles)
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200456
457 status=m.group(5)
458 passed=0
459
460 # Convert status to number as used by formatter.
461 if status=="Y":
462 passed = 1
463 if status=="N":
464 passed = 0
465 # Compute path to this node
466 newPath=path.copy()
467 newPath.append(theId)
468 # Find the node in the Tree
469 elem = findItem(root,newPath)
470
471
Christophe Favergeon4f462732019-11-13 14:11:14 +0100472 state = ERRORDESC
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200473 else:
474 if re.match(r'^%sp.*$' % prefix,l):
475 if benchFile:
476 benchFile.close()
477 benchFile=None
478 path.pop()
479 formatter.pop()
480 if re.match(r'^%s[t][ ]*$' % prefix,l):
481 state = INTEST
482 else:
483 state = NORMAL
Christophe Favergeon4f462732019-11-13 14:11:14 +0100484 elif state == ERRORDESC:
485 if len(l) > 0:
486 if re.match(r'^.*E:.*$',l):
487 if re.match(r'^.*E:[ ].*$',l):
488 m = re.match(r'^.*E:[ ](.*)$',l)
489 errorDetail = m.group(1)
490 else:
491 errorDetail = ""
492 state = TESTPARAM
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200493 else:
494 if len(l) > 0:
495 state = INTEST
496 params=""
497 if re.match(r'^.*b[ ]+([0-9,]+)$',l):
498 m=re.match(r'^.*b[ ]+([0-9,]+)$',l)
499 params=m.group(1).strip()
500 # Format the node
501 #print(elem.fullPath())
502 #createMissingDir(destPath)
503 writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config)
504 else:
505 params=""
506 writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config)
507 # Format the node
Christophe Favergeon4f462732019-11-13 14:11:14 +0100508 formatter.printTest(elem,theId,theError,errorDetail,theLine,passed,cycles,params)
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200509
510
511 formatter.end()
512
513
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100514def analyze(root,results,args,trace):
Christophe Favergeon5cacf9d2019-08-14 10:41:17 +0200515 # currentConfig.csv should be in the same place
516 resultPath=os.path.dirname(args.r)
517
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100518 if args.c:
Christophe Favergeon5cacf9d2019-08-14 10:41:17 +0200519 analyseResult(resultPath,root,results,args.e,args.b,trace,CSVFormatter())
Christophe Favergeone972cbd2019-11-19 15:54:13 +0100520 elif args.html:
521 analyseResult(resultPath,root,results,args.e,args.b,trace,HTMLFormatter())
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100522 elif args.m:
Christophe Favergeon5cacf9d2019-08-14 10:41:17 +0200523 analyseResult(resultPath,root,results,args.e,args.b,trace,MathematicaFormatter())
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100524 else:
Christophe Favergeon5cacf9d2019-08-14 10:41:17 +0200525 analyseResult(resultPath,root,results,args.e,args.b,trace,TextFormatter())
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100526
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200527parser = argparse.ArgumentParser(description='Parse test description')
528
Christophe Favergeon6f8eee92019-10-09 12:21:27 +0100529parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="Test description file path")
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200530# Where the result file can be found
531parser.add_argument('-r', nargs='?',type = str, default=None, help="Result file path")
532parser.add_argument('-c', action='store_true', help="CSV output")
Christophe Favergeone972cbd2019-11-19 15:54:13 +0100533parser.add_argument('-html', action='store_true', help="HTML output")
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200534parser.add_argument('-e', action='store_true', help="Embedded test")
535# -o needed when -e is true to know where to extract the output files
536parser.add_argument('-o', nargs='?',type = str, default="Output", help="Output dir path")
537
538parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path")
539parser.add_argument('-m', action='store_true', help="Mathematica output")
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100540parser.add_argument('-t', nargs='?',type = str, default=None, help="External trace file")
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200541
542args = parser.parse_args()
543
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100544
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200545if args.f is not None:
Christophe Favergeon6f8eee92019-10-09 12:21:27 +0100546 #p = parse.Parser()
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200547 # Parse the test description file
Christophe Favergeon6f8eee92019-10-09 12:21:27 +0100548 #root = p.parse(args.f)
549 root=parse.loadRoot(args.f)
Christophe Favergeonf76a8032019-08-09 09:15:50 +0100550 if args.t:
551 with open(args.t,"r") as trace:
552 with open(args.r,"r") as results:
553 analyze(root,results,args,iter(trace))
554 else:
555 with open(args.r,"r") as results:
556 analyze(root,results,args,None)
Christophe Favergeon3b2a0ee2019-06-12 13:29:14 +0200557 if args.e:
558 # In FPGA mode, extract output files from stdout (result file)
559 with open(args.r,"r") as results:
560 extractDataFiles(results,args.o)
561
562else:
563 parser.print_help()