CMSIS/DSP/Testing/processResult.py

# Process the test results
# Test status (like passed, or failed with error code)

import argparse
import re 
import TestScripts.NewParser as parse
import TestScripts.CodeGen
from collections import deque
import os.path
import csv

def findItem(root,path):
        """ Find a node in a tree
      
        Args:
          path (list) : A list of node ID
            This list is describing a path in the tree.
            By starting from the root and following this path,
            we can find the node in the tree.
        Raises:
          Nothing 
        Returns:
          TreeItem : A node
        """
        # The list is converted into a queue.
        q = deque(path) 
        q.popleft()
        c = root
        while q:
            n = q.popleft() 
            # We get the children based on its ID and continue
            c = c[n-1]
        return(c)

def joinit(iterable, delimiter):
    # Intersperse a delimiter between element of a list
    it = iter(iterable)
    yield next(it)
    for x in it:
        yield delimiter
        yield x

# Return test result as a text tree
class TextFormatter:
      def start(self):
          None 

      def printGroup(self,elem,theId):
        if elem is None:
           elem = root
        message=elem.data["message"]
        if not elem.data["deprecated"]:
           kind = "Suite"
           ident = " " * elem.ident
           if elem.kind == TestScripts.Parser.TreeElem.GROUP:
              kind = "Group"
           #print(elem.path)
           print("%s%s : %s (%d)" % (ident,kind,message,theId))

      def printTest(self,elem, theId, theError,theLine,passed,cycles,params):
          message=elem.data["message"]
          if not elem.data["deprecated"]:
             kind = "Test"
             ident = " " * elem.ident
             p="FAILED"
             if passed == 1:
                p="PASSED"
             print("%s%s (%d) : %s (cycles = %d)" % (ident,message,theId,p,cycles))
             if params:
                print("%s %s" % (ident,params))
             if passed != 1:
                print("%s Error = %d at line %d" % (ident, theError, theLine))

      def pop(self):
          None

      def end(self):
        None

# Return test result as a CSV
class CSVFormatter:

      def __init__(self):
        self.name=[]
        self._start=True

      def start(self):
          print("CATEGORY,NAME,ID,STATUS,CYCLES,PARAMS") 
          
      def printGroup(self,elem,theId):
        if elem is None:
           elem = root
        # Remove Root from category name in CSV file.
        if not self._start:
           self.name.append(elem.data["class"])
        else:
            self._start=False
        message=elem.data["message"]
        if not elem.data["deprecated"]:
           kind = "Suite"
           ident = " " * elem.ident
           if elem.kind == TestScripts.Parser.TreeElem.GROUP:
              kind = "Group"

      def printTest(self,elem, theId, theError, theLine,passed,cycles,params):
          message=elem.data["message"]
          if not elem.data["deprecated"]:
             kind = "Test"
             name=elem.data["class"] 
             category= "".join(list(joinit(self.name,":")))
             print("%s,%s,%d,%d,%d,\"%s\"" % (category,name,theId,passed,cycles,params))

      def pop(self):
         if self.name:
            self.name.pop()

      def end(self):
        None

class MathematicaFormatter:

      def __init__(self):
        self._hasContent=[False]
        self._toPop=[]

      def start(self):
          None

      def printGroup(self,elem,theId):
        if self._hasContent[len(self._hasContent)-1]:
           print(",",end="")
        
        print("<|") 
        self._hasContent[len(self._hasContent)-1] = True
        self._hasContent.append(False)
        if elem is None:
           elem = root
        message=elem.data["message"]
        if not elem.data["deprecated"]:

           kind = "Suite"
           ident = " " * elem.ident
           if elem.kind == TestScripts.Parser.TreeElem.GROUP:
              kind = "Group"
           print("\"%s\" ->" % (message))
           #if kind == "Suite":
           print("{",end="")
           self._toPop.append("}")
           #else:
           #   self._toPop.append("")

      def printTest(self,elem, theId, theError,theLine,passed,cycles,params):
          message=elem.data["message"]
          if not elem.data["deprecated"]:
             kind = "Test"
             ident = " " * elem.ident
             p="FAILED"
             if passed == 1:
                p="PASSED"
             parameters=""
             if params:
                parameters = "%s" % params
             if self._hasContent[len(self._hasContent)-1]:
               print(",",end="")
             print("<|\"NAME\" -> \"%s\",\"ID\" -> %d,\"STATUS\" -> \"%s\",\"CYCLES\" -> %d,\"PARAMS\" -> \"%s\"|>" % (message,theId,p,cycles,parameters))
             self._hasContent[len(self._hasContent)-1] = True
             #if passed != 1:
             #   print("%s Error = %d at line %d" % (ident, theError, theLine))

      def pop(self):
          print(self._toPop.pop(),end="")
          print("|>")
          self._hasContent.pop()

      def end(self):
        None

NORMAL = 1 
INTEST = 2
TESTPARAM = 3

def createMissingDir(destPath):
  theDir=os.path.normpath(os.path.dirname(destPath))
  if not os.path.exists(theDir):
      os.makedirs(theDir)

def correctPath(path):
  while (path[0]=="/") or (path[0] == "\\"):
      path = path[1:]
  return(path)

def extractDataFiles(results,outputDir):
  infile = False
  f = None
  for l in results:
      if re.match(r'^.*D:[ ].*$',l):
          if infile:
            if re.match(r'^.*D:[ ]END$',l):
               infile = False 
               if f:
                 f.close()
            else:
              if f:
                m = re.match(r'^.*D:[ ](.*)$',l)
                data = m.group(1)
                f.write(data)
                f.write("\n")

          else:
            m = re.match(r'^.*D:[ ](.*)$',l)
            path = str(m.group(1))
            infile = True 
            destPath = os.path.join(outputDir,correctPath(path))
            createMissingDir(destPath)
            f = open(destPath,"w")

         

def writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config):
  if benchFile:
    name=elem.data["class"] 
    category= elem.categoryDesc()
    benchFile.write("\"%s\",\"%s\",%d,%s,%d,%s\n" % (category,name,theId,params,cycles,config))

def analyseResult(root,results,embedded,benchmark,formatter):
    formatter.start()
    path = []
    state = NORMAL
    prefix=""
    elem=None
    theId=None
    theError=None
    theLine=None
    passed=0
    cycles=None
    benchFile = None
    config=""
    if embedded:
       prefix = ".*S:[ ]"

    # Parse the result file.
    # NORMAL mode is when we are parsing suite or group.
    # Otherwise we are parsing a test and we need to analyse the
    # test result.
    # TESTPARAM is used to read parameters of the test.
    # Format of output is:
    #node ident : s id or g id or t or u
    #test status : id error linenb status Y or N (Y when passing)
    #param for this test b x,x,x,x or b alone if not param
    #node end : p
    # In FPGA mode:
    #Prefix S:[ ] before driver dump
    # D:[ ] before data dump (output patterns)

    for l in results:
        l = l.strip() 
        if not re.match(r'^.*D:[ ].*$',l):
           if state == NORMAL:
              if len(l) > 0:
                 # Line starting with g or s is a suite or group.
                 # In FPGA mode, those line are prefixed with 'S: '
                 # and data file with 'D: '
                 if re.match(r'^%s[gs][ ]+[0-9]+.*$' % prefix,l):
                    # Extract the test id
                    theId=re.sub(r'^%s[gs][ ]+([0-9]+).*$' % prefix,r'\1',l)
                    theId=int(theId)
                    path.append(theId)
                    # From a list of id, find the TreeElem in the Parsed tree
                    # to know what is the node.
                    elem = findItem(root,path)
                    # Display formatted output for this node
                    if elem.params:
                       #print(elem.params.full)
                       benchPath = os.path.join(benchmark,elem.fullPath(),"fullBenchmark.csv")
                       createMissingDir(benchPath)
                       if benchFile:
                          printf("ERROR BENCH FILE %s ALREADY OPEN" % benchPath)
                          benchFile.close()
                          benchFile=None
                       benchFile=open(benchPath,"w")
                       header = "".join(list(joinit(elem.params.full,",")))
                       # A test and a benchmark are different
                       # so we don't dump a status and error
                       # A status and error in a benchmark would
                       # impact the cycles since the test
                       # would be taken into account in the measurement
                       # So benchmark are always passing and contain no test
                       #benchFile.write("ID,%s,PASSED,ERROR,CYCLES\n" % header)
                       csvheaders = ""

                       with open('currentConfig.csv', 'r') as f:
                          reader = csv.reader(f)
                          csvheaders = next(reader, None)
                          configList = list(reader)
                          #print(configList)
                          config = "".join(list(joinit(configList[0],",")))
                          configHeaders = "".join(list(joinit(csvheaders,",")))
                       benchFile.write("CATEGORY,NAME,ID,%s,CYCLES,%s\n" % (header,configHeaders))
   
                    formatter.printGroup(elem,theId)
      
                 # If we have detected a test, we switch to test mode
                 if re.match(r'^%s[t][ ]*$' % prefix,l):
                    state = INTEST
                 
      
                 # Pop
                 # End of suite or group
                 if re.match(r'^%sp.*$' % prefix,l):
                   if benchFile:
                      benchFile.close()
                      benchFile=None
                   path.pop()
                   formatter.pop()
           elif state == INTEST:
             if len(l) > 0:
               # In test mode, we are looking for test status.
               # A line starting with S
               # (There may be empty lines or line for data files)
               passRe = r'^%s([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([YN]).*$'  % prefix
               if re.match(passRe,l):
                    # If we have found a test status then we will start again
                    # in normal mode after this.
                    
                    m = re.match(passRe,l)
                    
                    # Extract test ID, test error code, line number and status
                    theId=m.group(1)
                    theId=int(theId)
      
                    theError=m.group(2)
                    theError=int(theError)
      
                    theLine=m.group(3)
                    theLine=int(theLine)
      
                    cycles = int(m.group(4))
   
                    status=m.group(5)
                    passed=0
      
                    # Convert status to number as used by formatter.
                    if status=="Y":
                       passed = 1
                    if status=="N":
                       passed = 0
                    # Compute path to this node
                    newPath=path.copy()
                    newPath.append(theId)
                    # Find the node in the Tree
                    elem = findItem(root,newPath)
   
                    
                    state = TESTPARAM
               else:
                 if re.match(r'^%sp.*$' % prefix,l):
                   if benchFile:
                      benchFile.close()
                      benchFile=None
                   path.pop()
                   formatter.pop()
                 if re.match(r'^%s[t][ ]*$' % prefix,l):
                    state = INTEST
                 else:
                    state = NORMAL
           else:
             if len(l) > 0:
                state = INTEST 
                params=""
                if re.match(r'^.*b[ ]+([0-9,]+)$',l):
                   m=re.match(r'^.*b[ ]+([0-9,]+)$',l)
                   params=m.group(1).strip()
                   # Format the node
                   #print(elem.fullPath())
                   #createMissingDir(destPath)
                   writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config)
                else:
                   params=""
                   writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config)
                   # Format the node
                formatter.printTest(elem,theId,theError,theLine,passed,cycles,params)

             
    formatter.end()          


parser = argparse.ArgumentParser(description='Parse test description')

parser.add_argument('-f', nargs='?',type = str, default=None, help="Test description file path")
# Where the result file can be found
parser.add_argument('-r', nargs='?',type = str, default=None, help="Result file path")
parser.add_argument('-c', action='store_true', help="CSV output")
parser.add_argument('-e', action='store_true', help="Embedded test")
# -o needed when -e is true to know where to extract the output files
parser.add_argument('-o', nargs='?',type = str, default="Output", help="Output dir path")

parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path")
parser.add_argument('-m', action='store_true', help="Mathematica output")

args = parser.parse_args()

if args.f is not None:
    p = parse.Parser()
    # Parse the test description file
    root = p.parse(args.f)
    with open(args.r,"r") as results:
        if args.c:
           analyseResult(root,results,args.e,args.b,CSVFormatter())
        elif args.m:
           analyseResult(root,results,args.e,args.b,MathematicaFormatter())
        else:
           analyseResult(root,results,args.e,args.b,TextFormatter())
    if args.e:
       # In FPGA mode, extract output files from stdout (result file)
       with open(args.r,"r") as results:
          extractDataFiles(results,args.o)
    
else:
    parser.print_help()