CMSIS-DSP: New testing framework
(For our internal use. In short term, we won't give support about it).
CMSIS-DSP: Update to cmake build for the testing framework
CMSIS-NN:Implementation of arm_fully_connected_s8
Use API and quantization compatible with TF Lite.
diff --git a/CMSIS/DSP/Testing/processResult.py b/CMSIS/DSP/Testing/processResult.py
new file mode 100644
index 0000000..d186eb6
--- /dev/null
+++ b/CMSIS/DSP/Testing/processResult.py
@@ -0,0 +1,419 @@
+# Process the test results
+# Test status (like passed, or failed with error code)
+
+import argparse
+import re
+import TestScripts.NewParser as parse
+import TestScripts.CodeGen
+from collections import deque
+import os.path
+import csv
+
+def findItem(root,path):
+ """ Find a node in a tree
+
+ Args:
+ path (list) : A list of node ID
+ This list is describing a path in the tree.
+ By starting from the root and following this path,
+ we can find the node in the tree.
+ Raises:
+ Nothing
+ Returns:
+ TreeItem : A node
+ """
+ # The list is converted into a queue.
+ q = deque(path)
+ q.popleft()
+ c = root
+ while q:
+ n = q.popleft()
+ # We get the children based on its ID and continue
+ c = c[n-1]
+ return(c)
+
+def joinit(iterable, delimiter):
+ # Intersperse a delimiter between element of a list
+ it = iter(iterable)
+ yield next(it)
+ for x in it:
+ yield delimiter
+ yield x
+
+# Return test result as a text tree
+class TextFormatter:
+ def start(self):
+ None
+
+ def printGroup(self,elem,theId):
+ if elem is None:
+ elem = root
+ message=elem.data["message"]
+ if not elem.data["deprecated"]:
+ kind = "Suite"
+ ident = " " * elem.ident
+ if elem.kind == TestScripts.Parser.TreeElem.GROUP:
+ kind = "Group"
+ #print(elem.path)
+ print("%s%s : %s (%d)" % (ident,kind,message,theId))
+
+ def printTest(self,elem, theId, theError,theLine,passed,cycles,params):
+ message=elem.data["message"]
+ if not elem.data["deprecated"]:
+ kind = "Test"
+ ident = " " * elem.ident
+ p="FAILED"
+ if passed == 1:
+ p="PASSED"
+ print("%s%s (%d) : %s (cycles = %d)" % (ident,message,theId,p,cycles))
+ if params:
+ print("%s %s" % (ident,params))
+ if passed != 1:
+ print("%s Error = %d at line %d" % (ident, theError, theLine))
+
+ def pop(self):
+ None
+
+ def end(self):
+ None
+
+# Return test result as a CSV
+class CSVFormatter:
+
+ def __init__(self):
+ self.name=[]
+ self._start=True
+
+ def start(self):
+ print("CATEGORY,NAME,ID,STATUS,CYCLES,PARAMS")
+
+ def printGroup(self,elem,theId):
+ if elem is None:
+ elem = root
+ # Remove Root from category name in CSV file.
+ if not self._start:
+ self.name.append(elem.data["class"])
+ else:
+ self._start=False
+ message=elem.data["message"]
+ if not elem.data["deprecated"]:
+ kind = "Suite"
+ ident = " " * elem.ident
+ if elem.kind == TestScripts.Parser.TreeElem.GROUP:
+ kind = "Group"
+
+ def printTest(self,elem, theId, theError, theLine,passed,cycles,params):
+ message=elem.data["message"]
+ if not elem.data["deprecated"]:
+ kind = "Test"
+ name=elem.data["class"]
+ category= "".join(list(joinit(self.name,":")))
+ print("%s,%s,%d,%d,%d,\"%s\"" % (category,name,theId,passed,cycles,params))
+
+ def pop(self):
+ if self.name:
+ self.name.pop()
+
+ def end(self):
+ None
+
+class MathematicaFormatter:
+
+ def __init__(self):
+ self._hasContent=[False]
+ self._toPop=[]
+
+ def start(self):
+ None
+
+ def printGroup(self,elem,theId):
+ if self._hasContent[len(self._hasContent)-1]:
+ print(",",end="")
+
+ print("<|")
+ self._hasContent[len(self._hasContent)-1] = True
+ self._hasContent.append(False)
+ if elem is None:
+ elem = root
+ message=elem.data["message"]
+ if not elem.data["deprecated"]:
+
+ kind = "Suite"
+ ident = " " * elem.ident
+ if elem.kind == TestScripts.Parser.TreeElem.GROUP:
+ kind = "Group"
+ print("\"%s\" ->" % (message))
+ #if kind == "Suite":
+ print("{",end="")
+ self._toPop.append("}")
+ #else:
+ # self._toPop.append("")
+
+ def printTest(self,elem, theId, theError,theLine,passed,cycles,params):
+ message=elem.data["message"]
+ if not elem.data["deprecated"]:
+ kind = "Test"
+ ident = " " * elem.ident
+ p="FAILED"
+ if passed == 1:
+ p="PASSED"
+ parameters=""
+ if params:
+ parameters = "%s" % params
+ if self._hasContent[len(self._hasContent)-1]:
+ print(",",end="")
+ print("<|\"NAME\" -> \"%s\",\"ID\" -> %d,\"STATUS\" -> \"%s\",\"CYCLES\" -> %d,\"PARAMS\" -> \"%s\"|>" % (message,theId,p,cycles,parameters))
+ self._hasContent[len(self._hasContent)-1] = True
+ #if passed != 1:
+ # print("%s Error = %d at line %d" % (ident, theError, theLine))
+
+ def pop(self):
+ print(self._toPop.pop(),end="")
+ print("|>")
+ self._hasContent.pop()
+
+ def end(self):
+ None
+
+NORMAL = 1
+INTEST = 2
+TESTPARAM = 3
+
+def createMissingDir(destPath):
+ theDir=os.path.normpath(os.path.dirname(destPath))
+ if not os.path.exists(theDir):
+ os.makedirs(theDir)
+
+def correctPath(path):
+ while (path[0]=="/") or (path[0] == "\\"):
+ path = path[1:]
+ return(path)
+
+def extractDataFiles(results,outputDir):
+ infile = False
+ f = None
+ for l in results:
+ if re.match(r'^.*D:[ ].*$',l):
+ if infile:
+ if re.match(r'^.*D:[ ]END$',l):
+ infile = False
+ if f:
+ f.close()
+ else:
+ if f:
+ m = re.match(r'^.*D:[ ](.*)$',l)
+ data = m.group(1)
+ f.write(data)
+ f.write("\n")
+
+ else:
+ m = re.match(r'^.*D:[ ](.*)$',l)
+ path = str(m.group(1))
+ infile = True
+ destPath = os.path.join(outputDir,correctPath(path))
+ createMissingDir(destPath)
+ f = open(destPath,"w")
+
+
+
+def writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config):
+ if benchFile:
+ name=elem.data["class"]
+ category= elem.categoryDesc()
+ benchFile.write("\"%s\",\"%s\",%d,%s,%d,%s\n" % (category,name,theId,params,cycles,config))
+
+def analyseResult(root,results,embedded,benchmark,formatter):
+ formatter.start()
+ path = []
+ state = NORMAL
+ prefix=""
+ elem=None
+ theId=None
+ theError=None
+ theLine=None
+ passed=0
+ cycles=None
+ benchFile = None
+ config=""
+ if embedded:
+ prefix = ".*S:[ ]"
+
+ # Parse the result file.
+ # NORMAL mode is when we are parsing suite or group.
+ # Otherwise we are parsing a test and we need to analyse the
+ # test result.
+ # TESTPARAM is used to read parameters of the test.
+ # Format of output is:
+ #node ident : s id or g id or t or u
+ #test status : id error linenb status Y or N (Y when passing)
+ #param for this test b x,x,x,x or b alone if not param
+ #node end : p
+ # In FPGA mode:
+ #Prefix S:[ ] before driver dump
+ # D:[ ] before data dump (output patterns)
+
+ for l in results:
+ l = l.strip()
+ if not re.match(r'^.*D:[ ].*$',l):
+ if state == NORMAL:
+ if len(l) > 0:
+ # Line starting with g or s is a suite or group.
+ # In FPGA mode, those line are prefixed with 'S: '
+ # and data file with 'D: '
+ if re.match(r'^%s[gs][ ]+[0-9]+.*$' % prefix,l):
+ # Extract the test id
+ theId=re.sub(r'^%s[gs][ ]+([0-9]+).*$' % prefix,r'\1',l)
+ theId=int(theId)
+ path.append(theId)
+ # From a list of id, find the TreeElem in the Parsed tree
+ # to know what is the node.
+ elem = findItem(root,path)
+ # Display formatted output for this node
+ if elem.params:
+ #print(elem.params.full)
+ benchPath = os.path.join(benchmark,elem.fullPath(),"fullBenchmark.csv")
+ createMissingDir(benchPath)
+ if benchFile:
+ printf("ERROR BENCH FILE %s ALREADY OPEN" % benchPath)
+ benchFile.close()
+ benchFile=None
+ benchFile=open(benchPath,"w")
+ header = "".join(list(joinit(elem.params.full,",")))
+ # A test and a benchmark are different
+ # so we don't dump a status and error
+ # A status and error in a benchmark would
+ # impact the cycles since the test
+ # would be taken into account in the measurement
+ # So benchmark are always passing and contain no test
+ #benchFile.write("ID,%s,PASSED,ERROR,CYCLES\n" % header)
+ csvheaders = ""
+
+ with open('currentConfig.csv', 'r') as f:
+ reader = csv.reader(f)
+ csvheaders = next(reader, None)
+ configList = list(reader)
+ #print(configList)
+ config = "".join(list(joinit(configList[0],",")))
+ configHeaders = "".join(list(joinit(csvheaders,",")))
+ benchFile.write("CATEGORY,NAME,ID,%s,CYCLES,%s\n" % (header,configHeaders))
+
+ formatter.printGroup(elem,theId)
+
+ # If we have detected a test, we switch to test mode
+ if re.match(r'^%s[t][ ]*$' % prefix,l):
+ state = INTEST
+
+
+ # Pop
+ # End of suite or group
+ if re.match(r'^%sp.*$' % prefix,l):
+ if benchFile:
+ benchFile.close()
+ benchFile=None
+ path.pop()
+ formatter.pop()
+ elif state == INTEST:
+ if len(l) > 0:
+ # In test mode, we are looking for test status.
+ # A line starting with S
+ # (There may be empty lines or line for data files)
+ passRe = r'^%s([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([YN]).*$' % prefix
+ if re.match(passRe,l):
+ # If we have found a test status then we will start again
+ # in normal mode after this.
+
+ m = re.match(passRe,l)
+
+ # Extract test ID, test error code, line number and status
+ theId=m.group(1)
+ theId=int(theId)
+
+ theError=m.group(2)
+ theError=int(theError)
+
+ theLine=m.group(3)
+ theLine=int(theLine)
+
+ cycles = int(m.group(4))
+
+ status=m.group(5)
+ passed=0
+
+ # Convert status to number as used by formatter.
+ if status=="Y":
+ passed = 1
+ if status=="N":
+ passed = 0
+ # Compute path to this node
+ newPath=path.copy()
+ newPath.append(theId)
+ # Find the node in the Tree
+ elem = findItem(root,newPath)
+
+
+ state = TESTPARAM
+ else:
+ if re.match(r'^%sp.*$' % prefix,l):
+ if benchFile:
+ benchFile.close()
+ benchFile=None
+ path.pop()
+ formatter.pop()
+ if re.match(r'^%s[t][ ]*$' % prefix,l):
+ state = INTEST
+ else:
+ state = NORMAL
+ else:
+ if len(l) > 0:
+ state = INTEST
+ params=""
+ if re.match(r'^.*b[ ]+([0-9,]+)$',l):
+ m=re.match(r'^.*b[ ]+([0-9,]+)$',l)
+ params=m.group(1).strip()
+ # Format the node
+ #print(elem.fullPath())
+ #createMissingDir(destPath)
+ writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config)
+ else:
+ params=""
+ writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config)
+ # Format the node
+ formatter.printTest(elem,theId,theError,theLine,passed,cycles,params)
+
+
+ formatter.end()
+
+
+parser = argparse.ArgumentParser(description='Parse test description')
+
+parser.add_argument('-f', nargs='?',type = str, default=None, help="Test description file path")
+# Where the result file can be found
+parser.add_argument('-r', nargs='?',type = str, default=None, help="Result file path")
+parser.add_argument('-c', action='store_true', help="CSV output")
+parser.add_argument('-e', action='store_true', help="Embedded test")
+# -o needed when -e is true to know where to extract the output files
+parser.add_argument('-o', nargs='?',type = str, default="Output", help="Output dir path")
+
+parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path")
+parser.add_argument('-m', action='store_true', help="Mathematica output")
+
+args = parser.parse_args()
+
+if args.f is not None:
+ p = parse.Parser()
+ # Parse the test description file
+ root = p.parse(args.f)
+ with open(args.r,"r") as results:
+ if args.c:
+ analyseResult(root,results,args.e,args.b,CSVFormatter())
+ elif args.m:
+ analyseResult(root,results,args.e,args.b,MathematicaFormatter())
+ else:
+ analyseResult(root,results,args.e,args.b,TextFormatter())
+ if args.e:
+ # In FPGA mode, extract output files from stdout (result file)
+ with open(args.r,"r") as results:
+ extractDataFiles(results,args.o)
+
+else:
+ parser.print_help()
\ No newline at end of file