CMSIS-DSP: Testing framework
Added support for external trace files for benchmark measurements.
diff --git a/CMSIS/DSP/Testing/CMakeLists.txt b/CMSIS/DSP/Testing/CMakeLists.txt
index 9752a56..365075b 100644
--- a/CMSIS/DSP/Testing/CMakeLists.txt
+++ b/CMSIS/DSP/Testing/CMakeLists.txt
@@ -61,6 +61,7 @@
endfunction()
option(BENCHMARK "Benchmarking compiled" OFF)
+option(EXTBENCH "Benchmarking with external traces" OFF)
project(Testing)
@@ -153,6 +154,10 @@
target_sources(FrameworkLib PRIVATE ${FRAMEWORKSRC})
+if (EXTBENCH)
+ target_compile_definitions(FrameworkLib PUBLIC EXTBENCH)
+endif()
+
### Includes
target_link_libraries(TestingLib PRIVATE CMSISDSP)
target_link_libraries(TestingLib PRIVATE CMSISNN)
diff --git a/CMSIS/DSP/Testing/FrameworkInclude/Timing.h b/CMSIS/DSP/Testing/FrameworkInclude/Timing.h
index 1169921..ac82bc6 100644
--- a/CMSIS/DSP/Testing/FrameworkInclude/Timing.h
+++ b/CMSIS/DSP/Testing/FrameworkInclude/Timing.h
@@ -9,4 +9,85 @@
Testing::cycles_t getCycles();
+#ifdef EXTBENCH
+extern unsigned long sectionCounter;
+
+#if defined ( __CC_ARM )
+ #define dbgInst(imm) __asm volatile{ DBG (imm) }
+#elif defined ( __GNUC__ ) || defined ( __llvm__ )
+ #define dbgInst(imm) __asm volatile("DBG %0\n\t" : :"Ir" ((imm)) )
+#else
+ #error "Unsupported compiler"
+#endif
+#define startSectionNB(num) dbgInst(((num) & 0x7) | 0x8)
+#define stopSectionNB(num) dbgInst(((num) & 0x7) | 0x0)
+
+static inline void startSection() {
+ switch(sectionCounter & 0x7)
+ {
+ case 0:
+ startSectionNB(0);
+ break;
+ case 1:
+ startSectionNB(1);
+ break;
+ case 2:
+ startSectionNB(2);
+ break;
+ case 3:
+ startSectionNB(3);
+ break;
+ case 4:
+ startSectionNB(4);
+ break;
+ case 5:
+ startSectionNB(5);
+ break;
+ case 6:
+ startSectionNB(6);
+ break;
+ case 7:
+ startSectionNB(7);
+ break;
+ default:
+ startSectionNB(0);
+ }
+}
+
+static inline void stopSection() {
+ switch(sectionCounter & 0x7)
+ {
+ case 0:
+ stopSectionNB(0);
+ break;
+ case 1:
+ stopSectionNB(1);
+ break;
+ case 2:
+ stopSectionNB(2);
+ break;
+ case 3:
+ stopSectionNB(3);
+ break;
+ case 4:
+ stopSectionNB(4);
+ break;
+ case 5:
+ stopSectionNB(5);
+ break;
+ case 6:
+ stopSectionNB(6);
+ break;
+ case 7:
+ stopSectionNB(7);
+ break;
+ default:
+ stopSectionNB(0);
+ }
+
+ sectionCounter++;
+}
+
+#endif
+
#endif
\ No newline at end of file
diff --git a/CMSIS/DSP/Testing/FrameworkSource/FPGA.cpp b/CMSIS/DSP/Testing/FrameworkSource/FPGA.cpp
index 098f158..a75936d 100644
--- a/CMSIS/DSP/Testing/FrameworkSource/FPGA.cpp
+++ b/CMSIS/DSP/Testing/FrameworkSource/FPGA.cpp
@@ -260,7 +260,11 @@
}
else
{
+#ifdef EXTBENCH
+ printf("S: %ld 0 0 t Y\n",this->currentId);
+#else
printf("S: %ld 0 0 %u Y\n",this->currentId, cycles);
+#endif
}
}
diff --git a/CMSIS/DSP/Testing/FrameworkSource/IORunner.cpp b/CMSIS/DSP/Testing/FrameworkSource/IORunner.cpp
index fb0849f..9ef7ab5 100644
--- a/CMSIS/DSP/Testing/FrameworkSource/IORunner.cpp
+++ b/CMSIS/DSP/Testing/FrameworkSource/IORunner.cpp
@@ -144,8 +144,16 @@
s->setUp(m_io->CurrentTestID(),params,m_mgr);
// Run the test
cycleMeasurementStart();
+#ifdef EXTBENCH
+ startSection();
+#endif
(s->*t)();
+#ifdef EXTBENCH
+ stopSection();
+#endif
+#ifndef EXTBENCH
cycles=getCycles();
+#endif
cycleMeasurementStop();
}
catch(Error &ex)
diff --git a/CMSIS/DSP/Testing/FrameworkSource/Semihosting.cpp b/CMSIS/DSP/Testing/FrameworkSource/Semihosting.cpp
index 09dee16..29ef390 100644
--- a/CMSIS/DSP/Testing/FrameworkSource/Semihosting.cpp
+++ b/CMSIS/DSP/Testing/FrameworkSource/Semihosting.cpp
@@ -364,7 +364,11 @@
}
else
{
+#ifdef EXTBENCH
+ printf("%ld 0 0 t Y\n",this->currentId);
+#else
printf("%ld 0 0 %u Y\n",this->currentId,cycles);
+#endif
}
}
diff --git a/CMSIS/DSP/Testing/FrameworkSource/Timing.cpp b/CMSIS/DSP/Testing/FrameworkSource/Timing.cpp
index a5259df..8b6737c 100644
--- a/CMSIS/DSP/Testing/FrameworkSource/Timing.cpp
+++ b/CMSIS/DSP/Testing/FrameworkSource/Timing.cpp
@@ -60,6 +60,10 @@
#define ENABLE_DIVIDER 0
#endif
+#ifdef EXTBENCH
+unsigned long sectionCounter=0;
+#endif
+
void initCycleMeasurement()
{
#ifdef CORTEXM
@@ -98,6 +102,7 @@
void cycleMeasurementStart()
{
+#ifndef EXTBENCH
#ifdef CORTEXM
SysTick->CTRL = SysTick_CTRL_CLKSOURCE_Msk;
SysTick->LOAD = SYSTICK_INITIAL_VALUE;
@@ -118,14 +123,18 @@
__get_CP(15, 0, value, 9, 13, 0);
startCycles = value;
#endif
+#endif
+
}
void cycleMeasurementStop()
{
+#ifndef EXTBENCH
#ifdef CORTEXM
SysTick->CTRL = SysTick_CTRL_CLKSOURCE_Msk;
SysTick->LOAD = SYSTICK_INITIAL_VALUE;
#endif
+#endif
}
Testing::cycles_t getCycles()
diff --git a/CMSIS/DSP/Testing/TestScripts/ParseTrace.py b/CMSIS/DSP/Testing/TestScripts/ParseTrace.py
new file mode 100755
index 0000000..2a42c62
--- /dev/null
+++ b/CMSIS/DSP/Testing/TestScripts/ParseTrace.py
@@ -0,0 +1,36 @@
+import re
+
+parseRe = re.compile('(.*)\s+([0-9]+):([0-9a-f]+):(.*)')
+
+dbgCnt=0
+
+clk0=0
+clk1=0
+
+def getCycles(t):
+ global dbgCnt
+ global clk0
+ global clk1
+ while(True):
+ try:
+ line = next(t)
+ if line:
+ m = parseRe.match(line)
+ if m:
+ if (('OP_HINT_DBG_32' in line) or ('DBG' in line)):
+ curClk = int(m.group(2))
+ if dbgCnt==0:
+ clk0 =curClk
+ if dbgCnt == 1:
+ clk1 = curClk
+ dbgCnt += 1
+ if dbgCnt == 2:
+ dbgCnt = 0
+ return(clk1 - clk0)
+ except StopIteration:
+ dbgCnt = 0
+ return(0)
+
+
+
+
\ No newline at end of file
diff --git a/CMSIS/DSP/Testing/processResult.py b/CMSIS/DSP/Testing/processResult.py
index 543c36b..43b85f4 100644
--- a/CMSIS/DSP/Testing/processResult.py
+++ b/CMSIS/DSP/Testing/processResult.py
@@ -8,6 +8,8 @@
from collections import deque
import os.path
import csv
+import TestScripts.ParseTrace
+
def findItem(root,path):
""" Find a node in a tree
@@ -226,7 +228,13 @@
old=elem.data["testData"]["oldID"]
benchFile.write("\"%s\",\"%s\",%d,\"%s\",%s,%d,%s\n" % (category,name,theId,old,params,cycles,config))
-def analyseResult(root,results,embedded,benchmark,formatter):
+def getCyclesFromTrace(trace):
+ if not trace:
+ return(0)
+ else:
+ return(TestScripts.ParseTrace.getCycles(trace))
+
+def analyseResult(root,results,embedded,benchmark,trace,formatter):
formatter.start()
path = []
state = NORMAL
@@ -321,7 +329,7 @@
# In test mode, we are looking for test status.
# A line starting with S
# (There may be empty lines or line for data files)
- passRe = r'^%s([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([YN]).*$' % prefix
+ passRe = r'^%s([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([t0-9]+)[ ]+([YN]).*$' % prefix
if re.match(passRe,l):
# If we have found a test status then we will start again
# in normal mode after this.
@@ -338,7 +346,11 @@
theLine=m.group(3)
theLine=int(theLine)
- cycles = int(m.group(4))
+ maybeCycles = m.group(4)
+ if maybeCycles == "t":
+ cycles = getCyclesFromTrace(trace)
+ else:
+ cycles = int(maybeCycles)
status=m.group(5)
passed=0
@@ -388,6 +400,14 @@
formatter.end()
+def analyze(root,results,args,trace):
+ if args.c:
+ analyseResult(root,results,args.e,args.b,trace,CSVFormatter())
+ elif args.m:
+ analyseResult(root,results,args.e,args.b,trace,MathematicaFormatter())
+ else:
+ analyseResult(root,results,args.e,args.b,trace,TextFormatter())
+
parser = argparse.ArgumentParser(description='Parse test description')
parser.add_argument('-f', nargs='?',type = str, default=None, help="Test description file path")
@@ -400,20 +420,22 @@
parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path")
parser.add_argument('-m', action='store_true', help="Mathematica output")
+parser.add_argument('-t', nargs='?',type = str, default=None, help="External trace file")
args = parser.parse_args()
+
if args.f is not None:
p = parse.Parser()
# Parse the test description file
root = p.parse(args.f)
- with open(args.r,"r") as results:
- if args.c:
- analyseResult(root,results,args.e,args.b,CSVFormatter())
- elif args.m:
- analyseResult(root,results,args.e,args.b,MathematicaFormatter())
- else:
- analyseResult(root,results,args.e,args.b,TextFormatter())
+ if args.t:
+ with open(args.t,"r") as trace:
+ with open(args.r,"r") as results:
+ analyze(root,results,args,iter(trace))
+ else:
+ with open(args.r,"r") as results:
+ analyze(root,results,args,None)
if args.e:
# In FPGA mode, extract output files from stdout (result file)
with open(args.r,"r") as results: