| import argparse |
| import sqlite3 |
| import re |
| import pandas as pd |
| import numpy as np |
| from TestScripts.doc.Structure import * |
| from TestScripts.doc.Format import * |
| |
| |
| |
| |
| |
| |
| # Command to get last runid |
| lastID="""SELECT runid FROM RUN ORDER BY runid DESC LIMIT 1 |
| """ |
| |
| # Command to get last runid and date |
| lastIDAndDate="""SELECT date FROM RUN WHERE runid=? |
| """ |
| |
| def getLastRunID(): |
| r=c.execute(lastID) |
| return(int(r.fetchone()[0])) |
| |
| def getrunIDDate(forID): |
| r=c.execute(lastIDAndDate,(forID,)) |
| return(r.fetchone()[0]) |
| |
| runid = 1 |
| |
| parser = argparse.ArgumentParser(description='Generate summary benchmarks') |
| |
| parser.add_argument('-b', nargs='?',type = str, default="bench.db", help="Benchmark database") |
| parser.add_argument('-o', nargs='?',type = str, default="full.md", help="Full summary") |
| parser.add_argument('-r', action='store_true', help="Regression database") |
| parser.add_argument('-t', nargs='?',type = str, default="md", help="md,html") |
| |
| # For runid or runid range |
| parser.add_argument('others', nargs=argparse.REMAINDER,help="Run ID") |
| |
| args = parser.parse_args() |
| |
| c = sqlite3.connect(args.b) |
| |
| if args.others: |
| runid=int(args.others[0]) |
| else: |
| runid=getLastRunID() |
| |
| # We extract data only from data tables |
| # Those tables below are used for descriptions |
| REMOVETABLES=['TESTNAME','TESTDATE','RUN','CORE', 'PLATFORM', 'COMPILERKIND', 'COMPILER', 'TYPE', 'CATEGORY', 'CONFIG'] |
| |
| # This is assuming the database is generated by the regression script |
| # So platform is the same for all benchmarks. |
| # Category and type is coming from the test name in the yaml |
| # So no need to add this information here |
| # Name is removed here because it is added at the beginning |
| REMOVECOLUMNS=['runid','name','type','platform','category','coredef','OPTIMIZED','HARDFP','FASTMATH','NEON','HELIUM','UNROLL','ROUNDING','DATE','compilerkindid','date','categoryid', 'ID', 'platformid', 'coreid', 'compilerid', 'typeid'] |
| |
| # Get existing benchmark tables |
| def getBenchTables(): |
| r=c.execute("SELECT name FROM sqlite_master WHERE type='table'") |
| benchtables=[] |
| for table in r: |
| if not table[0] in REMOVETABLES: |
| benchtables.append(table[0]) |
| return(benchtables) |
| |
| # get existing types in a table |
| def getExistingTypes(benchTable): |
| r=c.execute("select distinct typeid from %s order by typeid desc" % benchTable).fetchall() |
| result=[x[0] for x in r] |
| return(result) |
| |
| # Get compilers from specific type and table |
| allCompilers="""select distinct compilerid from %s WHERE typeid=?""" |
| |
| compilerDesc="""select compiler,version from COMPILER |
| INNER JOIN COMPILERKIND USING(compilerkindid) WHERE compilerid=?""" |
| |
| # Get existing compiler in a table for a specific type |
| # (In case report is structured by types) |
| def getExistingCompiler(benchTable,typeid): |
| r=c.execute(allCompilers % benchTable,(typeid,)).fetchall() |
| return([x[0] for x in r]) |
| |
| def getCompilerDesc(compilerid): |
| r=c.execute(compilerDesc,(compilerid,)).fetchone() |
| return(r) |
| |
| # Get type name from type id |
| def getTypeName(typeid): |
| r=c.execute("select type from TYPE where typeid=?",(typeid,)).fetchone() |
| return(r[0]) |
| |
| # Diff of 2 lists |
| def diff(first, second): |
| second = set(second) |
| return [item for item in first if item not in second] |
| |
| |
| # Command to get data for specific compiler |
| # and type |
| benchCmd="""select %s from %s |
| INNER JOIN CATEGORY USING(categoryid) |
| INNER JOIN PLATFORM USING(platformid) |
| INNER JOIN CORE USING(coreid) |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TYPE USING(typeid) |
| INNER JOIN TESTNAME USING(testnameid) |
| WHERE compilerid=? AND typeid = ? AND runid = ? |
| """ |
| |
| |
| # Command to get test names for specific compiler |
| # and type |
| benchNames="""select distinct name from %s |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TYPE USING(typeid) |
| INNER JOIN TESTNAME USING(testnameid) |
| WHERE compilerid=? AND typeid = ? AND runid = ? |
| """ |
| |
| # Command to get columns for specific table |
| benchCmdColumns="""select * from %s |
| INNER JOIN CATEGORY USING(categoryid) |
| INNER JOIN PLATFORM USING(platformid) |
| INNER JOIN CORE USING(coreid) |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TESTNAME USING(testnameid) |
| INNER JOIN TYPE USING(typeid) |
| """ |
| |
| def joinit(iterable, delimiter): |
| it = iter(iterable) |
| yield next(it) |
| for x in it: |
| yield delimiter |
| yield x |
| |
| # Is not a column name finishing by id |
| # (often primary key for thetable) |
| def isNotIDColumn(col): |
| if re.match(r'^.*id$',col): |
| return(False) |
| else: |
| return(True) |
| |
| # Get test names |
| # for specific typeid and compiler (for the data) |
| def getTestNames(benchTable,comp,typeid): |
| vals=(comp,typeid,runid) |
| result=c.execute(benchNames % benchTable,vals).fetchall() |
| return([x[0] for x in list(result)]) |
| |
| # Command to get data for specific compiler |
| # and type |
| nbElemsInBenchAndTypeAndCompilerCmd="""select count(*) from %s |
| WHERE compilerid=? AND typeid = ? AND runid = ? |
| """ |
| |
| nbElemsInBenchAndTypeCmd="""select count(*) from %s |
| WHERE typeid = ? AND runid = ? |
| """ |
| |
| nbElemsInBenchCmd="""select count(*) from %s |
| WHERE runid = ? |
| """ |
| |
| categoryName="""select distinct category from %s |
| INNER JOIN CATEGORY USING(categoryid) |
| WHERE runid = ? |
| """ |
| |
| def getCategoryName(benchTable,runid): |
| result=c.execute(categoryName % benchTable,(runid,)).fetchone() |
| return(result[0]) |
| |
| # Get nb elems in a table |
| def getNbElemsInBenchAndTypeAndCompilerCmd(benchTable,comp,typeid): |
| vals=(comp,typeid,runid) |
| result=c.execute(nbElemsInBenchAndTypeAndCompilerCmd % benchTable,vals).fetchone() |
| return(result[0]) |
| |
| def getNbElemsInBenchAndTypeCmd(benchTable,typeid): |
| vals=(typeid,runid) |
| result=c.execute(nbElemsInBenchAndTypeCmd % benchTable,vals).fetchone() |
| return(result[0]) |
| |
| def getNbElemsInBenchCmd(benchTable): |
| vals=(runid,) |
| result=c.execute(nbElemsInBenchCmd % benchTable,vals).fetchone() |
| return(result[0]) |
| |
| # Get names of columns and data for a table |
| # for specific typeid and compiler (for the data) |
| def getColNamesAndData(benchTable,comp,typeid): |
| cursor=c.cursor() |
| result=cursor.execute(benchCmdColumns % (benchTable)) |
| cols= [member[0] for member in cursor.description] |
| keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)] |
| keepColsStr = "".join(joinit(keepCols,",")) |
| vals=(comp,typeid,runid) |
| result=cursor.execute(benchCmd % (keepColsStr,benchTable),vals) |
| vals =np.array([list(x) for x in list(result)]) |
| return(keepCols,vals) |
| |
| |
| |
| PARAMS=["NB","NumTaps", "NBA", "NBB", "Factor", "NumStages","VECDIM","NBR","NBC","NBI","IFFT", "BITREV"] |
| |
| def regressionTableFor(name,section,ref,toSort,indexCols,field): |
| data=ref.pivot_table(index=indexCols, columns='core', |
| values=[field], aggfunc='first') |
| |
| data=data.sort_values(toSort) |
| |
| cores = [c[1] for c in list(data.columns)] |
| columns = diff(indexCols,['name']) |
| |
| dataTable=Table(columns,cores) |
| section.addTable(dataTable) |
| |
| dataForFunc=data.loc[name] |
| if type(dataForFunc) is pd.DataFrame: |
| for row in dataForFunc.itertuples(): |
| row=list(row) |
| if type(row[0]) is int: |
| row=[row[0]] + row[1:] |
| else: |
| row=list(row[0]) + row[1:] |
| if field=="MAXREGCOEF": |
| row=[("%.3f" % x) for x in row] |
| dataTable.addRow(row) |
| else: |
| if field=="MAXREGCOEF": |
| dataForFunc=[("%.3f" % x) for x in dataForFunc] |
| dataTable.addRow(dataForFunc) |
| |
| def formatTableByCore(typeSection,testNames,cols,vals): |
| if vals.size != 0: |
| ref=pd.DataFrame(vals,columns=cols) |
| toSort=["name"] |
| |
| for param in PARAMS: |
| if param in ref.columns: |
| ref[param]=pd.to_numeric(ref[param]) |
| toSort.append(param) |
| if args.r: |
| # Regression table |
| ref['MAX']=pd.to_numeric(ref['MAX']) |
| ref['MAXREGCOEF']=pd.to_numeric(ref['MAXREGCOEF']) |
| |
| indexCols=diff(cols,['core','Regression','MAXREGCOEF','MAX','version','compiler']) |
| valList = ['Regression'] |
| else: |
| ref['CYCLES']=pd.to_numeric(ref['CYCLES']) |
| |
| indexCols=diff(cols,['core','CYCLES','version','compiler']) |
| valList = ['CYCLES'] |
| |
| |
| |
| for name in testNames: |
| if args.r: |
| testSection = Section(name) |
| typeSection.addSection(testSection) |
| |
| regressionSection = Section("Regression") |
| testSection.addSection(regressionSection) |
| regressionTableFor(name,regressionSection,ref,toSort,indexCols,'Regression') |
| |
| maxCyclesSection = Section("Max cycles") |
| testSection.addSection(maxCyclesSection) |
| regressionTableFor(name,maxCyclesSection,ref,toSort,indexCols,'MAX') |
| |
| maxRegCoefSection = Section("Max Reg Coef") |
| testSection.addSection(maxRegCoefSection) |
| regressionTableFor(name,maxRegCoefSection,ref,toSort,indexCols,'MAXREGCOEF') |
| |
| else: |
| data=ref.pivot_table(index=indexCols, columns='core', |
| values=valList, aggfunc='first') |
| |
| data=data.sort_values(toSort) |
| |
| cores = [c[1] for c in list(data.columns)] |
| columns = diff(indexCols,['name']) |
| |
| testSection = Section(name) |
| typeSection.addSection(testSection) |
| |
| dataTable=Table(columns,cores) |
| testSection.addTable(dataTable) |
| |
| dataForFunc=data.loc[name] |
| if type(dataForFunc) is pd.DataFrame: |
| for row in dataForFunc.itertuples(): |
| row=list(row) |
| if type(row[0]) is int: |
| row=[row[0]] + row[1:] |
| else: |
| row=list(row[0]) + row[1:] |
| dataTable.addRow(row) |
| else: |
| dataTable.addRow(dataForFunc) |
| |
| # Add a report for each table |
| def addReportFor(document,runid,benchName): |
| nbElems = getNbElemsInBenchCmd(benchName) |
| if nbElems > 0: |
| categoryName = getCategoryName(benchName,runid) |
| benchSection = Section(categoryName) |
| document.addSection(benchSection) |
| print("Process %s\n" % benchName) |
| allTypes = getExistingTypes(benchName) |
| # Add report for each type |
| for aTypeID in allTypes: |
| nbElems = getNbElemsInBenchAndTypeCmd(benchName,aTypeID) |
| if nbElems > 0: |
| typeName = getTypeName(aTypeID) |
| typeSection = Section(typeName) |
| benchSection.addSection(typeSection) |
| ## Add report for each compiler |
| allCompilers = getExistingCompiler(benchName,aTypeID) |
| for compiler in allCompilers: |
| #print(compiler) |
| nbElems = getNbElemsInBenchAndTypeAndCompilerCmd(benchName,compiler,aTypeID) |
| # Print test results for table, type, compiler |
| if nbElems > 0: |
| compilerName,version=getCompilerDesc(compiler) |
| compilerSection = Section("%s (%s)" % (compilerName,version)) |
| typeSection.addSection(compilerSection) |
| cols,vals=getColNamesAndData(benchName,compiler,aTypeID) |
| names=getTestNames(benchName,compiler,aTypeID) |
| formatTableByCore(compilerSection,names,cols,vals) |
| |
| |
| |
| |
| toc=[Hierarchy("BasicMathsBenchmarks"), |
| Hierarchy("ComplexMathsBenchmarks"), |
| Hierarchy("FastMath"), |
| Hierarchy("Filters", |
| [Hierarchy("FIR"), |
| Hierarchy("BIQUAD"), |
| Hierarchy("DECIM"), |
| Hierarchy("MISC")]), |
| |
| Hierarchy("Support Functions", |
| [Hierarchy("Support"), |
| Hierarchy("SupportBar")]), |
| |
| Hierarchy("Matrix Operations" , |
| [Hierarchy("Binary"), |
| Hierarchy("Unary")]), |
| Hierarchy("Transform"), |
| |
| ] |
| |
| processed=[] |
| |
| def createDoc(document,sections,benchtables): |
| global processed |
| for s in sections: |
| if s.name in benchtables: |
| addReportFor(document,runid,s.name) |
| processed.append(s.name) |
| else: |
| section=Section(s.name) |
| document.addSection(section) |
| createDoc(section,s.sections,benchtables) |
| |
| try: |
| benchtables=getBenchTables() |
| theDate = getrunIDDate(runid) |
| document = Document(runid,theDate) |
| createDoc(document,toc,benchtables) |
| misc=Section("Miscellaneous") |
| document.addSection(misc) |
| remaining=diff(benchtables,processed) |
| for bench in remaining: |
| addReportFor(misc,runid,bench) |
| |
| #for bench in benchtables: |
| # addReportFor(document,bench) |
| with open(args.o,"w") as output: |
| if args.t=="md": |
| document.accept(Markdown(output)) |
| if args.t=="html": |
| document.accept(HTML(output,args.r)) |
| |
| finally: |
| c.close() |
| |
| |
| |
| |