| import argparse |
| import sqlite3 |
| import re |
| import pandas as pd |
| import numpy as np |
| from TestScripts.doc.Structure import * |
| from TestScripts.doc.Format import * |
| import os.path |
| |
| refCoreName="" |
| |
| runidCMD = "runid = ?" |
| |
| # Command to get last runid |
| lastID="""SELECT runid FROM RUN ORDER BY runid DESC LIMIT 1 |
| """ |
| |
| # Command to get last runid and date |
| lastIDAndDate="""SELECT date FROM RUN WHERE runid=? |
| """ |
| |
| # Command to get last runid |
| runIDDetails="""SELECT distinct core FROM %s |
| INNER JOIN CORE USING(coreid) |
| WHERE %s |
| """ |
| |
| def joinit(iterable, delimiter): |
| # Intersperse a delimiter between element of a list |
| it = iter(iterable) |
| yield next(it) |
| for x in it: |
| yield delimiter |
| yield x |
| |
| |
| def getLastRunID(): |
| r=c.execute(lastID) |
| return(int(r.fetchone()[0])) |
| |
| def getrunIDDate(forID): |
| r=c.execute(lastIDAndDate,(forID,)) |
| return(r.fetchone()[0]) |
| |
| |
| |
| |
| parser = argparse.ArgumentParser(description='Generate summary benchmarks') |
| |
| parser.add_argument('-b', nargs='?',type = str, default="bench.db", help="Database") |
| parser.add_argument('-o', nargs='?',type = str, default="full.md", help="Full summary") |
| parser.add_argument('-r', action='store_true', help="Regression database") |
| parser.add_argument('-t', nargs='?',type = str, default="md", help="type md or html") |
| parser.add_argument('-byc', action='store_true', help="Result oganized by Compiler") |
| parser.add_argument('-g', action='store_true', help="Include graphs in regression report") |
| |
| parser.add_argument('-details', action='store_true', help="Details about runids") |
| parser.add_argument('-lastid', action='store_true', help="Get last ID") |
| parser.add_argument('-comments', nargs='?',type = str, default="comments.txt", help="Comment section") |
| parser.add_argument('-byd', action='store_true', help="Result oganized by datatype") |
| parser.add_argument('-ratio', action='store_true', help="Compute ratios for regression by core instead of cycles") |
| parser.add_argument('-ref', nargs='?',type = str, default="M55", help="Reference COREDEF for ratio in db") |
| parser.add_argument('-clampval', nargs='?',type = float, default=8.0, help="Clamp for ratio") |
| parser.add_argument('-clamp', action='store_true', help="Clamp enabled for ratio") |
| parser.add_argument('-keep', nargs='?',type = str, help="Core to keep for ratio") |
| |
| # For runid or runid range |
| parser.add_argument('others', nargs=argparse.REMAINDER,help="Run ID") |
| |
| args = parser.parse_args() |
| |
| c = sqlite3.connect(args.b) |
| |
| if args.others: |
| vals=[] |
| runidCMD=[] |
| runidHeader=[] |
| runidVIEWcmd=[] |
| for t in args.others: |
| if re.search(r'-',t): |
| bounds=[int(x) for x in t.split("-")] |
| vals += bounds |
| runidHeader += ["%d <= runid <= %d" % tuple(bounds)] |
| runidCMD += ["(runid >= ? AND runid <= ?)"] |
| runidVIEWcmd += ["(runid >= %d AND runid <= %d) " % tuple(bounds)] |
| else: |
| theid=int(t) |
| runidHeader += ["runid == %d" % theid] |
| runidCMD += ["runid == ?"] |
| runidVIEWcmd += ["runid == %d" % theid] |
| vals.append(theid) |
| |
| runidval = tuple(vals) |
| runidHeader = "(" + "".join(joinit(runidHeader," OR ")) + ")" |
| runidCMD = "(" + "".join(joinit(runidCMD," OR ")) + ")" |
| runidVIEWcmd = "(" + "".join(joinit(runidVIEWcmd," OR ")) + ")" |
| else: |
| theid=getLastRunID() |
| print("Last run ID = %d\n" % theid) |
| runidval=(theid,) |
| runidHeader="%d" % theid |
| runidVIEWcmd="(runid = %d)" % theid |
| |
| |
| # We extract data only from data tables |
| # Those tables below are used for descriptions |
| REMOVETABLES=['TESTNAME','TESTDATE','RUN','CORE', 'PLATFORM', 'COMPILERKIND', 'COMPILER', 'TYPE', 'CATEGORY', 'CONFIG'] |
| |
| # This is assuming the database is generated by the regression script |
| # So platform is the same for all benchmarks. |
| # Category and type is coming from the test name in the yaml |
| # So no need to add this information here |
| # Name is removed here because it is added at the beginning |
| REMOVECOLUMNS=['runid','name','type','platform','category','coredef','OPTIMIZED','HARDFP','FASTMATH','NEON','HELIUM','UNROLL','ROUNDING','DATE','compilerkindid','date','categoryid', 'ID', 'platformid', 'coreid', 'compilerid', 'typeid'] |
| |
| REMOVECOLUMNSFORHISTORY=['Regression','MAXREGCOEF','name','type','platform','category','coredef','OPTIMIZED','HARDFP','FASTMATH','NEON','HELIUM','UNROLL','ROUNDING','DATE','compilerkindid','date','categoryid', 'ID', 'platformid', 'coreid', 'compilerid', 'typeid'] |
| |
| # Get existing benchmark tables |
| def getBenchTables(): |
| r=c.execute("SELECT name FROM sqlite_master WHERE type='table'") |
| benchtables=[] |
| for table in r: |
| if not table[0] in REMOVETABLES: |
| benchtables.append(table[0]) |
| return(benchtables) |
| |
| # get existing types in a table |
| def getExistingTypes(benchTable): |
| r=c.execute("select distinct typeid from %s WHERE %s order by typeid desc " % (benchTable,runidCMD),runidval).fetchall() |
| result=[x[0] for x in r] |
| return(result) |
| |
| # Get existing cores in a table |
| def getAllExistingCores(benchTable): |
| r=c.execute("select distinct coreid from %s WHERE %s order by coreid desc " % (benchTable,runidCMD),runidval).fetchall() |
| result=[x[0] for x in r] |
| return(result) |
| |
| def getrunIDDetails(): |
| tables=getBenchTables() |
| r=[] |
| for table in tables: |
| r += [x[0] for x in c.execute(runIDDetails % (table,runidCMD),runidval).fetchall()] |
| r=list(set(r)) |
| print(r) |
| |
| if args.lastid: |
| quit() |
| |
| if args.details: |
| getrunIDDetails() |
| quit() |
| |
| |
| # Get compilers from specific type and table |
| allCompilers="""select distinct compilerid from %s WHERE typeid=?""" |
| |
| # Get compilers from specific type and table |
| allCompilerForCore="""select distinct compilerid from %s WHERE coreid=?""" |
| |
| # Get compilers from specific type and table |
| allCores="""select distinct coreid from %s WHERE typeid=? AND (%s)""" |
| |
| |
| compilerDesc="""select compiler,version from COMPILER |
| INNER JOIN COMPILERKIND USING(compilerkindid) WHERE compilerid=?""" |
| |
| coreDesc="""select core from CORE WHERE coreid=?""" |
| |
| # Get existing compiler in a table for a specific type |
| # (In case report is structured by types) |
| def getExistingCompiler(benchTable,typeid): |
| r=c.execute(allCompilers % benchTable,(typeid,)).fetchall() |
| return([x[0] for x in r]) |
| |
| |
| # Get existing compiler in a table for a specific core |
| # (In case report is structured by core) |
| def getExistingCompilerForCore(benchTable,coreid): |
| r=c.execute(allCompilerForCore % benchTable,(coreid,)).fetchall() |
| return([x[0] for x in r]) |
| |
| def getExistingCores(benchTable,typeid): |
| vals = (typeid,) + runidval |
| r=c.execute(allCores % (benchTable,runidCMD),vals).fetchall() |
| return([x[0] for x in r]) |
| |
| |
| |
| def getCoreDesc(coreid): |
| r=c.execute(coreDesc,(coreid,)).fetchone() |
| return(r) |
| |
| def getCompilerDesc(compilerid): |
| r=c.execute(compilerDesc,(compilerid,)).fetchone() |
| return(r) |
| |
| # Get type name from type id |
| def getTypeName(typeid): |
| r=c.execute("select type from TYPE where typeid=?",(typeid,)).fetchone() |
| return(r[0]) |
| |
| # Diff of 2 lists |
| def diff(first, second): |
| second = set(second) |
| return [item for item in first if item not in second] |
| |
| |
| # Command to get data for specific compiler |
| # and type |
| benchCmdForCoreCompiler="""select %s from %s |
| INNER JOIN CATEGORY USING(categoryid) |
| INNER JOIN PLATFORM USING(platformid) |
| INNER JOIN CORE USING(coreid) |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TYPE USING(typeid) |
| INNER JOIN TESTNAME USING(testnameid) |
| WHERE coreid=? AND compilerid = ? AND (%s) |
| """ |
| |
| # Command to get data for specific core |
| # and type |
| historyCmd="""select %s from %s |
| INNER JOIN CATEGORY USING(categoryid) |
| INNER JOIN PLATFORM USING(platformid) |
| INNER JOIN CORE USING(coreid) |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TYPE USING(typeid) |
| INNER JOIN TESTNAME USING(testnameid) |
| WHERE compilerid=? AND coreid=? AND typeid = ? AND ID = ? AND runid > (? - 10) |
| """ |
| |
| compilersForHistory="""select distinct compilerid,compiler,version from %s |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| WHERE coreid=? AND typeid = ? AND ID = ? AND runid > (? - 10) |
| """ |
| |
| # Command to get data for specific core |
| # and type |
| benchCmdForCore="""select %s from %s |
| INNER JOIN CATEGORY USING(categoryid) |
| INNER JOIN PLATFORM USING(platformid) |
| INNER JOIN CORE USING(coreid) |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TYPE USING(typeid) |
| INNER JOIN TESTNAME USING(testnameid) |
| WHERE coreid=? AND typeid = ? AND (%s) |
| """ |
| |
| coresForHistory="""select distinct coreid,core from %s |
| INNER JOIN CORE USING(coreid) |
| WHERE compilerid=? AND typeid = ? AND ID = ? AND runid > (? - 10) |
| """ |
| |
| # Command to get data for specific compiler |
| # and type |
| benchCmdForCompiler="""select %s from %s |
| INNER JOIN CATEGORY USING(categoryid) |
| INNER JOIN PLATFORM USING(platformid) |
| INNER JOIN CORE USING(coreid) |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TYPE USING(typeid) |
| INNER JOIN TESTNAME USING(testnameid) |
| WHERE compilerid=? AND typeid = ? AND (%s) |
| """ |
| |
| # Command to get test names for specific compiler |
| # and type |
| benchNamesForCore="""select distinct name from %s |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TYPE USING(typeid) |
| INNER JOIN TESTNAME USING(testnameid) |
| WHERE coreid=? AND typeid = ? AND (%s) |
| """ |
| |
| # Command to get test names for specific core |
| # and compiler |
| benchNamesForCoreCompiler="""select distinct name from %s |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TYPE USING(typeid) |
| INNER JOIN TESTNAME USING(testnameid) |
| WHERE coreid=? AND compilerid = ? AND (%s) |
| """ |
| |
| # Command to get test names for specific compiler |
| # and type |
| benchNamesForCompiler="""select distinct name from %s |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TYPE USING(typeid) |
| INNER JOIN TESTNAME USING(testnameid) |
| WHERE compilerid=? AND typeid = ? AND (%s) |
| """ |
| |
| # Command to get columns for specific table |
| benchCmdColumns="""select * from %s |
| INNER JOIN CATEGORY USING(categoryid) |
| INNER JOIN PLATFORM USING(platformid) |
| INNER JOIN CORE USING(coreid) |
| INNER JOIN COMPILER USING(compilerid) |
| INNER JOIN COMPILERKIND USING(compilerkindid) |
| INNER JOIN TESTNAME USING(testnameid) |
| INNER JOIN TYPE USING(typeid) |
| """ |
| |
| def joinit(iterable, delimiter): |
| it = iter(iterable) |
| yield next(it) |
| for x in it: |
| yield delimiter |
| yield x |
| |
| # Is not a column name finishing by id |
| # (often primary key for thetable) |
| def isNotIDColumn(col): |
| if re.match(r'^.*id$',col): |
| return(False) |
| else: |
| return(True) |
| |
| # Get test names |
| # for specific core and compiler (for the data) |
| def getTestNamesForCoreCompiler(benchTable,compilerid,core): |
| vals=(core,compilerid) + runidval |
| result=c.execute(benchNamesForCoreCompiler % (benchTable,runidCMD),vals).fetchall() |
| names=[x[0] for x in list(result)] |
| return(names) |
| |
| # Get test names |
| # for specific typeid and core (for the data) |
| def getTestNamesForCore(benchTable,core,typeid): |
| vals=(core,typeid) + runidval |
| result=c.execute(benchNamesForCore % (benchTable,runidCMD),vals).fetchall() |
| names=[x[0] for x in list(result)] |
| return(names) |
| |
| # Get test names |
| # for specific typeid and compiler (for the data) |
| def getTestNamesForCompiler(benchTable,comp,typeid): |
| vals=(comp,typeid) + runidval |
| result=c.execute(benchNamesForCompiler % (benchTable,runidCMD),vals).fetchall() |
| names=[x[0] for x in list(result)] |
| return(names) |
| |
| # Command to get data for specific core |
| # and type |
| nbElemsInBenchAndTypeAndCoreCmd="""select count(*) from %s |
| WHERE coreid=? AND typeid = ? AND (%s) |
| """ |
| |
| # Command to get data for specific compiler |
| # and type |
| nbElemsInBenchAndTypeAndCompilerCmd="""select count(*) from %s |
| WHERE compilerid=? AND typeid = ? AND (%s) |
| """ |
| |
| # Command to get data for specific compiler |
| # and type |
| nbElemsInBenchAndCoreAndCompilerCmd="""select count(*) from %s |
| WHERE compilerid=? AND coreid = ? AND (%s) |
| """ |
| |
| nbElemsInBenchAndTypeCmd="""select count(*) from %s |
| WHERE typeid = ? AND (%s) |
| """ |
| nbElemsInBenchAndCoreCmd="""select count(*) from %s |
| WHERE coreid = ? AND (%s) |
| """ |
| |
| nbElemsInBenchCmd="""select count(*) from %s |
| WHERE %s |
| """ |
| |
| categoryName="""select distinct category from %s |
| INNER JOIN CATEGORY USING(categoryid) |
| WHERE %s |
| """ |
| |
| def getCategoryName(benchTable): |
| result=c.execute(categoryName % (benchTable,runidCMD),runidval).fetchone() |
| return(result[0]) |
| |
| # Get nb elems in a table |
| def getNbElemsInBenchAndTypeAndCoreCmd(benchTable,coreid,typeid): |
| vals=(coreid,typeid) + runidval |
| result=c.execute(nbElemsInBenchAndTypeAndCoreCmd % (benchTable,runidCMD),vals).fetchone() |
| return(result[0]) |
| |
| # Get nb elems in a table |
| def getNbElemsInBenchAndTypeAndCompilerCmd(benchTable,comp,typeid): |
| vals=(comp,typeid) + runidval |
| result=c.execute(nbElemsInBenchAndTypeAndCompilerCmd % (benchTable,runidCMD),vals).fetchone() |
| return(result[0]) |
| |
| # Get nb elems in a table |
| def getNbElemsInBenchAndCoreAndCompilerCmd(benchTable,comp,coreid): |
| vals=(comp,coreid) + runidval |
| result=c.execute(nbElemsInBenchAndCoreAndCompilerCmd % (benchTable,runidCMD),vals).fetchone() |
| return(result[0]) |
| |
| def getNbElemsInBenchAndTypeCmd(benchTable,typeid): |
| vals=(typeid,) + runidval |
| result=c.execute(nbElemsInBenchAndTypeCmd % (benchTable,runidCMD),vals).fetchone() |
| return(result[0]) |
| |
| def getNbElemsInBenchAndCoreCmd(benchTable,coreid): |
| vals=(coreid,) + runidval |
| result=c.execute(nbElemsInBenchAndCoreCmd % (benchTable,runidCMD),vals).fetchone() |
| return(result[0]) |
| |
| def getNbElemsInBenchCmd(benchTable): |
| result=c.execute(nbElemsInBenchCmd % (benchTable,runidCMD),runidval).fetchone() |
| return(result[0]) |
| |
| # Get names of columns and data for a table |
| # for specific typeid and coreid (for the data) |
| def getColNamesAndHistory(benchTable,compiler,core,typeid,testid): |
| cursor=c.cursor() |
| result=cursor.execute(benchCmdColumns % (benchTable)) |
| cols= [member[0] for member in cursor.description] |
| keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNSFORHISTORY) if isNotIDColumn(c)] |
| keepColsStr = "".join(joinit(keepCols,",")) |
| vals=(compiler,core,typeid,testid,runid) |
| result=cursor.execute(historyCmd % (keepColsStr,benchTable),vals) |
| vals =np.array([list(x) for x in list(result)]) |
| return(keepCols,vals) |
| |
| # Get names of columns and data for a table |
| # for specific typeid and coreid (for the data) |
| def getColNamesAndDataForCore(benchTable,core,typeid): |
| cursor=c.cursor() |
| result=cursor.execute(benchCmdColumns % (benchTable)) |
| cols= [member[0] for member in cursor.description] |
| keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)] |
| keepColsStr = "".join(joinit(keepCols,",")) |
| vals=(core,typeid) + runidval |
| result=cursor.execute(benchCmdForCore % (keepColsStr,benchTable,runidCMD),vals) |
| vals =np.array([list(x) for x in list(result)]) |
| return(keepCols,vals) |
| |
| # Get names of columns and data for a table |
| # for specific coreid and compilerid (for the data) |
| def getColNamesAndDataForCoreCompiler(benchTable,compilerid,core): |
| cursor=c.cursor() |
| result=cursor.execute(benchCmdColumns % (benchTable)) |
| cols= [member[0] for member in cursor.description] |
| keepCols = ['name','type'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)] |
| keepColsStr = "".join(joinit(keepCols,",")) |
| vals=(core,compilerid) + runidval |
| result=cursor.execute(benchCmdForCoreCompiler % (keepColsStr,benchTable,runidCMD),vals) |
| vals =np.array([list(x) for x in list(result)]) |
| return(keepCols,vals) |
| |
| # Get names of columns and data for a table |
| # for specific typeid and compiler (for the data) |
| def getColNamesAndDataForCompiler(benchTable,comp,typeid): |
| cursor=c.cursor() |
| result=cursor.execute(benchCmdColumns % (benchTable)) |
| cols= [member[0] for member in cursor.description] |
| keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)] |
| keepColsStr = "".join(joinit(keepCols,",")) |
| vals=(comp,typeid) + runidval |
| result=cursor.execute(benchCmdForCompiler % (keepColsStr,benchTable,runidCMD),vals) |
| vals =np.array([list(x) for x in list(result)]) |
| return(keepCols,vals) |
| |
| |
| |
| PARAMS=["NB","NumTaps", "NBA", "NBB", "Factor", "NumStages","VECDIM","NBR","NBC","NBI","IFFT", "BITREV"] |
| |
| def regressionTableFor(byname,name,section,ref,toSort,indexCols,field): |
| data=ref.pivot_table(index=indexCols, columns=byname, |
| values=[field], aggfunc='first') |
| |
| data=data.sort_values(toSort) |
| |
| cores = [c[1] for c in list(data.columns)] |
| columns = diff(indexCols,['name']) |
| |
| dataTable=Table(columns,cores) |
| section.addContent(dataTable) |
| |
| dataForFunc=data.loc[name] |
| |
| if type(dataForFunc) is pd.DataFrame: |
| bars={'cols':columns,'cores':cores,'data':[]} |
| for row in dataForFunc.itertuples(): |
| row=list(row) |
| if type(row[0]) is int: |
| row=[row[0]] + row[1:] |
| else: |
| row=list(row[0]) + row[1:] |
| if field=="MAXREGCOEF": |
| newrow = row |
| newrow[len(columns):] = [("%.3f" % x) for x in row[len(columns):]] |
| row=newrow |
| dataTable.addRow(row) |
| bars['data'].append(row) |
| return(bars) |
| else: |
| if field=="MAXREGCOEF": |
| dataForFunc=[("%.3f" % x) for x in dataForFunc] |
| dataTable.addRow(dataForFunc) |
| return(list(zip(cores,dataForFunc))) |
| |
| def formatColumnName(c): |
| return("".join(joinit(c,":"))) |
| |
| def getCoresForHistory(benchTable,compilerid,typeid,testid,runid): |
| vals=(compilerid,typeid,testid,runid) |
| result=c.execute(coresForHistory % benchTable,vals).fetchall() |
| ids=[(x[0],x[1]) for x in list(result)] |
| return(ids) |
| |
| def getCompilerForHistory(benchTable,coreid,typeid,testid,runid): |
| vals=(coreid,typeid,testid,runid) |
| result=c.execute(compilersForHistory % benchTable,vals).fetchall() |
| ids=[(x[0],x[1],x[2]) for x in list(result)] |
| return(ids) |
| |
| def getHistory(desc,testid,indexCols): |
| benchName,sectionID,typeid,runid = desc |
| |
| #print(benchName) |
| #print(sectionID) |
| #print(typeid) |
| #print(testid) |
| columns = diff(indexCols,['name']) |
| #print(columns) |
| if args.byc: |
| coreid=sectionID |
| compilerids=getCompilerForHistory(benchName,coreid,typeid,testid,runid) |
| series={} |
| for compilerid,compilername,version in compilerids: |
| result=getColNamesAndHistory(benchName,compilerid,coreid,typeid,testid) |
| #print("%s:%s" % (compilername,version)) |
| maxpos = result[0].index('MAX') |
| lrunid = result[0].index('runid') |
| r=[[int(x[lrunid]),int(x[maxpos])] for x in result[1:][0]] |
| series[corename]=r |
| hist=History(series,runid) |
| return(hist) |
| else: |
| compilerid=sectionID |
| coreids = getCoresForHistory(benchName,compilerid,typeid,testid,runid) |
| series={} |
| for coreid,corename in coreids: |
| result=getColNamesAndHistory(benchName,compilerid,coreid,typeid,testid) |
| #print(corename) |
| maxpos = result[0].index('MAX') |
| corepos = result[0].index('core') |
| lrunid = result[0].index('runid') |
| r=[[int(x[lrunid]),int(x[maxpos])] for x in result[1:][0]] |
| series[corename]=r |
| hist=History(series,runid) |
| return(hist) |
| |
| def convertRowToInt(r): |
| result=[] |
| for e in r: |
| if type(e) is float: |
| result.append(int(e)) |
| else: |
| result.append(e) |
| |
| return(result) |
| |
| def formatTableBy(desc,byname,section,typeSection,testNames,cols,vals): |
| if vals.size != 0: |
| ref=pd.DataFrame(vals,columns=cols) |
| toSort=["name"] |
| |
| for param in PARAMS: |
| if param in ref.columns: |
| ref[param]=pd.to_numeric(ref[param]) |
| toSort.append(param) |
| if args.r: |
| # Regression table |
| ref['MAX']=pd.to_numeric(ref['MAX']) |
| ref['MAXREGCOEF']=pd.to_numeric(ref['MAXREGCOEF']) |
| |
| indexCols=diff(cols,byname + ['Regression','MAXREGCOEF','MAX'] + section) |
| valList = ['Regression'] |
| |
| else: |
| ref['CYCLES']=pd.to_numeric(ref['CYCLES']).round(decimals=0) |
| |
| indexCols=diff(cols,byname + ['CYCLES'] + section) |
| valList = ['CYCLES'] |
| |
| |
| for name in testNames: |
| if args.r: |
| testSection = Section(name) |
| typeSection.addSection(testSection) |
| |
| maxCyclesSection = Section("Max cycles") |
| testSection.addSection(maxCyclesSection) |
| theCycles=regressionTableFor(byname,name,maxCyclesSection,ref,toSort,indexCols,'MAX') |
| if args.g: |
| if type(theCycles) is dict: |
| nbParams=len(theCycles['cols']) |
| for bar in theCycles['data']: |
| params=bar[0:nbParams] |
| values=bar[nbParams:] |
| title=[("%s=%s" % x) for x in list(zip(theCycles['cols'],params))] |
| title="".join(joinit(title," ")) |
| sec=Section(title) |
| maxCyclesSection.addSection(sec) |
| values=list(zip(theCycles['cores'],values)) |
| barChart=BarChart(values) |
| sec.addContent(barChart) |
| else: |
| #print(theCycles) |
| sec=Section("Graph") |
| maxCyclesSection.addSection(sec) |
| barChart=BarChart(theCycles) |
| sec.addContent(barChart) |
| |
| #history=getHistory(desc,testid,indexCols) |
| #testSection.addContent(history) |
| |
| regressionSection = Section("Regression") |
| testSection.addSection(regressionSection) |
| regressionTableFor(byname,name,regressionSection,ref,toSort,indexCols,'Regression') |
| |
| |
| maxRegCoefSection = Section("Max Reg Coef") |
| testSection.addSection(maxRegCoefSection) |
| regressionTableFor(byname,name,maxRegCoefSection,ref,toSort,indexCols,'MAXREGCOEF') |
| |
| else: |
| data=ref.pivot_table(index=indexCols, columns=byname, |
| values=valList, aggfunc='first') |
| |
| data=data.sort_values(toSort) |
| |
| #print(list(data.columns)) |
| |
| testSection = Section(name) |
| typeSection.addSection(testSection) |
| |
| dataForFunc=data.loc[name] |
| dataForFunc = dataForFunc.dropna(axis=1) |
| |
| columnsID = [formatColumnName(c[1:]) for c in list(dataForFunc.columns)] |
| columns = diff(indexCols,['name']) |
| |
| dataTable=Table(columns,columnsID) |
| testSection.addContent(dataTable) |
| |
| |
| if type(dataForFunc) is pd.DataFrame: |
| for row in dataForFunc.itertuples(): |
| if type(row[0]) is int: |
| row=list([row[0]] + list(row[1:])) |
| else: |
| row=list(row[0]) + list(row[1:]) |
| dataTable.addRow(convertRowToInt(row)) |
| else: |
| dataTable.addRow(convertRowToInt(dataForFunc)) |
| |
| referenceCoreID = None |
| |
| |
| |
| coreidSQL="select distinct coreid from CORE where coredef==?" |
| |
| def getCoreID(corename): |
| r=c.execute(coreidSQL,(corename,)) |
| t=r.fetchone() |
| if t is None: |
| print("Unrecognized reference core") |
| quit() |
| return(t[0]) |
| |
| refCore="""CREATE TEMP VIEW if not exists refCore AS |
| select * from %s where (coreid = %s) and (typeid = %s) |
| and %s |
| and compilerid = %s""" |
| |
| allOtherCores="""CREATE TEMP VIEW if not exists otherCore AS |
| select * from %s where (coreid != %s) and (typeid = %s) |
| and %s |
| and compilerid = %s""" |
| |
| otherCore="""CREATE TEMP VIEW if not exists otherCore AS |
| select * from %s where (coreid = %s) and (typeid = %s) |
| and %s |
| and compilerid = %s""" |
| |
| refCoreAllTypes="""CREATE TEMP VIEW if not exists refCore AS |
| select * from %s where (coreid = %s) |
| and %s |
| and compilerid = %s""" |
| |
| otherCoreAllTypes="""CREATE TEMP VIEW if not exists otherCore AS |
| select * from %s where (coreid = %s) |
| and %s |
| and compilerid = %s""" |
| |
| |
| ratioSQL="""select name,otherCore.compilerid as compilerid,CORE.core as core,%s(CAST(otherCore.MAX as FLOAT) / CAST(refCore.MAX AS FLOAT)) AS RATIO |
| from otherCore |
| INNER JOIN refCore ON (refCore.categoryid = otherCore.categoryid |
| AND refCore.testnameid = otherCore.testnameid |
| AND refCore.typeid = otherCore.typeid |
| AND refCore.runid = otherCore.runid |
| AND refCore.compilerid = otherCore.compilerid |
| ) |
| INNER JOIN TESTNAME ON TESTNAME.testnameid = refCore.testnameid |
| INNER JOIN CORE USING(coreid) |
| %s""" |
| |
| ratioSQLAllTypes="""select name,otherCore.compilerid as compilerid,TYPE.type as type,%s(CAST(otherCore.MAX as FLOAT) / CAST(refCore.MAX AS FLOAT)) AS RATIO |
| from otherCore |
| INNER JOIN refCore ON (refCore.categoryid = otherCore.categoryid |
| AND refCore.testnameid = otherCore.testnameid |
| AND refCore.typeid = otherCore.typeid |
| AND refCore.runid = otherCore.runid |
| AND refCore.compilerid = otherCore.compilerid |
| ) |
| INNER JOIN TESTNAME ON TESTNAME.testnameid = refCore.testnameid |
| INNER JOIN TYPE USING(typeid) |
| %s""" |
| |
| ratioTestNamesSQL="""select distinct TESTNAME.name |
| from otherCore |
| INNER JOIN refCore ON (refCore.categoryid = otherCore.categoryid |
| AND refCore.testnameid = otherCore.testnameid |
| AND refCore.typeid = otherCore.typeid |
| AND refCore.runid = otherCore.runid |
| AND refCore.compilerid = otherCore.compilerid |
| ) |
| INNER JOIN TESTNAME ON TESTNAME.testnameid = refCore.testnameid |
| INNER JOIN CORE USING(coreid) |
| %s""" |
| |
| |
| dropViewsRef="""drop view refCore""" |
| |
| dropViewsOther="""drop view otherCore""" |
| |
| def getTableParams(benchTable): |
| cursor=c.cursor() |
| result=cursor.execute("select * from %s limit 1" % (benchTable)) |
| cols= [member[0] for member in cursor.description] |
| params=[] |
| for x in cols: |
| if x in PARAMS: |
| params.append(x) |
| return(params) |
| |
| def computeRatio(benchName,viewParams,refMkViewCmd,otherMkViewCmd,byd): |
| params = getTableParams(benchName) |
| cols=["ratio"] |
| paramscmd="" |
| paramscols="" |
| paramsnames = ["refCore.%s as %s" % (x,x) for x in params] |
| paramseq = ["refCore.%s = otherCore.%s" % (x,x) for x in params] |
| if len(params) > 0: |
| cols = ["%s" % x for x in params] |
| cols.append("ratio") |
| paramscols= ("".join(joinit(paramsnames," , ")) + ",") |
| paramscmd = "WHERE " + "".join(joinit(paramseq," AND ")) |
| if byd: |
| ratioCmd = ratioSQLAllTypes % (paramscols,paramscmd) |
| else: |
| ratioCmd = ratioSQL % (paramscols,paramscmd) |
| ratioTestNames = ratioTestNamesSQL % (paramscmd) |
| |
| #print(refMkViewCmd) |
| #print(otherMkViewCmd) |
| # |
| #print(ratioCmd) |
| # |
| #print(dropViewsRef) |
| #print(dropViewsOther) |
| #quit() |
| |
| c.execute(refMkViewCmd) |
| c.execute(otherMkViewCmd) |
| |
| ratio=c.execute(ratioCmd).fetchall() |
| testNames=c.execute(ratioTestNames).fetchall() |
| testNames=[x[0] for x in testNames] |
| |
| c.execute(dropViewsRef) |
| c.execute(dropViewsOther) |
| |
| #print(ratio) |
| #quit() |
| if byd: |
| return(['name','compilerid','type'] + cols,params,ratio,testNames) |
| else: |
| return(['name','compilerid','core'] + cols,params,ratio,testNames) |
| |
| # Compute for all core for a given type |
| def computeRatioTable(benchName,referenceCore,typeID,compiler): |
| viewParams = (benchName,referenceCore,typeID,runidVIEWcmd,compiler) |
| refMkViewCmd = refCore % viewParams |
| otherMkViewCmd = allOtherCores % viewParams |
| if args.keep: |
| keepCoreID = getCoreID(args.keep) |
| otherParams = (benchName,keepCoreID,typeID,runidVIEWcmd,compiler) |
| otherMkViewCmd = otherCore % otherParams |
| return(computeRatio(benchName,viewParams,refMkViewCmd,otherMkViewCmd,False)) |
| |
| |
| def computeRatioTableForCore(benchName,referenceCore,otherCoreID,compiler): |
| viewParams = (benchName,referenceCore,runidVIEWcmd,compiler) |
| refMkViewCmd = refCoreAllTypes % viewParams |
| otherParams = (benchName,otherCoreID,runidVIEWcmd,compiler) |
| otherMkViewCmd = otherCoreAllTypes % otherParams |
| return(computeRatio(benchName,viewParams,refMkViewCmd,otherMkViewCmd,True)) |
| |
| def formatPerfRatio(s): |
| result=[] |
| for t in s: |
| if type(t) is float: |
| if args.clamp: |
| if t > args.clampval: |
| t = args.clampval |
| if t < 0.0: |
| result.append("NA") |
| else: |
| result.append(("%.3f" % t)) |
| else: |
| result.append(s) |
| |
| return(result) |
| |
| def addRatioTable(cols,params,data,section,testNames,byd): |
| ref=pd.DataFrame(data,columns=cols) |
| toSort=["name"] + params |
| for param in PARAMS: |
| if param in ref.columns: |
| ref[param]=pd.to_numeric(ref[param]) |
| |
| #print(testNames) |
| for name in testNames: |
| testSection = Section(name) |
| section.addSection(testSection) |
| |
| ratioSection = Section("Ratios") |
| testSection.addSection(ratioSection) |
| |
| #print(toSort) |
| #print(ref) |
| |
| if byd: |
| data=ref.pivot_table(index=toSort, columns=['type'], |
| values=["ratio"], aggfunc='first',fill_value=-1.0) |
| else: |
| data=ref.pivot_table(index=toSort, columns=['core'], |
| values=["ratio"], aggfunc='first') |
| |
| data=data.sort_values(toSort) |
| |
| #print(data) |
| dataForFunc=data.loc[name] |
| |
| cores = [c[1] for c in list(data.columns)] |
| |
| dataTable=Table(params,cores) |
| |
| if args.g: |
| if type(dataForFunc) is not pd.DataFrame: |
| sec=Section("Graph") |
| testSection.addSection(sec) |
| |
| barChart=BarChart(zip(cores,dataForFunc)) |
| sec.addContent(barChart) |
| |
| ratioSection.addContent(Text("A bigger ratio means the reference core \"%s\" is better" % refCoreName)) |
| |
| ratioSection.addContent(dataTable) |
| |
| if type(dataForFunc) is pd.DataFrame: |
| for row in dataForFunc.itertuples(): |
| row=list(row) |
| if type(row[0]) is int: |
| row=[row[0]] + formatPerfRatio(row[1:]) |
| else: |
| row=list(row[0]) + formatPerfRatio(row[1:]) |
| dataTable.addRow(row) |
| else: |
| row=list(dataForFunc) |
| dataTable.addRow(formatPerfRatio(row)) |
| |
| |
| |
| |
| |
| |
| # Add a report for each table |
| def addReportFor(document,benchName): |
| nbElems = getNbElemsInBenchCmd(benchName) |
| if nbElems > 0: |
| categoryName = getCategoryName(benchName) |
| benchSection = Section(categoryName) |
| document.addSection(benchSection) |
| print("Process %s\n" % benchName) |
| if args.byd: |
| allCores=getAllExistingCores(benchName) |
| if args.ratio: |
| allCores.remove(referenceCoreID) |
| if args.keep: |
| allCores=[getCoreID(args.keep)] |
| for aCoreID in allCores: |
| nbElems = getNbElemsInBenchAndCoreCmd(benchName,aCoreID) |
| if nbElems > 0: |
| coreName=getCoreDesc(aCoreID) |
| coreSection = Section("%s" % coreName) |
| benchSection.addSection(coreSection) |
| allCompilers = getExistingCompilerForCore(benchName,aCoreID) |
| for compiler in allCompilers: |
| #print(compiler) |
| if args.ratio: |
| cols,params,ratios,testNames=computeRatioTableForCore(benchName,referenceCoreID,aCoreID,compiler) |
| #print(cols) |
| #print(ratios) |
| #print(" ") |
| if len(ratios)>0: |
| compilerName,version=getCompilerDesc(compiler) |
| compilerSection = Section("%s (%s)" % (compilerName,version)) |
| coreSection.addSection(compilerSection) |
| addRatioTable(cols,params,ratios,compilerSection,testNames,True) |
| |
| else: |
| nbElems = getNbElemsInBenchAndCoreAndCompilerCmd(benchName,compiler,aCoreID) |
| |
| # Print test results for table, type, compiler |
| if nbElems > 0: |
| compilerName,version=getCompilerDesc(compiler) |
| compilerSection = Section("%s (%s)" % (compilerName,version)) |
| coreSection.addSection(compilerSection) |
| cols,vals=getColNamesAndDataForCoreCompiler(benchName,compiler,aCoreID) |
| desc=(benchName,compiler,aCoreID) |
| names=getTestNamesForCoreCompiler(benchName,compiler,aCoreID) |
| |
| formatTableBy(desc,['type'],['core','version','compiler'],compilerSection,names,cols,vals) |
| |
| else: |
| allTypes = getExistingTypes(benchName) |
| # Add report for each type |
| for aTypeID in allTypes: |
| nbElems = getNbElemsInBenchAndTypeCmd(benchName,aTypeID) |
| if nbElems > 0: |
| typeName = getTypeName(aTypeID) |
| typeSection = Section(typeName) |
| benchSection.addSection(typeSection) |
| if args.byc: |
| ## Add report for each core |
| allCores = getExistingCores(benchName,aTypeID) |
| for core in allCores: |
| #print(core) |
| nbElems = getNbElemsInBenchAndTypeAndCoreCmd(benchName,core,aTypeID) |
| # Print test results for table, type, compiler |
| if nbElems > 0: |
| coreName=getCoreDesc(core) |
| coreSection = Section("%s" % coreName) |
| typeSection.addSection(coreSection) |
| cols,vals=getColNamesAndDataForCore(benchName,core,aTypeID) |
| desc=(benchName,core,aTypeID) |
| names=getTestNamesForCore(benchName,core,aTypeID) |
| formatTableBy(desc,['compiler','version'],['core'],coreSection,names,cols,vals) |
| elif args.ratio: |
| allCompilers = getExistingCompiler(benchName,aTypeID) |
| for compiler in allCompilers: |
| cols,params,ratios,testNames=computeRatioTable(benchName,referenceCoreID,aTypeID,compiler) |
| #print(cols) |
| #print(ratios) |
| #print(" ") |
| if len(ratios)>0: |
| compilerName,version=getCompilerDesc(compiler) |
| compilerSection = Section("%s (%s)" % (compilerName,version)) |
| typeSection.addSection(compilerSection) |
| addRatioTable(cols,params,ratios,compilerSection,testNames,False) |
| |
| else: |
| ## Add report for each compiler |
| allCompilers = getExistingCompiler(benchName,aTypeID) |
| for compiler in allCompilers: |
| #print(compiler) |
| nbElems = getNbElemsInBenchAndTypeAndCompilerCmd(benchName,compiler,aTypeID) |
| # Print test results for table, type, compiler |
| if nbElems > 0: |
| compilerName,version=getCompilerDesc(compiler) |
| compilerSection = Section("%s (%s)" % (compilerName,version)) |
| typeSection.addSection(compilerSection) |
| cols,vals=getColNamesAndDataForCompiler(benchName,compiler,aTypeID) |
| desc=(benchName,compiler,aTypeID) |
| names=getTestNamesForCompiler(benchName,compiler,aTypeID) |
| formatTableBy(desc,['core'],['version','compiler'],compilerSection,names,cols,vals) |
| |
| |
| |
| |
| toc=[Hierarchy("BasicMathsBenchmarks"), |
| Hierarchy("ComplexMathsBenchmarks"), |
| Hierarchy("FastMath"), |
| Hierarchy("Filters", |
| [Hierarchy("FIR"), |
| Hierarchy("BIQUAD"), |
| Hierarchy("DECIM"), |
| Hierarchy("MISC")]), |
| |
| Hierarchy("Support Functions", |
| [Hierarchy("Support"), |
| Hierarchy("SupportBar")]), |
| |
| Hierarchy("Matrix Operations" , |
| [Hierarchy("Binary"), |
| Hierarchy("Unary")]), |
| Hierarchy("Transform"), |
| Hierarchy("Stats"), |
| Hierarchy("Classical ML",[ |
| Hierarchy("Bayes"), |
| Hierarchy("SVM"), |
| Hierarchy("Distance"), |
| ]), |
| |
| ] |
| |
| processed=[] |
| |
| def addComments(document): |
| if os.path.exists(args.comments): |
| section=Section("Measurement Context") |
| |
| document.addSection(section) |
| para="" |
| with open(args.comments,"r") as r: |
| for l in r: |
| if l.strip(): |
| para += l |
| else: |
| section.addContent(Text(para)) |
| para="" |
| if para: |
| section.addContent(Text(para)) |
| |
| if args.ratio: |
| section.addContent(Text("Reference core for the ratio is %s" % refCoreName)) |
| section.addContent(Text("A bigger ratio means the reference code is better")) |
| |
| |
| |
| def createDoc(document,sections,benchtables): |
| global processed,referenceCoreID |
| |
| for s in sections: |
| if s.name in benchtables: |
| addReportFor(document,s.name) |
| processed.append(s.name) |
| else: |
| section=Section(s.name) |
| document.addSection(section) |
| createDoc(section,s.sections,benchtables) |
| |
| try: |
| benchtables=getBenchTables() |
| document = Document(runidHeader) |
| |
| if args.ratio: |
| referenceCoreID= getCoreID(args.ref) |
| refCoreName=getCoreDesc(referenceCoreID) |
| |
| addComments(document) |
| |
| |
| |
| createDoc(document,toc,benchtables) |
| |
| misc=Section("Miscellaneous") |
| document.addSection(misc) |
| remaining=diff(benchtables,processed) |
| for bench in remaining: |
| addReportFor(misc,bench) |
| |
| #for bench in benchtables: |
| # addReportFor(document,bench) |
| with open(args.o,"w") as output: |
| if args.t=="md": |
| document.accept(Markdown(output)) |
| if args.t=="html": |
| reorder=NORMALFORMAT |
| if args.byc: |
| reorder=BYCFORMAT |
| if args.byd: |
| reorder=BYDFORMAT |
| document.accept(HTML(output,args.r,args.ratio,reorder)) |
| |
| finally: |
| c.close() |
| |
| |
| |
| |