CMSIS/DSP/Testing/addToRegDB.py

# Process the test results
# Test status (like passed, or failed with error code)

import argparse
import re 
import TestScripts.NewParser as parse
import TestScripts.CodeGen
from collections import deque
import os.path
import numpy as np
import pandas as pd
import statsmodels.api as sm
import statsmodels.formula.api as smf
import csv
import TestScripts.Deprecate as d
import sqlite3
import datetime, time
import re 

# For sql table creation
MKSTRFIELD=['Regression']
MKBOOLFIELD=['HARDFP', 'FASTMATH', 'NEON', 'HELIUM','UNROLL', 'ROUNDING','OPTIMIZED']
MKINTFIELD=['ID','MAX']
MKREALFIELD=['MAXREGCOEF']
MKDATEFIELD=[]
MKKEYFIELD=['DATE','NAME','CATEGORY', 'PLATFORM', 'CORE', 'COMPILER','TYPE','RUN']
MKKEYFIELDID={'CATEGORY':'categoryid', 
   'NAME':'testnameid',
   'DATE':'testdateid',
   'PLATFORM':'platformid', 
   'CORE':'coreid', 
   'COMPILER':'compilerid',
   'TYPE':'typeid',
   'RUN':'runid'}

# For csv table value extraction
VALSTRFIELD=['TESTNAME','VERSION','Regression']
VALBOOLFIELD=['HARDFP', 'FASTMATH', 'NEON', 'HELIUM','UNROLL', 'ROUNDING','OPTIMIZED']
VALINTFIELD=['ID', 'MAX']
VALREALFIELD=['MAXREGCOEF']
VALDATEFIELD=[]
# Some of those fields may be created by the parsing of other fields
VALKEYFIELD=['DATE','NAME','CATEGORY', 'PLATFORM', 'CORE', 'COMPILER','TYPE']

def joinit(iterable, delimiter):
    it = iter(iterable)
    yield next(it)
    for x in it:
        yield delimiter
        yield x

def tableExists(c,tableName):
   req=(tableName,)
   r=c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?",req)
   return(r.fetchone() != None)

def diff(first, second):
        second = set(second)
        return [item for item in first if item not in second]

def getColumns(elem,full):
  colsToKeep=[]
  cols = list(full.columns)
  params=diff(elem.params.full , elem.params.summary)
  common = diff(cols + ["TYPE","RUN"] , ['OLDID'] + params)  
 
  for field in common:
       if field in MKSTRFIELD:
          colsToKeep.append(field)
       if field in MKINTFIELD:
          colsToKeep.append(field)
       if field in MKREALFIELD:
          colsToKeep.append(field)
       if field in MKKEYFIELD:
          colsToKeep.append(field)
       if field in MKDATEFIELD:
          colsToKeep.append(field)
       if field in MKBOOLFIELD:
          colsToKeep.append(field)
  return(colsToKeep)

def createTableIfMissing(conn,elem,tableName,full):
   if not tableExists(conn,tableName):
     sql = "CREATE TABLE %s (" % tableName
     cols = list(full.columns)
     params=diff(elem.params.full , elem.params.summary)
     common = diff(cols + ["TYPE","RUN"] , ['OLDID'] + params)

     sql += "%sid INTEGER PRIMARY KEY"  % (tableName)
     start = ","   

     for field in params:
       sql += " %s\n  %s INTEGER"  % (start,field)
       start = ","

     for field in common:
       if field in MKSTRFIELD:
          sql += "%s\n  %s TEXT"  % (start,field)
       if field in MKINTFIELD:
          sql += "%s\n  %s INTEGER"  % (start,field)
       if field in MKREALFIELD:
          sql += "%s\n  %s REAL"  % (start,field)
       if field in MKKEYFIELD:
          sql += "%s\n  %s INTEGER"  % (start,MKKEYFIELDID[field])
       if field in MKDATEFIELD:
          sql += "%s\n  %s TEXT"  % (start,field)
       if field in MKBOOLFIELD:
          sql += "%s\n  %s INTEGER"  % (start,field)
       start = ","
     # Create foreign keys
     sql += "%sFOREIGN KEY(typeid) REFERENCES TYPE(typeid)," % start
     sql += "FOREIGN KEY(categoryid) REFERENCES CATEGORY(categoryid),"
     sql += "FOREIGN KEY(testnameid) REFERENCES TESTNAME(testnameid),"
     sql += "FOREIGN KEY(testdateid) REFERENCES TESTDATE(testdateid),"
     sql += "FOREIGN KEY(platformid) REFERENCES PLATFORM(platformid),"
     sql += "FOREIGN KEY(coreid) REFERENCES CORE(coreid),"
     sql += "FOREIGN KEY(compilerid) REFERENCES COMPILER(compilerid)"
     sql += "FOREIGN KEY(runid) REFERENCES RUN(runid)"
     sql += "  )"
     conn.execute(sql)

# Find the key or add it in a table
def findInTable(conn,table,keystr,strv,key):
    #print(sql)
    r = conn.execute("select %s from %s where %s=?" % (key,table,keystr),(strv,))
    result=r.fetchone()
    if result != None:
      return(result[0])
    else:
      conn.execute("INSERT INTO %s(%s) VALUES(?)" % (table,keystr),(strv,))
      conn.commit()
      r = conn.execute("select %s from %s where %s=?" % (key,table,keystr),(strv,))
      result=r.fetchone()
      if result != None:
         #print(result)
         return(result[0])
      else:
         return(None)

def findInCompilerTable(conn,kind,version):
    #print(sql)
    r = conn.execute("select compilerid from COMPILER where compilerkindid=? AND version=?"  , (kind,version))
    result=r.fetchone()
    if result != None:
      return(result[0])
    else:
      fullDate = datetime.datetime.now()
      dateid = findInTable(conn,"TESTDATE","date",str(fullDate),"testdateid")
      conn.execute("INSERT INTO COMPILER(compilerkindid,version,testdateid) VALUES(?,?,?)" ,(kind,version,dateid))
      conn.commit()
      r = conn.execute("select compilerid from COMPILER where compilerkindid=? AND version=? AND testdateid=?"  , (kind,version,dateid))
      result=r.fetchone()
      if result != None:
         #print(result)
         return(result[0])
      else:
         return(None)


def addRows(conn,elem,tableName,full,runid=0):
   # List of columns we have in DB which is
   # different from the columns in the table
   compilerid = 0
   platformid = 0 
   coreid = 0
   keep = getColumns(elem,full)
   cols = list(full.columns)
   params=diff(elem.params.full , elem.params.summary)
   common = diff(["TYPE"] + cols , ['OLDID'] + params)  
   colNameList = [] 
   for c in params + keep:
      if c in MKKEYFIELD:
          colNameList.append(MKKEYFIELDID[c])
      else:
         colNameList.append(c)
   colNames = "".join(joinit(colNameList,","))
   #print(colNameList)
   #print(colNames)
   #print(full)
   for index, row in full.iterrows():
       sql = "INSERT INTO %s(%s) VALUES(" % (tableName,colNames)
       keys = {}

       # Get data from columns
       for field in common:
        if field in VALSTRFIELD:
            keys[field]=row[field]
            if field == "NAME":
                name = row[field]
            if field == "TESTNAME":
                testname = row[field]
                if re.match(r'^.*_f64',testname):
                  keys["TYPE"] = "f64"
                if re.match(r'^.*_f32',testname):
                  keys["TYPE"] = "f32"
                if re.match(r'^.*_f16',testname):
                  keys["TYPE"] = "f16"
                if re.match(r'^.*_q31',testname):
                  keys["TYPE"] = "q31"
                if re.match(r'^.*_q15',testname):
                  keys["TYPE"] = "q15"
                if re.match(r'^.*_q7',testname):
                  keys["TYPE"] = "q7"

                if re.match(r'^.*_s8',testname):
                  keys["TYPE"] = "s8"
                if re.match(r'^.*_u8',testname):
                  keys["TYPE"] = "u8"
                if re.match(r'^.*_s16',testname):
                  keys["TYPE"] = "s16"
                if re.match(r'^.*_u16',testname):
                  keys["TYPE"] = "u16"
                if re.match(r'^.*_s32',testname):
                  keys["TYPE"] = "s32"
                if re.match(r'^.*_u32',testname):
                  keys["TYPE"] = "u32"
                if re.match(r'^.*_s64',testname):
                  keys["TYPE"] = "s64"
                if re.match(r'^.*_u64',testname):
                  keys["TYPE"] = "u64"
            
        if field in VALINTFIELD:
            keys[field]=row[field]
        if field in VALREALFIELD:
            keys[field]=row[field]
        if field in VALDATEFIELD:
            keys[field]=row[field]
        if field in VALBOOLFIELD:
            keys[field]=row[field]
        
       keys['RUN']=runid
       # Get foreign keys and create missing data
       for field in common:
        if field in VALKEYFIELD:
            if field == "CATEGORY":
              # Remove type extension to get category name so that
              # all types are maped to same category which will
              # help for post processing.
              testField=re.sub(r'^(.*)[:]([^:]+)(F16|F32|F64|Q31|Q15|Q7)$',r'\1',row[field])
              val = findInTable(conn,"CATEGORY","category",testField,"categoryid")
              keys[field]=val
            if field == "NAME":
              val = findInTable(conn,"TESTNAME","name",row[field],"testnameid")
              keys[field]=val
            if field == "DATE":
              val = findInTable(conn,"TESTDATE","date",str(row[field]),"testdateid")
              keys[field]=val
            if field == "CORE":
              val = findInTable(conn,"CORE","coredef",row[field],"coreid")
              keys[field]=val
              coreid = val
            if field == "PLATFORM":
              val = findInTable(conn,"PLATFORM","platform",row[field],"platformid")
              keys[field]=val
              platformid = val
            if field == "TYPE":
              val = findInTable(conn,"TYPE","type",keys["TYPE"],"typeid")
              keys[field]=val
            if field == "COMPILER":
              compilerkind = findInTable(conn,"COMPILERKIND","compiler",row[field],"compilerkindid")
              compiler = findInCompilerTable(conn,compilerkind,keys["VERSION"])
              keys[field]=compiler
              compilerid = compiler

       # Generate sql command
       start = ""  
       for field in params:
         sql += " %s\n  %d"  % (start,row[field])
         start = ","
         
       for field in keep:
         if field in MKSTRFIELD or field in MKDATEFIELD:
            sql += " %s\n  \"%s\""  % (start,keys[field])
         elif field in keep:
            if field in VALREALFIELD:
              sql += " %s\n  %f"  % (start,keys[field])
            else:
              sql += " %s\n  %d"  % (start,keys[field])
         start = ","

       sql += "  )"
       #print(sql)
       conn.execute(sql)  
   conn.commit() 
   return({'compilerid':compilerid,'platformid':platformid,'coreid':coreid})

def addConfig(conn,config,fullDate):
  dateid = findInTable(conn,"TESTDATE","date",str(fullDate),"testdateid")
  conn.execute("INSERT INTO CONFIG(compilerid,platformid,coreid,testdateid) VALUES(?,?,?,?)" ,(config['compilerid'],config['platformid'],config['coreid'],dateid))
  conn.commit()

def getGroup(a):
    return(re.sub(r'^(.+)(F64|F32|F16|Q31|Q15|Q7|U32|U16|U8|S32|S16|S8)$',r'\1',a))

def addOneBenchmark(elem,fullPath,db,group,runid):
   if os.path.isfile(fullPath):
      full=pd.read_csv(fullPath,dtype={'OLDID': str} ,keep_default_na = False)
      fullDate = datetime.datetime.now()
      full['DATE'] = fullDate
      if group:
         tableName = getGroup(group)
      else:
         tableName = getGroup(elem.data["class"])
      conn = sqlite3.connect(db)
      createTableIfMissing(conn,elem,tableName,full)
      config = addRows(conn,elem,tableName,full,runid)
      addConfig(conn,config,fullDate)
      conn.close()


def addToDB(benchmark,dbpath,elem,group,runid):
  if not elem.data["deprecated"]:
     if elem.params:
         benchPath = os.path.join(benchmark,elem.fullPath(),"regression.csv")
         print("Processing %s" % benchPath)
         addOneBenchmark(elem,benchPath,dbpath,group,runid)
         
     for c in elem.children:
       addToDB(benchmark,dbpath,c,group,runid)



parser = argparse.ArgumentParser(description='Generate summary benchmarks')

parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="Pickle path")
parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path")
#parser.add_argument('-e', action='store_true', help="Embedded test")
parser.add_argument('-o', nargs='?',type = str, default="reg.db", help="Regression benchmark database")
parser.add_argument('-r', nargs='?',type = int, default=0, help="Run ID")

parser.add_argument('others', nargs=argparse.REMAINDER, help="Suite class")

args = parser.parse_args()

if args.f is not None:
    #p = parse.Parser()
    # Parse the test description file
    #root = p.parse(args.f)
    root=parse.loadRoot(args.f)
    d.deprecate(root,args.others)
    if args.others:
      group=args.others[0] 
    else:
      group=None
    addToDB(args.b,args.o,root,group,args.r)
    
else:
    parser.print_help()