Manifest: Use Jinja2 for template generation
Use Jinja2 template engine to generate code from templates. This makes
the custom generator scripts in the tools directory obsolete.
For more info on Jina please see http://jinja.pocoo.org/
Details:
- rewrite template files to use jinja2 syntax
- remove fixmes for limitations of the template generator script
- fix crypto and attest partition manifest yaml file
- remove obsolete template generator scripts
- rewrite tfm_parse_manifest_list.py to use the jinja2 template engine
- update documentation
Change-Id: Iccceae7a5e14e5c5d6c6b5bcdf08a53137d81458
Signed-off-by: Mate Toth-Pal <mate.toth-pal@arm.com>
diff --git a/tools/generate_from_template.py b/tools/generate_from_template.py
deleted file mode 100644
index a1ab137..0000000
--- a/tools/generate_from_template.py
+++ /dev/null
@@ -1,115 +0,0 @@
-#-------------------------------------------------------------------------------
-# Copyright (c) 2018, Arm Limited. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-#
-#-------------------------------------------------------------------------------
-
-from __future__ import unicode_literals
-import os, re, io
-from keyword_substitution import keyword_substitute, Verbosity, log_print, REkeychain
-
-VERBOSITY = Verbosity.warning
-log_print(Verbosity.debug, "Setting verbosity to", VERBOSITY, verbosity=VERBOSITY)
-
-controlsymbol = "@!GENERATOR"
-controlblockstart = "@!GENERATOR_BLOCK_START!@"
-controlblockend = "@!GENERATOR_BLOCK_END!@"
-controlconditionstart = "@!GENERATOR_CONDITIONAL_START!@"
-controlconditionelse = "@!GENERATOR_CONDITIONAL_ELSE!@"
-controlconditionend = "@!GENERATOR_CONDITIONAL_END!@"
-control_print_iteration_counter = "@!GENERATOR_ITERATION_COUNTER!@"
-control_print_donotedit_warning = "@!GENERATOR_DONOTEDIT_WARNING!@"
-
-donotedit_warning = "/*********** " + \
- "WARNING: This is an auto-generated file. Do not edit!" + \
- " ***********/\n"
-
-# All operations assume tf-m repo root as active working directory
-
-# Functions
-def substitute(manifest, line, MISSING_KEYS_ACTION):
- outlist = keyword_substitute(manifest, line, MISSING_KEYS_ACTION)
- outstring = ""
- for outline in outlist:
- outstring += ''.join(outline)
- log_print(Verbosity.info, outstring)
- return outstring
-
-def generate(db, outfile_name):
- outfile = io.open(outfile_name, "w", newline='\n')
- with io.open(outfile_name + '.template', "r") as template_file:
- template = template_file.readlines()
-
- output = []
- blocks = []
- blocklines = []
- MISSING_KEYS_ACTION = 'replace False'
- inblock = False
- iteration_counter = 0
- for lineno, line in enumerate(template):
- if controlblockstart in line:
- inblock = True
- log_print(Verbosity.info, "Blockstart:", str(lineno))
- blocklines = []
- elif controlblockend in line:
- inblock = False
- iteration_counter = 0
- log_print(Verbosity.info, "Blocklines:", str(blocklines))
- for manifest in db:
- ignore_line = False
- for line in blocklines:
- if controlconditionstart in line:
- outstring = substitute(manifest, line, MISSING_KEYS_ACTION)
- if 'False' in outstring:
- log_print(Verbosity.info, "PRINT BLOCKED")
- ignore_line = True
- elif controlconditionend in line:
- log_print(Verbosity.info, "PRINT ENABLED")
- ignore_line = False
- elif controlconditionelse in line:
- log_print(Verbosity.info, "PRINT " + str(ignore_line))
- ignore_line = not ignore_line
- elif not ignore_line:
- if re.search(REkeychain, line):
- outstring = substitute(manifest, line, MISSING_KEYS_ACTION)
- else:
- outstring = line
- if control_print_iteration_counter in outstring:
- outstring = outstring.replace(
- control_print_iteration_counter,
- str(iteration_counter))
- elif controlsymbol in outstring:
- print "Invalid control symbol:", outstring
- print "exiting"
- exit(1)
- outfile.write(outstring)
- iteration_counter += 1
- # end for manifest in db
- blocks.append(blocklines)
- elif inblock:
- # inside a generator block
- # append line to blocklines to be processed by template generator
- blocklines.append(line)
- else:
- # outside a generator block
- if control_print_donotedit_warning in line:
- # print do not edit warning - Note: ignore rest of input line
- line = donotedit_warning
- elif control_print_iteration_counter in line:
- # line contains an iteration counter request, replace with value
- line = line.replace(control_print_iteration_counter, str(iteration_counter))
- elif controlsymbol in line:
- print "Invalid control symbol:", line
- print "exiting"
- exit(1)
- outfile.write(line)
- log_print(Verbosity.debug, "Blocks:" + str(blocks))
- outfile.close()
-# def generate()
-
-def generate_from_template_file(db, file_list):
- for file in file_list:
- outfile = file["output"]
- print "Generating", outfile
- generate(db, outfile)
diff --git a/tools/keyword_substitution.py b/tools/keyword_substitution.py
deleted file mode 100644
index 1687338..0000000
--- a/tools/keyword_substitution.py
+++ /dev/null
@@ -1,154 +0,0 @@
-#-------------------------------------------------------------------------------
-# Copyright (c) 2018, Arm Limited. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-#
-#-------------------------------------------------------------------------------
-
-import re
-try:
- from enum import Enum
-except ImportError as e:
- print e, "To install it, type:"
- print "pip install enum34"
- exit(1)
-
-class Verbosity(Enum):
- debug = 3
- info = 2
- warning = 1
- error = 0
-
-# Set default value
-VERBOSITY = Verbosity.info
-
-REkeychain = "@@\w+[\.\w+]*@@"
-REfirstkeyword = "@@\w+\.?"
-emptychain = "@@@@"
-
-MISSING_KEYS_ACTION = 'halt'
-
-def log_print(level, *args, **kwargs):
- global VERBOSITY
- if kwargs.get("verbosity"):
- VERBOSITY = kwargs["verbosity"]
- if level.value <= VERBOSITY.value:
- string = ' '.join(map(str, args))
- print string
-
-def leaftype(x):
- return isinstance(x, str) or isinstance(x, unicode) or \
- isinstance(x, type(1)) or isinstance(x, type(True))
-
-def substitute(templist, chains, db, depth):
- depth += 1
- log_print(Verbosity.info, "substitute(",templist, chains, db, depth,")")
- if isinstance(db, type([])):
- # db is list
- outlist = []
- for instance in db:
- log_print(Verbosity.info, "Going deeper at", depth, "for db list instance", instance)
- outlist.extend(substitute(templist, chains, instance, depth))
- log_print(Verbosity.info, "substitute", depth, "returning from list with", outlist)
- return outlist
-
- transientlist = list(templist)
- if leaftype(db):
- # db is leaf
- for chain in chains:
- if templist[chain] == emptychain:
- transientlist[chain] = str(db)
- else:
- print "keychain not empty but db is"
- transientlist[chain] = str(db) + templist[chain]
- continue
- chains = []
- log_print(Verbosity.info, "substitute", depth, "returning from leaf with", transientlist)
- return transientlist
-
- # db is dict
- # find chain groups with same key
- chaingroups = {"chains": [], "keys": []}
- for chain in chains:
- key = re.search('\w+', templist[chain])
- if not key:
- # chain does not define a leaf
- print "chain", chain, "out of keys before reaching leaf"
- continue
- key = key.group(0) # convert MatchObj to string
- for idx, groupkey in enumerate(chaingroups["keys"]):
- if key == groupkey:
- # insert chain in group
- chaingroups["chains"][idx].append(chain)
- break
- else:
- # key not yet in the list
- chaingroups["keys"].append(key)
- chaingroups["chains"].append([chain])
-
- log_print(Verbosity.debug, chaingroups)
-
- for groupidx, key in enumerate(chaingroups["keys"]):
- log_print(Verbosity.info, "key lookup in", db, "for", key)
- if key in db.keys():
- if leaftype(db[key]):
- # db entry value is leaf
- for chain in chaingroups["chains"][groupidx]:
- transientlist[chain] = str(db[key])
- chaingroups["chains"][groupidx] = []
- else:
- # db node is branch
- for chain in chaingroups["chains"][groupidx]:
- transientlist[chain] = re.sub(REfirstkeyword, "@@", templist[chain])
- else:
- # key not found in database
- if MISSING_KEYS_ACTION == 'report' or MISSING_KEYS_ACTION == 'halt':
- print "key", key, "not found, invalid chains", chaingroups["chains"][groupidx]
- placeholder = False
- if MISSING_KEYS_ACTION == 'hide':
- placeholder = ""
- if 'replace' in MISSING_KEYS_ACTION:
- placeholder = MISSING_KEYS_ACTION.replace("replace ", "")
- for chain in chaingroups["chains"][groupidx]:
- transientlist[chain] = placeholder
- chaingroups["chains"][groupidx] = []
- if MISSING_KEYS_ACTION == 'halt':
- exit(1)
- log_print(Verbosity.debug, "**** after selection chaingroups:", chaingroups, transientlist)
- forked = False
- for chidx, chains in enumerate(chaingroups["chains"]):
- if chains:
- key = chaingroups["keys"][chidx]
- log_print(Verbosity.info, "Going deeper at", depth, "for chains", chains, "in dict", db[key])
- if not forked:
- outlist = substitute(transientlist, chains, db[key], depth)
- forked = True
- else:
- transientlist = list(outlist)
- outlist = list()
- for item in transientlist:
- outlist.extend(substitute(item, chains, db[key], depth))
-
- if not forked:
- outlist = [transientlist]
-
- log_print(Verbosity.info, "substitute", depth, "returning from dict/leaf with", outlist)
- return outlist
-
-def keyword_substitute(db, line, missing_keys):
- global MISSING_KEYS_ACTION
- chains = []
- MISSING_KEYS_ACTION = missing_keys
- log_print(Verbosity.info, line)
- templist = re.split("(" + REkeychain + ")", line)
- for idx, item in enumerate(templist):
- if re.match(REkeychain, item):
- chains.append(idx)
- log_print(Verbosity.info, "Keychain:", item)
- outlist = substitute(templist, chains, db, 0)
- outstring = ""
- for outline in outlist:
- outstring += ''.join(outline) + "\n"
- log_print(Verbosity.info, "generator returns with:")
- log_print(Verbosity.info, outstring + "<<")
- return outlist
diff --git a/tools/keyword_substitution_test_vectors.yaml b/tools/keyword_substitution_test_vectors.yaml
deleted file mode 100644
index b6c975a..0000000
--- a/tools/keyword_substitution_test_vectors.yaml
+++ /dev/null
@@ -1,126 +0,0 @@
-#-------------------------------------------------------------------------------
-# Copyright (c) 2018, Arm Limited. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-#
-#-------------------------------------------------------------------------------
-
-{
- "test_list": [{
- "description": "Test no substitution",
- "template": "quickbrownfox",
- "db": {"a": "b", "c": "d"},
- "expected": "quickbrownfox\n"
- },
- {
- "description": "Test simple substitution",
- "template": "@@a@@",
- "db": {"a": "b", "c": "d"},
- "expected": "b\n"
- },
- {
- "template": "a123",
- "db": [{"c": "d", "a": {"a1": "lollipop", "a2": "PPAP"}},
- {"c": "x", "a": {"a1": "candyshop", "a2": "GitS: SAC"}}],
- "expected": "a123\na123\n"},
- {
- "description": "Test two independent base symbols with two substitutions",
- "template": "@@a@@ @@c@@",
- "db": [{"a": "b", "c": "d"}, {"a": "x", "c": "x"}],
- "expected": ["b d\n",
- "x x\n"]},
- {
- "template": "@@a@@@ @b@@c@@",
- "db": [{"a": "b", "c": "d"}, {"a": "x", "c": "x"}],
- "expected": ["b@ @bd\n",
- "x@ @bx\n"]},
- {
- "template": "@@a.a1@@",
- "db": [{"a": {"a1": "lollipop", "a2": "PPAP"}, "c": "d"}],
- "expected": "lollipop\n"},
- {
- "template": "@@a.a1@@ @@a.a2@@",
- "db": [{"a": {"a1": "lollipop", "a2": "PPAP"}, "c": "d"}, {"a": "x", "c": "x"}],
- "expected": ["lollipop PPAP\n",
- "x x\n"]},
- {
- "template": "@@a.a1@@ @@a.a2@@",
- "db": [{"a": {"a1": "lollipop", "a2": "PPAP"}, "c": "d"},
- {"a": {"a1": "candyshop", "a2": "GitS: SAC"}, "c": "x"}],
- "expected": ["lollipop PPAP\n",
- "candyshop GitS: SAC\n"]},
- {
- "template": "@@a.a1@@ @@a.a2@@ @@a@@",
- "db": [{"a": {"a1": "lollipop", "a2": "PPAP"}, "c": "d"}, {"a": {"a1": "candyshop", "a2": "GitS: SAC"}, "c": "x"}],
- "expected": ["lollipop PPAP @@@@\n",
- "candyshop GitS: SAC @@@@\n"]
- },
- {
- "description": "List in dict",
- "template": "@@a.name@@",
- "db": [{"a": [{"name": "inst1"}, {"name": "inst2"}]}],
- "expected": ["inst1\n",
- "inst2\n"]
- },
- {
- "description": "List in dict in list",
- "template": "@@name@@ subinstance: @@subinstlist.name@@",
- "db": [{"name": "inst1", "subinstlist": [{"name": "subinst1"}, {"name": "subinst2"}]},
- {"name": "inst2", "subinstlist": [{"name": "subinst3"}, {"name": "subinst4"}]}],
- "expected": ["inst1 subinstance: subinst1\n",
- "inst1 subinstance: subinst2\n",
- "inst2 subinstance: subinst3\n",
- "inst2 subinstance: subinst4\n"]
- },
- {
- "description": "Chain forks: multiple diverging chains",
- "template": "@@name@@ subinstance: @@subinstlist.name@@ @@subinst2.name@@",
- "db": {"name": "inst1", "subinstlist": [{"name": "subinst1"}, {"name": "subinst2"}],
- "subinst2": [{"name": "xxx"}, {"name": "yyy"}]},
- "expected": ["inst1 subinstance: subinst1 xxx\n",
- "inst1 subinstance: subinst1 yyy\n",
- "inst1 subinstance: subinst2 xxx\n",
- "inst1 subinstance: subinst2 yyy\n"]
- },
- {
- "description": "Chain forks: multiple diverging chains",
- "template": "@@name@@: @@subinstlist.name@@ @@subinstlist2.name@@ @@subinstlist3.name@@",
- "db": {"name": "inst1", "subinstlist1": [{"name": "subinst1"}, {"name": "subinst2"}],
- "subinstlist2": [{"name": "xxx"}, {"name": "yyy"}],
- "subinstlist3": [{"name": "aaa"}, {"name": "bbb"}]},
- "expected": ["inst1: @@subinstlist.name@@ xxx aaa\n",
- "inst1: @@subinstlist.name@@ xxx bbb\n",
- "inst1: @@subinstlist.name@@ yyy aaa\n",
- "inst1: @@subinstlist.name@@ yyy bbb\n"]
- },
- {
- "description": "Chain forks: multiple diverging chains, array of root instances",
- "template": "@@name@@ subinstance: @@subinstlist.name@@ @@subinst2.name@@",
- "db": [{"name": "inst1", "subinstlist": [{"name": "subinst1"}, {"name": "subinst2"}],
- "subinst2": [{"name": "xxx"}, {"name": "yyy"}]},
- {"name": "inst2", "subinstlist": [{"name": "subinst3"}, {"name": "subinst4"}],
- "subinst2": [{"name": "inst2subinstlist2subinst1"}, {"name": "inst2subinstlist2subinst2"}]}],
- "expected": ["inst1 subinstance: subinst1 xxx\n",
- "inst1 subinstance: subinst1 yyy\n",
- "inst1 subinstance: subinst2 xxx\n",
- "inst1 subinstance: subinst2 yyy\n",
- "inst2 subinstance: subinst3 inst2subinstlist2subinst1\n",
- "inst2 subinstance: subinst3 inst2subinstlist2subinst2\n",
- "inst2 subinstance: subinst4 inst2subinstlist2subinst1\n",
- "inst2 subinstance: subinst4 inst2subinstlist2subinst2\n"]
- },
- {
- "description": "Chain forks: multiple diverging chains with missing subinstance",
- "template": "@@name@@ subinstance: @@subinstlist1.name@@ @@subinstlist2.name@@",
- "db": [{"name": "inst1", "subinstlist1": [{"name": "subinst1"}, {"name": "subinst2"}],
- "subinstlist2": [{"name": "xxx"}, {"name": "yyy"}]},
- {"name": "inst2", "subinstlist1": [{"name": "subinst3"}, {"name": "subinst4"}]}],
- "expected": ["inst1 subinstance: subinst1 xxx\n",
- "inst1 subinstance: subinst1 yyy\n",
- "inst1 subinstance: subinst2 xxx\n",
- "inst1 subinstance: subinst2 yyy\n",
- "inst2 subinstance: subinst3 @@subinstlist2.name@@\n",
- "inst2 subinstance: subinst4 @@subinstlist2.name@@\n"]
- }
- ]
-}
diff --git a/tools/keyword_substitution_tests.py b/tools/keyword_substitution_tests.py
deleted file mode 100644
index 21ebc17..0000000
--- a/tools/keyword_substitution_tests.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#-------------------------------------------------------------------------------
-# Copyright (c) 2018, Arm Limited. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-#
-#-------------------------------------------------------------------------------
-
-from keyword_substitution import keyword_substitute, log_print, Verbosity
-
-try:
- import yaml
-except ImportError as e:
- print e, "To install it, type:"
- print "pip install PyYAML"
- exit(1)
-
-VERBOSITY = Verbosity.warning
-log_print(Verbosity.debug, "Setting verbosity to", VERBOSITY, verbosity=VERBOSITY)
-
-def testsuite():
- with open('keyword_substitution_test_vectors.yaml', 'r') as test_vectors_file:
- test_yaml = yaml.load(test_vectors_file)
- test_list = test_yaml["test_list"]
-
- for tcidx, test in enumerate(test_list):
- expected = ''.join(test["expected"])
- print
- log_print(Verbosity.debug, "template:", test["template"])
- log_print(Verbosity.debug, "db:", test["db"])
- outlist = keyword_substitute(test["db"], test["template"], "report")
- log_print(Verbosity.debug, outlist)
- outstring = ""
- for outline in outlist:
- outstring += ''.join(outline) + "\n"
- log_print(Verbosity.info, "Got:")
- log_print(Verbosity.info, outstring)
- if outstring == expected:
- print "Test", tcidx, "PASSED"
- test["result"] = "PASSED"
- else:
- print "Test", tcidx, "FAILED, expected:"
- print expected
- test["result"] = "FAILED"
-
- print
- print "Test summary:"
- for idx, test in enumerate(test_list):
- print "Test", idx, test["result"]
-
-def main():
- testsuite()
-
-if __name__ == "__main__":
- main()
diff --git a/tools/tfm_parse_manifest_list.py b/tools/tfm_parse_manifest_list.py
index 3129bde..ea6d118 100644
--- a/tools/tfm_parse_manifest_list.py
+++ b/tools/tfm_parse_manifest_list.py
@@ -1,28 +1,61 @@
#-------------------------------------------------------------------------------
-# Copyright (c) 2018, Arm Limited. All rights reserved.
+# Copyright (c) 2018-2019, Arm Limited. All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
#
#-------------------------------------------------------------------------------
import os
-from keyword_substitution import Verbosity, log_print
-from generate_from_template import generate_from_template_file
+import io
+from jinja2 import Environment, BaseLoader, select_autoescape
try:
import yaml
except ImportError as e:
- print e, "To install it, type:"
- print "pip install PyYAML"
+ print (e + " To install it, type:")
+ print ("pip install PyYAML")
exit(1)
-VERBOSITY = Verbosity.warning
-log_print(Verbosity.debug, "Setting verbosity to", VERBOSITY, verbosity=VERBOSITY)
+class TemplateLoader(BaseLoader):
+ """
+ Template loader class.
-# All operations assume tf-m repo root as active working directory
+ An instance of this class is passed to the template engine. It is
+ responsible for reading the template file
+ """
+ def __init__(self):
+ pass
-# Functions
+ def get_source(self, environment, template):
+ """
+ This function reads the template files.
+ For detailed documentation see:
+ http://jinja.pocoo.org/docs/2.10/api/#jinja2.BaseLoader.get_source
+
+ Please note that this function always return 'false' as 'uptodate'
+ value, so the output file will always be generated.
+ """
+ if not os.path.isfile(template):
+ raise TemplateNotFound(template)
+ with open(template) as f:
+ source = f.read()
+ return source, template, False
+
def load_manifest_list(file):
+ """
+ Load the substitution data from the manifests.
+
+ Parameters
+ ----------
+ file : file
+ A yaml file containing the manifest list
+
+ Returns
+ -------
+ list
+ The list of the contents of the manifest files, as generated by the yaml
+ parser.
+ """
db = []
manifest_list = yaml.load(file)
for item in manifest_list["manifest_list"]:
@@ -32,24 +65,59 @@
manifest = yaml.load(file)
db.append({"manifest": manifest, "attr": item})
except IOError:
- print "Manifest for "+item['name']+" cannot be opened at path "+item['manifest']
- return db
-# def load_manifest_list
+ raise Exception ("Manifest for "+item['name']+" cannot be opened at path "+item['manifest'])
-# main
+ return db
+
+
def main():
+ """
+ The entry point of the script.
+
+ Generates the output files based on the templates and the manifests.
+ """
+ donotedit_warning = \
+ "/*********** " + \
+ "WARNING: This is an auto-generated file. Do not edit!" + \
+ " ***********/"
+
+ env = Environment(
+ loader = TemplateLoader(),
+ autoescape = select_autoescape(['html', 'xml']),
+ lstrip_blocks = True,
+ trim_blocks = True,
+ keep_trailing_newline = True
+ )
+
with open(os.path.join('tools', 'tfm_manifest_list.yaml')) \
as manifest_list_yaml_file:
# Read manifest list file, build database
db = load_manifest_list(manifest_list_yaml_file)
+ utilities = {}
+ utilities['donotedit_warning']=donotedit_warning
+
+ context = {}
+ context['manifests'] = db
+ context['utilities'] = utilities
+
with open(os.path.join('tools', 'tfm_generated_file_list.yaml')) \
as file_list_yaml_file:
# read list of files that need to be generated from templates using db
file_list_yaml = yaml.load(file_list_yaml_file)
file_list = file_list_yaml["file_list"]
- generate_from_template_file(db, file_list)
- print "Generation of files done"
+ for file in file_list:
+ outfile_name = file["output"]
+ templatefile_name = outfile_name + '.template'
+
+ print ("Generating " + outfile_name)
+
+ template = env.get_template(templatefile_name)
+
+ outfile = io.open(outfile_name, "w", newline='\n')
+ outfile.write(template.render(context))
+ outfile.close()
+ print ("Generation of files done")
if __name__ == "__main__":
main()