Add initial version of c-picker
Introduce the following features to c-picker:
* Picking of the elements from C source files:
* Include directives
* Functions
* Variables
* Removing 'static' keyword from declarations
* Mapping coverage to the original source
* Documentation of the system
Signed-off-by: Imre Kis <imre.kis@arm.com>
Change-Id: Ia5cb90d3096b16b15aafb86363b8cabfe7d2ab72
diff --git a/c_picker/picker.py b/c_picker/picker.py
new file mode 100644
index 0000000..963a313
--- /dev/null
+++ b/c_picker/picker.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019-2021, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""
+This module can fetch elements (include directives, functions, etc) from C source codes.
+The main purpose of this module is help unit testing by isolating functions from the
+rest of the code.
+"""
+
+import enum
+import os
+import sys
+
+# pylint exceptions are used because of the necessary Config.set_library_path call.
+from clang.cindex import Config
+if "CLANG_LIBRARY_PATH" in os.environ:
+ Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
+import clang.cindex # pylint: disable=wrong-import-position
+
+from c_picker.coverage import MappingDescriptor # pylint: disable=wrong-import-position
+
+class CPicker:
+ """ CPicker can fetch C source element from a file matching the parameters and options. """
+
+ class Type(enum.Enum):
+ """ C source element type """
+ include = clang.cindex.CursorKind.INCLUSION_DIRECTIVE
+ function = clang.cindex.CursorKind.FUNCTION_DECL
+ variable = clang.cindex.CursorKind.VAR_DECL
+
+ class Option(enum.Enum):
+ """ Parameter for modifying the behaviour of the fetcher """
+
+ class RemoveStaticProcessor:
+ """ Removes 'static' from function declaration """
+
+ @staticmethod
+ def is_matching(node):
+ """ Checks if the storage class STATIC """
+ return node.storage_class == clang.cindex.StorageClass.STATIC
+
+ @staticmethod
+ def process(lines):
+ """ Removes 'static' before the function body """
+ processed_lines = []
+ function_body_started = False
+ for line in lines:
+ if not function_body_started:
+ processed_lines.append(line.replace("static ", ""))
+ function_body_started = "{" in line
+ else:
+ processed_lines.append(line)
+ return processed_lines
+
+ remove_static = RemoveStaticProcessor
+
+ class ElementDescriptor:
+ """ Data structure of matching parameters """
+ def __init__(self, file_name, element_type, element_name=None):
+ self.file_name = file_name
+ self.element_type = element_type
+ self.element_name = element_name
+ self.args = None
+ self.options = None
+
+ def set_args(self, args):
+ """ Setting arguments of the parser """
+ self.args = args
+
+ def set_options(self, options):
+ """ Setting options for the fetcher """
+ self.options = options
+
+ def is_single_matching_type(self):
+ """ There's only a single matching element """
+ return bool(self.element_name)
+
+ def is_matching(self, node):
+ """ The node is matching the element defined in this instance """
+ if self.element_type == CPicker.Type.include:
+ return ((node.kind == self.element_type.value) and
+ (str(node.location.file) == self.file_name))
+
+ if self.element_type == CPicker.Type.function:
+ return ((node.kind == self.element_type.value) and
+ (str(node.location.file) == self.file_name) and
+ (node.spelling == self.element_name) and
+ node.is_definition())
+ if self.element_type == CPicker.Type.variable:
+ return ((node.kind == self.element_type.value) and
+ (str(node.location.file) == self.file_name) and
+ (node.spelling == self.element_name))
+ raise Exception("Invalid element type")
+
+ def get_option_processors(self, node):
+ """ Get processor function for matching options """
+ processors = []
+ for option in self.options:
+ if option.value.is_matching(node):
+ processors.append(option.value)
+ return processors
+
+ def __init__(self, output=None, print_dependencies=False):
+ self.output = output if output else sys.stdout
+ self.print_dependencies = print_dependencies
+
+ def generate_header_comment(self):
+ """ Warning comment in the first line of the code """
+ self.output.write("/* DO NOT MODIFY! Generated by c-picker. */\n")
+
+ def generate_dependencies(self, elements):
+ """ Print the set of files included in the elements list """
+ dependencies = set()
+
+ for element in elements:
+ dependencies.add(os.path.abspath(element.file_name))
+
+ self.output.write(";".join(dependencies))
+
+ def fetch(self, element_descriptor):
+ """ Fetching C source element from the file """
+
+ # Parsing file with clang
+ try:
+ parser_options = clang.cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD
+ index = clang.cindex.Index.create()
+ translation_unit = index.parse(element_descriptor.file_name,
+ args=element_descriptor.args,
+ options=parser_options)
+ except clang.cindex.TranslationUnitLoadError as exception:
+ raise Exception("Failed to parse " + element_descriptor.file_name + ": "
+ + str(exception))
+
+ # Element comment
+ if element_descriptor.element_name:
+ element_text = element_descriptor.element_name
+ else:
+ element_text = element_descriptor.element_type.name
+
+ self.output.write("\n/* %s from %s */\n" % (element_text, element_descriptor.file_name))
+
+ # Searching for matching elements
+ for node in translation_unit.cursor.walk_preorder():
+ if element_descriptor.is_matching(node):
+ self.dump(node, element_descriptor.get_option_processors(node))
+ if element_descriptor.is_single_matching_type():
+ break
+ else:
+ if element_descriptor.is_single_matching_type():
+ raise Exception("%s not found in %s"
+ % (element_descriptor.element_name, element_descriptor.file_name))
+
+ def dump(self, node, options_processors):
+ """ Dump the contents of a node to the specified output """
+ with open(str(node.location.file), "r") as source_file:
+ source_lines = source_file.readlines()
+ source_lines = source_lines[node.extent.start.line - 1 : node.extent.end.line]
+
+ if len(source_lines) == 1:
+ source_lines = [source_lines[0][node.extent.start.column - 1 :
+ node.extent.end.column - 1]]
+ elif len(source_lines) > 1:
+ source_lines = ([source_lines[0][node.extent.start.column - 1 :]] +
+ source_lines[1:-1] +
+ [source_lines[-1][: node.extent.end.column - 1]])
+
+ for processor in options_processors:
+ source_lines = processor.process(source_lines)
+
+ # Mapping information of the original source
+ mapping_descriptor = MappingDescriptor(str(node.location.file), node.extent.start.line)
+ self.output.write(MappingDescriptor.serialize(mapping_descriptor) + "\n")
+
+ for source_line in source_lines:
+ self.output.write(source_line)
+ if node.kind == CPicker.Type.variable.value:
+ self.output.write(";")
+ self.output.write("\n")
+
+ def process(self, elements):
+ """ Processes element list """
+ try:
+ if not self.print_dependencies:
+ self.generate_header_comment()
+
+ for element in elements:
+ self.fetch(element)
+ else:
+ self.generate_dependencies(elements)
+ except clang.cindex.LibclangError as _:
+ raise Exception("Please ensure you have the correct version of libclang installed" +
+ " and the CLANG_LIBRARY_PATH environment variable set.")