| #!/usr/bin/env python3 |
| # |
| # Copyright The Mbed TLS Contributors |
| # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later |
| |
| """ |
| This script confirms that the naming of all symbols and identifiers in Mbed TLS |
| are consistent with the house style and are also self-consistent. It only runs |
| on Linux and macOS since it depends on nm. |
| |
| It contains two major Python classes, CodeParser and NameChecker. They both have |
| a comprehensive "run-all" function (comprehensive_parse() and perform_checks()) |
| but the individual functions can also be used for specific needs. |
| |
| CodeParser makes heavy use of regular expressions to parse the code, and is |
| dependent on the current code formatting. Many Python C parser libraries require |
| preprocessed C code, which means no macro parsing. Compiler tools are also not |
| very helpful when we want the exact location in the original source (which |
| becomes impossible when e.g. comments are stripped). |
| |
| NameChecker performs the following checks: |
| |
| - All exported and available symbols in the library object files, are explicitly |
| declared in the header files. This uses the nm command. |
| - All macros, constants, and identifiers (function names, struct names, etc) |
| follow the required regex pattern. |
| - Typo checking: All words that begin with MBED|PSA exist as macros or constants. |
| |
| The script returns 0 on success, 1 on test failure, and 2 if there is a script |
| error. It must be run from Mbed TLS root. |
| """ |
| |
| import abc |
| import argparse |
| import fnmatch |
| import glob |
| import textwrap |
| import os |
| import sys |
| import traceback |
| import re |
| import enum |
| import shutil |
| import subprocess |
| import logging |
| |
| import scripts_path # pylint: disable=unused-import |
| from mbedtls_dev import build_tree |
| |
| |
| # Naming patterns to check against. These are defined outside the NameCheck |
| # class for ease of modification. |
| PUBLIC_MACRO_PATTERN = r"^(MBEDTLS|PSA)_[0-9A-Z_]*[0-9A-Z]$" |
| INTERNAL_MACRO_PATTERN = r"^[0-9A-Za-z_]*[0-9A-Z]$" |
| CONSTANTS_PATTERN = PUBLIC_MACRO_PATTERN |
| IDENTIFIER_PATTERN = r"^(mbedtls|psa)_[0-9a-z_]*[0-9a-z]$" |
| |
| class Match(): # pylint: disable=too-few-public-methods |
| """ |
| A class representing a match, together with its found position. |
| |
| Fields: |
| * filename: the file that the match was in. |
| * line: the full line containing the match. |
| * line_no: the line number. |
| * pos: a tuple of (start, end) positions on the line where the match is. |
| * name: the match itself. |
| """ |
| def __init__(self, filename, line, line_no, pos, name): |
| # pylint: disable=too-many-arguments |
| self.filename = filename |
| self.line = line |
| self.line_no = line_no |
| self.pos = pos |
| self.name = name |
| |
| def __str__(self): |
| """ |
| Return a formatted code listing representation of the erroneous line. |
| """ |
| gutter = format(self.line_no, "4d") |
| underline = self.pos[0] * " " + (self.pos[1] - self.pos[0]) * "^" |
| |
| return ( |
| " {0} |\n".format(" " * len(gutter)) + |
| " {0} | {1}".format(gutter, self.line) + |
| " {0} | {1}\n".format(" " * len(gutter), underline) |
| ) |
| |
| class Problem(abc.ABC): # pylint: disable=too-few-public-methods |
| """ |
| An abstract parent class representing a form of static analysis error. |
| It extends an Abstract Base Class, which means it is not instantiable, and |
| it also mandates certain abstract methods to be implemented in subclasses. |
| """ |
| # Class variable to control the quietness of all problems |
| quiet = False |
| def __init__(self): |
| self.textwrapper = textwrap.TextWrapper() |
| self.textwrapper.width = 80 |
| self.textwrapper.initial_indent = " > " |
| self.textwrapper.subsequent_indent = " " |
| |
| def __str__(self): |
| """ |
| Unified string representation method for all Problems. |
| """ |
| if self.__class__.quiet: |
| return self.quiet_output() |
| return self.verbose_output() |
| |
| @abc.abstractmethod |
| def quiet_output(self): |
| """ |
| The output when --quiet is enabled. |
| """ |
| pass |
| |
| @abc.abstractmethod |
| def verbose_output(self): |
| """ |
| The default output with explanation and code snippet if appropriate. |
| """ |
| pass |
| |
| class SymbolNotInHeader(Problem): # pylint: disable=too-few-public-methods |
| """ |
| A problem that occurs when an exported/available symbol in the object file |
| is not explicitly declared in header files. Created with |
| NameCheck.check_symbols_declared_in_header() |
| |
| Fields: |
| * symbol_name: the name of the symbol. |
| """ |
| def __init__(self, symbol_name): |
| self.symbol_name = symbol_name |
| Problem.__init__(self) |
| |
| def quiet_output(self): |
| return "{0}".format(self.symbol_name) |
| |
| def verbose_output(self): |
| return self.textwrapper.fill( |
| "'{0}' was found as an available symbol in the output of nm, " |
| "however it was not declared in any header files." |
| .format(self.symbol_name)) |
| |
| class PatternMismatch(Problem): # pylint: disable=too-few-public-methods |
| """ |
| A problem that occurs when something doesn't match the expected pattern. |
| Created with NameCheck.check_match_pattern() |
| |
| Fields: |
| * pattern: the expected regex pattern |
| * match: the Match object in question |
| """ |
| def __init__(self, pattern, match): |
| self.pattern = pattern |
| self.match = match |
| Problem.__init__(self) |
| |
| |
| def quiet_output(self): |
| return ( |
| "{0}:{1}:{2}" |
| .format(self.match.filename, self.match.line_no, self.match.name) |
| ) |
| |
| def verbose_output(self): |
| return self.textwrapper.fill( |
| "{0}:{1}: '{2}' does not match the required pattern '{3}'." |
| .format( |
| self.match.filename, |
| self.match.line_no, |
| self.match.name, |
| self.pattern |
| ) |
| ) + "\n" + str(self.match) |
| |
| class Typo(Problem): # pylint: disable=too-few-public-methods |
| """ |
| A problem that occurs when a word using MBED or PSA doesn't |
| appear to be defined as constants nor enum values. Created with |
| NameCheck.check_for_typos() |
| |
| Fields: |
| * match: the Match object of the MBED|PSA name in question. |
| """ |
| def __init__(self, match): |
| self.match = match |
| Problem.__init__(self) |
| |
| def quiet_output(self): |
| return ( |
| "{0}:{1}:{2}" |
| .format(self.match.filename, self.match.line_no, self.match.name) |
| ) |
| |
| def verbose_output(self): |
| return self.textwrapper.fill( |
| "{0}:{1}: '{2}' looks like a typo. It was not found in any " |
| "macros or any enums. If this is not a typo, put " |
| "//no-check-names after it." |
| .format(self.match.filename, self.match.line_no, self.match.name) |
| ) + "\n" + str(self.match) |
| |
| class CodeParser(): |
| """ |
| Class for retrieving files and parsing the code. This can be used |
| independently of the checks that NameChecker performs, for example for |
| list_internal_identifiers.py. |
| """ |
| def __init__(self, log): |
| self.log = log |
| build_tree.check_repo_path() |
| |
| # Memo for storing "glob expression": set(filepaths) |
| self.files = {} |
| |
| # Globally excluded filenames. |
| # Note that "*" can match directory separators in exclude lists. |
| self.excluded_files = ["*/bn_mul", "*/compat-2.x.h"] |
| |
| def comprehensive_parse(self): |
| """ |
| Comprehensive ("default") function to call each parsing function and |
| retrieve various elements of the code, together with the source location. |
| |
| Returns a dict of parsed item key to the corresponding List of Matches. |
| """ |
| self.log.info("Parsing source code...") |
| self.log.debug( |
| "The following files are excluded from the search: {}" |
| .format(str(self.excluded_files)) |
| ) |
| |
| all_macros = {"public": [], "internal": [], "private":[]} |
| all_macros["public"] = self.parse_macros([ |
| "include/mbedtls/*.h", |
| "include/psa/*.h", |
| "3rdparty/everest/include/everest/everest.h", |
| "3rdparty/everest/include/everest/x25519.h" |
| ]) |
| all_macros["internal"] = self.parse_macros([ |
| "library/*.h", |
| "tests/include/test/drivers/*.h", |
| ]) |
| all_macros["private"] = self.parse_macros([ |
| "library/*.c", |
| ]) |
| enum_consts = self.parse_enum_consts([ |
| "include/mbedtls/*.h", |
| "include/psa/*.h", |
| "library/*.h", |
| "library/*.c", |
| "3rdparty/everest/include/everest/everest.h", |
| "3rdparty/everest/include/everest/x25519.h" |
| ]) |
| identifiers, excluded_identifiers = self.parse_identifiers([ |
| "include/mbedtls/*.h", |
| "include/psa/*.h", |
| "library/*.h", |
| "3rdparty/everest/include/everest/everest.h", |
| "3rdparty/everest/include/everest/x25519.h" |
| ], ["3rdparty/p256-m/p256-m/p256-m.h"]) |
| mbed_psa_words = self.parse_mbed_psa_words([ |
| "include/mbedtls/*.h", |
| "include/psa/*.h", |
| "library/*.h", |
| "3rdparty/everest/include/everest/everest.h", |
| "3rdparty/everest/include/everest/x25519.h", |
| "library/*.c", |
| "3rdparty/everest/library/everest.c", |
| "3rdparty/everest/library/x25519.c" |
| ], ["library/psa_crypto_driver_wrappers.h"]) |
| symbols = self.parse_symbols() |
| |
| # Remove identifier macros like mbedtls_printf or mbedtls_calloc |
| identifiers_justname = [x.name for x in identifiers] |
| actual_macros = {"public": [], "internal": []} |
| for scope in actual_macros: |
| for macro in all_macros[scope]: |
| if macro.name not in identifiers_justname: |
| actual_macros[scope].append(macro) |
| |
| self.log.debug("Found:") |
| # Aligns the counts on the assumption that none exceeds 4 digits |
| for scope in actual_macros: |
| self.log.debug(" {:4} Total {} Macros" |
| .format(len(all_macros[scope]), scope)) |
| self.log.debug(" {:4} {} Non-identifier Macros" |
| .format(len(actual_macros[scope]), scope)) |
| self.log.debug(" {:4} Enum Constants".format(len(enum_consts))) |
| self.log.debug(" {:4} Identifiers".format(len(identifiers))) |
| self.log.debug(" {:4} Exported Symbols".format(len(symbols))) |
| return { |
| "public_macros": actual_macros["public"], |
| "internal_macros": actual_macros["internal"], |
| "private_macros": all_macros["private"], |
| "enum_consts": enum_consts, |
| "identifiers": identifiers, |
| "excluded_identifiers": excluded_identifiers, |
| "symbols": symbols, |
| "mbed_psa_words": mbed_psa_words |
| } |
| |
| def is_file_excluded(self, path, exclude_wildcards): |
| """Whether the given file path is excluded.""" |
| # exclude_wildcards may be None. Also, consider the global exclusions. |
| exclude_wildcards = (exclude_wildcards or []) + self.excluded_files |
| for pattern in exclude_wildcards: |
| if fnmatch.fnmatch(path, pattern): |
| return True |
| return False |
| |
| def get_all_files(self, include_wildcards, exclude_wildcards): |
| """ |
| Get all files that match any of the included UNIX-style wildcards |
| and filter them into included and excluded lists. |
| While the check_names script is designed only for use on UNIX/macOS |
| (due to nm), this function alone will work fine on Windows even with |
| forward slashes in the wildcard. |
| |
| Args: |
| * include_wildcards: a List of shell-style wildcards to match filepaths. |
| * exclude_wildcards: a List of shell-style wildcards to exclude. |
| |
| Returns: |
| * inc_files: A List of relative filepaths for included files. |
| * exc_files: A List of relative filepaths for excluded files. |
| """ |
| accumulator = set() |
| all_wildcards = include_wildcards + (exclude_wildcards or []) |
| for wildcard in all_wildcards: |
| accumulator = accumulator.union(glob.iglob(wildcard)) |
| |
| inc_files = [] |
| exc_files = [] |
| for path in accumulator: |
| if self.is_file_excluded(path, exclude_wildcards): |
| exc_files.append(path) |
| else: |
| inc_files.append(path) |
| return (inc_files, exc_files) |
| |
| def get_included_files(self, include_wildcards, exclude_wildcards): |
| """ |
| Get all files that match any of the included UNIX-style wildcards. |
| While the check_names script is designed only for use on UNIX/macOS |
| (due to nm), this function alone will work fine on Windows even with |
| forward slashes in the wildcard. |
| |
| Args: |
| * include_wildcards: a List of shell-style wildcards to match filepaths. |
| * exclude_wildcards: a List of shell-style wildcards to exclude. |
| |
| Returns a List of relative filepaths. |
| """ |
| accumulator = set() |
| |
| for include_wildcard in include_wildcards: |
| accumulator = accumulator.union(glob.iglob(include_wildcard)) |
| |
| return list(path for path in accumulator |
| if not self.is_file_excluded(path, exclude_wildcards)) |
| |
| def parse_macros(self, include, exclude=None): |
| """ |
| Parse all macros defined by #define preprocessor directives. |
| |
| Args: |
| * include: A List of glob expressions to look for files through. |
| * exclude: A List of glob expressions for excluding files. |
| |
| Returns a List of Match objects for the found macros. |
| """ |
| macro_regex = re.compile(r"# *define +(?P<macro>\w+)") |
| exclusions = ( |
| "asm", "inline", "EMIT", "_CRT_SECURE_NO_DEPRECATE", "MULADDC_" |
| ) |
| |
| files = self.get_included_files(include, exclude) |
| self.log.debug("Looking for macros in {} files".format(len(files))) |
| |
| macros = [] |
| for header_file in files: |
| with open(header_file, "r", encoding="utf-8") as header: |
| for line_no, line in enumerate(header): |
| for macro in macro_regex.finditer(line): |
| if macro.group("macro").startswith(exclusions): |
| continue |
| |
| macros.append(Match( |
| header_file, |
| line, |
| line_no, |
| macro.span("macro"), |
| macro.group("macro"))) |
| |
| return macros |
| |
| def parse_mbed_psa_words(self, include, exclude=None): |
| """ |
| Parse all words in the file that begin with MBED|PSA, in and out of |
| macros, comments, anything. |
| |
| Args: |
| * include: A List of glob expressions to look for files through. |
| * exclude: A List of glob expressions for excluding files. |
| |
| Returns a List of Match objects for words beginning with MBED|PSA. |
| """ |
| # Typos of TLS are common, hence the broader check below than MBEDTLS. |
| mbed_regex = re.compile(r"\b(MBED.+?|PSA)_[A-Z0-9_]*") |
| exclusions = re.compile(r"// *no-check-names|#error") |
| |
| files = self.get_included_files(include, exclude) |
| self.log.debug( |
| "Looking for MBED|PSA words in {} files" |
| .format(len(files)) |
| ) |
| |
| mbed_psa_words = [] |
| for filename in files: |
| with open(filename, "r", encoding="utf-8") as fp: |
| for line_no, line in enumerate(fp): |
| if exclusions.search(line): |
| continue |
| |
| for name in mbed_regex.finditer(line): |
| mbed_psa_words.append(Match( |
| filename, |
| line, |
| line_no, |
| name.span(0), |
| name.group(0))) |
| |
| return mbed_psa_words |
| |
| def parse_enum_consts(self, include, exclude=None): |
| """ |
| Parse all enum value constants that are declared. |
| |
| Args: |
| * include: A List of glob expressions to look for files through. |
| * exclude: A List of glob expressions for excluding files. |
| |
| Returns a List of Match objects for the findings. |
| """ |
| files = self.get_included_files(include, exclude) |
| self.log.debug("Looking for enum consts in {} files".format(len(files))) |
| |
| # Emulate a finite state machine to parse enum declarations. |
| # OUTSIDE_KEYWORD = outside the enum keyword |
| # IN_BRACES = inside enum opening braces |
| # IN_BETWEEN = between enum keyword and opening braces |
| states = enum.Enum("FSM", ["OUTSIDE_KEYWORD", "IN_BRACES", "IN_BETWEEN"]) |
| enum_consts = [] |
| for header_file in files: |
| state = states.OUTSIDE_KEYWORD |
| with open(header_file, "r", encoding="utf-8") as header: |
| for line_no, line in enumerate(header): |
| # Match typedefs and brackets only when they are at the |
| # beginning of the line -- if they are indented, they might |
| # be sub-structures within structs, etc. |
| optional_c_identifier = r"([_a-zA-Z][_a-zA-Z0-9]*)?" |
| if (state == states.OUTSIDE_KEYWORD and |
| re.search(r"^(typedef +)?enum " + \ |
| optional_c_identifier + \ |
| r" *{", line)): |
| state = states.IN_BRACES |
| elif (state == states.OUTSIDE_KEYWORD and |
| re.search(r"^(typedef +)?enum", line)): |
| state = states.IN_BETWEEN |
| elif (state == states.IN_BETWEEN and |
| re.search(r"^{", line)): |
| state = states.IN_BRACES |
| elif (state == states.IN_BRACES and |
| re.search(r"^}", line)): |
| state = states.OUTSIDE_KEYWORD |
| elif (state == states.IN_BRACES and |
| not re.search(r"^ *#", line)): |
| enum_const = re.search(r"^ *(?P<enum_const>\w+)", line) |
| if not enum_const: |
| continue |
| |
| enum_consts.append(Match( |
| header_file, |
| line, |
| line_no, |
| enum_const.span("enum_const"), |
| enum_const.group("enum_const"))) |
| |
| return enum_consts |
| |
| IGNORED_CHUNK_REGEX = re.compile('|'.join([ |
| r'/\*.*?\*/', # block comment entirely on one line |
| r'//.*', # line comment |
| r'(?P<string>")(?:[^\\\"]|\\.)*"', # string literal |
| ])) |
| |
| def strip_comments_and_literals(self, line, in_block_comment): |
| """Strip comments and string literals from line. |
| |
| Continuation lines are not supported. |
| |
| If in_block_comment is true, assume that the line starts inside a |
| block comment. |
| |
| Return updated values of (line, in_block_comment) where: |
| * Comments in line have been replaced by a space (or nothing at the |
| start or end of the line). |
| * String contents have been removed. |
| * in_block_comment indicates whether the line ends inside a block |
| comment that continues on the next line. |
| """ |
| |
| # Terminate current multiline comment? |
| if in_block_comment: |
| m = re.search(r"\*/", line) |
| if m: |
| in_block_comment = False |
| line = line[m.end(0):] |
| else: |
| return '', True |
| |
| # Remove full comments and string literals. |
| # Do it all together to handle cases like "/*" correctly. |
| # Note that continuation lines are not supported. |
| line = re.sub(self.IGNORED_CHUNK_REGEX, |
| lambda s: '""' if s.group('string') else ' ', |
| line) |
| |
| # Start an unfinished comment? |
| # (If `/*` was part of a complete comment, it's already been removed.) |
| m = re.search(r"/\*", line) |
| if m: |
| in_block_comment = True |
| line = line[:m.start(0)] |
| |
| return line, in_block_comment |
| |
| IDENTIFIER_REGEX = re.compile('|'.join([ |
| # Match " something(a" or " *something(a". Functions. |
| # Assumptions: |
| # - function definition from return type to one of its arguments is |
| # all on one line |
| # - function definition line only contains alphanumeric, asterisk, |
| # underscore, and open bracket |
| r".* \**(\w+) *\( *\w", |
| # Match "(*something)(". |
| r".*\( *\* *(\w+) *\) *\(", |
| # Match names of named data structures. |
| r"(?:typedef +)?(?:struct|union|enum) +(\w+)(?: *{)?$", |
| # Match names of typedef instances, after closing bracket. |
| r"}? *(\w+)[;[].*", |
| ])) |
| # The regex below is indented for clarity. |
| EXCLUSION_LINES = re.compile("|".join([ |
| r"extern +\"C\"", |
| r"(typedef +)?(struct|union|enum)( *{)?$", |
| r"} *;?$", |
| r"$", |
| r"//", |
| r"#", |
| ])) |
| |
| def parse_identifiers_in_file(self, header_file, identifiers): |
| """ |
| Parse all lines of a header where a function/enum/struct/union/typedef |
| identifier is declared, based on some regex and heuristics. Highly |
| dependent on formatting style. |
| |
| Append found matches to the list ``identifiers``. |
| """ |
| |
| with open(header_file, "r", encoding="utf-8") as header: |
| in_block_comment = False |
| # The previous line variable is used for concatenating lines |
| # when identifiers are formatted and spread across multiple |
| # lines. |
| previous_line = "" |
| |
| for line_no, line in enumerate(header): |
| line, in_block_comment = \ |
| self.strip_comments_and_literals(line, in_block_comment) |
| |
| if self.EXCLUSION_LINES.match(line): |
| previous_line = "" |
| continue |
| |
| # If the line contains only space-separated alphanumeric |
| # characters (or underscore, asterisk, or open parenthesis), |
| # and nothing else, high chance it's a declaration that |
| # continues on the next line |
| if re.search(r"^([\w\*\(]+\s+)+$", line): |
| previous_line += line |
| continue |
| |
| # If previous line seemed to start an unfinished declaration |
| # (as above), concat and treat them as one. |
| if previous_line: |
| line = previous_line.strip() + " " + line.strip() + "\n" |
| previous_line = "" |
| |
| # Skip parsing if line has a space in front = heuristic to |
| # skip function argument lines (highly subject to formatting |
| # changes) |
| if line[0] == " ": |
| continue |
| |
| identifier = self.IDENTIFIER_REGEX.search(line) |
| |
| if not identifier: |
| continue |
| |
| # Find the group that matched, and append it |
| for group in identifier.groups(): |
| if not group: |
| continue |
| |
| identifiers.append(Match( |
| header_file, |
| line, |
| line_no, |
| identifier.span(), |
| group)) |
| |
| def parse_identifiers(self, include, exclude=None): |
| """ |
| Parse all lines of a header where a function/enum/struct/union/typedef |
| identifier is declared, based on some regex and heuristics. Highly |
| dependent on formatting style. Identifiers in excluded files are still |
| parsed |
| |
| Args: |
| * include: A List of glob expressions to look for files through. |
| * exclude: A List of glob expressions for excluding files. |
| |
| Returns: a Tuple of two Lists of Match objects with identifiers. |
| * included_identifiers: A List of Match objects with identifiers from |
| included files. |
| * excluded_identifiers: A List of Match objects with identifiers from |
| excluded files. |
| """ |
| |
| included_files, excluded_files = \ |
| self.get_all_files(include, exclude) |
| |
| self.log.debug("Looking for included identifiers in {} files".format \ |
| (len(included_files))) |
| |
| included_identifiers = [] |
| excluded_identifiers = [] |
| for header_file in included_files: |
| self.parse_identifiers_in_file(header_file, included_identifiers) |
| for header_file in excluded_files: |
| self.parse_identifiers_in_file(header_file, excluded_identifiers) |
| |
| return (included_identifiers, excluded_identifiers) |
| |
| def parse_symbols(self): |
| """ |
| Compile the Mbed TLS libraries, and parse the TLS, Crypto, and x509 |
| object files using nm to retrieve the list of referenced symbols. |
| Exceptions thrown here are rethrown because they would be critical |
| errors that void several tests, and thus needs to halt the program. This |
| is explicitly done for clarity. |
| |
| Returns a List of unique symbols defined and used in the libraries. |
| """ |
| self.log.info("Compiling...") |
| symbols = [] |
| |
| # Back up the config and atomically compile with the full configuration. |
| shutil.copy( |
| "include/mbedtls/mbedtls_config.h", |
| "include/mbedtls/mbedtls_config.h.bak" |
| ) |
| try: |
| # Use check=True in all subprocess calls so that failures are raised |
| # as exceptions and logged. |
| subprocess.run( |
| ["python3", "scripts/config.py", "full"], |
| universal_newlines=True, |
| check=True |
| ) |
| my_environment = os.environ.copy() |
| my_environment["CFLAGS"] = "-fno-asynchronous-unwind-tables" |
| # Run make clean separately to lib to prevent unwanted behavior when |
| # make is invoked with parallelism. |
| subprocess.run( |
| ["make", "clean"], |
| universal_newlines=True, |
| check=True |
| ) |
| subprocess.run( |
| ["make", "lib"], |
| env=my_environment, |
| universal_newlines=True, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT, |
| check=True |
| ) |
| |
| # Perform object file analysis using nm |
| symbols = self.parse_symbols_from_nm([ |
| "library/libmbedcrypto.a", |
| "library/libmbedtls.a", |
| "library/libmbedx509.a" |
| ]) |
| |
| subprocess.run( |
| ["make", "clean"], |
| universal_newlines=True, |
| check=True |
| ) |
| except subprocess.CalledProcessError as error: |
| self.log.debug(error.output) |
| raise error |
| finally: |
| # Put back the original config regardless of there being errors. |
| # Works also for keyboard interrupts. |
| shutil.move( |
| "include/mbedtls/mbedtls_config.h.bak", |
| "include/mbedtls/mbedtls_config.h" |
| ) |
| |
| return symbols |
| |
| def parse_symbols_from_nm(self, object_files): |
| """ |
| Run nm to retrieve the list of referenced symbols in each object file. |
| Does not return the position data since it is of no use. |
| |
| Args: |
| * object_files: a List of compiled object filepaths to search through. |
| |
| Returns a List of unique symbols defined and used in any of the object |
| files. |
| """ |
| nm_undefined_regex = re.compile(r"^\S+: +U |^$|^\S+:$") |
| nm_valid_regex = re.compile(r"^\S+( [0-9A-Fa-f]+)* . _*(?P<symbol>\w+)") |
| exclusions = ("FStar", "Hacl") |
| |
| symbols = [] |
| |
| # Gather all outputs of nm |
| nm_output = "" |
| for lib in object_files: |
| nm_output += subprocess.run( |
| ["nm", "-og", lib], |
| universal_newlines=True, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT, |
| check=True |
| ).stdout |
| |
| for line in nm_output.splitlines(): |
| if not nm_undefined_regex.search(line): |
| symbol = nm_valid_regex.search(line) |
| if (symbol and not symbol.group("symbol").startswith(exclusions)): |
| symbols.append(symbol.group("symbol")) |
| else: |
| self.log.error(line) |
| |
| return symbols |
| |
| class NameChecker(): |
| """ |
| Representation of the core name checking operation performed by this script. |
| """ |
| def __init__(self, parse_result, log): |
| self.parse_result = parse_result |
| self.log = log |
| |
| def perform_checks(self, quiet=False): |
| """ |
| A comprehensive checker that performs each check in order, and outputs |
| a final verdict. |
| |
| Args: |
| * quiet: whether to hide detailed problem explanation. |
| """ |
| self.log.info("=============") |
| Problem.quiet = quiet |
| problems = 0 |
| problems += self.check_symbols_declared_in_header() |
| |
| pattern_checks = [ |
| ("public_macros", PUBLIC_MACRO_PATTERN), |
| ("internal_macros", INTERNAL_MACRO_PATTERN), |
| ("enum_consts", CONSTANTS_PATTERN), |
| ("identifiers", IDENTIFIER_PATTERN) |
| ] |
| for group, check_pattern in pattern_checks: |
| problems += self.check_match_pattern(group, check_pattern) |
| |
| problems += self.check_for_typos() |
| |
| self.log.info("=============") |
| if problems > 0: |
| self.log.info("FAIL: {0} problem(s) to fix".format(str(problems))) |
| if quiet: |
| self.log.info("Remove --quiet to see explanations.") |
| else: |
| self.log.info("Use --quiet for minimal output.") |
| return 1 |
| else: |
| self.log.info("PASS") |
| return 0 |
| |
| def check_symbols_declared_in_header(self): |
| """ |
| Perform a check that all detected symbols in the library object files |
| are properly declared in headers. |
| Assumes parse_names_in_source() was called before this. |
| |
| Returns the number of problems that need fixing. |
| """ |
| problems = [] |
| all_identifiers = self.parse_result["identifiers"] + \ |
| self.parse_result["excluded_identifiers"] |
| |
| for symbol in self.parse_result["symbols"]: |
| found_symbol_declared = False |
| for identifier_match in all_identifiers: |
| if symbol == identifier_match.name: |
| found_symbol_declared = True |
| break |
| |
| if not found_symbol_declared: |
| problems.append(SymbolNotInHeader(symbol)) |
| |
| self.output_check_result("All symbols in header", problems) |
| return len(problems) |
| |
| def check_match_pattern(self, group_to_check, check_pattern): |
| """ |
| Perform a check that all items of a group conform to a regex pattern. |
| Assumes parse_names_in_source() was called before this. |
| |
| Args: |
| * group_to_check: string key to index into self.parse_result. |
| * check_pattern: the regex to check against. |
| |
| Returns the number of problems that need fixing. |
| """ |
| problems = [] |
| |
| for item_match in self.parse_result[group_to_check]: |
| if not re.search(check_pattern, item_match.name): |
| problems.append(PatternMismatch(check_pattern, item_match)) |
| # Double underscore should not be used for names |
| if re.search(r".*__.*", item_match.name): |
| problems.append( |
| PatternMismatch("no double underscore allowed", item_match)) |
| |
| self.output_check_result( |
| "Naming patterns of {}".format(group_to_check), |
| problems) |
| return len(problems) |
| |
| def check_for_typos(self): |
| """ |
| Perform a check that all words in the source code beginning with MBED are |
| either defined as macros, or as enum constants. |
| Assumes parse_names_in_source() was called before this. |
| |
| Returns the number of problems that need fixing. |
| """ |
| problems = [] |
| |
| # Set comprehension, equivalent to a list comprehension wrapped by set() |
| all_caps_names = { |
| match.name |
| for match |
| in self.parse_result["public_macros"] + |
| self.parse_result["internal_macros"] + |
| self.parse_result["private_macros"] + |
| self.parse_result["enum_consts"] |
| } |
| typo_exclusion = re.compile(r"XXX|__|_$|^MBEDTLS_.*CONFIG_FILE$|" |
| r"MBEDTLS_TEST_LIBTESTDRIVER*|" |
| r"PSA_CRYPTO_DRIVER_TEST") |
| |
| for name_match in self.parse_result["mbed_psa_words"]: |
| found = name_match.name in all_caps_names |
| |
| # Since MBEDTLS_PSA_ACCEL_XXX defines are defined by the |
| # PSA driver, they will not exist as macros. However, they |
| # should still be checked for typos using the equivalent |
| # BUILTINs that exist. |
| if "MBEDTLS_PSA_ACCEL_" in name_match.name: |
| found = name_match.name.replace( |
| "MBEDTLS_PSA_ACCEL_", |
| "MBEDTLS_PSA_BUILTIN_") in all_caps_names |
| |
| if not found and not typo_exclusion.search(name_match.name): |
| problems.append(Typo(name_match)) |
| |
| self.output_check_result("Likely typos", problems) |
| return len(problems) |
| |
| def output_check_result(self, name, problems): |
| """ |
| Write out the PASS/FAIL status of a performed check depending on whether |
| there were problems. |
| |
| Args: |
| * name: the name of the test |
| * problems: a List of encountered Problems |
| """ |
| if problems: |
| self.log.info("{}: FAIL\n".format(name)) |
| for problem in problems: |
| self.log.warning(str(problem)) |
| else: |
| self.log.info("{}: PASS".format(name)) |
| |
| def main(): |
| """ |
| Perform argument parsing, and create an instance of CodeParser and |
| NameChecker to begin the core operation. |
| """ |
| parser = argparse.ArgumentParser( |
| formatter_class=argparse.RawDescriptionHelpFormatter, |
| description=( |
| "This script confirms that the naming of all symbols and identifiers " |
| "in Mbed TLS are consistent with the house style and are also " |
| "self-consistent.\n\n" |
| "Expected to be run from the Mbed TLS root directory.") |
| ) |
| parser.add_argument( |
| "-v", "--verbose", |
| action="store_true", |
| help="show parse results" |
| ) |
| parser.add_argument( |
| "-q", "--quiet", |
| action="store_true", |
| help="hide unnecessary text, explanations, and highlights" |
| ) |
| |
| args = parser.parse_args() |
| |
| # Configure the global logger, which is then passed to the classes below |
| log = logging.getLogger() |
| log.setLevel(logging.DEBUG if args.verbose else logging.INFO) |
| log.addHandler(logging.StreamHandler()) |
| |
| try: |
| code_parser = CodeParser(log) |
| parse_result = code_parser.comprehensive_parse() |
| except Exception: # pylint: disable=broad-except |
| traceback.print_exc() |
| sys.exit(2) |
| |
| name_checker = NameChecker(parse_result, log) |
| return_code = name_checker.perform_checks(quiet=args.quiet) |
| |
| sys.exit(return_code) |
| |
| if __name__ == "__main__": |
| main() |