| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 | 
|  | 2 | # | 
|  | 3 | # Copyright The Mbed TLS Contributors | 
|  | 4 | # SPDX-License-Identifier: Apache-2.0 | 
|  | 5 | # | 
|  | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may | 
|  | 7 | # not use this file except in compliance with the License. | 
|  | 8 | # You may obtain a copy of the License at | 
|  | 9 | # | 
|  | 10 | # http://www.apache.org/licenses/LICENSE-2.0 | 
|  | 11 | # | 
|  | 12 | # Unless required by applicable law or agreed to in writing, software | 
|  | 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | 
|  | 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 
|  | 15 | # See the License for the specific language governing permissions and | 
|  | 16 | # limitations under the License. | 
|  | 17 |  | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 18 | """ | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 19 | This script confirms that the naming of all symbols and identifiers in Mbed TLS | 
| Yuto Takano | 159255a | 2021-08-06 17:00:28 +0100 | [diff] [blame] | 20 | are consistent with the house style and are also self-consistent. It only runs | 
|  | 21 | on Linux and macOS since it depends on nm. | 
|  | 22 |  | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 23 | It contains two major Python classes, CodeParser and NameChecker. They both have | 
|  | 24 | a comprehensive "run-all" function (comprehensive_parse() and perform_checks()) | 
|  | 25 | but the individual functions can also be used for specific needs. | 
|  | 26 |  | 
|  | 27 | CodeParser makes heavy use of regular expressions to parse the code, and is | 
|  | 28 | dependent on the current code formatting. Many Python C parser libraries require | 
|  | 29 | preprocessed C code, which means no macro parsing. Compiler tools are also not | 
|  | 30 | very helpful when we want the exact location in the original source (which | 
|  | 31 | becomes impossible when e.g. comments are stripped). | 
|  | 32 |  | 
|  | 33 | NameChecker performs the following checks: | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 34 |  | 
|  | 35 | - All exported and available symbols in the library object files, are explicitly | 
| Yuto Takano | 159255a | 2021-08-06 17:00:28 +0100 | [diff] [blame] | 36 | declared in the header files. This uses the nm command. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 37 | - All macros, constants, and identifiers (function names, struct names, etc) | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 38 | follow the required regex pattern. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 39 | - Typo checking: All words that begin with MBED exist as macros or constants. | 
| Yuto Takano | fc54dfb | 2021-08-07 17:18:28 +0100 | [diff] [blame] | 40 |  | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 41 | The script returns 0 on success, 1 on test failure, and 2 if there is a script | 
| Yuto Takano | 8246eb8 | 2021-08-16 10:37:24 +0100 | [diff] [blame] | 42 | error. It must be run from Mbed TLS root. | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 43 | """ | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 44 |  | 
| Yuto Takano | fc1e9ff | 2021-08-23 13:54:56 +0100 | [diff] [blame] | 45 | import abc | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 46 | import argparse | 
| Gilles Peskine | 89458d1 | 2021-09-27 19:20:17 +0200 | [diff] [blame] | 47 | import fnmatch | 
| Yuto Takano | 977e07f | 2021-08-09 11:56:15 +0100 | [diff] [blame] | 48 | import glob | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 49 | import textwrap | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 50 | import os | 
|  | 51 | import sys | 
|  | 52 | import traceback | 
|  | 53 | import re | 
| Yuto Takano | b1417b4 | 2021-08-17 10:30:20 +0100 | [diff] [blame] | 54 | import enum | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 55 | import shutil | 
|  | 56 | import subprocess | 
|  | 57 | import logging | 
|  | 58 |  | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 59 | # Naming patterns to check against. These are defined outside the NameCheck | 
|  | 60 | # class for ease of modification. | 
| Yuto Takano | bb7dca4 | 2021-08-05 19:57:58 +0100 | [diff] [blame] | 61 | MACRO_PATTERN = r"^(MBEDTLS|PSA)_[0-9A-Z_]*[0-9A-Z]$" | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 62 | CONSTANTS_PATTERN = MACRO_PATTERN | 
| Yuto Takano | c183893 | 2021-08-05 19:52:09 +0100 | [diff] [blame] | 63 | IDENTIFIER_PATTERN = r"^(mbedtls|psa)_[0-9a-z_]*[0-9a-z]$" | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 64 |  | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 65 | class Match(): # pylint: disable=too-few-public-methods | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 66 | """ | 
|  | 67 | A class representing a match, together with its found position. | 
|  | 68 |  | 
|  | 69 | Fields: | 
|  | 70 | * filename: the file that the match was in. | 
|  | 71 | * line: the full line containing the match. | 
| Yuto Takano | 704b0f7 | 2021-08-17 10:41:23 +0100 | [diff] [blame] | 72 | * line_no: the line number. | 
|  | 73 | * pos: a tuple of (start, end) positions on the line where the match is. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 74 | * name: the match itself. | 
|  | 75 | """ | 
| Yuto Takano | 704b0f7 | 2021-08-17 10:41:23 +0100 | [diff] [blame] | 76 | def __init__(self, filename, line, line_no, pos, name): | 
|  | 77 | # pylint: disable=too-many-arguments | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 78 | self.filename = filename | 
|  | 79 | self.line = line | 
| Yuto Takano | 704b0f7 | 2021-08-17 10:41:23 +0100 | [diff] [blame] | 80 | self.line_no = line_no | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 81 | self.pos = pos | 
|  | 82 | self.name = name | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 83 |  | 
| Yuto Takano | a4e7512 | 2021-08-06 17:23:28 +0100 | [diff] [blame] | 84 | def __str__(self): | 
| Yuto Takano | fb86ac7 | 2021-08-16 10:32:40 +0100 | [diff] [blame] | 85 | """ | 
|  | 86 | Return a formatted code listing representation of the erroneous line. | 
|  | 87 | """ | 
| Yuto Takano | 704b0f7 | 2021-08-17 10:41:23 +0100 | [diff] [blame] | 88 | gutter = format(self.line_no, "4d") | 
|  | 89 | underline = self.pos[0] * " " + (self.pos[1] - self.pos[0]) * "^" | 
| Yuto Takano | 381fda8 | 2021-08-06 23:37:20 +0100 | [diff] [blame] | 90 |  | 
| Yuto Takano | a4e7512 | 2021-08-06 17:23:28 +0100 | [diff] [blame] | 91 | return ( | 
| Yuto Takano | fb86ac7 | 2021-08-16 10:32:40 +0100 | [diff] [blame] | 92 | " {0} |\n".format(" " * len(gutter)) + | 
| Yuto Takano | 381fda8 | 2021-08-06 23:37:20 +0100 | [diff] [blame] | 93 | " {0} | {1}".format(gutter, self.line) + | 
| Yuto Takano | fb86ac7 | 2021-08-16 10:32:40 +0100 | [diff] [blame] | 94 | " {0} | {1}\n".format(" " * len(gutter), underline) | 
| Yuto Takano | a4e7512 | 2021-08-06 17:23:28 +0100 | [diff] [blame] | 95 | ) | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 96 |  | 
| Yuto Takano | fc1e9ff | 2021-08-23 13:54:56 +0100 | [diff] [blame] | 97 | class Problem(abc.ABC): # pylint: disable=too-few-public-methods | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 98 | """ | 
| Yuto Takano | fc1e9ff | 2021-08-23 13:54:56 +0100 | [diff] [blame] | 99 | An abstract parent class representing a form of static analysis error. | 
|  | 100 | It extends an Abstract Base Class, which means it is not instantiable, and | 
|  | 101 | it also mandates certain abstract methods to be implemented in subclasses. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 102 | """ | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 103 | # Class variable to control the quietness of all problems | 
|  | 104 | quiet = False | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 105 | def __init__(self): | 
|  | 106 | self.textwrapper = textwrap.TextWrapper() | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 107 | self.textwrapper.width = 80 | 
| Yuto Takano | a4e7512 | 2021-08-06 17:23:28 +0100 | [diff] [blame] | 108 | self.textwrapper.initial_indent = "    > " | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 109 | self.textwrapper.subsequent_indent = "      " | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 110 |  | 
| Yuto Takano | fc1e9ff | 2021-08-23 13:54:56 +0100 | [diff] [blame] | 111 | def __str__(self): | 
|  | 112 | """ | 
|  | 113 | Unified string representation method for all Problems. | 
|  | 114 | """ | 
|  | 115 | if self.__class__.quiet: | 
|  | 116 | return self.quiet_output() | 
|  | 117 | return self.verbose_output() | 
|  | 118 |  | 
|  | 119 | @abc.abstractmethod | 
|  | 120 | def quiet_output(self): | 
|  | 121 | """ | 
|  | 122 | The output when --quiet is enabled. | 
|  | 123 | """ | 
|  | 124 | pass | 
|  | 125 |  | 
|  | 126 | @abc.abstractmethod | 
|  | 127 | def verbose_output(self): | 
|  | 128 | """ | 
|  | 129 | The default output with explanation and code snippet if appropriate. | 
|  | 130 | """ | 
|  | 131 | pass | 
|  | 132 |  | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 133 | class SymbolNotInHeader(Problem): # pylint: disable=too-few-public-methods | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 134 | """ | 
|  | 135 | A problem that occurs when an exported/available symbol in the object file | 
|  | 136 | is not explicitly declared in header files. Created with | 
|  | 137 | NameCheck.check_symbols_declared_in_header() | 
|  | 138 |  | 
|  | 139 | Fields: | 
|  | 140 | * symbol_name: the name of the symbol. | 
|  | 141 | """ | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 142 | def __init__(self, symbol_name): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 143 | self.symbol_name = symbol_name | 
|  | 144 | Problem.__init__(self) | 
|  | 145 |  | 
| Yuto Takano | fc1e9ff | 2021-08-23 13:54:56 +0100 | [diff] [blame] | 146 | def quiet_output(self): | 
|  | 147 | return "{0}".format(self.symbol_name) | 
| Yuto Takano | 55614b5 | 2021-08-07 01:00:18 +0100 | [diff] [blame] | 148 |  | 
| Yuto Takano | fc1e9ff | 2021-08-23 13:54:56 +0100 | [diff] [blame] | 149 | def verbose_output(self): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 150 | return self.textwrapper.fill( | 
|  | 151 | "'{0}' was found as an available symbol in the output of nm, " | 
|  | 152 | "however it was not declared in any header files." | 
|  | 153 | .format(self.symbol_name)) | 
|  | 154 |  | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 155 | class PatternMismatch(Problem): # pylint: disable=too-few-public-methods | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 156 | """ | 
|  | 157 | A problem that occurs when something doesn't match the expected pattern. | 
|  | 158 | Created with NameCheck.check_match_pattern() | 
|  | 159 |  | 
|  | 160 | Fields: | 
|  | 161 | * pattern: the expected regex pattern | 
|  | 162 | * match: the Match object in question | 
|  | 163 | """ | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 164 | def __init__(self, pattern, match): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 165 | self.pattern = pattern | 
|  | 166 | self.match = match | 
|  | 167 | Problem.__init__(self) | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 168 |  | 
| Yuto Takano | 55614b5 | 2021-08-07 01:00:18 +0100 | [diff] [blame] | 169 |  | 
| Yuto Takano | fc1e9ff | 2021-08-23 13:54:56 +0100 | [diff] [blame] | 170 | def quiet_output(self): | 
|  | 171 | return ( | 
|  | 172 | "{0}:{1}:{2}" | 
|  | 173 | .format(self.match.filename, self.match.line_no, self.match.name) | 
|  | 174 | ) | 
|  | 175 |  | 
|  | 176 | def verbose_output(self): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 177 | return self.textwrapper.fill( | 
| Yuto Takano | a4e7512 | 2021-08-06 17:23:28 +0100 | [diff] [blame] | 178 | "{0}:{1}: '{2}' does not match the required pattern '{3}'." | 
|  | 179 | .format( | 
|  | 180 | self.match.filename, | 
| Yuto Takano | 5f83171 | 2021-08-18 18:03:24 +0100 | [diff] [blame] | 181 | self.match.line_no, | 
| Yuto Takano | a4e7512 | 2021-08-06 17:23:28 +0100 | [diff] [blame] | 182 | self.match.name, | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 183 | self.pattern | 
|  | 184 | ) | 
|  | 185 | ) + "\n" + str(self.match) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 186 |  | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 187 | class Typo(Problem): # pylint: disable=too-few-public-methods | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 188 | """ | 
|  | 189 | A problem that occurs when a word using MBED doesn't appear to be defined as | 
|  | 190 | constants nor enum values. Created with NameCheck.check_for_typos() | 
|  | 191 |  | 
|  | 192 | Fields: | 
|  | 193 | * match: the Match object of the MBED name in question. | 
|  | 194 | """ | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 195 | def __init__(self, match): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 196 | self.match = match | 
|  | 197 | Problem.__init__(self) | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 198 |  | 
| Yuto Takano | fc1e9ff | 2021-08-23 13:54:56 +0100 | [diff] [blame] | 199 | def quiet_output(self): | 
|  | 200 | return ( | 
|  | 201 | "{0}:{1}:{2}" | 
|  | 202 | .format(self.match.filename, self.match.line_no, self.match.name) | 
|  | 203 | ) | 
| Yuto Takano | 55614b5 | 2021-08-07 01:00:18 +0100 | [diff] [blame] | 204 |  | 
| Yuto Takano | fc1e9ff | 2021-08-23 13:54:56 +0100 | [diff] [blame] | 205 | def verbose_output(self): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 206 | return self.textwrapper.fill( | 
| Yuto Takano | a4e7512 | 2021-08-06 17:23:28 +0100 | [diff] [blame] | 207 | "{0}:{1}: '{2}' looks like a typo. It was not found in any " | 
|  | 208 | "macros or any enums. If this is not a typo, put " | 
|  | 209 | "//no-check-names after it." | 
| Yuto Takano | 5f83171 | 2021-08-18 18:03:24 +0100 | [diff] [blame] | 210 | .format(self.match.filename, self.match.line_no, self.match.name) | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 211 | ) + "\n" + str(self.match) | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 212 |  | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 213 | class CodeParser(): | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 214 | """ | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 215 | Class for retrieving files and parsing the code. This can be used | 
|  | 216 | independently of the checks that NameChecker performs, for example for | 
|  | 217 | list_internal_identifiers.py. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 218 | """ | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 219 | def __init__(self, log): | 
|  | 220 | self.log = log | 
| Yuto Takano | fc54dfb | 2021-08-07 17:18:28 +0100 | [diff] [blame] | 221 | self.check_repo_path() | 
| Yuto Takano | 977e07f | 2021-08-09 11:56:15 +0100 | [diff] [blame] | 222 |  | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 223 | # Memo for storing "glob expression": set(filepaths) | 
|  | 224 | self.files = {} | 
|  | 225 |  | 
| Gilles Peskine | 89458d1 | 2021-09-27 19:20:17 +0200 | [diff] [blame] | 226 | # Globally excluded filenames. | 
|  | 227 | # Note that "*" can match directory separators in exclude lists. | 
|  | 228 | self.excluded_files = ["*/bn_mul", "*/compat-2.x.h"] | 
| Yuto Takano | 977e07f | 2021-08-09 11:56:15 +0100 | [diff] [blame] | 229 |  | 
| Yuto Takano | fc54dfb | 2021-08-07 17:18:28 +0100 | [diff] [blame] | 230 | @staticmethod | 
|  | 231 | def check_repo_path(): | 
|  | 232 | """ | 
|  | 233 | Check that the current working directory is the project root, and throw | 
|  | 234 | an exception if not. | 
|  | 235 | """ | 
|  | 236 | if not all(os.path.isdir(d) for d in ["include", "library", "tests"]): | 
|  | 237 | raise Exception("This script must be run from Mbed TLS root") | 
|  | 238 |  | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 239 | def comprehensive_parse(self): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 240 | """ | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 241 | Comprehensive ("default") function to call each parsing function and | 
|  | 242 | retrieve various elements of the code, together with the source location. | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 243 |  | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 244 | Returns a dict of parsed item key to the corresponding List of Matches. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 245 | """ | 
|  | 246 | self.log.info("Parsing source code...") | 
| Yuto Takano | d24e037 | 2021-08-06 16:42:33 +0100 | [diff] [blame] | 247 | self.log.debug( | 
| Yuto Takano | 5095343 | 2021-08-09 14:54:36 +0100 | [diff] [blame] | 248 | "The following files are excluded from the search: {}" | 
| Yuto Takano | d24e037 | 2021-08-06 16:42:33 +0100 | [diff] [blame] | 249 | .format(str(self.excluded_files)) | 
|  | 250 | ) | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 251 |  | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 252 | all_macros = self.parse_macros([ | 
|  | 253 | "include/mbedtls/*.h", | 
|  | 254 | "include/psa/*.h", | 
|  | 255 | "library/*.h", | 
|  | 256 | "tests/include/test/drivers/*.h", | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 257 | "3rdparty/everest/include/everest/everest.h", | 
|  | 258 | "3rdparty/everest/include/everest/x25519.h" | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 259 | ]) | 
|  | 260 | enum_consts = self.parse_enum_consts([ | 
|  | 261 | "include/mbedtls/*.h", | 
|  | 262 | "library/*.h", | 
|  | 263 | "3rdparty/everest/include/everest/everest.h", | 
|  | 264 | "3rdparty/everest/include/everest/x25519.h" | 
|  | 265 | ]) | 
|  | 266 | identifiers = self.parse_identifiers([ | 
|  | 267 | "include/mbedtls/*.h", | 
|  | 268 | "include/psa/*.h", | 
|  | 269 | "library/*.h", | 
|  | 270 | "3rdparty/everest/include/everest/everest.h", | 
|  | 271 | "3rdparty/everest/include/everest/x25519.h" | 
|  | 272 | ]) | 
|  | 273 | mbed_words = self.parse_mbed_words([ | 
|  | 274 | "include/mbedtls/*.h", | 
|  | 275 | "include/psa/*.h", | 
|  | 276 | "library/*.h", | 
|  | 277 | "3rdparty/everest/include/everest/everest.h", | 
|  | 278 | "3rdparty/everest/include/everest/x25519.h", | 
|  | 279 | "library/*.c", | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 280 | "3rdparty/everest/library/everest.c", | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 281 | "3rdparty/everest/library/x25519.c" | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 282 | ]) | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 283 | symbols = self.parse_symbols() | 
|  | 284 |  | 
|  | 285 | # Remove identifier macros like mbedtls_printf or mbedtls_calloc | 
|  | 286 | identifiers_justname = [x.name for x in identifiers] | 
|  | 287 | actual_macros = [] | 
|  | 288 | for macro in all_macros: | 
|  | 289 | if macro.name not in identifiers_justname: | 
|  | 290 | actual_macros.append(macro) | 
|  | 291 |  | 
|  | 292 | self.log.debug("Found:") | 
| Yuto Takano | 9d9c6dc | 2021-08-16 10:43:45 +0100 | [diff] [blame] | 293 | # Aligns the counts on the assumption that none exceeds 4 digits | 
|  | 294 | self.log.debug("  {:4} Total Macros".format(len(all_macros))) | 
|  | 295 | self.log.debug("  {:4} Non-identifier Macros".format(len(actual_macros))) | 
|  | 296 | self.log.debug("  {:4} Enum Constants".format(len(enum_consts))) | 
|  | 297 | self.log.debug("  {:4} Identifiers".format(len(identifiers))) | 
|  | 298 | self.log.debug("  {:4} Exported Symbols".format(len(symbols))) | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 299 | return { | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 300 | "macros": actual_macros, | 
|  | 301 | "enum_consts": enum_consts, | 
|  | 302 | "identifiers": identifiers, | 
|  | 303 | "symbols": symbols, | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 304 | "mbed_words": mbed_words | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 305 | } | 
|  | 306 |  | 
| Gilles Peskine | 89458d1 | 2021-09-27 19:20:17 +0200 | [diff] [blame] | 307 | def is_file_excluded(self, path, exclude_wildcards): | 
| Gilles Peskine | 8a83224 | 2021-09-28 10:12:49 +0200 | [diff] [blame] | 308 | """Whether the given file path is excluded.""" | 
| Gilles Peskine | 89458d1 | 2021-09-27 19:20:17 +0200 | [diff] [blame] | 309 | # exclude_wildcards may be None. Also, consider the global exclusions. | 
|  | 310 | exclude_wildcards = (exclude_wildcards or []) + self.excluded_files | 
|  | 311 | for pattern in exclude_wildcards: | 
|  | 312 | if fnmatch.fnmatch(path, pattern): | 
|  | 313 | return True | 
|  | 314 | return False | 
|  | 315 |  | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 316 | def get_files(self, include_wildcards, exclude_wildcards): | 
|  | 317 | """ | 
|  | 318 | Get all files that match any of the UNIX-style wildcards. While the | 
|  | 319 | check_names script is designed only for use on UNIX/macOS (due to nm), | 
|  | 320 | this function alone would work fine on Windows even with forward slashes | 
|  | 321 | in the wildcard. | 
|  | 322 |  | 
|  | 323 | Args: | 
|  | 324 | * include_wildcards: a List of shell-style wildcards to match filepaths. | 
|  | 325 | * exclude_wildcards: a List of shell-style wildcards to exclude. | 
|  | 326 |  | 
|  | 327 | Returns a List of relative filepaths. | 
|  | 328 | """ | 
|  | 329 | accumulator = set() | 
|  | 330 |  | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 331 | for include_wildcard in include_wildcards: | 
| Gilles Peskine | 89458d1 | 2021-09-27 19:20:17 +0200 | [diff] [blame] | 332 | accumulator = accumulator.union(glob.iglob(include_wildcard)) | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 333 |  | 
| Gilles Peskine | 89458d1 | 2021-09-27 19:20:17 +0200 | [diff] [blame] | 334 | return list(path for path in accumulator | 
|  | 335 | if not self.is_file_excluded(path, exclude_wildcards)) | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 336 |  | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 337 | def parse_macros(self, include, exclude=None): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 338 | """ | 
|  | 339 | Parse all macros defined by #define preprocessor directives. | 
|  | 340 |  | 
|  | 341 | Args: | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 342 | * include: A List of glob expressions to look for files through. | 
|  | 343 | * exclude: A List of glob expressions for excluding files. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 344 |  | 
|  | 345 | Returns a List of Match objects for the found macros. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 346 | """ | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 347 | macro_regex = re.compile(r"# *define +(?P<macro>\w+)") | 
|  | 348 | exclusions = ( | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 349 | "asm", "inline", "EMIT", "_CRT_SECURE_NO_DEPRECATE", "MULADDC_" | 
|  | 350 | ) | 
|  | 351 |  | 
| Yuto Takano | 5095343 | 2021-08-09 14:54:36 +0100 | [diff] [blame] | 352 | files = self.get_files(include, exclude) | 
|  | 353 | self.log.debug("Looking for macros in {} files".format(len(files))) | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 354 |  | 
| Yuto Takano | 5095343 | 2021-08-09 14:54:36 +0100 | [diff] [blame] | 355 | macros = [] | 
|  | 356 | for header_file in files: | 
| Yuto Takano | a083d15 | 2021-08-07 00:25:59 +0100 | [diff] [blame] | 357 | with open(header_file, "r", encoding="utf-8") as header: | 
| Yuto Takano | 8f457cf | 2021-08-06 17:54:58 +0100 | [diff] [blame] | 358 | for line_no, line in enumerate(header): | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 359 | for macro in macro_regex.finditer(line): | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 360 | if macro.group("macro").startswith(exclusions): | 
|  | 361 | continue | 
|  | 362 |  | 
|  | 363 | macros.append(Match( | 
|  | 364 | header_file, | 
|  | 365 | line, | 
| Yuto Takano | 704b0f7 | 2021-08-17 10:41:23 +0100 | [diff] [blame] | 366 | line_no, | 
|  | 367 | macro.span("macro"), | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 368 | macro.group("macro"))) | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 369 |  | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 370 | return macros | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 371 |  | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 372 | def parse_mbed_words(self, include, exclude=None): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 373 | """ | 
| Yuto Takano | b47b504 | 2021-08-07 00:42:54 +0100 | [diff] [blame] | 374 | Parse all words in the file that begin with MBED, in and out of macros, | 
|  | 375 | comments, anything. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 376 |  | 
|  | 377 | Args: | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 378 | * include: A List of glob expressions to look for files through. | 
|  | 379 | * exclude: A List of glob expressions for excluding files. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 380 |  | 
|  | 381 | Returns a List of Match objects for words beginning with MBED. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 382 | """ | 
| Yuto Takano | b47b504 | 2021-08-07 00:42:54 +0100 | [diff] [blame] | 383 | # Typos of TLS are common, hence the broader check below than MBEDTLS. | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 384 | mbed_regex = re.compile(r"\bMBED.+?_[A-Z0-9_]*") | 
|  | 385 | exclusions = re.compile(r"// *no-check-names|#error") | 
|  | 386 |  | 
| Yuto Takano | 5095343 | 2021-08-09 14:54:36 +0100 | [diff] [blame] | 387 | files = self.get_files(include, exclude) | 
|  | 388 | self.log.debug("Looking for MBED words in {} files".format(len(files))) | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 389 |  | 
| Yuto Takano | 5095343 | 2021-08-09 14:54:36 +0100 | [diff] [blame] | 390 | mbed_words = [] | 
|  | 391 | for filename in files: | 
| Yuto Takano | a083d15 | 2021-08-07 00:25:59 +0100 | [diff] [blame] | 392 | with open(filename, "r", encoding="utf-8") as fp: | 
| Yuto Takano | 8f457cf | 2021-08-06 17:54:58 +0100 | [diff] [blame] | 393 | for line_no, line in enumerate(fp): | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 394 | if exclusions.search(line): | 
| Yuto Takano | c62b408 | 2021-08-05 20:17:07 +0100 | [diff] [blame] | 395 | continue | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 396 |  | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 397 | for name in mbed_regex.finditer(line): | 
|  | 398 | mbed_words.append(Match( | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 399 | filename, | 
|  | 400 | line, | 
| Yuto Takano | 704b0f7 | 2021-08-17 10:41:23 +0100 | [diff] [blame] | 401 | line_no, | 
|  | 402 | name.span(0), | 
|  | 403 | name.group(0))) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 404 |  | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 405 | return mbed_words | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 406 |  | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 407 | def parse_enum_consts(self, include, exclude=None): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 408 | """ | 
|  | 409 | Parse all enum value constants that are declared. | 
|  | 410 |  | 
|  | 411 | Args: | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 412 | * include: A List of glob expressions to look for files through. | 
|  | 413 | * exclude: A List of glob expressions for excluding files. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 414 |  | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 415 | Returns a List of Match objects for the findings. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 416 | """ | 
| Yuto Takano | 5095343 | 2021-08-09 14:54:36 +0100 | [diff] [blame] | 417 | files = self.get_files(include, exclude) | 
|  | 418 | self.log.debug("Looking for enum consts in {} files".format(len(files))) | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 419 |  | 
| Yuto Takano | b1417b4 | 2021-08-17 10:30:20 +0100 | [diff] [blame] | 420 | # Emulate a finite state machine to parse enum declarations. | 
|  | 421 | # OUTSIDE_KEYWORD = outside the enum keyword | 
|  | 422 | # IN_BRACES = inside enum opening braces | 
|  | 423 | # IN_BETWEEN = between enum keyword and opening braces | 
|  | 424 | states = enum.Enum("FSM", ["OUTSIDE_KEYWORD", "IN_BRACES", "IN_BETWEEN"]) | 
| Yuto Takano | 5095343 | 2021-08-09 14:54:36 +0100 | [diff] [blame] | 425 | enum_consts = [] | 
|  | 426 | for header_file in files: | 
| Yuto Takano | b1417b4 | 2021-08-17 10:30:20 +0100 | [diff] [blame] | 427 | state = states.OUTSIDE_KEYWORD | 
| Yuto Takano | a083d15 | 2021-08-07 00:25:59 +0100 | [diff] [blame] | 428 | with open(header_file, "r", encoding="utf-8") as header: | 
| Yuto Takano | 8f457cf | 2021-08-06 17:54:58 +0100 | [diff] [blame] | 429 | for line_no, line in enumerate(header): | 
| Yuto Takano | 13ecd99 | 2021-08-06 16:56:52 +0100 | [diff] [blame] | 430 | # Match typedefs and brackets only when they are at the | 
|  | 431 | # beginning of the line -- if they are indented, they might | 
|  | 432 | # be sub-structures within structs, etc. | 
| Yuto Takano | b1417b4 | 2021-08-17 10:30:20 +0100 | [diff] [blame] | 433 | if (state == states.OUTSIDE_KEYWORD and | 
| Yuto Takano | 3590691 | 2021-08-17 11:05:43 +0100 | [diff] [blame] | 434 | re.search(r"^(typedef +)?enum +{", line)): | 
| Yuto Takano | b1417b4 | 2021-08-17 10:30:20 +0100 | [diff] [blame] | 435 | state = states.IN_BRACES | 
|  | 436 | elif (state == states.OUTSIDE_KEYWORD and | 
|  | 437 | re.search(r"^(typedef +)?enum", line)): | 
|  | 438 | state = states.IN_BETWEEN | 
|  | 439 | elif (state == states.IN_BETWEEN and | 
|  | 440 | re.search(r"^{", line)): | 
|  | 441 | state = states.IN_BRACES | 
|  | 442 | elif (state == states.IN_BRACES and | 
|  | 443 | re.search(r"^}", line)): | 
|  | 444 | state = states.OUTSIDE_KEYWORD | 
|  | 445 | elif (state == states.IN_BRACES and | 
|  | 446 | not re.search(r"^ *#", line)): | 
| Yuto Takano | 90bc026 | 2021-08-16 11:34:10 +0100 | [diff] [blame] | 447 | enum_const = re.search(r"^ *(?P<enum_const>\w+)", line) | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 448 | if not enum_const: | 
|  | 449 | continue | 
|  | 450 |  | 
|  | 451 | enum_consts.append(Match( | 
|  | 452 | header_file, | 
|  | 453 | line, | 
| Yuto Takano | 704b0f7 | 2021-08-17 10:41:23 +0100 | [diff] [blame] | 454 | line_no, | 
|  | 455 | enum_const.span("enum_const"), | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 456 | enum_const.group("enum_const"))) | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 457 |  | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 458 | return enum_consts | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 459 |  | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 460 | def parse_identifiers(self, include, exclude=None): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 461 | """ | 
| Yuto Takano | 8246eb8 | 2021-08-16 10:37:24 +0100 | [diff] [blame] | 462 | Parse all lines of a header where a function/enum/struct/union/typedef | 
| Yuto Takano | b1417b4 | 2021-08-17 10:30:20 +0100 | [diff] [blame] | 463 | identifier is declared, based on some regex and heuristics. Highly | 
|  | 464 | dependent on formatting style. | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 465 |  | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 466 | Args: | 
| Yuto Takano | 8e9a219 | 2021-08-09 14:48:53 +0100 | [diff] [blame] | 467 | * include: A List of glob expressions to look for files through. | 
|  | 468 | * exclude: A List of glob expressions for excluding files. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 469 |  | 
|  | 470 | Returns a List of Match objects with identifiers. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 471 | """ | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 472 | identifier_regex = re.compile( | 
|  | 473 | # Match " something(a" or " *something(a". Functions. | 
|  | 474 | # Assumptions: | 
|  | 475 | # - function definition from return type to one of its arguments is | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 476 | #   all on one line | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 477 | # - function definition line only contains alphanumeric, asterisk, | 
|  | 478 | #   underscore, and open bracket | 
|  | 479 | r".* \**(\w+) *\( *\w|" | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 480 | # Match "(*something)(". | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 481 | r".*\( *\* *(\w+) *\) *\(|" | 
|  | 482 | # Match names of named data structures. | 
|  | 483 | r"(?:typedef +)?(?:struct|union|enum) +(\w+)(?: *{)?$|" | 
|  | 484 | # Match names of typedef instances, after closing bracket. | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 485 | r"}? *(\w+)[;[].*" | 
|  | 486 | ) | 
| Yuto Takano | 3590691 | 2021-08-17 11:05:43 +0100 | [diff] [blame] | 487 | # The regex below is indented for clarity. | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 488 | exclusion_lines = re.compile( | 
|  | 489 | r"^(" | 
| Yuto Takano | 3590691 | 2021-08-17 11:05:43 +0100 | [diff] [blame] | 490 | r"extern +\"C\"|" # pylint: disable=bad-continuation | 
| Yuto Takano | 90bc026 | 2021-08-16 11:34:10 +0100 | [diff] [blame] | 491 | r"(typedef +)?(struct|union|enum)( *{)?$|" | 
|  | 492 | r"} *;?$|" | 
|  | 493 | r"$|" | 
|  | 494 | r"//|" | 
|  | 495 | r"#" | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 496 | r")" | 
|  | 497 | ) | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 498 |  | 
| Yuto Takano | 5095343 | 2021-08-09 14:54:36 +0100 | [diff] [blame] | 499 | files = self.get_files(include, exclude) | 
|  | 500 | self.log.debug("Looking for identifiers in {} files".format(len(files))) | 
|  | 501 |  | 
|  | 502 | identifiers = [] | 
|  | 503 | for header_file in files: | 
| Yuto Takano | a083d15 | 2021-08-07 00:25:59 +0100 | [diff] [blame] | 504 | with open(header_file, "r", encoding="utf-8") as header: | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 505 | in_block_comment = False | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 506 | # The previous line variable is used for concatenating lines | 
| Yuto Takano | b1417b4 | 2021-08-17 10:30:20 +0100 | [diff] [blame] | 507 | # when identifiers are formatted and spread across multiple | 
|  | 508 | # lines. | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 509 | previous_line = "" | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 510 |  | 
| Yuto Takano | 8f457cf | 2021-08-06 17:54:58 +0100 | [diff] [blame] | 511 | for line_no, line in enumerate(header): | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 512 | # Skip parsing this line if a block comment ends on it, | 
|  | 513 | # but don't skip if it has just started -- there is a chance | 
|  | 514 | # it ends on the same line. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 515 | if re.search(r"/\*", line): | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 516 | in_block_comment = not in_block_comment | 
|  | 517 | if re.search(r"\*/", line): | 
|  | 518 | in_block_comment = not in_block_comment | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 519 | continue | 
|  | 520 |  | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 521 | if in_block_comment: | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 522 | previous_line = "" | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 523 | continue | 
|  | 524 |  | 
| Yuto Takano | 90bc026 | 2021-08-16 11:34:10 +0100 | [diff] [blame] | 525 | if exclusion_lines.search(line): | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 526 | previous_line = "" | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 527 | continue | 
|  | 528 |  | 
| Yuto Takano | cfc9e4a | 2021-08-06 20:02:32 +0100 | [diff] [blame] | 529 | # If the line contains only space-separated alphanumeric | 
|  | 530 | # characters (or underscore, asterisk, or, open bracket), | 
|  | 531 | # and nothing else, high chance it's a declaration that | 
|  | 532 | # continues on the next line | 
| Yuto Takano | 90bc026 | 2021-08-16 11:34:10 +0100 | [diff] [blame] | 533 | if re.search(r"^([\w\*\(]+\s+)+$", line): | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 534 | previous_line += line | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 535 | continue | 
|  | 536 |  | 
|  | 537 | # If previous line seemed to start an unfinished declaration | 
| Yuto Takano | cfc9e4a | 2021-08-06 20:02:32 +0100 | [diff] [blame] | 538 | # (as above), concat and treat them as one. | 
|  | 539 | if previous_line: | 
| Yuto Takano | 90bc026 | 2021-08-16 11:34:10 +0100 | [diff] [blame] | 540 | line = previous_line.strip() + " " + line.strip() + "\n" | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 541 | previous_line = "" | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 542 |  | 
| Yuto Takano | 8246eb8 | 2021-08-16 10:37:24 +0100 | [diff] [blame] | 543 | # Skip parsing if line has a space in front = heuristic to | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 544 | # skip function argument lines (highly subject to formatting | 
|  | 545 | # changes) | 
|  | 546 | if line[0] == " ": | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 547 | continue | 
| Yuto Takano | 6f38ab3 | 2021-08-05 21:07:14 +0100 | [diff] [blame] | 548 |  | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 549 | identifier = identifier_regex.search(line) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 550 |  | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 551 | if not identifier: | 
|  | 552 | continue | 
|  | 553 |  | 
|  | 554 | # Find the group that matched, and append it | 
|  | 555 | for group in identifier.groups(): | 
|  | 556 | if not group: | 
|  | 557 | continue | 
|  | 558 |  | 
|  | 559 | identifiers.append(Match( | 
|  | 560 | header_file, | 
|  | 561 | line, | 
| Yuto Takano | 704b0f7 | 2021-08-17 10:41:23 +0100 | [diff] [blame] | 562 | line_no, | 
|  | 563 | identifier.span(), | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 564 | group)) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 565 |  | 
|  | 566 | return identifiers | 
|  | 567 |  | 
|  | 568 | def parse_symbols(self): | 
|  | 569 | """ | 
|  | 570 | Compile the Mbed TLS libraries, and parse the TLS, Crypto, and x509 | 
|  | 571 | object files using nm to retrieve the list of referenced symbols. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 572 | Exceptions thrown here are rethrown because they would be critical | 
|  | 573 | errors that void several tests, and thus needs to halt the program. This | 
|  | 574 | is explicitly done for clarity. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 575 |  | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 576 | Returns a List of unique symbols defined and used in the libraries. | 
|  | 577 | """ | 
|  | 578 | self.log.info("Compiling...") | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 579 | symbols = [] | 
|  | 580 |  | 
|  | 581 | # Back up the config and atomically compile with the full configratuion. | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 582 | shutil.copy( | 
|  | 583 | "include/mbedtls/mbedtls_config.h", | 
|  | 584 | "include/mbedtls/mbedtls_config.h.bak" | 
|  | 585 | ) | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 586 | try: | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 587 | # Use check=True in all subprocess calls so that failures are raised | 
|  | 588 | # as exceptions and logged. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 589 | subprocess.run( | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 590 | ["python3", "scripts/config.py", "full"], | 
| Yuto Takano | bcc3d99 | 2021-08-06 23:14:58 +0100 | [diff] [blame] | 591 | universal_newlines=True, | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 592 | check=True | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 593 | ) | 
|  | 594 | my_environment = os.environ.copy() | 
|  | 595 | my_environment["CFLAGS"] = "-fno-asynchronous-unwind-tables" | 
| Yuto Takano | 4b7d23d | 2021-08-17 10:48:22 +0100 | [diff] [blame] | 596 | # Run make clean separately to lib to prevent unwanted behavior when | 
|  | 597 | # make is invoked with parallelism. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 598 | subprocess.run( | 
| Yuto Takano | 4b7d23d | 2021-08-17 10:48:22 +0100 | [diff] [blame] | 599 | ["make", "clean"], | 
|  | 600 | universal_newlines=True, | 
|  | 601 | check=True | 
|  | 602 | ) | 
|  | 603 | subprocess.run( | 
|  | 604 | ["make", "lib"], | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 605 | env=my_environment, | 
| Yuto Takano | bcc3d99 | 2021-08-06 23:14:58 +0100 | [diff] [blame] | 606 | universal_newlines=True, | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 607 | stdout=subprocess.PIPE, | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 608 | stderr=subprocess.STDOUT, | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 609 | check=True | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 610 | ) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 611 |  | 
|  | 612 | # Perform object file analysis using nm | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 613 | symbols = self.parse_symbols_from_nm([ | 
|  | 614 | "library/libmbedcrypto.a", | 
|  | 615 | "library/libmbedtls.a", | 
|  | 616 | "library/libmbedx509.a" | 
|  | 617 | ]) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 618 |  | 
|  | 619 | subprocess.run( | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 620 | ["make", "clean"], | 
| Yuto Takano | bcc3d99 | 2021-08-06 23:14:58 +0100 | [diff] [blame] | 621 | universal_newlines=True, | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 622 | check=True | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 623 | ) | 
|  | 624 | except subprocess.CalledProcessError as error: | 
| Yuto Takano | 25eeb7b | 2021-08-06 21:27:59 +0100 | [diff] [blame] | 625 | self.log.debug(error.output) | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 626 | raise error | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 627 | finally: | 
| Yuto Takano | 6fececf | 2021-08-07 17:28:23 +0100 | [diff] [blame] | 628 | # Put back the original config regardless of there being errors. | 
|  | 629 | # Works also for keyboard interrupts. | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 630 | shutil.move( | 
|  | 631 | "include/mbedtls/mbedtls_config.h.bak", | 
|  | 632 | "include/mbedtls/mbedtls_config.h" | 
|  | 633 | ) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 634 |  | 
|  | 635 | return symbols | 
|  | 636 |  | 
|  | 637 | def parse_symbols_from_nm(self, object_files): | 
|  | 638 | """ | 
|  | 639 | Run nm to retrieve the list of referenced symbols in each object file. | 
|  | 640 | Does not return the position data since it is of no use. | 
|  | 641 |  | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 642 | Args: | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 643 | * object_files: a List of compiled object filepaths to search through. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 644 |  | 
|  | 645 | Returns a List of unique symbols defined and used in any of the object | 
|  | 646 | files. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 647 | """ | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 648 | nm_undefined_regex = re.compile(r"^\S+: +U |^$|^\S+:$") | 
|  | 649 | nm_valid_regex = re.compile(r"^\S+( [0-9A-Fa-f]+)* . _*(?P<symbol>\w+)") | 
| Yuto Takano | 12a7ecd | 2021-08-07 00:40:29 +0100 | [diff] [blame] | 650 | exclusions = ("FStar", "Hacl") | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 651 |  | 
|  | 652 | symbols = [] | 
|  | 653 |  | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 654 | # Gather all outputs of nm | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 655 | nm_output = "" | 
|  | 656 | for lib in object_files: | 
|  | 657 | nm_output += subprocess.run( | 
|  | 658 | ["nm", "-og", lib], | 
| Yuto Takano | bcc3d99 | 2021-08-06 23:14:58 +0100 | [diff] [blame] | 659 | universal_newlines=True, | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 660 | stdout=subprocess.PIPE, | 
|  | 661 | stderr=subprocess.STDOUT, | 
|  | 662 | check=True | 
|  | 663 | ).stdout | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 664 |  | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 665 | for line in nm_output.splitlines(): | 
| Yuto Takano | 90bc026 | 2021-08-16 11:34:10 +0100 | [diff] [blame] | 666 | if not nm_undefined_regex.search(line): | 
|  | 667 | symbol = nm_valid_regex.search(line) | 
| Yuto Takano | 12a7ecd | 2021-08-07 00:40:29 +0100 | [diff] [blame] | 668 | if (symbol and not symbol.group("symbol").startswith(exclusions)): | 
| Yuto Takano | e77f699 | 2021-08-05 20:22:59 +0100 | [diff] [blame] | 669 | symbols.append(symbol.group("symbol")) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 670 | else: | 
|  | 671 | self.log.error(line) | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 672 |  | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 673 | return symbols | 
|  | 674 |  | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 675 | class NameChecker(): | 
|  | 676 | """ | 
|  | 677 | Representation of the core name checking operation performed by this script. | 
|  | 678 | """ | 
|  | 679 | def __init__(self, parse_result, log): | 
|  | 680 | self.parse_result = parse_result | 
|  | 681 | self.log = log | 
|  | 682 |  | 
| Yuto Takano | 55614b5 | 2021-08-07 01:00:18 +0100 | [diff] [blame] | 683 | def perform_checks(self, quiet=False): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 684 | """ | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 685 | A comprehensive checker that performs each check in order, and outputs | 
|  | 686 | a final verdict. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 687 |  | 
|  | 688 | Args: | 
| Yuto Takano | 55614b5 | 2021-08-07 01:00:18 +0100 | [diff] [blame] | 689 | * quiet: whether to hide detailed problem explanation. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 690 | """ | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 691 | self.log.info("=============") | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 692 | Problem.quiet = quiet | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 693 | problems = 0 | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 694 | problems += self.check_symbols_declared_in_header() | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 695 |  | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 696 | pattern_checks = [ | 
|  | 697 | ("macros", MACRO_PATTERN), | 
|  | 698 | ("enum_consts", CONSTANTS_PATTERN), | 
|  | 699 | ("identifiers", IDENTIFIER_PATTERN) | 
|  | 700 | ] | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 701 | for group, check_pattern in pattern_checks: | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 702 | problems += self.check_match_pattern(group, check_pattern) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 703 |  | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 704 | problems += self.check_for_typos() | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 705 |  | 
|  | 706 | self.log.info("=============") | 
|  | 707 | if problems > 0: | 
|  | 708 | self.log.info("FAIL: {0} problem(s) to fix".format(str(problems))) | 
| Yuto Takano | 55614b5 | 2021-08-07 01:00:18 +0100 | [diff] [blame] | 709 | if quiet: | 
|  | 710 | self.log.info("Remove --quiet to see explanations.") | 
| Yuto Takano | fc54dfb | 2021-08-07 17:18:28 +0100 | [diff] [blame] | 711 | else: | 
|  | 712 | self.log.info("Use --quiet for minimal output.") | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 713 | return 1 | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 714 | else: | 
|  | 715 | self.log.info("PASS") | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 716 | return 0 | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 717 |  | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 718 | def check_symbols_declared_in_header(self): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 719 | """ | 
|  | 720 | Perform a check that all detected symbols in the library object files | 
|  | 721 | are properly declared in headers. | 
| Yuto Takano | 977e07f | 2021-08-09 11:56:15 +0100 | [diff] [blame] | 722 | Assumes parse_names_in_source() was called before this. | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 723 |  | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 724 | Returns the number of problems that need fixing. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 725 | """ | 
|  | 726 | problems = [] | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 727 |  | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 728 | for symbol in self.parse_result["symbols"]: | 
|  | 729 | found_symbol_declared = False | 
|  | 730 | for identifier_match in self.parse_result["identifiers"]: | 
|  | 731 | if symbol == identifier_match.name: | 
|  | 732 | found_symbol_declared = True | 
|  | 733 | break | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 734 |  | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 735 | if not found_symbol_declared: | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 736 | problems.append(SymbolNotInHeader(symbol)) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 737 |  | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 738 | self.output_check_result("All symbols in header", problems) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 739 | return len(problems) | 
|  | 740 |  | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 741 | def check_match_pattern(self, group_to_check, check_pattern): | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 742 | """ | 
|  | 743 | Perform a check that all items of a group conform to a regex pattern. | 
| Yuto Takano | 977e07f | 2021-08-09 11:56:15 +0100 | [diff] [blame] | 744 | Assumes parse_names_in_source() was called before this. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 745 |  | 
|  | 746 | Args: | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 747 | * group_to_check: string key to index into self.parse_result. | 
|  | 748 | * check_pattern: the regex to check against. | 
|  | 749 |  | 
|  | 750 | Returns the number of problems that need fixing. | 
|  | 751 | """ | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 752 | problems = [] | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 753 |  | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 754 | for item_match in self.parse_result[group_to_check]: | 
| Yuto Takano | 90bc026 | 2021-08-16 11:34:10 +0100 | [diff] [blame] | 755 | if not re.search(check_pattern, item_match.name): | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 756 | problems.append(PatternMismatch(check_pattern, item_match)) | 
| Yuto Takano | 90bc026 | 2021-08-16 11:34:10 +0100 | [diff] [blame] | 757 | # Double underscore should not be used for names | 
|  | 758 | if re.search(r".*__.*", item_match.name): | 
| Yuto Takano | 704b0f7 | 2021-08-17 10:41:23 +0100 | [diff] [blame] | 759 | problems.append( | 
|  | 760 | PatternMismatch("no double underscore allowed", item_match)) | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 761 |  | 
|  | 762 | self.output_check_result( | 
|  | 763 | "Naming patterns of {}".format(group_to_check), | 
| Yuto Takano | 55614b5 | 2021-08-07 01:00:18 +0100 | [diff] [blame] | 764 | problems) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 765 | return len(problems) | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 766 |  | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 767 | def check_for_typos(self): | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 768 | """ | 
|  | 769 | Perform a check that all words in the soure code beginning with MBED are | 
|  | 770 | either defined as macros, or as enum constants. | 
| Yuto Takano | 977e07f | 2021-08-09 11:56:15 +0100 | [diff] [blame] | 771 | Assumes parse_names_in_source() was called before this. | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 772 |  | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 773 | Returns the number of problems that need fixing. | 
|  | 774 | """ | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 775 | problems = [] | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 776 |  | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 777 | # Set comprehension, equivalent to a list comprehension wrapped by set() | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 778 | all_caps_names = { | 
|  | 779 | match.name | 
|  | 780 | for match | 
|  | 781 | in self.parse_result["macros"] + self.parse_result["enum_consts"]} | 
|  | 782 | typo_exclusion = re.compile(r"XXX|__|_$|^MBEDTLS_.*CONFIG_FILE$") | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 783 |  | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 784 | for name_match in self.parse_result["mbed_words"]: | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 785 | found = name_match.name in all_caps_names | 
|  | 786 |  | 
|  | 787 | # Since MBEDTLS_PSA_ACCEL_XXX defines are defined by the | 
|  | 788 | # PSA driver, they will not exist as macros. However, they | 
|  | 789 | # should still be checked for typos using the equivalent | 
|  | 790 | # BUILTINs that exist. | 
|  | 791 | if "MBEDTLS_PSA_ACCEL_" in name_match.name: | 
|  | 792 | found = name_match.name.replace( | 
|  | 793 | "MBEDTLS_PSA_ACCEL_", | 
|  | 794 | "MBEDTLS_PSA_BUILTIN_") in all_caps_names | 
|  | 795 |  | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 796 | if not found and not typo_exclusion.search(name_match.name): | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 797 | problems.append(Typo(name_match)) | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 798 |  | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 799 | self.output_check_result("Likely typos", problems) | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 800 | return len(problems) | 
|  | 801 |  | 
| Yuto Takano | 5473be2 | 2021-08-17 10:14:01 +0100 | [diff] [blame] | 802 | def output_check_result(self, name, problems): | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 803 | """ | 
|  | 804 | Write out the PASS/FAIL status of a performed check depending on whether | 
|  | 805 | there were problems. | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 806 |  | 
|  | 807 | Args: | 
| Yuto Takano | d70d446 | 2021-08-09 12:45:51 +0100 | [diff] [blame] | 808 | * name: the name of the test | 
|  | 809 | * problems: a List of encountered Problems | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 810 | """ | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 811 | if problems: | 
| Yuto Takano | 55614b5 | 2021-08-07 01:00:18 +0100 | [diff] [blame] | 812 | self.log.info("{}: FAIL\n".format(name)) | 
|  | 813 | for problem in problems: | 
|  | 814 | self.log.warning(str(problem)) | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 815 | else: | 
| Yuto Takano | 81528c0 | 2021-08-06 16:22:06 +0100 | [diff] [blame] | 816 | self.log.info("{}: PASS".format(name)) | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 817 |  | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 818 | def main(): | 
|  | 819 | """ | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 820 | Perform argument parsing, and create an instance of CodeParser and | 
|  | 821 | NameChecker to begin the core operation. | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 822 | """ | 
| Yuto Takano | f005c33 | 2021-08-09 13:56:36 +0100 | [diff] [blame] | 823 | parser = argparse.ArgumentParser( | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 824 | formatter_class=argparse.RawDescriptionHelpFormatter, | 
|  | 825 | description=( | 
|  | 826 | "This script confirms that the naming of all symbols and identifiers " | 
|  | 827 | "in Mbed TLS are consistent with the house style and are also " | 
|  | 828 | "self-consistent.\n\n" | 
| Yuto Takano | f005c33 | 2021-08-09 13:56:36 +0100 | [diff] [blame] | 829 | "Expected to be run from the MbedTLS root directory.") | 
|  | 830 | ) | 
|  | 831 | parser.add_argument( | 
|  | 832 | "-v", "--verbose", | 
|  | 833 | action="store_true", | 
|  | 834 | help="show parse results" | 
|  | 835 | ) | 
|  | 836 | parser.add_argument( | 
|  | 837 | "-q", "--quiet", | 
|  | 838 | action="store_true", | 
|  | 839 | help="hide unnecessary text, explanations, and highlighs" | 
|  | 840 | ) | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 841 |  | 
| Yuto Takano | f005c33 | 2021-08-09 13:56:36 +0100 | [diff] [blame] | 842 | args = parser.parse_args() | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 843 |  | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 844 | # Configure the global logger, which is then passed to the classes below | 
|  | 845 | log = logging.getLogger() | 
|  | 846 | log.setLevel(logging.DEBUG if args.verbose else logging.INFO) | 
|  | 847 | log.addHandler(logging.StreamHandler()) | 
|  | 848 |  | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 849 | try: | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 850 | code_parser = CodeParser(log) | 
|  | 851 | parse_result = code_parser.comprehensive_parse() | 
| Yuto Takano | d93fa37 | 2021-08-06 23:05:55 +0100 | [diff] [blame] | 852 | except Exception: # pylint: disable=broad-except | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 853 | traceback.print_exc() | 
|  | 854 | sys.exit(2) | 
|  | 855 |  | 
| Yuto Takano | 55c6c87 | 2021-08-09 15:35:19 +0100 | [diff] [blame] | 856 | name_checker = NameChecker(parse_result, log) | 
|  | 857 | return_code = name_checker.perform_checks(quiet=args.quiet) | 
|  | 858 |  | 
|  | 859 | sys.exit(return_code) | 
|  | 860 |  | 
| Darryl Green | d580292 | 2018-05-08 15:30:59 +0100 | [diff] [blame] | 861 | if __name__ == "__main__": | 
| Yuto Takano | 3963967 | 2021-08-05 19:47:48 +0100 | [diff] [blame] | 862 | main() |