Merge pull request #6093 from wernerlewis/bignum_test_script
Add bignum test case generation script
diff --git a/scripts/make_generated_files.bat b/scripts/make_generated_files.bat
index 662da98..e9d9275 100644
--- a/scripts/make_generated_files.bat
+++ b/scripts/make_generated_files.bat
@@ -10,4 +10,5 @@
python scripts\generate_ssl_debug_helpers.py || exit /b 1
perl scripts\generate_visualc_files.pl || exit /b 1
python scripts\generate_psa_constants.py || exit /b 1
+python tests\scripts\generate_bignum_tests.py || exit /b 1
python tests\scripts\generate_psa_tests.py || exit /b 1
diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py
new file mode 100644
index 0000000..a88425f
--- /dev/null
+++ b/scripts/mbedtls_dev/test_generation.py
@@ -0,0 +1,219 @@
+"""Common test generation classes and main function.
+
+These are used both by generate_psa_tests.py and generate_bignum_tests.py.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import posixpath
+import re
+
+from abc import ABCMeta, abstractmethod
+from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar
+
+from mbedtls_dev import build_tree
+from mbedtls_dev import test_case
+
+T = TypeVar('T') #pylint: disable=invalid-name
+
+
+class BaseTarget(metaclass=ABCMeta):
+ """Base target for test case generation.
+
+ Child classes of this class represent an output file, and can be referred
+ to as file targets. These indicate where test cases will be written to for
+ all subclasses of the file target, which is set by `target_basename`.
+
+ Attributes:
+ count: Counter for test cases from this class.
+ case_description: Short description of the test case. This may be
+ automatically generated using the class, or manually set.
+ dependencies: A list of dependencies required for the test case.
+ show_test_count: Toggle for inclusion of `count` in the test description.
+ target_basename: Basename of file to write generated tests to. This
+ should be specified in a child class of BaseTarget.
+ test_function: Test function which the class generates cases for.
+ test_name: A common name or description of the test function. This can
+ be `test_function`, a clearer equivalent, or a short summary of the
+ test function's purpose.
+ """
+ count = 0
+ case_description = ""
+ dependencies = [] # type: List[str]
+ show_test_count = True
+ target_basename = ""
+ test_function = ""
+ test_name = ""
+
+ def __new__(cls, *args, **kwargs):
+ # pylint: disable=unused-argument
+ cls.count += 1
+ return super().__new__(cls)
+
+ @abstractmethod
+ def arguments(self) -> List[str]:
+ """Get the list of arguments for the test case.
+
+ Override this method to provide the list of arguments required for
+ the `test_function`.
+
+ Returns:
+ List of arguments required for the test function.
+ """
+ raise NotImplementedError
+
+ def description(self) -> str:
+ """Create a test case description.
+
+ Creates a description of the test case, including a name for the test
+ function, an optional case count, and a description of the specific
+ test case. This should inform a reader what is being tested, and
+ provide context for the test case.
+
+ Returns:
+ Description for the test case.
+ """
+ if self.show_test_count:
+ return "{} #{} {}".format(
+ self.test_name, self.count, self.case_description
+ ).strip()
+ else:
+ return "{} {}".format(self.test_name, self.case_description).strip()
+
+
+ def create_test_case(self) -> test_case.TestCase:
+ """Generate TestCase from the instance."""
+ tc = test_case.TestCase()
+ tc.set_description(self.description())
+ tc.set_function(self.test_function)
+ tc.set_arguments(self.arguments())
+ tc.set_dependencies(self.dependencies)
+
+ return tc
+
+ @classmethod
+ @abstractmethod
+ def generate_function_tests(cls) -> Iterator[test_case.TestCase]:
+ """Generate test cases for the class test function.
+
+ This will be called in classes where `test_function` is set.
+ Implementations should yield TestCase objects, by creating instances
+ of the class with appropriate input data, and then calling
+ `create_test_case()` on each.
+ """
+ raise NotImplementedError
+
+ @classmethod
+ def generate_tests(cls) -> Iterator[test_case.TestCase]:
+ """Generate test cases for the class and its subclasses.
+
+ In classes with `test_function` set, `generate_function_tests()` is
+ called to generate test cases first.
+
+ In all classes, this method will iterate over its subclasses, and
+ yield from `generate_tests()` in each. Calling this method on a class X
+ will yield test cases from all classes derived from X.
+ """
+ if cls.test_function:
+ yield from cls.generate_function_tests()
+ for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__):
+ yield from subclass.generate_tests()
+
+
+class TestGenerator:
+ """Generate test cases and write to data files."""
+ def __init__(self, options) -> None:
+ self.test_suite_directory = self.get_option(options, 'directory',
+ 'tests/suites')
+ # Update `targets` with an entry for each child class of BaseTarget.
+ # Each entry represents a file generated by the BaseTarget framework,
+ # and enables generating the .data files using the CLI.
+ self.targets.update({
+ subclass.target_basename: subclass.generate_tests
+ for subclass in BaseTarget.__subclasses__()
+ })
+
+ @staticmethod
+ def get_option(options, name: str, default: T) -> T:
+ value = getattr(options, name, None)
+ return default if value is None else value
+
+ def filename_for(self, basename: str) -> str:
+ """The location of the data file with the specified base name."""
+ return posixpath.join(self.test_suite_directory, basename + '.data')
+
+ def write_test_data_file(self, basename: str,
+ test_cases: Iterable[test_case.TestCase]) -> None:
+ """Write the test cases to a .data file.
+
+ The output file is ``basename + '.data'`` in the test suite directory.
+ """
+ filename = self.filename_for(basename)
+ test_case.write_data_file(filename, test_cases)
+
+ # Note that targets whose names contain 'test_format' have their content
+ # validated by `abi_check.py`.
+ targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]]
+
+ def generate_target(self, name: str, *target_args) -> None:
+ """Generate cases and write to data file for a target.
+
+ For target callables which require arguments, override this function
+ and pass these arguments using super() (see PSATestGenerator).
+ """
+ test_cases = self.targets[name](*target_args)
+ self.write_test_data_file(name, test_cases)
+
+def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator):
+ """Command line entry point."""
+ parser = argparse.ArgumentParser(description=description)
+ parser.add_argument('--list', action='store_true',
+ help='List available targets and exit')
+ parser.add_argument('--list-for-cmake', action='store_true',
+ help='Print \';\'-separated list of available targets and exit')
+ parser.add_argument('--directory', metavar='DIR',
+ help='Output directory (default: tests/suites)')
+ # The `--directory` option is interpreted relative to the directory from
+ # which the script is invoked, but the default is relative to the root of
+ # the mbedtls tree. The default should not be set above, but instead after
+ # `build_tree.chdir_to_root()` is called.
+ parser.add_argument('targets', nargs='*', metavar='TARGET',
+ help='Target file to generate (default: all; "-": none)')
+ options = parser.parse_args(args)
+ build_tree.chdir_to_root()
+ generator = generator_class(options)
+ if options.list:
+ for name in sorted(generator.targets):
+ print(generator.filename_for(name))
+ return
+ # List in a cmake list format (i.e. ';'-separated)
+ if options.list_for_cmake:
+ print(';'.join(generator.filename_for(name)
+ for name in sorted(generator.targets)), end='')
+ return
+ if options.targets:
+ # Allow "-" as a special case so you can run
+ # ``generate_xxx_tests.py - $targets`` and it works uniformly whether
+ # ``$targets`` is empty or not.
+ options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target))
+ for target in options.targets
+ if target != '-']
+ else:
+ options.targets = sorted(generator.targets)
+ for target in options.targets:
+ generator.generate_target(target)
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 6049b74..57cf977 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -20,24 +20,54 @@
execute_process(
COMMAND
${MBEDTLS_PYTHON_EXECUTABLE}
+ ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py
+ --list-for-cmake
+ --directory suites
+ WORKING_DIRECTORY
+ ${CMAKE_CURRENT_SOURCE_DIR}/..
+ OUTPUT_VARIABLE
+ base_bignum_generated_data_files)
+
+execute_process(
+ COMMAND
+ ${MBEDTLS_PYTHON_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_psa_tests.py
--list-for-cmake
--directory suites
WORKING_DIRECTORY
${CMAKE_CURRENT_SOURCE_DIR}/..
OUTPUT_VARIABLE
- base_generated_data_files)
+ base_psa_generated_data_files)
# Derive generated file paths in the build directory
-set(generated_data_files "")
-foreach(file ${base_generated_data_files})
- list(APPEND generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file})
+set(base_generated_data_files ${base_bignum_generated_data_files} ${base_psa_generated_data_files})
+set(bignum_generated_data_files "")
+set(psa_generated_data_files "")
+foreach(file ${base_bignum_generated_data_files})
+ list(APPEND bignum_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file})
+endforeach()
+foreach(file ${base_psa_generated_data_files})
+ list(APPEND psa_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file})
endforeach()
if(GEN_FILES)
add_custom_command(
OUTPUT
- ${generated_data_files}
+ ${bignum_generated_data_files}
+ WORKING_DIRECTORY
+ ${CMAKE_CURRENT_SOURCE_DIR}/..
+ COMMAND
+ ${MBEDTLS_PYTHON_EXECUTABLE}
+ ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py
+ --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+ DEPENDS
+ ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_bignum_tests.py
+ ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_case.py
+ ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_generation.py
+ )
+ add_custom_command(
+ OUTPUT
+ ${psa_generated_data_files}
WORKING_DIRECTORY
${CMAKE_CURRENT_SOURCE_DIR}/..
COMMAND
@@ -50,6 +80,7 @@
${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/macro_collector.py
${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/psa_storage.py
${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_case.py
+ ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/mbedtls_dev/test_generation.py
${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_config.h
${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_values.h
${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_extra.h
@@ -65,7 +96,8 @@
# they can cause race conditions in parallel builds.
# With this line, only 4 sub-makefiles include the above command, that reduces
# the risk of a race.
-add_custom_target(test_suite_generated_data DEPENDS ${generated_data_files})
+add_custom_target(test_suite_bignum_generated_data DEPENDS ${bignum_generated_data_files})
+add_custom_target(test_suite_psa_generated_data DEPENDS ${psa_generated_data_files})
# Test suites caught by SKIP_TEST_SUITES are built but not executed.
# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
# but not "test_suite_foobar".
@@ -82,23 +114,39 @@
# Get the test names of the tests with generated .data files
# from the generated_data_files list in parent scope.
- set(generated_data_names "")
- foreach(generated_data_file ${generated_data_files})
+ set(bignum_generated_data_names "")
+ set(psa_generated_data_names "")
+ foreach(generated_data_file ${bignum_generated_data_files})
# Get the plain filename
get_filename_component(generated_data_name ${generated_data_file} NAME)
# Remove the ".data" extension
get_name_without_last_ext(generated_data_name ${generated_data_name})
# Remove leading "test_suite_"
string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
- list(APPEND generated_data_names ${generated_data_name})
+ list(APPEND bignum_generated_data_names ${generated_data_name})
+ endforeach()
+ foreach(generated_data_file ${psa_generated_data_files})
+ # Get the plain filename
+ get_filename_component(generated_data_name ${generated_data_file} NAME)
+ # Remove the ".data" extension
+ get_name_without_last_ext(generated_data_name ${generated_data_name})
+ # Remove leading "test_suite_"
+ string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
+ list(APPEND psa_generated_data_names ${generated_data_name})
endforeach()
- if(";${generated_data_names};" MATCHES ";${data_name};")
+ if(";${bignum_generated_data_names};" MATCHES ";${data_name};")
set(data_file
${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
+ set(dependency test_suite_bignum_generated_data)
+ elseif(";${psa_generated_data_names};" MATCHES ";${data_name};")
+ set(data_file
+ ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
+ set(dependency test_suite_psa_generated_data)
else()
set(data_file
${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data)
+ set(dependency test_suite_bignum_generated_data test_suite_psa_generated_data)
endif()
add_custom_command(
@@ -129,7 +177,7 @@
)
add_executable(test_suite_${data_name} test_suite_${data_name}.c $<TARGET_OBJECTS:mbedtls_test>)
- add_dependencies(test_suite_${data_name} test_suite_generated_data)
+ add_dependencies(test_suite_${data_name} ${dependency})
target_link_libraries(test_suite_${data_name} ${libs})
# Include test-specific header files from ./include and private header
# files (used by some invasive tests) from ../library. Public header
diff --git a/tests/Makefile b/tests/Makefile
index 0d08f84..8777ae9 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -66,28 +66,45 @@
endif
.PHONY: generated_files
-GENERATED_DATA_FILES := $(patsubst tests/%,%,$(shell \
+GENERATED_BIGNUM_DATA_FILES := $(patsubst tests/%,%,$(shell \
+ $(PYTHON) scripts/generate_bignum_tests.py --list || \
+ echo FAILED \
+))
+ifeq ($(GENERATED_BIGNUM_DATA_FILES),FAILED)
+$(error "$(PYTHON) scripts/generate_bignum_tests.py --list" failed)
+endif
+GENERATED_PSA_DATA_FILES := $(patsubst tests/%,%,$(shell \
$(PYTHON) scripts/generate_psa_tests.py --list || \
echo FAILED \
))
-ifeq ($(GENERATED_DATA_FILES),FAILED)
+ifeq ($(GENERATED_PSA_DATA_FILES),FAILED)
$(error "$(PYTHON) scripts/generate_psa_tests.py --list" failed)
endif
-GENERATED_FILES := $(GENERATED_DATA_FILES)
+GENERATED_FILES := $(GENERATED_PSA_DATA_FILES) $(GENERATED_BIGNUM_DATA_FILES)
generated_files: $(GENERATED_FILES)
-# generate_psa_tests.py spends more time analyzing inputs than generating
-# outputs. Its inputs are the same no matter which files are being generated.
+# generate_bignum_tests.py and generate_psa_tests.py spend more time analyzing
+# inputs than generating outputs. Its inputs are the same no matter which files
+# are being generated.
# It's rare not to want all the outputs. So always generate all of its outputs.
# Use an intermediate phony dependency so that parallel builds don't run
# a separate instance of the recipe for each output file.
-.SECONDARY: generated_psa_test_data
-$(GENERATED_DATA_FILES): generated_psa_test_data
+.SECONDARY: generated_bignum_test_data generated_psa_test_data
+$(GENERATED_BIGNUM_DATA_FILES): generated_bignum_test_data
+generated_bignum_test_data: scripts/generate_bignum_tests.py
+generated_bignum_test_data: ../scripts/mbedtls_dev/test_case.py
+generated_bignum_test_data: ../scripts/mbedtls_dev/test_generation.py
+generated_bignum_test_data:
+ echo " Gen $(GENERATED_BIGNUM_DATA_FILES)"
+ $(PYTHON) scripts/generate_bignum_tests.py
+
+$(GENERATED_PSA_DATA_FILES): generated_psa_test_data
generated_psa_test_data: scripts/generate_psa_tests.py
generated_psa_test_data: ../scripts/mbedtls_dev/crypto_knowledge.py
generated_psa_test_data: ../scripts/mbedtls_dev/macro_collector.py
generated_psa_test_data: ../scripts/mbedtls_dev/psa_storage.py
generated_psa_test_data: ../scripts/mbedtls_dev/test_case.py
+generated_psa_test_data: ../scripts/mbedtls_dev/test_generation.py
## The generated file only depends on the options that are present in
## crypto_config.h, not on which options are set. To avoid regenerating this
## file all the time when switching between configurations, don't declare
@@ -98,7 +115,7 @@
generated_psa_test_data: ../include/psa/crypto_extra.h
generated_psa_test_data: suites/test_suite_psa_crypto_metadata.data
generated_psa_test_data:
- echo " Gen $(GENERATED_DATA_FILES) ..."
+ echo " Gen $(GENERATED_PSA_DATA_FILES) ..."
$(PYTHON) scripts/generate_psa_tests.py
# A test application is built for each suites/test_suite_*.data file.
@@ -107,7 +124,7 @@
DATA_FILES := $(wildcard suites/test_suite_*.data)
# Make sure that generated data files are included even if they don't
# exist yet when the makefile is parsed.
-DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_DATA_FILES))
+DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_FILES))
APPS = $(basename $(subst suites/,,$(DATA_FILES)))
# Construct executable name by adding OS specific suffix $(EXEXT).
diff --git a/tests/scripts/check-generated-files.sh b/tests/scripts/check-generated-files.sh
index 1736f24..3006ec7 100755
--- a/tests/scripts/check-generated-files.sh
+++ b/tests/scripts/check-generated-files.sh
@@ -126,4 +126,5 @@
# the step that creates or updates these files.
check scripts/generate_visualc_files.pl visualc/VS2010
check scripts/generate_psa_constants.py programs/psa/psa_constant_names_generated.c
+check tests/scripts/generate_bignum_tests.py $(tests/scripts/generate_bignum_tests.py --list)
check tests/scripts/generate_psa_tests.py $(tests/scripts/generate_psa_tests.py --list)
diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py
new file mode 100755
index 0000000..ceafa4a
--- /dev/null
+++ b/tests/scripts/generate_bignum_tests.py
@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""Generate test data for bignum functions.
+
+With no arguments, generate all test data. With non-option arguments,
+generate only the specified files.
+
+Class structure:
+
+Child classes of test_generation.BaseTarget (file targets) represent an output
+file. These indicate where test cases will be written to, for all subclasses of
+this target. Multiple file targets should not reuse a `target_basename`.
+
+Each subclass derived from a file target can either be:
+ - A concrete class, representing a test function, which generates test cases.
+ - An abstract class containing shared methods and attributes, not associated
+ with a test function. An example is BignumOperation, which provides
+ common features used for bignum binary operations.
+
+Both concrete and abstract subclasses can be derived from, to implement
+additional test cases (see BignumCmp and BignumCmpAbs for examples of deriving
+from abstract and concrete classes).
+
+
+Adding test case generation for a function:
+
+A subclass representing the test function should be added, deriving from a
+file target such as BignumTarget. This test class must set/implement the
+following:
+ - test_function: the function name from the associated .function file.
+ - test_name: a descriptive name or brief summary to refer to the test
+ function.
+ - arguments(): a method to generate the list of arguments required for the
+ test_function.
+ - generate_function_test(): a method to generate TestCases for the function.
+ This should create instances of the class with required input data, and
+ call `.create_test_case()` to yield the TestCase.
+
+Additional details and other attributes/methods are given in the documentation
+of BaseTarget in test_generation.py.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+import sys
+import typing
+
+from abc import ABCMeta, abstractmethod
+from typing import Iterator, List, Tuple, TypeVar
+
+import scripts_path # pylint: disable=unused-import
+from mbedtls_dev import test_case
+from mbedtls_dev import test_generation
+
+T = TypeVar('T') #pylint: disable=invalid-name
+
+def hex_to_int(val: str) -> int:
+ return int(val, 16) if val else 0
+
+def quote_str(val) -> str:
+ return "\"{}\"".format(val)
+
+def combination_pairs(values: List[T]) -> List[Tuple[T, T]]:
+ """Return all pair combinations from input values.
+
+ The return value is cast, as older versions of mypy are unable to derive
+ the specific type returned by itertools.combinations_with_replacement.
+ """
+ return typing.cast(
+ List[Tuple[T, T]],
+ list(itertools.combinations_with_replacement(values, 2))
+ )
+
+
+class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta):
+ #pylint: disable=abstract-method
+ """Target for bignum (mpi) test case generation."""
+ target_basename = 'test_suite_mpi.generated'
+
+
+class BignumOperation(BignumTarget, metaclass=ABCMeta):
+ """Common features for bignum binary operations.
+
+ This adds functionality common in binary operation tests. This includes
+ generation of case descriptions, using descriptions of values and symbols
+ to represent the operation or result.
+
+ Attributes:
+ symbol: Symbol used for the operation in case description.
+ input_values: List of values to use as test case inputs. These are
+ combined to produce pairs of values.
+ input_cases: List of tuples containing pairs of test case inputs. This
+ can be used to implement specific pairs of inputs.
+ """
+ symbol = ""
+ input_values = [
+ "", "0", "7b", "-7b",
+ "0000000000000000123", "-0000000000000000123",
+ "1230000000000000000", "-1230000000000000000"
+ ] # type: List[str]
+ input_cases = [] # type: List[Tuple[str, str]]
+
+ def __init__(self, val_a: str, val_b: str) -> None:
+ self.arg_a = val_a
+ self.arg_b = val_b
+ self.int_a = hex_to_int(val_a)
+ self.int_b = hex_to_int(val_b)
+
+ def arguments(self) -> List[str]:
+ return [quote_str(self.arg_a), quote_str(self.arg_b), self.result()]
+
+ def description(self) -> str:
+ """Generate a description for the test case.
+
+ If not set, case_description uses the form A `symbol` B, where symbol
+ is used to represent the operation. Descriptions of each value are
+ generated to provide some context to the test case.
+ """
+ if not self.case_description:
+ self.case_description = "{} {} {}".format(
+ self.value_description(self.arg_a),
+ self.symbol,
+ self.value_description(self.arg_b)
+ )
+ return super().description()
+
+ @abstractmethod
+ def result(self) -> str:
+ """Get the result of the operation.
+
+ This could be calculated during initialization and stored as `_result`
+ and then returned, or calculated when the method is called.
+ """
+ raise NotImplementedError
+
+ @staticmethod
+ def value_description(val) -> str:
+ """Generate a description of the argument val.
+
+ This produces a simple description of the value, which is used in test
+ case naming to add context.
+ """
+ if val == "":
+ return "0 (null)"
+ if val == "0":
+ return "0 (1 limb)"
+
+ if val[0] == "-":
+ tmp = "negative"
+ val = val[1:]
+ else:
+ tmp = "positive"
+ if val[0] == "0":
+ tmp += " with leading zero limb"
+ elif len(val) > 10:
+ tmp = "large " + tmp
+ return tmp
+
+ @classmethod
+ def get_value_pairs(cls) -> Iterator[Tuple[str, str]]:
+ """Generator to yield pairs of inputs.
+
+ Combinations are first generated from all input values, and then
+ specific cases provided.
+ """
+ yield from combination_pairs(cls.input_values)
+ yield from cls.input_cases
+
+ @classmethod
+ def generate_function_tests(cls) -> Iterator[test_case.TestCase]:
+ for a_value, b_value in cls.get_value_pairs():
+ cur_op = cls(a_value, b_value)
+ yield cur_op.create_test_case()
+
+
+class BignumCmp(BignumOperation):
+ """Test cases for bignum value comparison."""
+ count = 0
+ test_function = "mbedtls_mpi_cmp_mpi"
+ test_name = "MPI compare"
+ input_cases = [
+ ("-2", "-3"),
+ ("-2", "-2"),
+ ("2b4", "2b5"),
+ ("2b5", "2b6")
+ ]
+
+ def __init__(self, val_a, val_b) -> None:
+ super().__init__(val_a, val_b)
+ self._result = int(self.int_a > self.int_b) - int(self.int_a < self.int_b)
+ self.symbol = ["<", "==", ">"][self._result + 1]
+
+ def result(self) -> str:
+ return str(self._result)
+
+
+class BignumCmpAbs(BignumCmp):
+ """Test cases for absolute bignum value comparison."""
+ count = 0
+ test_function = "mbedtls_mpi_cmp_abs"
+ test_name = "MPI compare (abs)"
+
+ def __init__(self, val_a, val_b) -> None:
+ super().__init__(val_a.strip("-"), val_b.strip("-"))
+
+
+class BignumAdd(BignumOperation):
+ """Test cases for bignum value addition."""
+ count = 0
+ symbol = "+"
+ test_function = "mbedtls_mpi_add_mpi"
+ test_name = "MPI add"
+ input_cases = combination_pairs(
+ [
+ "1c67967269c6", "9cde3",
+ "-1c67967269c6", "-9cde3",
+ ]
+ )
+
+ def result(self) -> str:
+ return quote_str("{:x}".format(self.int_a + self.int_b))
+
+if __name__ == '__main__':
+ # Use the section of the docstring relevant to the CLI as description
+ test_generation.main(sys.argv[1:], "\n".join(__doc__.splitlines()[:4]))
diff --git a/tests/scripts/generate_psa_tests.py b/tests/scripts/generate_psa_tests.py
index 3d23edd..c788fd7 100755
--- a/tests/scripts/generate_psa_tests.py
+++ b/tests/scripts/generate_psa_tests.py
@@ -20,22 +20,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import argparse
import enum
-import os
-import posixpath
import re
import sys
-from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional, TypeVar
+from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional
import scripts_path # pylint: disable=unused-import
-from mbedtls_dev import build_tree
from mbedtls_dev import crypto_knowledge
from mbedtls_dev import macro_collector
from mbedtls_dev import psa_storage
from mbedtls_dev import test_case
-
-T = TypeVar('T') #pylint: disable=invalid-name
+from mbedtls_dev import test_generation
def psa_want_symbol(name: str) -> str:
@@ -897,35 +892,11 @@
yield from super().generate_all_keys()
yield from self.all_keys_for_implicit_usage()
-class TestGenerator:
- """Generate test data."""
-
- def __init__(self, options) -> None:
- self.test_suite_directory = self.get_option(options, 'directory',
- 'tests/suites')
- self.info = Information()
-
- @staticmethod
- def get_option(options, name: str, default: T) -> T:
- value = getattr(options, name, None)
- return default if value is None else value
-
- def filename_for(self, basename: str) -> str:
- """The location of the data file with the specified base name."""
- return posixpath.join(self.test_suite_directory, basename + '.data')
-
- def write_test_data_file(self, basename: str,
- test_cases: Iterable[test_case.TestCase]) -> None:
- """Write the test cases to a .data file.
-
- The output file is ``basename + '.data'`` in the test suite directory.
- """
- filename = self.filename_for(basename)
- test_case.write_data_file(filename, test_cases)
-
+class PSATestGenerator(test_generation.TestGenerator):
+ """Test generator subclass including PSA targets and info."""
# Note that targets whose names contain 'test_format' have their content
# validated by `abi_check.py`.
- TARGETS = {
+ targets = {
'test_suite_psa_crypto_generate_key.generated':
lambda info: KeyGenerate(info).test_cases_for_key_generation(),
'test_suite_psa_crypto_not_supported.generated':
@@ -938,44 +909,12 @@
lambda info: StorageFormatV0(info).all_test_cases(),
} #type: Dict[str, Callable[[Information], Iterable[test_case.TestCase]]]
- def generate_target(self, name: str) -> None:
- test_cases = self.TARGETS[name](self.info)
- self.write_test_data_file(name, test_cases)
+ def __init__(self, options):
+ super().__init__(options)
+ self.info = Information()
-def main(args):
- """Command line entry point."""
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument('--list', action='store_true',
- help='List available targets and exit')
- parser.add_argument('--list-for-cmake', action='store_true',
- help='Print \';\'-separated list of available targets and exit')
- parser.add_argument('--directory', metavar='DIR',
- help='Output directory (default: tests/suites)')
- parser.add_argument('targets', nargs='*', metavar='TARGET',
- help='Target file to generate (default: all; "-": none)')
- options = parser.parse_args(args)
- build_tree.chdir_to_root()
- generator = TestGenerator(options)
- if options.list:
- for name in sorted(generator.TARGETS):
- print(generator.filename_for(name))
- return
- # List in a cmake list format (i.e. ';'-separated)
- if options.list_for_cmake:
- print(';'.join(generator.filename_for(name)
- for name in sorted(generator.TARGETS)), end='')
- return
- if options.targets:
- # Allow "-" as a special case so you can run
- # ``generate_psa_tests.py - $targets`` and it works uniformly whether
- # ``$targets`` is empty or not.
- options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target))
- for target in options.targets
- if target != '-']
- else:
- options.targets = sorted(generator.TARGETS)
- for target in options.targets:
- generator.generate_target(target)
+ def generate_target(self, name: str, *target_args) -> None:
+ super().generate_target(name, self.info)
if __name__ == '__main__':
- main(sys.argv[1:])
+ test_generation.main(sys.argv[1:], __doc__, PSATestGenerator)