tf_fuzz: fix sst `check ... except fail` calls
Fix the error code modelling of sst calls with data value checks (`check
...`), and except fail.
Currently, demo/8.test erroneously fails:
----
set sst name snortwaggle data *;
read sst name snortwaggle check "almost certainly not *this*" expect fail;
----
When reading `expect fail`, the sst read call is marked as "should
fail". However, the intended behaviour is for the read to succeed and
the check to fail.
To fix this, this patch adds a template variable, $check_expect, that
can be used to modify the expected value of the check (similar to how
$expect is used for expected values of calls). Using this, separate
pass/fail values can be set for the check as well as the call itself.
Change-Id: Ia55b6f2fef5737d67540e98ac6fc5e7695a8515f
Signed-off-by: Nik Dewally <Nik.Dewally@arm.com>
diff --git a/tf_fuzz/lib/tfm_boilerplate.txt b/tf_fuzz/lib/tfm_boilerplate.txt
index 3634238..5b65684 100644
--- a/tf_fuzz/lib/tfm_boilerplate.txt
+++ b/tf_fuzz/lib/tfm_boilerplate.txt
@@ -169,7 +169,7 @@
}
/* Check that the data is correct */
if (memcmp($act_data, $exp_data,
- $length) != 0) {
+ $length) != $check_expect) {
TEST_FAIL("Read data should be equal to result data");
return;
}
diff --git a/tf_fuzz/tfz-cpp/calls/psa_call.cpp b/tf_fuzz/tfz-cpp/calls/psa_call.cpp
index 46af409..0e1680b 100644
--- a/tf_fuzz/tfz-cpp/calls/psa_call.cpp
+++ b/tf_fuzz/tfz-cpp/calls/psa_call.cpp
@@ -88,51 +88,104 @@
{
string formalized; // "proper" result string
- if (!exp_data.pf_nothing) {
- if (exp_data.pf_pass) {
- find_replace_all ("$expect",
- test_state->bplate->bplate_string[sst_pass_string],
- check_code);
- } else if (exp_data.pf_fail) {
- // Check for not-success:
- find_replace_1st ("!=", "==",
- check_code);
- find_replace_all ("$expect",
- test_state->bplate->bplate_string[sst_pass_string],
- check_code);
- find_replace_1st ("expected ", "expected not ",
- check_code);
+ if (exp_data.pf_nothing) { // do not generate result checks
+ return;
+
+ }
+ if (exp_data.pf_pass) { // expect pass
+ find_replace_all ("$expect",
+ test_state->bplate->bplate_string[sst_pass_string],
+ check_code);
+
+ // `check "foo"`
+ find_replace_all ("$check_expect",
+ "0",
+ check_code);
+
+ } else if (exp_data.pf_fail) { // expect fail
+
+ // If the command is `... check "foo" expect fail;`, the fail
+ // binds to the check, not the command itself.
+ if (exp_data.data_specified) {
+ // expect a pass for the sst call itself.
+ find_replace_all ("$expect",
+ test_state->bplate->bplate_string[sst_pass_string],
+ check_code);
+
+ // expect a failure for the check.
+ find_replace_all ("!= $check_expect",
+ "== 0",
+ check_code);
+
+ find_replace_1st ("should be equal", "should not be equal",
+ check_code);
} else {
- if (exp_data.pf_specified) {
- formalized = formalize (exp_data.pf_result_string, "PSA_ERROR_");
- find_replace_all ("$expect", formalized, check_code);
- } else {
- // Figure out what the message should read:
- switch (asset_info.how_asset_found) {
- case asset_search::found_active:
- case asset_search::created_new:
- find_replace_all ("$expect",
- test_state->bplate->
- bplate_string[sst_pass_string],
- check_code);
- break;
- case asset_search::found_deleted:
- case asset_search::not_found:
- find_replace_all ("$expect",
- test_state->bplate->
- bplate_string[sst_fail_removed],
- check_code);
- break;
- default:
- find_replace_1st ("!=", "==",
- check_code); // like "fail", just make sure...
- find_replace_all ("$expect",
- test_state->bplate->
- bplate_string[sst_pass_string],
- check_code); // ... it's *not* PSA_SUCCESS
- break;
- }
- }
+ // Check for not-success:
+ find_replace_1st ("!=", "==",
+ check_code);
+ find_replace_all ("$expect",
+ test_state->bplate->bplate_string[sst_pass_string],
+ check_code);
+ find_replace_1st ("expected ", "expected not ",
+ check_code);
+ }
+ }
+
+ else if (exp_data.pf_specified) { // expect <PSA_ERROR_CODE>
+ formalized = formalize (exp_data.pf_result_string, "PSA_ERROR_");
+ find_replace_all ("$expect", formalized, check_code);
+
+ // NOTE: Assumes that the variable used to store the actual
+ // value initialised to a different value than the expected
+ // value.
+ find_replace_all ("!= $check_expect","== 0",check_code);
+ find_replace_1st ("should be equal", "should not be equal",
+ check_code);
+ }
+
+ else { // no explicit expect -- simulate
+
+ // Figure out what the message should read:
+ switch (asset_info.how_asset_found) {
+ case asset_search::found_active:
+ case asset_search::created_new:
+ find_replace_all ("$expect",
+ test_state->bplate->
+ bplate_string[sst_pass_string],
+ check_code);
+
+ find_replace_all ("$check_expect","0",check_code);
+ break;
+ case asset_search::found_deleted:
+ case asset_search::not_found:
+ find_replace_all ("$expect",
+ test_state->bplate->
+ bplate_string[sst_fail_removed],
+ check_code);
+
+ // NOTE: Assumes that the variable used to store the actual
+ // value initialised to a different value than the expected
+ // value.
+ find_replace_all ("!= $check_expect","== 0",check_code);
+ find_replace_1st ("should be equal", "should not be equal",
+ check_code);
+ break;
+ default:
+ find_replace_1st ("!=", "==",
+ check_code); // like "fail", just make sure...
+
+ find_replace_all ("$expect",
+ test_state->bplate->
+ bplate_string[sst_pass_string],
+ check_code); // ... it's *not* PSA_SUCCESS
+
+ // NOTE: Assumes that the variable used to store the actual
+ // value initialised to a different value than the expected
+ // value.
+ find_replace_all ("!= $check_expect","== 0",check_code);
+ find_replace_1st ("should be equal", "should not be equal",
+ check_code);
+ break;
}
}
}
diff --git a/tf_fuzz/tfz-cpp/regression/000008_set_sst_name_rand_data_read_check_wrong/exp_test.c b/tf_fuzz/tfz-cpp/regression/000008_set_sst_name_rand_data_read_check_wrong/exp_test.c
index 400ac89..974b2ca 100644
--- a/tf_fuzz/tfz-cpp/regression/000008_set_sst_name_rand_data_read_check_wrong/exp_test.c
+++ b/tf_fuzz/tfz-cpp/regression/000008_set_sst_name_rand_data_read_check_wrong/exp_test.c
@@ -49,14 +49,14 @@
}
sst_status = psa_ps_get\(@@@001@@@, 0, @@@003@@@, snortwaggle_act_data
&snortwaggle_act_length);
- if (sst_status == PSA_SUCCESS) {
- TEST_FAIL("psa_ps_get() expected not PSA_SUCCESS.");
+ if (sst_status != PSA_SUCCESS) {
+ TEST_FAIL("psa_ps_get() expected PSA_SUCCESS.");
return;
}
/* Check that the data is correct */
if \(memcmp\(snortwaggle_act_data, snortwaggle_exp_data,
- snortwaggle_act_length\) != 0\) {
- TEST_FAIL("Read data should be equal to result data");
+ snortwaggle_act_length\) == 0\) {
+ TEST_FAIL("Read data should not be equal to result data");
return;
}
diff --git a/tf_fuzz/tfz-cpp/regression/000010_read_nonexistent_sst_check_string/exp_test.c b/tf_fuzz/tfz-cpp/regression/000010_read_nonexistent_sst_check_string/exp_test.c
index 72085dc..0cd77b8 100644
--- a/tf_fuzz/tfz-cpp/regression/000010_read_nonexistent_sst_check_string/exp_test.c
+++ b/tf_fuzz/tfz-cpp/regression/000010_read_nonexistent_sst_check_string/exp_test.c
@@ -46,8 +46,8 @@
}
/* Check that the data is correct */
if \(memcmp\(napoleon_act_data, napoleon_exp_data,
- napoleon_act_length\) != 0\) {
- TEST_FAIL("Read data should be equal to result data");
+ napoleon_act_length\) == 0\) {
+ TEST_FAIL("Read data should not be equal to result data");
return;
}
diff --git a/tf_fuzz/tfz-cpp/regression/000012_read_nonexistent_sst_check_string_expect_other/exp_test.c b/tf_fuzz/tfz-cpp/regression/000012_read_nonexistent_sst_check_string_expect_other/exp_test.c
index 74c1545..93e16b5 100644
--- a/tf_fuzz/tfz-cpp/regression/000012_read_nonexistent_sst_check_string_expect_other/exp_test.c
+++ b/tf_fuzz/tfz-cpp/regression/000012_read_nonexistent_sst_check_string_expect_other/exp_test.c
@@ -46,8 +46,8 @@
}
/* Check that the data is correct */
if \(memcmp\(napoleon_act_data, napoleon_exp_data,
- napoleon_act_length\) != 0\) \{
- TEST_FAIL("Read data should be equal to result data");
+ napoleon_act_length\) == 0\) \{
+ TEST_FAIL("Read data should not be equal to result data");
return;
}
diff --git a/tf_fuzz/tfz-cpp/regression/000017_read_sst_check_single_asset_multiple_times/exp_test.c b/tf_fuzz/tfz-cpp/regression/000017_read_sst_check_single_asset_multiple_times/exp_test.c
index 1abe615..a251a42 100644
--- a/tf_fuzz/tfz-cpp/regression/000017_read_sst_check_single_asset_multiple_times/exp_test.c
+++ b/tf_fuzz/tfz-cpp/regression/000017_read_sst_check_single_asset_multiple_times/exp_test.c
@@ -51,38 +51,38 @@
}
sst_status = psa_ps_get\(@@@001@@@, 0, \d+, just_checking_act_data,
&just_checking_act_length);
- if (sst_status == PSA_SUCCESS) {
- TEST_FAIL("psa_ps_get() expected not PSA_SUCCESS.");
+ if (sst_status != PSA_SUCCESS) {
+ TEST_FAIL("psa_ps_get() expected PSA_SUCCESS.");
return;
}
/* Check that the data is correct */
if \(memcmp\(just_checking_act_data, just_checking_exp_data,
- just_checking_act_length\) != 0\) {
- TEST_FAIL("Read data should be equal to result data");
+ just_checking_act_length\) == 0\) {
+ TEST_FAIL("Read data should not be equal to result data");
return;
}
sst_status = psa_ps_get\(@@@001@@@, 0, \d+, just_checking_act_data,
&just_checking_act_length);
- if (sst_status == PSA_SUCCESS) {
- TEST_FAIL("psa_ps_get() expected not PSA_SUCCESS.");
+ if (sst_status != PSA_SUCCESS) {
+ TEST_FAIL("psa_ps_get() expected PSA_SUCCESS.");
return;
}
/* Check that the data is correct */
if \(memcmp\(just_checking_act_data, just_checking_exp_data_1,
- just_checking_act_length\) != 0\) {
- TEST_FAIL("Read data should be equal to result data");
+ just_checking_act_length\) == 0\) {
+ TEST_FAIL("Read data should not be equal to result data");
return;
}
sst_status = psa_ps_get\(@@@001@@@, 0, \d+, just_checking_act_data,
&just_checking_act_length);
- if (sst_status == PSA_SUCCESS) {
- TEST_FAIL("psa_ps_get() expected not PSA_SUCCESS.");
+ if (sst_status != PSA_SUCCESS) {
+ TEST_FAIL("psa_ps_get() expected PSA_SUCCESS.");
return;
}
/* Check that the data is correct */
if \(memcmp\(just_checking_act_data, just_checking_exp_data_2,
- just_checking_act_length\) != 0\) {
- TEST_FAIL("Read data should be equal to result data");
+ just_checking_act_length\) == 0\) {
+ TEST_FAIL("Read data should not be equal to result data");
return;
}
sst_status = psa_ps_get\(@@@001@@@, 0, \d+, just_checking_act_data,
diff --git a/tf_fuzz/tfz-cpp/regression/000018_000016_and_000017/exp_test.c b/tf_fuzz/tfz-cpp/regression/000018_000016_and_000017/exp_test.c
index 353f3fb..06b82b4 100644
--- a/tf_fuzz/tfz-cpp/regression/000018_000016_and_000017/exp_test.c
+++ b/tf_fuzz/tfz-cpp/regression/000018_000016_and_000017/exp_test.c
@@ -61,14 +61,14 @@
}
sst_status = psa_ps_get\(@@@001@@@, 0, 11, indecisive_act_data,
&indecisive_act_length);
- if (sst_status == PSA_SUCCESS) {
- TEST_FAIL("psa_ps_get() expected not PSA_SUCCESS.");
+ if (sst_status != PSA_SUCCESS) {
+ TEST_FAIL("psa_ps_get() expected PSA_SUCCESS.");
return;
}
/* Check that the data is correct */
if (memcmp(indecisive_act_data, indecisive_exp_data,
- indecisive_act_length\) != 0\) {
- TEST_FAIL("Read data should be equal to result data");
+ indecisive_act_length\) == 0\) {
+ TEST_FAIL("Read data should not be equal to result data");
return;
}
/* Resetting SST asset "indecisive," with data "Second val...". */
@@ -80,14 +80,14 @@
}
sst_status = psa_ps_get\(@@@001@@@, 0, 12, indecisive_act_data,
&indecisive_act_length);
- if (sst_status == PSA_SUCCESS) {
- TEST_FAIL("psa_ps_get() expected not PSA_SUCCESS.");
+ if (sst_status != PSA_SUCCESS) {
+ TEST_FAIL("psa_ps_get() expected PSA_SUCCESS.");
return;
}
/* Check that the data is correct */
if (memcmp(indecisive_act_data, indecisive_exp_data_1,
- indecisive_act_length) != 0) {
- TEST_FAIL("Read data should be equal to result data");
+ indecisive_act_length) == 0) {
+ TEST_FAIL("Read data should not be equal to result data");
return;
}
/\* Resetting SST asset "indecisive," with data "@@001@10@@...". \*/
@@ -106,14 +106,14 @@
}
sst_status = psa_ps_get\(@@@001@@@, 0, 12, indecisive_act_data,
&indecisive_act_length);
- if (sst_status == PSA_SUCCESS) {
- TEST_FAIL("psa_ps_get() expected not PSA_SUCCESS.");
+ if (sst_status != PSA_SUCCESS) {
+ TEST_FAIL("psa_ps_get() expected PSA_SUCCESS.");
return;
}
/* Check that the data is correct */
if (memcmp(indecisive_act_data, indecisive_exp_data_2,
- indecisive_act_length) != 0) {
- TEST_FAIL("Read data should be equal to result data");
+ indecisive_act_length) == 0) {
+ TEST_FAIL("Read data should not be equal to result data");
return;
}
/\* Resetting SST asset "indecisive," with data "@@002@10@@...". \*/
diff --git a/tf_fuzz/tfz-cpp/regression/000021_abbreviated_result_codes/exp_test.c b/tf_fuzz/tfz-cpp/regression/000021_abbreviated_result_codes/exp_test.c
index 74c1545..93e16b5 100644
--- a/tf_fuzz/tfz-cpp/regression/000021_abbreviated_result_codes/exp_test.c
+++ b/tf_fuzz/tfz-cpp/regression/000021_abbreviated_result_codes/exp_test.c
@@ -46,8 +46,8 @@
}
/* Check that the data is correct */
if \(memcmp\(napoleon_act_data, napoleon_exp_data,
- napoleon_act_length\) != 0\) \{
- TEST_FAIL("Read data should be equal to result data");
+ napoleon_act_length\) == 0\) \{
+ TEST_FAIL("Read data should not be equal to result data");
return;
}
diff --git a/tf_fuzz/tfz-cpp/utility/data_blocks.hpp b/tf_fuzz/tfz-cpp/utility/data_blocks.hpp
index bee4594..723ee95 100644
--- a/tf_fuzz/tfz-cpp/utility/data_blocks.hpp
+++ b/tf_fuzz/tfz-cpp/utility/data_blocks.hpp
@@ -43,7 +43,7 @@
bool pf_nothing; // true to not generate results-check(s)
bool pf_pass; // if !expect.pf_nothing, then pass is expected
bool pf_fail; // if "expect fail" was specified
- bool pf_specified;
+ bool pf_specified; // if "expect <ERROR_CODE>" was specified
/* if !pf_nothing && !pf_pass, then
true == expected result was specified
false == tf_fuzz must model expected result, and
@@ -185,7 +185,7 @@
bool can_encrypt; // OK for encryption (fail other uses).
bool can_decrypt; // OK for decryption (fail other uses).
bool can_sign; // OK for signing (fail other operations).
- bool can_verify; // OK for verifing a message signature (fail other uses).
+ bool can_verify; // OK for verifying a message signature (fail other uses).
bool derivable; // OK for derive other keys (fail other uses).
bool persistent; // must be deleted at the end of test.
string usage_string;