Add sha384 and sha512 case

Signed-off-by: Jerry Yu <jerry.h.yu@arm.com>
diff --git a/library/ssl_tls13_generic.c b/library/ssl_tls13_generic.c
index 2338beb..4960c9a 100644
--- a/library/ssl_tls13_generic.c
+++ b/library/ssl_tls13_generic.c
@@ -1125,6 +1125,12 @@
                                            ( "unknown key size: %"
                                              MBEDTLS_PRINTF_SIZET " bits",
                                              own_key_size ) );
+                    MBEDTLS_SSL_DEBUG_MSG( 1,
+                        ( "signature algorithm not in "
+                            "received or offered list." ) );
+                    MBEDTLS_SSL_PEND_FATAL_ALERT(
+                                        MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER,
+                                        MBEDTLS_ERR_SSL_HANDSHAKE_FAILURE );
                     return( MBEDTLS_ERR_SSL_HANDSHAKE_FAILURE );
             }
             break;
@@ -1134,25 +1140,46 @@
         case MBEDTLS_SSL_SIG_RSA:
             /* Determine the size of the key */
             own_key_size = mbedtls_pk_get_bitlen( own_key );
-            switch( own_key_size )
+            if( own_key_size <= 2048 &&
+                mbedtls_ssl_sig_alg_is_received( ssl,
+                                    MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA256 ) )
             {
-                case 2048:
-                    md_alg  = MBEDTLS_MD_SHA256;
-                    algorithm = MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA256;
-                    break;
-                default:
-                    MBEDTLS_SSL_DEBUG_MSG( 3,
-                                           ( "unknown key size: %"
-                                             MBEDTLS_PRINTF_SIZET " bits",
-                                             own_key_size ) );
-                    return( MBEDTLS_ERR_SSL_HANDSHAKE_FAILURE );
+                md_alg  = MBEDTLS_MD_SHA256;
+                algorithm = MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA256;
             }
+            else if( own_key_size <= 3072 &&
+                     mbedtls_ssl_sig_alg_is_received( ssl,
+                                    MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA384 ) )
+            {
+                md_alg  = MBEDTLS_MD_SHA384;
+                algorithm = MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA384;
+            }
+            else if( own_key_size <= 4096 &&
+                     mbedtls_ssl_sig_alg_is_received( ssl,
+                                    MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA512 ) )
+            {
+                md_alg  = MBEDTLS_MD_SHA512;
+                algorithm = MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA512;
+            }
+            else
+            {
+                MBEDTLS_SSL_DEBUG_MSG( 3, ( "unknown key size: %"
+                                            MBEDTLS_PRINTF_SIZET " bits",
+                                            own_key_size ) );
+                MBEDTLS_SSL_DEBUG_MSG( 1,
+                    ( "signature algorithm not in received or offered list." ) );
+                MBEDTLS_SSL_PEND_FATAL_ALERT(
+                                        MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER,
+                                        MBEDTLS_ERR_SSL_HANDSHAKE_FAILURE );
+                return( MBEDTLS_ERR_SSL_HANDSHAKE_FAILURE );
+            }
+
             if( mbedtls_rsa_set_padding( mbedtls_pk_rsa( *own_key ),
                                          MBEDTLS_RSA_PKCS_V21,
                                          md_alg ) != 0 )
             {
                 MBEDTLS_SSL_DEBUG_MSG( 1, ( "Set RSA padding Fail" ) );
-                return( MBEDTLS_ERR_SSL_HANDSHAKE_FAILURE );
+                return( MBEDTLS_ERR_SSL_INTERNAL_ERROR );
             }
             break;
 #endif /* MBEDTLS_RSA_C && MBEDTLS_PKCS1_V21 */
@@ -1165,7 +1192,7 @@
     if( !mbedtls_ssl_sig_alg_is_received( ssl, algorithm ) )
     {
         MBEDTLS_SSL_DEBUG_MSG( 1,
-                               ( "signature algorithm not in received list." ) );
+                    ( "signature algorithm not in received or offered list." ) );
         MBEDTLS_SSL_PEND_FATAL_ALERT( MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER,
                                       MBEDTLS_ERR_SSL_HANDSHAKE_FAILURE );
         return( MBEDTLS_ERR_SSL_HANDSHAKE_FAILURE );
diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh
index 61c4407..36cbe37 100755
--- a/tests/ssl-opt.sh
+++ b/tests/ssl-opt.sh
@@ -9999,6 +9999,38 @@
             -c "client state: MBEDTLS_SSL_CLIENT_CERTIFICATE_VERIFY" \
             -c "Protocol is TLSv1.3"
 
+requires_openssl_tls1_3
+requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3
+requires_config_enabled MBEDTLS_DEBUG_C
+requires_config_enabled MBEDTLS_SSL_CLI_C
+requires_config_enabled MBEDTLS_RSA_C
+run_test    "TLS 1.3: Client authentication, client algorithm not in server list - openssl" \
+            "$O_NEXT_SRV -msg -tls1_3 -num_tickets 0 -no_resume_ephemeral -no_cache -Verify 10 -no_middlebox
+                -sigalgs ecdsa_secp256r1_sha256" \
+            "$P_CLI debug_level=4 force_version=tls13 crt_file=data_files/cert_sha256.crt \
+                    key_file=data_files/server1.key sig_algs=ecdsa_secp256r1_sha256,rsa_pss_rsae_sha256" \
+            1 \
+            -c "got a certificate request" \
+            -c "client state: MBEDTLS_SSL_CLIENT_CERTIFICATE" \
+            -c "client state: MBEDTLS_SSL_CLIENT_CERTIFICATE_VERIFY" \
+            -c "signature algorithm not in received or offered list."
+
+requires_gnutls_tls1_3
+requires_gnutls_next_no_ticket
+requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3
+requires_config_enabled MBEDTLS_DEBUG_C
+requires_config_enabled MBEDTLS_SSL_CLI_C
+requires_config_enabled MBEDTLS_RSA_C
+run_test    "TLS 1.3: Client authentication, client algorithm not in server list - gnutls" \
+            "$G_NEXT_SRV --debug=4 --priority=NORMAL:-VERS-ALL:+VERS-TLS1.3:+CIPHER-ALL:-SIGN-ALL:+SIGN-ECDSA-SECP256R1-SHA256:%NO_TICKETS:%DISABLE_TLS13_COMPAT_MODE" \
+            "$P_CLI debug_level=3 force_version=tls13 crt_file=data_files/server2-sha256.crt \
+                    key_file=data_files/server2.key sig_algs=ecdsa_secp256r1_sha256,rsa_pss_rsae_sha256" \
+            1 \
+            -c "got a certificate request" \
+            -c "client state: MBEDTLS_SSL_CLIENT_CERTIFICATE" \
+            -c "client state: MBEDTLS_SSL_CLIENT_CERTIFICATE_VERIFY" \
+            -c "signature algorithm not in received or offered list."
+
 requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3
 requires_config_enabled MBEDTLS_SSL_TLS1_3_COMPATIBILITY_MODE
 requires_config_enabled MBEDTLS_DEBUG_C