Add negative test for PSS key used for PKCS1 sig

Signed-off-by: Simo Sorce <[email protected]>

tests/trsapssam

@@ -9,6 +9,7 @@ sed -e "s/#pkcs11-module-encode-provider-uri-to-pem/pkcs11-module-encode-provide
     "${OPENSSL_CONF}" > "${OPENSSL_CONF}.rsapss_genpkey"
 OPENSSL_CONF=${OPENSSL_CONF}.rsapss_genpkey
 
+title PARA "Generate RSA PSS key restricted to SHA256"
 ossl '
 genpkey -propquery "?provider=pkcs11"
         -algorithm "RSA-PSS" -pkeyopt "rsa_pss_keygen_md:SHA256"
@@ -56,3 +57,30 @@ if [ $FAIL -ne 0 ]; then
     echo "Signature seem to have failed for unrelated reasons"
     exit 1
 fi
+
+title PARA "Generate RSA PSS key"
+ossl '
+genpkey -propquery "?provider=pkcs11"
+        -algorithm "RSA-PSS"
+        -pkeyopt "pkcs11_uri:pkcs11:object=Test-RSA-PSS-Only"'
+
+FAIL=0
+title PARA "Fail Signing with RSA PKCS1 mech and RSA-PSS key"
+ossl '
+pkeyutl -sign -inkey "pkcs11:object=Test-RSA-PSS-Only;type=private"
+              -digest sha256
+              -pkeyopt rsa_padding_mode:pkcs1
+              -in ${RAND64FILE}
+              -rawin
+              -out ${TMPPDIR}/sha384-rsa-not-rsapss-sig.bin 2>&1' "$helper_emit" || FAIL=1
+if [ $FAIL -eq 0 ]; then
+    echo "Signature should have failed due to PSS restrictions"
+    exit 1
+fi
+output="$helper_output"
+FAIL=0
+echo "$output" | grep "An invalid mechanism was specified to the cryptographic operation" > /dev/null 2>&1 || FAIL=1
+if [ $FAIL -ne 0 ]; then
+    echo "Signature seem to have failed for unrelated reasons"
+    exit 1
+fi