From 392c669164e8316d523b945ab476374d77850827 Mon Sep 17 00:00:00 2001 From: Jeff Anderson Date: Fri, 26 Sep 2025 18:52:53 -0400 Subject: [PATCH 1/3] implement OAESP params for asymmetric encryption --- src/lib/SoftHSM.cpp | 117 ++++++++++---- src/lib/crypto/AsymmetricAlgorithm.h | 6 +- src/lib/crypto/OSSLDH.cpp | 4 +- src/lib/crypto/OSSLDH.h | 4 +- src/lib/crypto/OSSLDSA.cpp | 4 +- src/lib/crypto/OSSLDSA.h | 4 +- src/lib/crypto/OSSLECDH.cpp | 4 +- src/lib/crypto/OSSLECDH.h | 4 +- src/lib/crypto/OSSLECDSA.cpp | 4 +- src/lib/crypto/OSSLECDSA.h | 4 +- src/lib/crypto/OSSLEDDSA.cpp | 4 +- src/lib/crypto/OSSLEDDSA.h | 4 +- src/lib/crypto/OSSLRSA.cpp | 220 +++++++++++++++++++++++---- src/lib/crypto/OSSLRSA.h | 4 +- 14 files changed, 302 insertions(+), 85 deletions(-) diff --git a/src/lib/SoftHSM.cpp b/src/lib/SoftHSM.cpp index 234fbfa18..234856fca 100644 --- a/src/lib/SoftHSM.cpp +++ b/src/lib/SoftHSM.cpp @@ -113,6 +113,24 @@ std::auto_ptr SoftHSM::instance(NULL); #endif +// Helper function to get hash length in bytes for OAEP parameters +static CK_ULONG getOAEPHashLength(CK_MECHANISM_TYPE hashAlg) { + switch (hashAlg) { + case CKM_SHA_1: + return 20; // 160 bits / 8 + case CKM_SHA224: + return 28; // 224 bits / 8 + case CKM_SHA256: + return 32; // 256 bits / 8 + case CKM_SHA384: + return 48; // 384 bits / 8 + case CKM_SHA512: + return 64; // 512 bits / 8 + default: + return 0; // Unsupported + } +} + static CK_RV newP11Object(CK_OBJECT_CLASS objClass, CK_KEY_TYPE keyType, CK_CERTIFICATE_TYPE certType, P11Object **p11object) { switch(objClass) { @@ -2514,6 +2532,7 @@ CK_RV SoftHSM::AsymEncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec session->setOpType(SESSION_OP_ENCRYPT); session->setAsymmetricCryptoOp(asymCrypto); session->setMechanism(mechanism); + session->setParameters(pMechanism->pParameter, pMechanism->ulParameterLen); session->setAllowMultiPartOp(false); session->setAllowSinglePartOp(true); session->setPublicKey(publicKey); @@ -2648,10 +2667,21 @@ static CK_RV AsymEncrypt(Session* session, CK_BYTE_PTR pData, CK_ULONG ulDataLen data += ByteString(pData, ulDataLen); // Encrypt the data - if (!asymCrypto->encrypt(publicKey,data,encryptedData,mechanism)) + // Get OAEP parameters from session if this is an OAEP operation + const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr; + if (mechanism == AsymMech::RSA_PKCS_OAEP) { + size_t paramLen; + void* param = session->getParameters(paramLen); + if (param != nullptr && paramLen == sizeof(CK_RSA_PKCS_OAEP_PARAMS)) { + oaepParams = (CK_RSA_PKCS_OAEP_PARAMS*)param; + } + } + + + if (!asymCrypto->encrypt(publicKey,data,encryptedData,mechanism,oaepParams)) { - session->resetOp(); - return CKR_GENERAL_ERROR; + session->resetOp(); + return CKR_GENERAL_ERROR; } // Check size @@ -3214,16 +3244,9 @@ CK_RV SoftHSM::AsymDecryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec DEBUG_MSG("pParameter must be of type CK_RSA_PKCS_OAEP_PARAMS"); return CKR_ARGUMENTS_BAD; } - if (CK_RSA_PKCS_OAEP_PARAMS_PTR(pMechanism->pParameter)->hashAlg != CKM_SHA_1) - { - DEBUG_MSG("hashAlg must be CKM_SHA_1"); - return CKR_ARGUMENTS_BAD; - } - if (CK_RSA_PKCS_OAEP_PARAMS_PTR(pMechanism->pParameter)->mgf != CKG_MGF1_SHA1) - { - DEBUG_MSG("mgf must be CKG_MGF1_SHA1"); - return CKR_ARGUMENTS_BAD; - } + rv = MechParamCheckRSAPKCSOAEP(pMechanism); + if (rv != CKR_OK) + return rv; mechanism = AsymMech::RSA_PKCS_OAEP; isRSA = true; @@ -3267,6 +3290,7 @@ CK_RV SoftHSM::AsymDecryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec session->setOpType(SESSION_OP_DECRYPT); session->setAsymmetricCryptoOp(asymCrypto); session->setMechanism(mechanism); + session->setParameters(pMechanism->pParameter, pMechanism->ulParameterLen); session->setAllowMultiPartOp(false); session->setAllowSinglePartOp(true); session->setPrivateKey(privateKey); @@ -3392,10 +3416,20 @@ static CK_RV AsymDecrypt(Session* session, CK_BYTE_PTR pEncryptedData, CK_ULONG ByteString data; // Decrypt the data - if (!asymCrypto->decrypt(privateKey,encryptedData,data,mechanism)) + // Get OAEP parameters from session if this is an OAEP operation + const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr; + if (mechanism == AsymMech::RSA_PKCS_OAEP) { + size_t paramLen; + void* param = session->getParameters(paramLen); + if (param != nullptr && paramLen == sizeof(CK_RSA_PKCS_OAEP_PARAMS)) { + oaepParams = (CK_RSA_PKCS_OAEP_PARAMS*)param; + } + } + + if (!asymCrypto->decrypt(privateKey,encryptedData,data,mechanism,oaepParams)) { - session->resetOp(); - return CKR_ENCRYPTED_DATA_INVALID; + session->resetOp(); + return CKR_ENCRYPTED_DATA_INVALID; } // Check size @@ -6474,12 +6508,21 @@ CK_RV SoftHSM::WrapKeyAsym break; case CKM_RSA_PKCS_OAEP: - mech = AsymMech::RSA_PKCS_OAEP; - // SHA-1 is the only supported option - // PKCS#11 2.40 draft 2 section 2.1.8: input length <= k-2-2hashLen - if (keydata.size() > modulus_length - 2 - 2 * 160 / 8) - return CKR_KEY_SIZE_RANGE; - break; + { + mech = AsymMech::RSA_PKCS_OAEP; + // Get hash length from OAEP parameters + CK_ULONG hashLen = 20; // Default to SHA-1 (160 bits / 8) + if (pMechanism->pParameter != NULL_PTR && + pMechanism->ulParameterLen == sizeof(CK_RSA_PKCS_OAEP_PARAMS)) { + CK_RSA_PKCS_OAEP_PARAMS_PTR oaepParams = (CK_RSA_PKCS_OAEP_PARAMS_PTR)pMechanism->pParameter; + hashLen = getOAEPHashLength(oaepParams->hashAlg); + if (hashLen == 0) return CKR_MECHANISM_PARAM_INVALID; + } + // PKCS#11 2.40 draft 2 section 2.1.8: input length <= k-2-2hashLen + if (keydata.size() > modulus_length - 2 - 2 * hashLen) + return CKR_KEY_SIZE_RANGE; + break; + } default: return CKR_MECHANISM_INVALID; @@ -13175,15 +13218,27 @@ CK_RV SoftHSM::MechParamCheckRSAPKCSOAEP(CK_MECHANISM_PTR pMechanism) } CK_RSA_PKCS_OAEP_PARAMS_PTR params = (CK_RSA_PKCS_OAEP_PARAMS_PTR)pMechanism->pParameter; - if (params->hashAlg != CKM_SHA_1) - { - ERROR_MSG("hashAlg must be CKM_SHA_1"); - return CKR_ARGUMENTS_BAD; - } - if (params->mgf != CKG_MGF1_SHA1) - { - ERROR_MSG("mgf must be CKG_MGF1_SHA1"); - return CKR_ARGUMENTS_BAD; + switch (params->hashAlg) { + case CKM_SHA_1: + case CKM_SHA256: + case CKM_SHA224: + case CKM_SHA384: + case CKM_SHA512: + break; // Valid hash algorithms + default: + ERROR_MSG("hashAlg must be CKM_SHA_1, CKM_SHA256, CKM_SHA224, CKM_SHA384, or CKM_SHA512"); + return CKR_ARGUMENTS_BAD; + } + switch (params->mgf) { + case CKG_MGF1_SHA1: + case CKG_MGF1_SHA224: + case CKG_MGF1_SHA256: + case CKG_MGF1_SHA384: + case CKG_MGF1_SHA512: + break; // Valid MGF types + default: + ERROR_MSG("mgf must be CKG_MGF1_SHA1, CKG_MGF1_SHA224, CKG_MGF1_SHA256, CKG_MGF1_SHA384, or CKG_MGF1_SHA512"); + return CKR_ARGUMENTS_BAD; } if (params->source != CKZ_DATA_SPECIFIED) { diff --git a/src/lib/crypto/AsymmetricAlgorithm.h b/src/lib/crypto/AsymmetricAlgorithm.h index 52519db65..15ba48338 100644 --- a/src/lib/crypto/AsymmetricAlgorithm.h +++ b/src/lib/crypto/AsymmetricAlgorithm.h @@ -41,6 +41,8 @@ #include "PrivateKey.h" #include "RNG.h" #include "SymmetricKey.h" +#include "pkcs11.h" // For CK_MECHANISM_TYPE and other types + struct AsymAlgo { @@ -138,10 +140,10 @@ class AsymmetricAlgorithm virtual bool verifyFinal(const ByteString& signature); // Encryption functions - virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding) = 0; + virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr) = 0; // Decryption functions - virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding) = 0; + virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr) = 0; // Wrap/Unwrap keys bool wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding); diff --git a/src/lib/crypto/OSSLDH.cpp b/src/lib/crypto/OSSLDH.cpp index ee61733cc..9d5a68a4e 100644 --- a/src/lib/crypto/OSSLDH.cpp +++ b/src/lib/crypto/OSSLDH.cpp @@ -91,7 +91,7 @@ bool OSSLDH::verifyFinal(const ByteString& /*signature*/) // Encryption functions bool OSSLDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, - ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/) + ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { ERROR_MSG("DH does not support encryption"); @@ -100,7 +100,7 @@ bool OSSLDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, // Decryption functions bool OSSLDH::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/, - ByteString& /*data*/, const AsymMech::Type /*padding*/) + ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { ERROR_MSG("DH does not support decryption"); diff --git a/src/lib/crypto/OSSLDH.h b/src/lib/crypto/OSSLDH.h index d611aee11..32bae32c3 100644 --- a/src/lib/crypto/OSSLDH.h +++ b/src/lib/crypto/OSSLDH.h @@ -54,10 +54,10 @@ class OSSLDH : public AsymmetricAlgorithm virtual bool verifyFinal(const ByteString& signature); // Encryption functions - virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding); + virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Decryption functions - virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding); + virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Key factory virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL); diff --git a/src/lib/crypto/OSSLDSA.cpp b/src/lib/crypto/OSSLDSA.cpp index 06b5d5016..40a2cf0d7 100644 --- a/src/lib/crypto/OSSLDSA.cpp +++ b/src/lib/crypto/OSSLDSA.cpp @@ -437,7 +437,7 @@ bool OSSLDSA::verifyFinal(const ByteString& signature) // Encryption functions bool OSSLDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, - ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/) + ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { ERROR_MSG("DSA does not support encryption"); @@ -446,7 +446,7 @@ bool OSSLDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, // Decryption functions bool OSSLDSA::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/, - ByteString& /*data*/, const AsymMech::Type /*padding*/) + ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { ERROR_MSG("DSA does not support decryption"); diff --git a/src/lib/crypto/OSSLDSA.h b/src/lib/crypto/OSSLDSA.h index 1fb2b1e5f..b5535c0fc 100644 --- a/src/lib/crypto/OSSLDSA.h +++ b/src/lib/crypto/OSSLDSA.h @@ -60,10 +60,10 @@ class OSSLDSA : public AsymmetricAlgorithm virtual bool verifyFinal(const ByteString& signature); // Encryption functions - virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding); + virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Decryption functions - virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding); + virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Key factory virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL); diff --git a/src/lib/crypto/OSSLECDH.cpp b/src/lib/crypto/OSSLECDH.cpp index e2abaeb24..8705dbc51 100644 --- a/src/lib/crypto/OSSLECDH.cpp +++ b/src/lib/crypto/OSSLECDH.cpp @@ -94,7 +94,7 @@ bool OSSLECDH::verifyFinal(const ByteString& /*signature*/) // Encryption functions bool OSSLECDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, - ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/) + ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { ERROR_MSG("ECDH does not support encryption"); @@ -103,7 +103,7 @@ bool OSSLECDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, // Decryption functions bool OSSLECDH::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/, - ByteString& /*data*/, const AsymMech::Type /*padding*/) + ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { ERROR_MSG("ECDH does not support decryption"); diff --git a/src/lib/crypto/OSSLECDH.h b/src/lib/crypto/OSSLECDH.h index 2cafa6fd2..eff9309c7 100644 --- a/src/lib/crypto/OSSLECDH.h +++ b/src/lib/crypto/OSSLECDH.h @@ -54,10 +54,10 @@ class OSSLECDH : public AsymmetricAlgorithm virtual bool verifyFinal(const ByteString& signature); // Encryption functions - virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding); + virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Decryption functions - virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding); + virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Key factory virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL); diff --git a/src/lib/crypto/OSSLECDSA.cpp b/src/lib/crypto/OSSLECDSA.cpp index 23139c1b2..07acc7134 100644 --- a/src/lib/crypto/OSSLECDSA.cpp +++ b/src/lib/crypto/OSSLECDSA.cpp @@ -331,7 +331,7 @@ bool OSSLECDSA::verifyFinal(const ByteString& /*signature*/) // Encryption functions bool OSSLECDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, - ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/) + ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { ERROR_MSG("ECDSA does not support encryption"); @@ -340,7 +340,7 @@ bool OSSLECDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, // Decryption functions bool OSSLECDSA::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/, - ByteString& /*data*/, const AsymMech::Type /*padding*/) + ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { ERROR_MSG("ECDSA does not support decryption"); diff --git a/src/lib/crypto/OSSLECDSA.h b/src/lib/crypto/OSSLECDSA.h index 992fa40b8..dad843d28 100644 --- a/src/lib/crypto/OSSLECDSA.h +++ b/src/lib/crypto/OSSLECDSA.h @@ -56,10 +56,10 @@ class OSSLECDSA : public AsymmetricAlgorithm virtual bool verifyFinal(const ByteString& signature); // Encryption functions - virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding); + virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Decryption functions - virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding); + virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Key factory virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL); diff --git a/src/lib/crypto/OSSLEDDSA.cpp b/src/lib/crypto/OSSLEDDSA.cpp index dc2f4cc39..358d72c1c 100644 --- a/src/lib/crypto/OSSLEDDSA.cpp +++ b/src/lib/crypto/OSSLEDDSA.cpp @@ -208,7 +208,7 @@ bool OSSLEDDSA::verifyFinal(const ByteString& /*signature*/) // Encryption functions bool OSSLEDDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, - ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/) + ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { ERROR_MSG("EDDSA does not support encryption"); @@ -217,7 +217,7 @@ bool OSSLEDDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, // Decryption functions bool OSSLEDDSA::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/, - ByteString& /*data*/, const AsymMech::Type /*padding*/) + ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { ERROR_MSG("EDDSA does not support decryption"); diff --git a/src/lib/crypto/OSSLEDDSA.h b/src/lib/crypto/OSSLEDDSA.h index 02b8a115a..c649ad6d4 100644 --- a/src/lib/crypto/OSSLEDDSA.h +++ b/src/lib/crypto/OSSLEDDSA.h @@ -56,10 +56,10 @@ class OSSLEDDSA : public AsymmetricAlgorithm virtual bool verifyFinal(const ByteString& signature); // Encryption functions - virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding); + virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Decryption functions - virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding); + virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Key factory virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL); diff --git a/src/lib/crypto/OSSLRSA.cpp b/src/lib/crypto/OSSLRSA.cpp index d36a1e682..828cf2f8b 100644 --- a/src/lib/crypto/OSSLRSA.cpp +++ b/src/lib/crypto/OSSLRSA.cpp @@ -42,6 +42,24 @@ #include #include +// Helper to convert PKCS#11 hash mechanism to OpenSSL EVP_MD +static const EVP_MD* getEVPMDFromCKM(CK_MECHANISM_TYPE hashAlg) { + switch (hashAlg) { + case 0x220: // CKM_SHA_1 + return EVP_sha1(); + case 0x250: // CKM_SHA256 + return EVP_sha256(); + case 0x240: // CKM_SHA224 + return EVP_sha224(); + case 0x260: // CKM_SHA384 + return EVP_sha384(); + case 0x270: // CKM_SHA512 + return EVP_sha512(); + default: + return EVP_sha1(); // Fallback to SHA-1 + } +} + // Constructor OSSLRSA::OSSLRSA() { @@ -1206,7 +1224,7 @@ bool OSSLRSA::verifyFinal(const ByteString& signature) // Encryption functions bool OSSLRSA::encrypt(PublicKey* publicKey, const ByteString& data, - ByteString& encryptedData, const AsymMech::Type padding) + ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { // Check if the public key is the right type if (!publicKey->isOfType(OSSLRSAPublicKey::type)) @@ -1237,16 +1255,81 @@ bool OSSLRSA::encrypt(PublicKey* publicKey, const ByteString& data, } else if (padding == AsymMech::RSA_PKCS_OAEP) { - // The size of the input data cannot be more than the modulus - // length of the key - 41 - if (data.size() > (size_t) (RSA_size(rsa) - 41)) - { - ERROR_MSG("Too much data supplied for RSA OAEP encryption"); - - return false; - } - - osslPadding = RSA_PKCS1_OAEP_PADDING; + // Use EVP API for OAEP to support configurable hash algorithms + EVP_PKEY* pkey = EVP_PKEY_new(); + EVP_PKEY_set1_RSA(pkey, rsa); + + EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new(pkey, NULL); + if (ctx == NULL) { + ERROR_MSG("Failed to create EVP context for OAEP encryption"); + EVP_PKEY_free(pkey); + return false; + } + + if (EVP_PKEY_encrypt_init(ctx) <= 0) { + ERROR_MSG("Failed to initialize EVP encryption"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + + if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0) { + ERROR_MSG("Failed to set OAEP padding"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + + // Get hash algorithm from OAEP parameters + const EVP_MD* hashMD = EVP_sha1(); // Default to SHA-1 + const EVP_MD* mgfMD = EVP_sha1(); // Default to SHA-1 + + if (oaepParams != nullptr) { + hashMD = getEVPMDFromCKM(oaepParams->hashAlg); + mgfMD = getEVPMDFromCKM(oaepParams->mgf == 1 ? 0x220 : // CKG_MGF1_SHA1 -> CKM_SHA_1 + oaepParams->mgf == 2 ? 0x250 : // CKG_MGF1_SHA256 -> CKM_SHA256 + oaepParams->mgf == 3 ? 0x240 : // CKG_MGF1_SHA224 -> CKM_SHA224 + oaepParams->mgf == 4 ? 0x260 : // CKG_MGF1_SHA384 -> CKM_SHA384 + oaepParams->mgf == 5 ? 0x270 : // CKG_MGF1_SHA512 -> CKM_SHA512 + 0x220); // Default to SHA-1 + } + + if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, hashMD) <= 0) { + ERROR_MSG("Failed to set OAEP hash algorithm"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, mgfMD) <= 0) { + ERROR_MSG("Failed to set MGF1 hash algorithm"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + + // Perform encryption + size_t outlen; + if (EVP_PKEY_encrypt(ctx, NULL, &outlen, data.const_byte_str(), data.size()) <= 0) { + ERROR_MSG("Failed to determine output length for OAEP encryption"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + + encryptedData.resize(outlen); + int encryptResult = EVP_PKEY_encrypt(ctx, &encryptedData[0], &outlen, data.const_byte_str(), data.size()); + + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + + if (encryptResult <= 0) { + ERROR_MSG("RSA OAEP encryption failed (0x%08X)", ERR_get_error()); + return false; + } + + encryptedData.resize(outlen); + return true; } else if (padding == AsymMech::RSA) { @@ -1267,14 +1350,16 @@ bool OSSLRSA::encrypt(PublicKey* publicKey, const ByteString& data, return false; } - // Perform the RSA operation - encryptedData.resize(RSA_size(rsa)); + // Perform the RSA operation (for non-OAEP cases only) + if (padding != AsymMech::RSA_PKCS_OAEP) { + encryptedData.resize(RSA_size(rsa)); - if (RSA_public_encrypt(data.size(), (unsigned char*) data.const_byte_str(), &encryptedData[0], rsa, osslPadding) == -1) - { - ERROR_MSG("RSA public key encryption failed (0x%08X)", ERR_get_error()); + if (RSA_public_encrypt(data.size(), (unsigned char*) data.const_byte_str(), &encryptedData[0], rsa, osslPadding) == -1) + { + ERROR_MSG("RSA public key encryption failed (0x%08X)", ERR_get_error()); - return false; + return false; + } } return true; @@ -1282,7 +1367,7 @@ bool OSSLRSA::encrypt(PublicKey* publicKey, const ByteString& data, // Decryption functions bool OSSLRSA::decrypt(PrivateKey* privateKey, const ByteString& encryptedData, - ByteString& data, const AsymMech::Type padding) + ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) { // Check if the private key is the right type if (!privateKey->isOfType(OSSLRSAPrivateKey::type)) @@ -1312,20 +1397,95 @@ bool OSSLRSA::decrypt(PrivateKey* privateKey, const ByteString& encryptedData, osslPadding = RSA_PKCS1_PADDING; break; case AsymMech::RSA_PKCS_OAEP: - osslPadding = RSA_PKCS1_OAEP_PADDING; - break; + { + // Use EVP API for OAEP to support configurable hash algorithms + EVP_PKEY* pkey = EVP_PKEY_new(); + EVP_PKEY_set1_RSA(pkey, rsa); + + EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new(pkey, NULL); + if (ctx == NULL) { + ERROR_MSG("Failed to create EVP context for OAEP decryption"); + EVP_PKEY_free(pkey); + return false; + } + + if (EVP_PKEY_decrypt_init(ctx) <= 0) { + ERROR_MSG("Failed to initialize EVP decryption"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + + if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0) { + ERROR_MSG("Failed to set OAEP padding"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + + // Get hash algorithm from OAEP parameters + const EVP_MD* hashMD = EVP_sha1(); // Default to SHA-1 + const EVP_MD* mgfMD = EVP_sha1(); // Default to SHA-1 + + if (oaepParams != nullptr) { + hashMD = getEVPMDFromCKM(oaepParams->hashAlg); + mgfMD = getEVPMDFromCKM(oaepParams->mgf == 1 ? 0x220 : // CKG_MGF1_SHA1 -> CKM_SHA_1 + oaepParams->mgf == 2 ? 0x250 : // CKG_MGF1_SHA256 -> CKM_SHA256 + oaepParams->mgf == 3 ? 0x240 : // CKG_MGF1_SHA224 -> CKM_SHA224 + oaepParams->mgf == 4 ? 0x260 : // CKG_MGF1_SHA384 -> CKM_SHA384 + oaepParams->mgf == 5 ? 0x270 : // CKG_MGF1_SHA512 -> CKM_SHA512 + 0x220); // Default to SHA-1 + } + + if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, hashMD) <= 0) { + ERROR_MSG("Failed to set OAEP hash algorithm"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, mgfMD) <= 0) { + ERROR_MSG("Failed to set MGF1 hash algorithm"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + + // Perform decryption + size_t outlen; + if (EVP_PKEY_decrypt(ctx, NULL, &outlen, encryptedData.const_byte_str(), encryptedData.size()) <= 0) { + ERROR_MSG("Failed to determine output length for OAEP decryption"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + + data.resize(outlen); + int decryptResult = EVP_PKEY_decrypt(ctx, &data[0], &outlen, encryptedData.const_byte_str(), encryptedData.size()); + + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + + if (decryptResult <= 0) { + ERROR_MSG("RSA OAEP decryption failed (0x%08X)", ERR_get_error()); + return false; + } + + data.resize(outlen); + return true; + } case AsymMech::RSA: - osslPadding = RSA_NO_PADDING; - break; + osslPadding = RSA_NO_PADDING; + break; default: - ERROR_MSG("Invalid padding mechanism supplied (%i)", padding); - return false; - } - - // Perform the RSA operation - data.resize(RSA_size(rsa)); - - int decSize = RSA_private_decrypt(encryptedData.size(), (unsigned char*) encryptedData.const_byte_str(), &data[0], rsa, osslPadding); + ERROR_MSG("Invalid padding mechanism supplied (%i)", padding); + return false; + } + + // Perform the RSA operation (for non-OAEP cases) + data.resize(RSA_size(rsa)); + + int decSize = RSA_private_decrypt(encryptedData.size(), (unsigned char*) encryptedData.const_byte_str(), &data[0], rsa, osslPadding); if (decSize == -1) { diff --git a/src/lib/crypto/OSSLRSA.h b/src/lib/crypto/OSSLRSA.h index 5b7db6d64..634c2c307 100644 --- a/src/lib/crypto/OSSLRSA.h +++ b/src/lib/crypto/OSSLRSA.h @@ -60,10 +60,10 @@ class OSSLRSA : public AsymmetricAlgorithm virtual bool verifyFinal(const ByteString& signature); // Encryption functions - virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding); + virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Decryption functions - virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding); + virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr); // Key factory virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL); From 7d494c0a93fdab5508f47f442e19aa758e6eb815 Mon Sep 17 00:00:00 2001 From: Jeff Anderson Date: Fri, 26 Sep 2025 21:56:40 -0400 Subject: [PATCH 2/3] Update src/lib/crypto/OSSLRSA.cpp Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/lib/crypto/OSSLRSA.cpp | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/lib/crypto/OSSLRSA.cpp b/src/lib/crypto/OSSLRSA.cpp index 828cf2f8b..c6197778f 100644 --- a/src/lib/crypto/OSSLRSA.cpp +++ b/src/lib/crypto/OSSLRSA.cpp @@ -1427,14 +1427,16 @@ bool OSSLRSA::decrypt(PrivateKey* privateKey, const ByteString& encryptedData, const EVP_MD* hashMD = EVP_sha1(); // Default to SHA-1 const EVP_MD* mgfMD = EVP_sha1(); // Default to SHA-1 - if (oaepParams != nullptr) { + if (oaepParams != NULL) { hashMD = getEVPMDFromCKM(oaepParams->hashAlg); - mgfMD = getEVPMDFromCKM(oaepParams->mgf == 1 ? 0x220 : // CKG_MGF1_SHA1 -> CKM_SHA_1 - oaepParams->mgf == 2 ? 0x250 : // CKG_MGF1_SHA256 -> CKM_SHA256 - oaepParams->mgf == 3 ? 0x240 : // CKG_MGF1_SHA224 -> CKM_SHA224 - oaepParams->mgf == 4 ? 0x260 : // CKG_MGF1_SHA384 -> CKM_SHA384 - oaepParams->mgf == 5 ? 0x270 : // CKG_MGF1_SHA512 -> CKM_SHA512 - 0x220); // Default to SHA-1 + switch (oaepParams->mgf) { + case CKG_MGF1_SHA1: mgfMD = EVP_sha1(); break; + case CKG_MGF1_SHA224: mgfMD = EVP_sha224(); break; + case CKG_MGF1_SHA256: mgfMD = EVP_sha256(); break; + case CKG_MGF1_SHA384: mgfMD = EVP_sha384(); break; + case CKG_MGF1_SHA512: mgfMD = EVP_sha512(); break; + default: /* keep default SHA-1 */ break; + } } if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, hashMD) <= 0) { @@ -1450,6 +1452,28 @@ bool OSSLRSA::decrypt(PrivateKey* privateKey, const ByteString& encryptedData, EVP_PKEY_free(pkey); return false; } + + // Optional: set OAEP label if provided + if (oaepParams != NULL && + oaepParams->source == CKZ_DATA_SPECIFIED && + oaepParams->pSourceData != NULL && + oaepParams->ulSourceDataLen > 0) { + unsigned char* label = (unsigned char*) OPENSSL_malloc(oaepParams->ulSourceDataLen); + if (label == NULL) { + ERROR_MSG("Failed to allocate memory for OAEP label"); + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + memcpy(label, oaepParams->pSourceData, oaepParams->ulSourceDataLen); + if (EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, label, (int)oaepParams->ulSourceDataLen) <= 0) { + ERROR_MSG("Failed to set OAEP label"); + OPENSSL_free(label); // free on failure; ownership transfers on success + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return false; + } + } // Perform decryption size_t outlen; From 5ce7b4b4c6cfbd2ee46c709c94f04c6561fe925e Mon Sep 17 00:00:00 2001 From: Jeff Anderson Date: Fri, 26 Sep 2025 22:20:47 -0400 Subject: [PATCH 3/3] incorporate feedback from code review --- src/lib/crypto/OSSLEDDSA.cpp | 4 +-- src/lib/crypto/OSSLRSA.cpp | 54 ++++++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/lib/crypto/OSSLEDDSA.cpp b/src/lib/crypto/OSSLEDDSA.cpp index 358d72c1c..39b7cfa70 100644 --- a/src/lib/crypto/OSSLEDDSA.cpp +++ b/src/lib/crypto/OSSLEDDSA.cpp @@ -208,7 +208,7 @@ bool OSSLEDDSA::verifyFinal(const ByteString& /*signature*/) // Encryption functions bool OSSLEDDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, - ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) + ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* /*oaepParams*/) { ERROR_MSG("EDDSA does not support encryption"); @@ -217,7 +217,7 @@ bool OSSLEDDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/, // Decryption functions bool OSSLEDDSA::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/, - ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams) + ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* /*oaepParams*/) { ERROR_MSG("EDDSA does not support decryption"); diff --git a/src/lib/crypto/OSSLRSA.cpp b/src/lib/crypto/OSSLRSA.cpp index c6197778f..af61c8af3 100644 --- a/src/lib/crypto/OSSLRSA.cpp +++ b/src/lib/crypto/OSSLRSA.cpp @@ -45,18 +45,12 @@ // Helper to convert PKCS#11 hash mechanism to OpenSSL EVP_MD static const EVP_MD* getEVPMDFromCKM(CK_MECHANISM_TYPE hashAlg) { switch (hashAlg) { - case 0x220: // CKM_SHA_1 - return EVP_sha1(); - case 0x250: // CKM_SHA256 - return EVP_sha256(); - case 0x240: // CKM_SHA224 - return EVP_sha224(); - case 0x260: // CKM_SHA384 - return EVP_sha384(); - case 0x270: // CKM_SHA512 - return EVP_sha512(); - default: - return EVP_sha1(); // Fallback to SHA-1 + case CKM_SHA_1: return EVP_sha1(); + case CKM_SHA224: return EVP_sha224(); + case CKM_SHA256: return EVP_sha256(); + case CKM_SHA384: return EVP_sha384(); + case CKM_SHA512: return EVP_sha512(); + default: return EVP_sha1(); // Fallback to SHA-1 } } @@ -1257,7 +1251,15 @@ bool OSSLRSA::encrypt(PublicKey* publicKey, const ByteString& data, { // Use EVP API for OAEP to support configurable hash algorithms EVP_PKEY* pkey = EVP_PKEY_new(); - EVP_PKEY_set1_RSA(pkey, rsa); + if (pkey == NULL) { + ERROR_MSG("Failed to allocate EVP_PKEY"); + return false; + } + if (EVP_PKEY_set1_RSA(pkey, rsa) <= 0) { + ERROR_MSG("Failed to set RSA key on EVP_PKEY"); + EVP_PKEY_free(pkey); + return false; + } EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new(pkey, NULL); if (ctx == NULL) { @@ -1284,14 +1286,16 @@ bool OSSLRSA::encrypt(PublicKey* publicKey, const ByteString& data, const EVP_MD* hashMD = EVP_sha1(); // Default to SHA-1 const EVP_MD* mgfMD = EVP_sha1(); // Default to SHA-1 - if (oaepParams != nullptr) { + if (oaepParams != NULL) { hashMD = getEVPMDFromCKM(oaepParams->hashAlg); - mgfMD = getEVPMDFromCKM(oaepParams->mgf == 1 ? 0x220 : // CKG_MGF1_SHA1 -> CKM_SHA_1 - oaepParams->mgf == 2 ? 0x250 : // CKG_MGF1_SHA256 -> CKM_SHA256 - oaepParams->mgf == 3 ? 0x240 : // CKG_MGF1_SHA224 -> CKM_SHA224 - oaepParams->mgf == 4 ? 0x260 : // CKG_MGF1_SHA384 -> CKM_SHA384 - oaepParams->mgf == 5 ? 0x270 : // CKG_MGF1_SHA512 -> CKM_SHA512 - 0x220); // Default to SHA-1 + switch (oaepParams->mgf) { + case CKG_MGF1_SHA1: mgfMD = EVP_sha1(); break; + case CKG_MGF1_SHA224: mgfMD = EVP_sha224(); break; + case CKG_MGF1_SHA256: mgfMD = EVP_sha256(); break; + case CKG_MGF1_SHA384: mgfMD = EVP_sha384(); break; + case CKG_MGF1_SHA512: mgfMD = EVP_sha512(); break; + default: /* keep default SHA-1 */ break; + } } if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, hashMD) <= 0) { @@ -1400,7 +1404,15 @@ bool OSSLRSA::decrypt(PrivateKey* privateKey, const ByteString& encryptedData, { // Use EVP API for OAEP to support configurable hash algorithms EVP_PKEY* pkey = EVP_PKEY_new(); - EVP_PKEY_set1_RSA(pkey, rsa); + if (pkey == NULL) { + ERROR_MSG("Failed to allocate EVP_PKEY"); + return false; + } + if (EVP_PKEY_set1_RSA(pkey, rsa) <= 0) { + ERROR_MSG("Failed to set RSA key on EVP_PKEY"); + EVP_PKEY_free(pkey); + return false; + } EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new(pkey, NULL); if (ctx == NULL) {