Skip to content

implement OAESP params for asymmetric encryption #820

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
programmerq wants to merge 3 commits into
base: main
Choose a base branch
from
Open

implement OAESP params for asymmetric encryption #820

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
392c669
implement OAESP params for asymmetric encryption
programmerq Sep 26, 2025
7d494c0
Update src/lib/crypto/OSSLRSA.cpp
programmerq Sep 27, 2025
5ce7b4b
incorporate feedback from code review
programmerq Sep 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 86 additions & 31 deletions src/lib/SoftHSM.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,24 @@ std::auto_ptr<SoftHSM> SoftHSM::instance(NULL);

#endif

// Helper function to get hash length in bytes for OAEP parameters
static CK_ULONG getOAEPHashLength(CK_MECHANISM_TYPE hashAlg) {
switch (hashAlg) {
case CKM_SHA_1:
return 20; // 160 bits / 8
case CKM_SHA224:
return 28; // 224 bits / 8
case CKM_SHA256:
return 32; // 256 bits / 8
case CKM_SHA384:
return 48; // 384 bits / 8
case CKM_SHA512:
return 64; // 512 bits / 8
default:
return 0; // Unsupported
}
}

static CK_RV newP11Object(CK_OBJECT_CLASS objClass, CK_KEY_TYPE keyType, CK_CERTIFICATE_TYPE certType, P11Object **p11object)
{
switch(objClass) {
Expand Down Expand Up @@ -2514,6 +2532,7 @@ CK_RV SoftHSM::AsymEncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec
session->setOpType(SESSION_OP_ENCRYPT);
session->setAsymmetricCryptoOp(asymCrypto);
session->setMechanism(mechanism);
session->setParameters(pMechanism->pParameter, pMechanism->ulParameterLen);
session->setAllowMultiPartOp(false);
session->setAllowSinglePartOp(true);
session->setPublicKey(publicKey);
Expand Down Expand Up @@ -2648,10 +2667,21 @@ static CK_RV AsymEncrypt(Session* session, CK_BYTE_PTR pData, CK_ULONG ulDataLen
data += ByteString(pData, ulDataLen);

// Encrypt the data
if (!asymCrypto->encrypt(publicKey,data,encryptedData,mechanism))
// Get OAEP parameters from session if this is an OAEP operation
const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr;
if (mechanism == AsymMech::RSA_PKCS_OAEP) {
size_t paramLen;
void* param = session->getParameters(paramLen);
if (param != nullptr && paramLen == sizeof(CK_RSA_PKCS_OAEP_PARAMS)) {
oaepParams = (CK_RSA_PKCS_OAEP_PARAMS*)param;
}
}


if (!asymCrypto->encrypt(publicKey,data,encryptedData,mechanism,oaepParams))
{
session->resetOp();
return CKR_GENERAL_ERROR;
session->resetOp();
return CKR_GENERAL_ERROR;
}

// Check size
Expand Down Expand Up @@ -3214,16 +3244,9 @@ CK_RV SoftHSM::AsymDecryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec
DEBUG_MSG("pParameter must be of type CK_RSA_PKCS_OAEP_PARAMS");
return CKR_ARGUMENTS_BAD;
}
if (CK_RSA_PKCS_OAEP_PARAMS_PTR(pMechanism->pParameter)->hashAlg != CKM_SHA_1)
{
DEBUG_MSG("hashAlg must be CKM_SHA_1");
return CKR_ARGUMENTS_BAD;
}
if (CK_RSA_PKCS_OAEP_PARAMS_PTR(pMechanism->pParameter)->mgf != CKG_MGF1_SHA1)
{
DEBUG_MSG("mgf must be CKG_MGF1_SHA1");
return CKR_ARGUMENTS_BAD;
}
rv = MechParamCheckRSAPKCSOAEP(pMechanism);
if (rv != CKR_OK)
return rv;

mechanism = AsymMech::RSA_PKCS_OAEP;
isRSA = true;
Expand Down Expand Up @@ -3267,6 +3290,7 @@ CK_RV SoftHSM::AsymDecryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec
session->setOpType(SESSION_OP_DECRYPT);
session->setAsymmetricCryptoOp(asymCrypto);
session->setMechanism(mechanism);
session->setParameters(pMechanism->pParameter, pMechanism->ulParameterLen);
session->setAllowMultiPartOp(false);
session->setAllowSinglePartOp(true);
session->setPrivateKey(privateKey);
Expand Down Expand Up @@ -3392,10 +3416,20 @@ static CK_RV AsymDecrypt(Session* session, CK_BYTE_PTR pEncryptedData, CK_ULONG
ByteString data;

// Decrypt the data
if (!asymCrypto->decrypt(privateKey,encryptedData,data,mechanism))
// Get OAEP parameters from session if this is an OAEP operation
const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr;
if (mechanism == AsymMech::RSA_PKCS_OAEP) {
size_t paramLen;
void* param = session->getParameters(paramLen);
if (param != nullptr && paramLen == sizeof(CK_RSA_PKCS_OAEP_PARAMS)) {
oaepParams = (CK_RSA_PKCS_OAEP_PARAMS*)param;
}
}

if (!asymCrypto->decrypt(privateKey,encryptedData,data,mechanism,oaepParams))
{
session->resetOp();
return CKR_ENCRYPTED_DATA_INVALID;
session->resetOp();
return CKR_ENCRYPTED_DATA_INVALID;
}

// Check size
Expand Down Expand Up @@ -6474,12 +6508,21 @@ CK_RV SoftHSM::WrapKeyAsym
break;

case CKM_RSA_PKCS_OAEP:
mech = AsymMech::RSA_PKCS_OAEP;
// SHA-1 is the only supported option
// PKCS#11 2.40 draft 2 section 2.1.8: input length <= k-2-2hashLen
if (keydata.size() > modulus_length - 2 - 2 * 160 / 8)
return CKR_KEY_SIZE_RANGE;
break;
{
mech = AsymMech::RSA_PKCS_OAEP;
// Get hash length from OAEP parameters
CK_ULONG hashLen = 20; // Default to SHA-1 (160 bits / 8)
if (pMechanism->pParameter != NULL_PTR &&
pMechanism->ulParameterLen == sizeof(CK_RSA_PKCS_OAEP_PARAMS)) {
CK_RSA_PKCS_OAEP_PARAMS_PTR oaepParams = (CK_RSA_PKCS_OAEP_PARAMS_PTR)pMechanism->pParameter;
hashLen = getOAEPHashLength(oaepParams->hashAlg);
if (hashLen == 0) return CKR_MECHANISM_PARAM_INVALID;
}
// PKCS#11 2.40 draft 2 section 2.1.8: input length <= k-2-2hashLen
if (keydata.size() > modulus_length - 2 - 2 * hashLen)
return CKR_KEY_SIZE_RANGE;
break;
}

default:
return CKR_MECHANISM_INVALID;
Expand Down Expand Up @@ -13175,15 +13218,27 @@ CK_RV SoftHSM::MechParamCheckRSAPKCSOAEP(CK_MECHANISM_PTR pMechanism)
}

CK_RSA_PKCS_OAEP_PARAMS_PTR params = (CK_RSA_PKCS_OAEP_PARAMS_PTR)pMechanism->pParameter;
if (params->hashAlg != CKM_SHA_1)
{
ERROR_MSG("hashAlg must be CKM_SHA_1");
return CKR_ARGUMENTS_BAD;
}
if (params->mgf != CKG_MGF1_SHA1)
{
ERROR_MSG("mgf must be CKG_MGF1_SHA1");
return CKR_ARGUMENTS_BAD;
switch (params->hashAlg) {
case CKM_SHA_1:
case CKM_SHA256:
case CKM_SHA224:
case CKM_SHA384:
case CKM_SHA512:
break; // Valid hash algorithms
default:
ERROR_MSG("hashAlg must be CKM_SHA_1, CKM_SHA256, CKM_SHA224, CKM_SHA384, or CKM_SHA512");
return CKR_ARGUMENTS_BAD;
}
switch (params->mgf) {
case CKG_MGF1_SHA1:
case CKG_MGF1_SHA224:
case CKG_MGF1_SHA256:
case CKG_MGF1_SHA384:
case CKG_MGF1_SHA512:
break; // Valid MGF types
default:
ERROR_MSG("mgf must be CKG_MGF1_SHA1, CKG_MGF1_SHA224, CKG_MGF1_SHA256, CKG_MGF1_SHA384, or CKG_MGF1_SHA512");
return CKR_ARGUMENTS_BAD;
}
if (params->source != CKZ_DATA_SPECIFIED)
{
Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/AsymmetricAlgorithm.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@
#include "PrivateKey.h"
#include "RNG.h"
#include "SymmetricKey.h"
#include "pkcs11.h" // For CK_MECHANISM_TYPE and other types


struct AsymAlgo
{
Expand Down Expand Up @@ -138,10 +140,10 @@ class AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding) = 0;
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr) = 0;

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding) = 0;
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr) = 0;

// Wrap/Unwrap keys
bool wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/OSSLDH.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ bool OSSLDH::verifyFinal(const ByteString& /*signature*/)

// Encryption functions
bool OSSLDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams)
{
ERROR_MSG("DH does not support encryption");

Expand All @@ -100,7 +100,7 @@ bool OSSLDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool OSSLDH::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams)
{
ERROR_MSG("DH does not support decryption");

Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/OSSLDH.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ class OSSLDH : public AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/OSSLDSA.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ bool OSSLDSA::verifyFinal(const ByteString& signature)

// Encryption functions
bool OSSLDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams)
{
ERROR_MSG("DSA does not support encryption");

Expand All @@ -446,7 +446,7 @@ bool OSSLDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool OSSLDSA::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams)
{
ERROR_MSG("DSA does not support decryption");

Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/OSSLDSA.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,10 @@ class OSSLDSA : public AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/OSSLECDH.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ bool OSSLECDH::verifyFinal(const ByteString& /*signature*/)

// Encryption functions
bool OSSLECDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams)
{
ERROR_MSG("ECDH does not support encryption");

Expand All @@ -103,7 +103,7 @@ bool OSSLECDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool OSSLECDH::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams)
{
ERROR_MSG("ECDH does not support decryption");

Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/OSSLECDH.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ class OSSLECDH : public AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/OSSLECDSA.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ bool OSSLECDSA::verifyFinal(const ByteString& /*signature*/)

// Encryption functions
bool OSSLECDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams)
{
ERROR_MSG("ECDSA does not support encryption");

Expand All @@ -340,7 +340,7 @@ bool OSSLECDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool OSSLECDSA::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams)
{
ERROR_MSG("ECDSA does not support decryption");

Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/OSSLECDSA.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ class OSSLECDSA : public AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/OSSLEDDSA.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ bool OSSLEDDSA::verifyFinal(const ByteString& /*signature*/)

// Encryption functions
bool OSSLEDDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* /*oaepParams*/)
{
ERROR_MSG("EDDSA does not support encryption");

Expand All @@ -217,7 +217,7 @@ bool OSSLEDDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool OSSLEDDSA::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/, const CK_RSA_PKCS_OAEP_PARAMS* /*oaepParams*/)
{
ERROR_MSG("EDDSA does not support decryption");

Expand Down
4 changes: 2 additions & 2 deletions src/lib/crypto/OSSLEDDSA.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ class OSSLEDDSA : public AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const CK_RSA_PKCS_OAEP_PARAMS* oaepParams = nullptr);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
Expand Down
Loading