Import RSA private key from pem file

See more:

C++

cryptography

TLS

Hello,

I need help. I'm trying to implement TLS1.3 on my server and I'm stuck importing a 2048-bit RSA private key. I want to import it from a .pem file. I manage to read it, then convert it from base64 to hex (binary) and I have it all in a string.

I use NCryptImportKey and I managed to import the key, in the sense that it doesn't give me an error, then I sign a hash (for the whole handshake), also without error. But when I test with openssl s_client ... it returns an error "bad signature".

Please give me an idea, something to help me. I'm 99% sure that I generated the hash correctly, so it only remains if the key is imported successfully, or the settings for the signing algorithm are ok.

Thank you very much.

p.s. here is a linke with some img

https://docs.microsoft.com/en-us/answers/questions/841041/import-rsa-private-key-from-pem-file.html[^]

What I have tried:

if (andSign == true)
	{
		DWORD policy = NCRYPT_ALLOW_EXPORT_FLAG | NCRYPT_ALLOW_PLAINTEXT_EXPORT_FLAG;
		LPCWSTR name = L"ImportedKey";
		BCryptBuffer cb[1];
		cb[0].BufferType = NCRYPTBUFFER_PKCS_KEY_NAME;
		cb[0].pvBuffer = (void*)name;
		cb[0].cbBuffer = lstrlenW(name) * 2 + 2;

		NCryptBufferDesc desc;
		desc.ulVersion = 0;
		desc.pBuffers = cb;
		desc.cBuffers = 1;
		DWORD keyLength = 2048;
	
		if (FAILED(secStatus = NCryptImportKey(hProvider, NULL, NCRYPT_PKCS8_PRIVATE_KEY_BLOB, &desc, &hKeyNew, (PBYTE)derKey.c_str(), derKey.length(), NCRYPT_DO_NOT_FINALIZE_FLAG)))
		{
			tmp.Format(_T("NCryptImportKey a generat eroarea 0x%x ..."), secStatus);
			AddLogToList(NULL, tmp, 0);
			ret = false;
			goto CleanUp2;
		}

		if (FAILED(secStatus = NCryptSetProperty(hKeyNew, NCRYPT_EXPORT_POLICY_PROPERTY, (PBYTE)(&policy), sizeof(policy), NCRYPT_PERSIST_FLAG)))
		{
			tmp.Format(_T("NCryptImportKey a generat eroarea 0x%x ..."), secStatus);
			AddLogToList(NULL, tmp, 0);
			ret = false;
			goto CleanUp2;
		}

		if (FAILED(secStatus = NCryptSetProperty(hKeyNew, NCRYPT_LENGTH_PROPERTY, (PBYTE)(&keyLength), sizeof(keyLength), NCRYPT_PERSIST_FLAG)))
		{
			tmp.Format(_T("NCryptImportKey a generat eroarea 0x%x ..."), secStatus);
			AddLogToList(NULL, tmp, 0);
			ret = false;
			goto CleanUp2;
		}
	
		if (FAILED(secStatus = NCryptFinalizeKey(hKeyNew, 0)))
		{
			tmp.Format(_T("NCryptImportKey a generat eroarea 0x%x ..."), secStatus);
			AddLogToList(NULL, tmp, 0);
			ret = false;
			goto CleanUp2;
		}

		BCRYPT_PSS_PADDING_INFO   paddingInfo = {};
		paddingInfo.pszAlgId = BCRYPT_SHA256_ALGORITHM;
		paddingInfo.cbSalt = 32;
		//BCRYPT_PKCS1_PADDING_INFO  paddingInfo = {};
		//paddingInfo.pszAlgId = BCRYPT_SHA256_ALGORITHM;

		string hasPadded = "";
		for (unsigned int i = 0; i < 64; i++) hasPadded += (char)0x20;
		hasPadded += "TLS 1.3, server CertificateVerify";
		hasPadded += (char)0x00;
		hasPadded += pContext->hash256;
		
		//sign the hash - get length
		if (FAILED(secStatus = NCryptSignHash(hKeyNew, &paddingInfo, (PUCHAR)hasPadded.c_str(), hasPadded.length(), NULL, 0, &cbHashSigned, BCRYPT_PAD_PSS)))//BCRYPT_PAD_PKCS1)))
		{
			tmp.Format(_T("Semnare hash - 'NCryptSignHash' a returnat eroarea 0x%x ..."), secStatus);
			AddLogToList(pContext, tmp, pContext->m_ID);
			ret = false;
			goto CleanUp2;
		}

		//allocate the signature buffer
		pbHashSigned = new char[cbHashSigned];
		if (FAILED(secStatus = NCryptSignHash(hKeyNew, &paddingInfo, (PUCHAR)hasPadded.c_str(), hasPadded.length(), (PUCHAR)pbHashSigned, cbHashSigned, &cbHashSigned, BCRYPT_PAD_PSS)))//
		{
			tmp.Format(_T("Semnare hash - 'NCryptSignHash' a returnat eroarea 0x%x ..."), secStatus);
			AddLogToList(pContext, tmp, pContext->m_ID);
			ret = false;
			goto CleanUp2;
		}
		
		if (FAILED(secStatus = NCryptVerifySignature(hKeyNew, &paddingInfo, (PUCHAR)hasPadded.c_str(), hasPadded.length(), (PUCHAR)pbHashSigned, cbHashSigned, BCRYPT_PAD_PSS)))//
		{
			tmp.Format(_T("Verificare Semnare hash - 'NCryptVerifySignature' a returnat eroarea 0x%x ..."), secStatus);
			AddLogToList(pContext, tmp, pContext->m_ID);
			ret = false;
			goto CleanUp2;
		}

		pContext->m_ContextLock.Lock();
		pContext->pbHashSigned = "";
		for (unsigned int i = 0; i < cbHashSigned; i++) pContext->pbHashSigned = pContext->pbHashSigned + pbHashSigned[i];
		pContext->m_ContextLock.Unlock();
	}