Cant get RC4 ciphertext to decrypt properly

See more:

C++

C

cryptography

I have been playing around with the Microsoft Crypto API. Using CALG_RC4 I am trying to encrypt a plaintext where the key is derived from an md5 hash of the plaintext.

I have verified that the "password" md5, ciphertext and lengths of both buffers are the same when I pass them to the decrypt function as they were after being encrypted.

CryptDecrypt however does not successfully decrypt the ciphertext back to plaintext.

Does anyone know why? I'm obviously doing something wrong.

An additional note: the context (HCRYPTPROV) is different between the two functions (they are in separate binaries). Not sure if that matters, I know it does for certain KeyBlob stuff.

Forgive the poor formatting of my copy paste:

BOOL _Encrypt( MessagePair * _write_data )
{

static  HCRYPTPROV			hCryptContext		= NULL;
	HCRYPTHASH			hHash				= 0;
	HCRYPTKEY			hKey				= 0;
	DWORD				data_len			= 4;			
	DWORD				crypto_len;		
	BOOL				retval				= false;
	DWORD error;

if( !hCryptContext )
{
	if( !CryptAcquireContextA( &hCryptContext,
                                         NULL, 
                                         MS_ENHANCED_PROV_A,
                                         PROV_RSA_FULL,  0 ) )
	{
			return false;
	}
}

_write_data->hash_key = (PBYTE) Get_MD5_Generic( _write_data->data,
                                                  _write_data->data_len,
                                                   hCryptContext );

_write_data->hash_len = HASHLEN;

if( !_write_data->hash_key ) return false;

//begin to hash data block
if( CryptCreateHash( hCryptContext, CALG_MD5, 0, 0, &hHash ) )
{
	if( CryptHashData( hHash, 
                           (BYTE *) _write_data->hash_key, 
                            _write_data->data_len, 
                             NULL ) )
	{

        if( CryptDeriveKey( hCryptContext, ENCRYPT_ALG, hHash, KEYLEN, &hKey ) )
	{
	      if( CryptEncrypt( hKey, 
                                NULL, 
                                true, 
                                NULL,
                               (PBYTE) _write_data->data,
                                (DWORD*) &_write_data->data_len, BUFFERSIZE ) )
		{
			retval = true;
		}
		else
		{
			error = GetLastError();
		}
		CryptDestroyKey( hKey );	
	}
		
}
CryptDestroyHash( hHash );
}

if( _write_data->hash_key && !retval )
{
	VirtualFree( _write_data->hash_key, 0, MEM_RELEASE );	
	_write_data->hash_key = NULL;
}

return retval;

}

bool _Decrypt( MessagePair * message )
{
static  HCRYPTPROV		hCryptContext;
	HCRYPTHASH		hHash;
	HCRYPTKEY		hKey;

	BOOL			retval = false;

	DWORD			decrypt_len = LOGBUFFERSIZE;
	DWORD			err = 4;
	DWORD			hashsize = 0x10;

	if( !hCryptContext )
	{
		if( !CryptAcquireContextA( &hCryptContext, NULL, MS_ENHANCED_PROV_A, PROV_RSA_FULL,  0 ) )
		{
			return NULL;
		}
	}

	if( CryptCreateHash( hCryptContext, CALG_MD5, 0, 0, &hHash ) )
	{

		if( CryptHashData( hHash, (BYTE *) message->hash_key, message->hash_len, 0 )  )
			{				

			if( CryptDeriveKey( hCryptContext, ENCRYPT_ALG, hHash, KEYLEN, &hKey ) )
			{
				if( CryptDecrypt( hKey, 
                                                   NULL, 
                                                   true, 
                                                   NULL, 
                                                   (BYTE *)message->data, 
                                                   (DWORD*)&message->data_len ) )
				{
					retval = true;
				}
				CryptDestroyKey( hKey );							
			}		
			else
			{
				err = GetLastError();
			}
		}		
	CryptDestroyHash( hHash );
}

return retval;
}