18126345 SSH1 server failed with "cipher_init: EVP_CipherInit failed for 3des" error
authorMisaki Miyashita <Misaki.Miyashita@Oracle.COM>
Mon, 27 Jan 2014 15:13:46 -0800
changeset 1665 9e945128be3b
parent 1664 128a1af2859e
child 1666 69d14d547e77
18126345 SSH1 server failed with "cipher_init: EVP_CipherInit failed for 3des" error
components/openssl/openssl-1.0.1-fips-140/engines/pkcs11/e_pk11.c
components/openssl/openssl-1.0.1-fips-140/engines/pkcs11/e_pk11_pub.c
components/openssl/openssl-1.0.1/engines/pkcs11/e_pk11.c
--- a/components/openssl/openssl-1.0.1-fips-140/engines/pkcs11/e_pk11.c	Mon Jan 27 09:25:05 2014 -0800
+++ b/components/openssl/openssl-1.0.1-fips-140/engines/pkcs11/e_pk11.c	Mon Jan 27 15:13:46 2014 -0800
@@ -3027,8 +3027,10 @@
 		pk11_choose_pubkey_slot(mech_info, token_info, current_slot,
 			rv, best_number_of_mechs, best_pubkey_slot_sofar);
 
-		pk11_choose_cipher_digest(&local_cipher_nids,
-			&local_digest_nids, pFuncList, current_slot);
+		(void) memset(local_cipher_nids, 0, sizeof (local_cipher_nids));
+		(void) memset(local_digest_nids, 0, sizeof (local_digest_nids));
+		pk11_choose_cipher_digest(local_cipher_nids,
+			local_digest_nids, pFuncList, current_slot);
 		}
 
 	if (best_number_of_mechs == 0)
@@ -3179,9 +3181,6 @@
 
 	DEBUG_SLOT_SEL("%s: checking cipher/digest\n", PK11_DBG);
 
-	(void) memset(local_cipher_nids, 0, sizeof (local_cipher_nids));
-	(void) memset(local_digest_nids, 0, sizeof (local_digest_nids));
-
 	pk11_find_symmetric_ciphers(pFuncList, current_slot,
 	    &current_slot_n_cipher, local_cipher_nids);
 
@@ -3206,10 +3205,12 @@
 		SLOTID = current_slot;
 		cipher_count = current_slot_n_cipher;
 		digest_count = current_slot_n_digest;
+		OPENSSL_assert(cipher_count <= PK11_CIPHER_MAX);
+		OPENSSL_assert(digest_count <= PK11_DIGEST_MAX);
 		(void) memcpy(cipher_nids, local_cipher_nids,
-			sizeof (local_cipher_nids));
+			sizeof (int) * cipher_count);
 		(void) memcpy(digest_nids, local_digest_nids,
-			sizeof (local_digest_nids));
+			sizeof (int) * digest_count);
 		}
 	}
 
@@ -3221,6 +3222,8 @@
 	static CK_RV rv;
 	static CK_MECHANISM_TYPE last_checked_mech = (CK_MECHANISM_TYPE)-1;
 
+	OPENSSL_assert(cipher->mech_type != (CK_MECHANISM_TYPE)-1);
+
 	DEBUG_SLOT_SEL("%s: checking mech: %x", PK11_DBG, cipher->mech_type);
 	if (cipher->mech_type != last_checked_mech)
 		{
--- a/components/openssl/openssl-1.0.1-fips-140/engines/pkcs11/e_pk11_pub.c	Mon Jan 27 09:25:05 2014 -0800
+++ b/components/openssl/openssl-1.0.1-fips-140/engines/pkcs11/e_pk11_pub.c	Mon Jan 27 15:13:46 2014 -0800
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2014, Oracle and/or its affiliates. All rights reserved.
  *
  */
 
@@ -127,7 +127,7 @@
 static int pk11_RSA_sign(int type, const unsigned char *m, unsigned int m_len,
 	unsigned char *sigret, unsigned int *siglen, const RSA *rsa);
 static int pk11_RSA_verify(int dtype, const unsigned char *m,
-	unsigned int m_len, unsigned char *sigbuf, unsigned int siglen,
+	unsigned int m_len, const unsigned char *sigbuf, unsigned int siglen,
 	const RSA *rsa);
 EVP_PKEY *pk11_load_privkey(ENGINE*, const char *privkey_id,
 	UI_METHOD *ui_method, void *callback_data);
@@ -1103,7 +1103,7 @@
 	}
 
 static int pk11_RSA_verify(int type, const unsigned char *m,
-	unsigned int m_len, unsigned char *sigbuf, unsigned int siglen,
+	unsigned int m_len, const unsigned char *sigbuf, unsigned int siglen,
 	const RSA *rsa)
 	{
 	X509_SIG sig;
@@ -1197,8 +1197,8 @@
 			    rv);
 			goto err;
 			}
-		rv = pFuncList->C_Verify(sp->session, s, i, sigbuf,
-			(CK_ULONG)siglen);
+		rv = pFuncList->C_Verify(sp->session, s, i,
+			(CK_BYTE_PTR)sigbuf, (CK_ULONG)siglen);
 
 		if (rv != CKR_OK)
 			{
--- a/components/openssl/openssl-1.0.1/engines/pkcs11/e_pk11.c	Mon Jan 27 09:25:05 2014 -0800
+++ b/components/openssl/openssl-1.0.1/engines/pkcs11/e_pk11.c	Mon Jan 27 15:13:46 2014 -0800
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2014, Oracle and/or its affiliates. All rights reserved.
  */
 
 /* crypto/engine/e_pk11.c */
@@ -3099,8 +3099,10 @@
 		pk11_choose_pubkey_slot(mech_info, token_info, current_slot,
 			rv, best_number_of_mechs, best_pubkey_slot_sofar);
 
-		pk11_choose_cipher_digest(&local_cipher_nids,
-			&local_digest_nids, pFuncList, current_slot);
+		(void) memset(local_cipher_nids, 0, sizeof (local_cipher_nids));
+		(void) memset(local_digest_nids, 0, sizeof (local_digest_nids));
+		pk11_choose_cipher_digest(local_cipher_nids,
+			local_digest_nids, pFuncList, current_slot);
 		}
 
 	if (best_number_of_mechs == 0)
@@ -3249,9 +3251,6 @@
 
 	DEBUG_SLOT_SEL("%s: checking cipher/digest\n", PK11_DBG);
 
-	(void) memset(local_cipher_nids, 0, sizeof (local_cipher_nids));
-	(void) memset(local_digest_nids, 0, sizeof (local_digest_nids));
-
 	pk11_find_symmetric_ciphers(pFuncList, current_slot,
 	    &current_slot_n_cipher, local_cipher_nids);
 
@@ -3275,10 +3274,12 @@
 		SLOTID = current_slot;
 		cipher_count = current_slot_n_cipher;
 		digest_count = current_slot_n_digest;
+		OPENSSL_assert(cipher_count <= PK11_CIPHER_MAX);
+		OPENSSL_assert(digest_count <= PK11_DIGEST_MAX);
 		(void) memcpy(cipher_nids, local_cipher_nids,
-			sizeof (local_cipher_nids));
+			sizeof (int) * cipher_count);
 		(void) memcpy(digest_nids, local_digest_nids,
-			sizeof (local_digest_nids));
+			sizeof (int) * digest_count);
 		}
 	}
 
@@ -3290,6 +3291,8 @@
 	static CK_RV rv;
 	static CK_MECHANISM_TYPE last_checked_mech = (CK_MECHANISM_TYPE)-1;
 
+	OPENSSL_assert(cipher->mech_type != (CK_MECHANISM_TYPE)-1);
+
 	DEBUG_SLOT_SEL("%s: checking mech: %x", PK11_DBG, cipher->mech_type);
 	if (cipher->mech_type != last_checked_mech)
 		{