7155371 "ldm init-system" on s11u1 hits libcrypto.so.1.0.0`pk11_get_session SEGV in mutex_lock_impl
authorTomas Kuthan <tomas.kuthan@oracle.com>
Mon, 11 Jun 2012 22:35:26 -0700
changeset 866 c6e2467e422e
parent 865 bc755eadb4db
child 867 181d993d52cf
7155371 "ldm init-system" on s11u1 hits libcrypto.so.1.0.0`pk11_get_session SEGV in mutex_lock_impl 6980830 Memory leaked when calling ENGINE_load_private_key()
components/openssl/openssl-0.9.8-fips-140/engines/pkcs11/hw_pk11_pub.c
components/openssl/openssl-1.0.0/engines/pkcs11/hw_pk11.c
components/openssl/openssl-1.0.0/engines/pkcs11/hw_pk11_pub.c
--- a/components/openssl/openssl-0.9.8-fips-140/engines/pkcs11/hw_pk11_pub.c	Mon Jun 11 13:32:59 2012 -0700
+++ b/components/openssl/openssl-0.9.8-fips-140/engines/pkcs11/hw_pk11_pub.c	Mon Jun 11 22:35:26 2012 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2012, Oracle and/or its affiliates. All rights reserved.
  *
  */
 
@@ -1864,11 +1864,14 @@
 	 * problem. If the application expects the private components to be read
 	 * from the keystore then that is not a supported way of usage.
 	 */
-	if (rsa->d != NULL && (sp->opdata_rsa_d_num = BN_dup(rsa->d)) == NULL)
+	if (rsa->d != NULL)
 		{
-		PK11err(PK11_F_GET_PRIV_RSA_KEY, PK11_R_MALLOC_FAILURE);
-		rollback = CK_TRUE;
-		goto err;
+		if ((sp->opdata_rsa_d_num = BN_dup(rsa->d)) == NULL)
+			{
+			PK11err(PK11_F_GET_PRIV_RSA_KEY, PK11_R_MALLOC_FAILURE);
+			rollback = CK_TRUE;
+			goto err;
+			}
 		}
 	else
 		sp->opdata_rsa_d_num = NULL;
--- a/components/openssl/openssl-1.0.0/engines/pkcs11/hw_pk11.c	Mon Jun 11 13:32:59 2012 -0700
+++ b/components/openssl/openssl-1.0.0/engines/pkcs11/hw_pk11.c	Mon Jun 11 22:35:26 2012 -0700
@@ -818,6 +818,7 @@
 static CK_BBOOL pk11_library_initialized = CK_FALSE;
 static CK_BBOOL pk11_atfork_initialized = CK_FALSE;
 static int pk11_pid = 0;
+static ENGINE* pk11_engine = NULL;
 
 static DSO *pk11_dso = NULL;
 
@@ -1032,12 +1033,33 @@
 	return (ret);
 	}
 
+int
+pk11_engine_loaded()
+	{
+	ENGINE *e;
+	int rtrn = 0;
+
+	if ((e = ENGINE_by_id(engine_pk11_id)) != NULL)
+		{
+		rtrn = 1;
+		ENGINE_free(e);
+		}
+	return (rtrn);
+	}
+
 void
 ENGINE_load_pk11(void)
 	{
 	ENGINE *e_pk11 = NULL;
 
 	/*
+	 * Do not attempt to load the engine twice!
+	 * Multiple instances would share static variables from this file.
+	 */
+	if (pk11_engine_loaded())
+		return;
+
+	/*
 	 * Do not use dynamic PKCS#11 library on Solaris due to
 	 * security reasons. We will link it in statically.
 	 */
@@ -1189,6 +1211,14 @@
 	int any_slot_found;
 	int i;
 
+	if (e != pk11_engine)
+		{
+		if (pk11_engine)
+			ENGINE_free(pk11_engine);
+		pk11_engine = e;
+		ENGINE_up_ref(e);
+		}
+
 	/*
 	 * pk11_library_initialized is set to 0 in pk11_finish() which is called
 	 * from ENGINE_finish(). However, if there is still at least one
@@ -1371,6 +1401,15 @@
 	{
 	int i;
 
+	/*
+	 * Make sure, right engine instance is being destroyed.
+	 * Engine e may be the wrong instance if
+	 * 	1) either someone calls ENGINE_load_pk11 twice
+	 * 	2) or last ref. to an already finished engine is being destroyed
+	 */
+	if (e != pk11_engine)
+		goto err;
+
 	if (pk11_dso == NULL)
 		{
 		PK11err(PK11_F_FINISH, PK11_R_NOT_LOADED);
@@ -1426,6 +1465,8 @@
 	pFuncList = NULL;
 	pk11_library_initialized = CK_FALSE;
 	pk11_pid = 0;
+	ENGINE_free(pk11_engine);
+	pk11_engine = NULL;
 	/*
 	 * There is no way how to unregister atfork handlers (other than
 	 * unloading the library) so we just free the locks. For this reason
--- a/components/openssl/openssl-1.0.0/engines/pkcs11/hw_pk11_pub.c	Mon Jun 11 13:32:59 2012 -0700
+++ b/components/openssl/openssl-1.0.0/engines/pkcs11/hw_pk11_pub.c	Mon Jun 11 22:35:26 2012 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2012, Oracle and/or its affiliates. All rights reserved.
  */
 
 /* crypto/engine/hw_pk11_pub.c */
@@ -1867,11 +1867,14 @@
 	 * problem. If the application expects the private components to be read
 	 * from the keystore then that is not a supported way of usage.
 	 */
-	if (rsa->d != NULL && (sp->opdata_rsa_d_num = BN_dup(rsa->d)) == NULL)
+	if (rsa->d != NULL)
 		{
-		PK11err(PK11_F_GET_PRIV_RSA_KEY, PK11_R_MALLOC_FAILURE);
-		rollback = CK_TRUE;
-		goto err;
+		if ((sp->opdata_rsa_d_num = BN_dup(rsa->d)) == NULL)
+			{
+			PK11err(PK11_F_GET_PRIV_RSA_KEY, PK11_R_MALLOC_FAILURE);
+			rollback = CK_TRUE;
+			goto err;
+			}
 		}
 	else
 		sp->opdata_rsa_d_num = NULL;