7155371 "ldm init-system" on s11u1 hits libcrypto.so.1.0.0`pk11_get_session SEGV in mutex_lock_impl
6980830 Memory leaked when calling ENGINE_load_private_key()
--- a/components/openssl/openssl-0.9.8-fips-140/engines/pkcs11/hw_pk11_pub.c Mon Jun 11 13:32:59 2012 -0700
+++ b/components/openssl/openssl-0.9.8-fips-140/engines/pkcs11/hw_pk11_pub.c Mon Jun 11 22:35:26 2012 -0700
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2012, Oracle and/or its affiliates. All rights reserved.
*
*/
@@ -1864,11 +1864,14 @@
* problem. If the application expects the private components to be read
* from the keystore then that is not a supported way of usage.
*/
- if (rsa->d != NULL && (sp->opdata_rsa_d_num = BN_dup(rsa->d)) == NULL)
+ if (rsa->d != NULL)
{
- PK11err(PK11_F_GET_PRIV_RSA_KEY, PK11_R_MALLOC_FAILURE);
- rollback = CK_TRUE;
- goto err;
+ if ((sp->opdata_rsa_d_num = BN_dup(rsa->d)) == NULL)
+ {
+ PK11err(PK11_F_GET_PRIV_RSA_KEY, PK11_R_MALLOC_FAILURE);
+ rollback = CK_TRUE;
+ goto err;
+ }
}
else
sp->opdata_rsa_d_num = NULL;
--- a/components/openssl/openssl-1.0.0/engines/pkcs11/hw_pk11.c Mon Jun 11 13:32:59 2012 -0700
+++ b/components/openssl/openssl-1.0.0/engines/pkcs11/hw_pk11.c Mon Jun 11 22:35:26 2012 -0700
@@ -818,6 +818,7 @@
static CK_BBOOL pk11_library_initialized = CK_FALSE;
static CK_BBOOL pk11_atfork_initialized = CK_FALSE;
static int pk11_pid = 0;
+static ENGINE* pk11_engine = NULL;
static DSO *pk11_dso = NULL;
@@ -1032,12 +1033,33 @@
return (ret);
}
+int
+pk11_engine_loaded()
+ {
+ ENGINE *e;
+ int rtrn = 0;
+
+ if ((e = ENGINE_by_id(engine_pk11_id)) != NULL)
+ {
+ rtrn = 1;
+ ENGINE_free(e);
+ }
+ return (rtrn);
+ }
+
void
ENGINE_load_pk11(void)
{
ENGINE *e_pk11 = NULL;
/*
+ * Do not attempt to load the engine twice!
+ * Multiple instances would share static variables from this file.
+ */
+ if (pk11_engine_loaded())
+ return;
+
+ /*
* Do not use dynamic PKCS#11 library on Solaris due to
* security reasons. We will link it in statically.
*/
@@ -1189,6 +1211,14 @@
int any_slot_found;
int i;
+ if (e != pk11_engine)
+ {
+ if (pk11_engine)
+ ENGINE_free(pk11_engine);
+ pk11_engine = e;
+ ENGINE_up_ref(e);
+ }
+
/*
* pk11_library_initialized is set to 0 in pk11_finish() which is called
* from ENGINE_finish(). However, if there is still at least one
@@ -1371,6 +1401,15 @@
{
int i;
+ /*
+ * Make sure, right engine instance is being destroyed.
+ * Engine e may be the wrong instance if
+ * 1) either someone calls ENGINE_load_pk11 twice
+ * 2) or last ref. to an already finished engine is being destroyed
+ */
+ if (e != pk11_engine)
+ goto err;
+
if (pk11_dso == NULL)
{
PK11err(PK11_F_FINISH, PK11_R_NOT_LOADED);
@@ -1426,6 +1465,8 @@
pFuncList = NULL;
pk11_library_initialized = CK_FALSE;
pk11_pid = 0;
+ ENGINE_free(pk11_engine);
+ pk11_engine = NULL;
/*
* There is no way how to unregister atfork handlers (other than
* unloading the library) so we just free the locks. For this reason
--- a/components/openssl/openssl-1.0.0/engines/pkcs11/hw_pk11_pub.c Mon Jun 11 13:32:59 2012 -0700
+++ b/components/openssl/openssl-1.0.0/engines/pkcs11/hw_pk11_pub.c Mon Jun 11 22:35:26 2012 -0700
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2012, Oracle and/or its affiliates. All rights reserved.
*/
/* crypto/engine/hw_pk11_pub.c */
@@ -1867,11 +1867,14 @@
* problem. If the application expects the private components to be read
* from the keystore then that is not a supported way of usage.
*/
- if (rsa->d != NULL && (sp->opdata_rsa_d_num = BN_dup(rsa->d)) == NULL)
+ if (rsa->d != NULL)
{
- PK11err(PK11_F_GET_PRIV_RSA_KEY, PK11_R_MALLOC_FAILURE);
- rollback = CK_TRUE;
- goto err;
+ if ((sp->opdata_rsa_d_num = BN_dup(rsa->d)) == NULL)
+ {
+ PK11err(PK11_F_GET_PRIV_RSA_KEY, PK11_R_MALLOC_FAILURE);
+ rollback = CK_TRUE;
+ goto err;
+ }
}
else
sp->opdata_rsa_d_num = NULL;