20358335 memory leak in libcrypto
authorMisaki Miyashita <Misaki.Miyashita@Oracle.COM>
Thu, 08 Sep 2016 08:47:55 -0700
changeset 6860 fbbec9b86c26
parent 6859 569bef81e3c4
child 6861 6110892450ff
20358335 memory leak in libcrypto 23285559 ssh libcrypto`solaris_locking_setup() atfork handler calls malloc()
components/openssl/common/patches/029-fork_safe.patch
components/openssl/openssl-default/patches/102-wanboot.patch
--- a/components/openssl/common/patches/029-fork_safe.patch	Wed Sep 07 18:09:38 2016 -0500
+++ b/components/openssl/common/patches/029-fork_safe.patch	Thu Sep 08 08:47:55 2016 -0700
@@ -5,8 +5,8 @@
 # This change was implemented in-house.  The issue was brought up to
 # the upstream engineers, but there was no commitment.
 #
---- openssl-1.0.1f/crypto/cryptlib.c.~1~	Fri Feb  7 10:41:36 2014
-+++ openssl-1.0.1f/crypto/cryptlib.c	Thu Feb  6 16:03:58 2014
+--- a/crypto/cryptlib.c	2016-05-03 06:44:42.000000000 -0700
++++ b/crypto/cryptlib.c	2016-09-02 08:47:23.640202700 -0700
 @@ -116,6 +116,7 @@
  
  #include "cryptlib.h"
@@ -34,7 +34,7 @@
 +     * setting of another locking callback.
 +     */
  }
-
+ 
  void CRYPTO_set_dynlock_lock_callback(void (*func) (int mode,
 @@ -382,7 +388,10 @@
                                                      const char *file,
@@ -46,7 +46,7 @@
 +     * setting of another locking callback.
 +     */
  }
-
+ 
  void CRYPTO_set_dynlock_destroy_callback(void (*func)
 @@ -389,7 +398,10 @@
                                            (struct CRYPTO_dynlock_value *l,
@@ -58,12 +58,12 @@
 +     * setting of another locking callback.
 +     */
  }
-
+ 
  void (*CRYPTO_get_locking_callback(void)) (int mode, int type,
[email protected]@ -402,6 +414,[email protected]@
[email protected]@ -402,6 +414,127 @@
      return (add_lock_callback);
  }
-
+ 
 +/*
 + * This is the locking callback function which all applications will be
 + * using when CRYPTO_lock() is called.
@@ -72,9 +72,9 @@
 +    int line)
 +{
 +    if (mode & CRYPTO_LOCK) {
-+        pthread_mutex_lock(&solaris_openssl_locks[type]);
++        (void) pthread_mutex_lock(&solaris_openssl_locks[type]);
 +    } else {
-+        pthread_mutex_unlock(&solaris_openssl_locks[type]);
++        (void) pthread_mutex_unlock(&solaris_openssl_locks[type]);
 +    }
 +}
 +
@@ -123,28 +123,37 @@
 +}
 +
 +
-+/*
-+ * This function is called when a child process is forked to setup its own
-+ * global locking callback function ptr and mutexes.
-+ */
-+static void solaris_fork_child(void)
++static void solaris_fork_prep(void)
++{
++    int i;
++
++    for (i = 0; i < CRYPTO_NUM_LOCKS; i++) {
++        (void) pthread_mutex_lock(&solaris_openssl_locks[i]);
++    }
++}
++
++static void solaris_fork_post(void)
 +{
-+    /*
-+     * clear locking_callback to indicate that locks should
-+     * be reinitialized.
-+     */
-+    locking_callback = NULL;
-+    solaris_locking_setup();
++    int i;
++
++    for (i = CRYPTO_NUM_LOCKS - 1; i >= 0; i--) {
++        (void) pthread_mutex_unlock(&solaris_openssl_locks[i]);
++    }
++
++    OPENSSL_assert(dynlock_create_callback == solaris_dynlock_create);
++    OPENSSL_assert(dynlock_lock_callback == solaris_dynlock_lock);
++    OPENSSL_assert(dynlock_destroy_callback == solaris_dynlock_destroy);
++    OPENSSL_assert(locking_callback == solaris_locking_callback);
 +}
 +
 +/*
-+ * This function allocates and initializes the global mutex array, and
-+ * sets the locking callback.
++ * This is called by the _init() function to setup locks used by OpenSSL
++ * and locking callback functions.
 + */
-+void solaris_locking_setup()
++void
++solaris_locking_setup()
 +{
 +    int i;
-+    int num_locks;
 +
 +    /* setup the dynlock callback if not already */
 +    if (dynlock_create_callback == NULL) {
@@ -156,40 +165,30 @@
 +    if (dynlock_destroy_callback == NULL) {
 +        dynlock_destroy_callback = solaris_dynlock_destroy;
 +    }
-+
-+    /* locking callback is already setup. Nothing to do */
-+    if (locking_callback != NULL) {
-+        return;
++    if (locking_callback == NULL) {
++	    locking_callback = solaris_locking_callback;
 +    }
 +
-+    /*
-+     * Set atfork handler so that child can setup its own mutexes and
-+     * locking callbacks when it is forked
-+     */
-+    (void) pthread_atfork(NULL, NULL, solaris_fork_child);
-+
-+    /* allocate locks needed by OpenSSL  */
-+    num_locks = CRYPTO_num_locks();
++    /* allocate and initialize locks needed by OpenSSL  */
 +    solaris_openssl_locks =
-+        OPENSSL_malloc(sizeof (pthread_mutex_t) * num_locks);
++        OPENSSL_malloc(sizeof (pthread_mutex_t) * CRYPTO_NUM_LOCKS);
 +    if (solaris_openssl_locks == NULL) {
 +        fprintf(stderr,
 +            "solaris_locking_setup: memory allocation failure.\n");
 +        abort();
 +    }
++    for (i = 0; i < CRYPTO_NUM_LOCKS; i++) {
++        (void) pthread_mutex_init(&solaris_openssl_locks[i], NULL);
++    }
 +
-+    /* initialize openssl mutexes */
-+    for (i = 0; i < num_locks; i++) {
-+        pthread_mutex_init(&solaris_openssl_locks[i], NULL);
-+    }
-+    locking_callback = solaris_locking_callback;
++    (void) pthread_atfork(solaris_fork_prep, solaris_fork_post, solaris_fork_post);
++}
 +
-+}
 +
  void CRYPTO_set_locking_callback(void (*func) (int mode, int type,
                                                 const char *file, int line))
  {
[email protected]@ -410,7 +486,11 @@
[email protected]@ -410,7 +541,11 @@
       * started.
       */
      OPENSSL_init();
@@ -202,7 +201,7 @@
  }
  
  void CRYPTO_set_add_lock_callback(int (*func) (int *num, int mount, int type,
[email protected]@ -471,9 +551,10 @@
[email protected]@ -471,9 +606,10 @@
  
  int CRYPTO_THREADID_set_callback(void (*func) (CRYPTO_THREADID *))
  {
@@ -216,7 +215,7 @@
      return 1;
  }
  
[email protected]@ -529,7 +669,10 @@
[email protected]@ -529,7 +665,10 @@
  
  void CRYPTO_set_id_callback(unsigned long (*func) (void))
  {
--- a/components/openssl/openssl-default/patches/102-wanboot.patch	Wed Sep 07 18:09:38 2016 -0500
+++ b/components/openssl/openssl-default/patches/102-wanboot.patch	Thu Sep 08 08:47:55 2016 -0700
@@ -35,16 +35,16 @@
  SHLIBDIRS= crypto ssl
 
 --- openssl-1.0.0e/crypto/cryptlib.c    2011-06-22 08:39:00.000000000 -0700
-+++ openssl-1.0.0e_patched/crypto/cryptlib.c    2011-12-12 06:17:45.422476900 -0800
++++ openssl-1.0.0e/crypto/cryptlib.c    2011-12-12 06:17:45.422476900 -0800
 @@ -421,11 +421,13 @@
  static void solaris_locking_callback(int mode, int type, const char *file,
      int line)
  {
 +#ifndef    _BOOT
      if (mode & CRYPTO_LOCK) {
-         pthread_mutex_lock(&solaris_openssl_locks[type]);
+         (void) pthread_mutex_lock(&solaris_openssl_locks[type]);
      } else {
-         pthread_mutex_unlock(&solaris_openssl_locks[type]);
+         (void) pthread_mutex_unlock(&solaris_openssl_locks[type]);
      }
 +#endif
  }
@@ -57,16 +57,16 @@
 +#ifndef    _BOOT
      int                        ret;
      pthread_mutex_t    *dynlock;
-
+ 
 @@ -446,6 +449,9 @@
      OPENSSL_assert(ret == 0);
-
+ 
      return ((struct CRYPTO_dynlock_value *)dynlock);
 +#else
 +    return (NULL);
 +#endif
  }
-
+ 
  static void
 @@ -452,6 +458,7 @@
  solaris_dynlock_lock(int mode, struct CRYPTO_dynlock_value *dynlock,
@@ -78,13 +78,13 @@
      if (mode & CRYPTO_LOCK) {
 @@ -461,6 +468,7 @@
      }
-
+ 
      OPENSSL_assert(ret == 0);
 +#endif
  }
-
+ 
  static void
[email protected]@ -467,9 +475,11 @@
[email protected]@ -467,23 +475,28 @@
  solaris_dynlock_destroy(struct CRYPTO_dynlock_value *dynlock,
      const char *file, int line)
  {
@@ -94,30 +94,54 @@
      OPENSSL_assert(ret == 0);
 +#endif
  }
-
-
[email protected]@ -513,6 +523,12 @@
+ 
+ 
+ static void solaris_fork_prep(void)
+ {
++#ifndef    _BOOT
+     int i;
+ 
+     for (i = 0; i < CRYPTO_NUM_LOCKS; i++) {
+         (void) pthread_mutex_lock(&solaris_openssl_locks[i]);
+     }
++#endif
+ }
+ 
+ static void solaris_fork_post(void)
+ {
++#ifndef	_BOOT
+     int i;
+ 
+     for (i = CRYPTO_NUM_LOCKS - 1; i >= 0; i--) {
[email protected]@ -494,6 +507,7 @@
+     OPENSSL_assert(dynlock_lock_callback == solaris_dynlock_lock);
+     OPENSSL_assert(dynlock_destroy_callback == solaris_dynlock_destroy);
+     OPENSSL_assert(locking_callback == solaris_locking_callback);
++#endif
+ }
+ 
+ /*
[email protected]@ -517,6 +531,12 @@
+ 	    locking_callback = solaris_locking_callback;
      }
  
-     /*
++    /*
 +     * pthread_* can't be used in wanboot.
 +     * wanboot needs not be thread-safe and mutexes and locking callback
 +     * function will not be setup for wanboot.
 +     */
 +#ifndef    _BOOT
-+    /*
-      * Set atfork handler so that child can setup its own mutexes and
-      * locking callbacks when it is forked
-      */
[email protected]@ -533,7 +549,7 @@
-         pthread_mutex_init(&solaris_openssl_locks[i], NULL);
+     /* allocate and initialize locks needed by OpenSSL  */
+     solaris_openssl_locks =
+         OPENSSL_malloc(sizeof (pthread_mutex_t) * CRYPTO_NUM_LOCKS);
[email protected]@ -530,6 +550,7 @@
      }
-     locking_callback = solaris_locking_callback;
--
+ 
+     (void) pthread_atfork(solaris_fork_prep, solaris_fork_post, solaris_fork_post);
 +#endif
  }
  
- void CRYPTO_set_locking_callback(void (*func) (int mode, int type,
+ 
 @@ -1104,6 +1120,12 @@
          MessageBox(NULL, buf, _T("OpenSSL: FATAL"), MB_OK | MB_ICONERROR);
  }
@@ -143,7 +167,7 @@
  
  void OpenSSLDie(const char *file, int line, const char *assertion)
  {
-+#ifndef    _BOOT        
++#ifndef _BOOT
      OPENSSL_showfatal
          ("%s(%d): OpenSSL internal error, assertion failed: %s\n", file, line,
           assertion);
@@ -151,7 +175,7 @@
 +    fprintf(stderr,
 +        "%s(%d): OpenSSL internal error, assertion failed: %s\n",
 +        file,line,assertion);
-+#endif    
++#endif
  #if !defined(_WIN32) || defined(__CYGWIN__)
      abort();
  #else