prevent endless recursion when trace API is used within OPENSSL_init_crypto()

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Paul Yang <kaishen.yy@antfin.com>
(Merged from https://github.com/openssl/openssl/pull/9567)

diff --git a/crypto/include/internal/cryptlib_int.h b/crypto/include/internal/cryptlib_int.h
index 673a004f0f6b8efca34392922b6f7971f0114c17..69d94be54a8a28d50e2978dedf9707869e7b1923 100644 (file)
--- a/crypto/include/internal/cryptlib_int.h
+++ b/crypto/include/internal/cryptlib_int.h
@@ -27,6 +27,5 @@ void ossl_ctx_thread_stop(void *arg);
 # define OPENSSL_INIT_ZLIB                   0x00010000L
 # define OPENSSL_INIT_BASE_ONLY              0x00040000L
 
-int ossl_trace_init(void);
 void ossl_trace_cleanup(void);
 void ossl_malloc_setup_failures(void);

diff --git a/crypto/init.c b/crypto/init.c
index 04fd33087a7bb9f79d9fc2aa143a9825b6dbce4a..d4da7b27e34bb989d6e0767166b541fcc8e6be34 100644 (file)
--- a/crypto/init.c
+++ b/crypto/init.c
@@ -44,8 +44,7 @@ static CRYPTO_ONCE base = CRYPTO_ONCE_STATIC_INIT;
 static int base_inited = 0;
 DEFINE_RUN_ONCE_STATIC(ossl_init_base)
 {
-    if (ossl_trace_init() == 0)
-        return 0;
+    /* no need to init trace */
 
     OSSL_TRACE(INIT, "ossl_init_base: setting up stop handlers\n");
 #ifndef OPENSSL_NO_CRYPTO_MDEBUG

diff --git a/crypto/trace.c b/crypto/trace.c
old mode 100644 (file)
new mode 100755 (executable)
index d9524da..cc99fff
--- a/crypto/trace.c
+++ b/crypto/trace.c
@@ -10,6 +10,7 @@
 #include <stdio.h>
 #include <string.h>
 
+#include "internal/thread_once.h"
 #include <openssl/bio.h>
 #include <openssl/crypto.h>
 #include <openssl/trace.h>
@@ -218,6 +219,13 @@ static int trace_detach_cb(int category, int type, const void *data)
     return 1;
 }
 
+static int do_ossl_trace_init(void);
+static CRYPTO_ONCE trace_inited = CRYPTO_ONCE_STATIC_INIT;
+DEFINE_RUN_ONCE_STATIC(ossl_trace_init)
+{
+    return do_ossl_trace_init();
+}
+
 static int set_trace_data(int category, int type, BIO **channel,
                           const char **prefix, const char **suffix,
                           int (*attach_cb)(int, int, const void *),
@@ -227,8 +235,9 @@ static int set_trace_data(int category, int type, BIO **channel,
     char *curr_prefix = NULL;
     char *curr_suffix = NULL;
 
-    /* Ensure ossl_trace_init() is called */
-    OPENSSL_init_crypto(0, NULL);
+    /* Ensure do_ossl_trace_init() is called once */
+    if (!RUN_ONCE(&trace_inited, ossl_trace_init))
+        return 0;
 
     curr_channel = trace_channels[category].bio;
     curr_prefix = trace_channels[category].prefix;
@@ -297,19 +306,15 @@ static int set_trace_data(int category, int type, BIO **channel,
 
     return 1;
 }
-#endif
 
-int ossl_trace_init(void)
+static int do_ossl_trace_init(void)
 {
-#ifndef OPENSSL_NO_TRACE
     trace_lock = CRYPTO_THREAD_lock_new();
-    if (trace_lock == NULL)
-        return 0;
-#endif
-
-    return 1;
+    return trace_lock != NULL;
 }
 
+#endif
+
 void ossl_trace_cleanup(void)
 {
 #ifndef OPENSSL_NO_TRACE