common: use recursive pthread mutex for library lock

This allows us to do nested locking within one thread avoiding a lockup
when remoting the p11-kit-proxy.so module:

  #0  0x00007f190f35838d in __lll_lock_wait () from /lib64/libpthread.so.0
  #1  0x00007f190f351e4d in pthread_mutex_lock () from /lib64/libpthread.so.0
  #2  0x00007f190f98657f in C_GetFunctionList (list=0x7ffe7ec3f798) at p11-kit/proxy.c:2355
  #3  0x00007f190f993cc9 in dlopen_and_get_function_list (funcs=0x7ffe7ec3f798, path=0x7ffe7ec40926 "/usr/local/lib/p11-kit-proxy.so", mod=0x249e3d0) at p11-kit/modules.c:337
  #4  load_module_from_file_inlock (name=name@entry=0x0, path=path@entry=0x7ffe7ec40926 "/usr/local/lib/p11-kit-proxy.so", result=result@entry=0x7ffe7ec3f7e8) at p11-kit/modules.c:382
  #5  0x00007f190f99587f in p11_kit_module_load (module_path=module_path@entry=0x7ffe7ec40926 "/usr/local/lib/p11-kit-proxy.so", flags=flags@entry=0) at p11-kit/modules.c:2427
  #6  0x0000000000401c4b in serve_module_from_file (file=0x7ffe7ec40926 "/usr/local/lib/p11-kit-proxy.so") at p11-kit/remote.c:105
  #7  main (argc=1, argv=<optimized out>) at p11-kit/remote.c:169

The Windows NT mutex is aready recursive by default.

diff --git a/common/compat.c b/common/compat.c
index 02e6408a057d23a2a824174ed83ac81ec3dea931..627411e40036974aceb0aa2aeeb5dba6351ac470 100644 (file)
--- a/common/compat.c
+++ b/common/compat.c
@@ -171,7 +171,7 @@ p11_mutex_init (p11_mutex_t *mutex)
        int ret;
 
        pthread_mutexattr_init (&attr);
-       pthread_mutexattr_settype (&attr, PTHREAD_MUTEX_DEFAULT);
+       pthread_mutexattr_settype (&attr, PTHREAD_MUTEX_RECURSIVE);
        ret = pthread_mutex_init (mutex, &attr);
        assert (ret == 0);
        pthread_mutexattr_destroy (&attr);