Major cleanup of internal mutex locking. Be more consistant in how we do

things, and avoid potential deadlocks caused when a thread holding a uClibc
internal lock get canceled and terminates without releasing the lock.  This
change also provides a single place, bits/uClibc_mutex.h, for thread libraries
to modify to change all instances of internal locking.

13 files changed, 237 insertions, 229 deletions

diff --git a/libc/stdlib/_atexit.c b/libc/stdlib/_atexit.c
index 236156001..5b290c928 100644
--- a/libc/stdlib/_atexit.c
+++ b/libc/stdlib/_atexit.c
@@ -20,7 +20,7 @@
  *      _stdio_term.
  *
  * Jul 2001          Steve Thayer
- * 
+ *
  *   Added an on_exit implementation (that now matches gnu libc definition.)
  *   Pulled atexit_handler out of the atexit object since it is now required by
  *   on_exit as well.  Renamed it to __exit_handler.
@@ -43,16 +43,12 @@
 #include <errno.h>
 #include <atomic.h>
 
+#include <bits/uClibc_mutex.h>
+__UCLIBC_MUTEX_EXTERN(__atexit_lock);
+
 libc_hidden_proto(exit)
 libc_hidden_proto(_exit)
 
-#ifdef __UCLIBC_HAS_THREADS__
-# include <pthread.h>
-extern pthread_mutex_t mylock;
-#endif
-#define LOCK	__pthread_mutex_lock(&mylock)
-#define UNLOCK	__pthread_mutex_unlock(&mylock)
-
 
 typedef void (*aefuncp) (void);         /* atexit function pointer */
 typedef void (*oefuncp) (int, void *);  /* on_exit function pointer */
@@ -137,7 +133,7 @@ weak_alias(old_atexit,atexit)
 int on_exit(oefuncp func, void *arg)
 {
     struct exit_function *efp;
-    
+
     if (func == NULL) {
         return 0;
     }
@@ -161,7 +157,7 @@ libc_hidden_proto(__cxa_atexit)
 int __cxa_atexit (cxaefuncp func, void *arg, void *dso_handle)
 {
     struct exit_function *efp;
-    
+
     if (func == NULL) {
         return 0;
     }
@@ -243,26 +239,25 @@ struct exit_function attribute_hidden *__new_exitfn(void)
 {
     struct exit_function *efp;
 
-    LOCK;
+    __UCLIBC_MUTEX_LOCK(__atexit_lock);
 
 #ifdef __UCLIBC_DYNAMIC_ATEXIT__
     /* If we are out of function table slots, make some more */
     if (__exit_slots < __exit_count+1) {
-        efp=realloc(__exit_function_table, 
+        efp=realloc(__exit_function_table,
                     (__exit_slots+20)*sizeof(struct exit_function));
         if (efp == NULL) {
-            UNLOCK;
             __set_errno(ENOMEM);
-            return 0;
+	    goto DONE;
         }
         __exit_function_table = efp;
         __exit_slots += 20;
     }
 #else
     if (__exit_count >= __UCLIBC_MAX_ATEXIT) {
-        UNLOCK;
         __set_errno(ENOMEM);
-        return 0;
+	efp = NULL;
+	goto DONE;
     }
 #endif
 
@@ -270,8 +265,8 @@ struct exit_function attribute_hidden *__new_exitfn(void)
     efp = &__exit_function_table[__exit_count++];
     efp->type = ef_in_use;
 
-    UNLOCK;
-
+DONE:
+    __UCLIBC_MUTEX_UNLOCK(__atexit_lock);
     return efp;
 }
 
@@ -302,7 +297,7 @@ void __exit_handler(int status)
 		}
 	}
 #ifdef __UCLIBC_DYNAMIC_ATEXIT__
-	/* Free up memory used by the __exit_function_table structure */ 
+	/* Free up memory used by the __exit_function_table structure */
 	if (__exit_function_table)
 	    free(__exit_function_table);
 #endif
@@ -312,9 +307,7 @@ void __exit_handler(int status)
 #ifdef L_exit
 extern void weak_function _stdio_term(void) attribute_hidden;
 attribute_hidden void (*__exit_cleanup) (int) = 0;
-#ifdef __UCLIBC_HAS_THREADS__
-pthread_mutex_t mylock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
-#endif
+__UCLIBC_MUTEX_INIT(__atexit_lock, PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP);
 
 extern void __uClibc_fini(void);
 libc_hidden_proto(__uClibc_fini)
@@ -325,11 +318,11 @@ libc_hidden_proto(__uClibc_fini)
 void exit(int rv)
 {
 	/* Perform exit-specific cleanup (atexit and on_exit) */
-	LOCK;
+	__UCLIBC_MUTEX_LOCK(__atexit_lock);
 	if (__exit_cleanup) {
 		__exit_cleanup(rv);
 	}
-	UNLOCK;
+	__UCLIBC_MUTEX_UNLOCK(__atexit_lock);
 
 	__uClibc_fini();
 
@@ -337,7 +330,7 @@ void exit(int rv)
 	 * this will attempt to commit all buffered writes.  It may also
 	 * unbuffer all writable files, or close them outright.
 	 * Check the stdio routines for details. */
-	if (_stdio_term) 
+	if (_stdio_term)
 	    _stdio_term();
 
 	_exit(rv);
diff --git a/libc/stdlib/abort.c b/libc/stdlib/abort.c
index a940768c0..6e1806eae 100644
--- a/libc/stdlib/abort.c
+++ b/libc/stdlib/abort.c
@@ -49,12 +49,8 @@ extern void weak_function _stdio_term(void) attribute_hidden;
 static int been_there_done_that = 0;
 
 /* Be prepared in case multiple threads try to abort() */
-#ifdef __UCLIBC_HAS_THREADS__
-# include <pthread.h>
-static pthread_mutex_t mylock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
-#endif
-#define LOCK	__pthread_mutex_lock(&mylock)
-#define UNLOCK	__pthread_mutex_unlock(&mylock)
+#include <bits/uClibc_mutex.h>
+__UCLIBC_MUTEX_STATIC(mylock, PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP);
 
 /* Cause an abnormal program termination with core-dump */
 void abort(void)
@@ -62,7 +58,7 @@ void abort(void)
 	sigset_t sigs;
 
 	/* Make sure we acquire the lock before proceeding */
-	LOCK;
+	__UCLIBC_MUTEX_LOCK_CANCEL_UNSAFE(mylock);
 
 	/* Unmask SIGABRT to be sure we can get it */
 	if (__sigemptyset(&sigs) == 0 && __sigaddset(&sigs, SIGABRT) == 0) {
@@ -85,9 +81,9 @@ void abort(void)
 #endif
 
 abort_it:
-			UNLOCK;
+			__UCLIBC_MUTEX_UNLOCK_CANCEL_UNSAFE(mylock);
 			raise(SIGABRT);
-			LOCK;
+			__UCLIBC_MUTEX_LOCK_CANCEL_UNSAFE(mylock);
 		}
 
 		/* Still here?  Try to remove any signal handlers */
diff --git a/libc/stdlib/malloc-simple/alloc.c b/libc/stdlib/malloc-simple/alloc.c
index e382cee55..0b842076d 100644
--- a/libc/stdlib/malloc-simple/alloc.c
+++ b/libc/stdlib/malloc-simple/alloc.c
@@ -113,12 +113,11 @@ void free(void *ptr)
 #endif
 
 #ifdef L_memalign
-#ifdef __UCLIBC_HAS_THREADS__
-# include <pthread.h>
-pthread_mutex_t __malloc_lock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
-#endif
-#define LOCK	__pthread_mutex_lock(&__malloc_lock)
-#define UNLOCK	__pthread_mutex_unlock(&__malloc_lock)
+
+#include <bits/uClibc_mutex.h>
+__UCLIBC_MUTEX_STATIC(__malloc_lock, PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP);
+#define __MALLOC_LOCK		__UCLIBC_MUTEX_LOCK(__malloc_lock)
+#define __MALLOC_UNLOCK		__UCLIBC_MUTEX_UNLOCK(__malloc_lock)
 
 /* List of blocks allocated with memalign or valloc */
 struct alignlist
@@ -137,7 +136,7 @@ int __libc_free_aligned(void *ptr)
 	if (ptr == NULL)
 		return 0;
 
-	LOCK;
+	__MALLOC_LOCK;
 	for (l = _aligned_blocks; l != NULL; l = l->next) {
 		if (l->aligned == ptr) {
 			/* Mark the block as free */
@@ -148,7 +147,7 @@ int __libc_free_aligned(void *ptr)
 			return 1;
 		}
 	}
-	UNLOCK;
+	__MALLOC_UNLOCK;
 	return 0;
 }
 void * memalign (size_t alignment, size_t size)
@@ -160,11 +159,10 @@ void * memalign (size_t alignment, size_t size)
 	if (result == NULL)
 		return NULL;
 
-	adj = (unsigned long int) ((unsigned long int) ((char *) result -
-	      (char *) NULL)) % alignment;
+	adj = (unsigned long int) ((unsigned long int) ((char *) result - (char *) NULL)) % alignment;
 	if (adj != 0) {
 		struct alignlist *l;
-		LOCK;
+		__MALLOC_LOCK;
 		for (l = _aligned_blocks; l != NULL; l = l->next)
 			if (l->aligned == NULL)
 				/* This slot is free.  Use it.  */
@@ -173,15 +171,16 @@ void * memalign (size_t alignment, size_t size)
 			l = (struct alignlist *) malloc (sizeof (struct alignlist));
 			if (l == NULL) {
 				free(result);
-				UNLOCK;
-				return NULL;
+				result = NULL;
+				goto DONE;
 			}
 			l->next = _aligned_blocks;
 			_aligned_blocks = l;
 		}
 		l->exact = result;
 		result = l->aligned = (char *) result + alignment - adj;
-		UNLOCK;
+DONE:
+		__MALLOC_UNLOCK;
 	}
 
 	return result;
diff --git a/libc/stdlib/malloc-standard/calloc.c b/libc/stdlib/malloc-standard/calloc.c
index 99e8884ad..b94fb372b 100644
--- a/libc/stdlib/malloc-standard/calloc.c
+++ b/libc/stdlib/malloc-standard/calloc.c
@@ -36,7 +36,7 @@ void* calloc(size_t n_elements, size_t elem_size)
 	return NULL;
     }
 
-    LOCK;
+    __MALLOC_LOCK;
     mem = malloc(size);
     if (mem != 0) {
 	p = mem2chunk(mem);
@@ -88,7 +88,7 @@ void* calloc(size_t n_elements, size_t elem_size)
 	}
 #endif
     }
-    UNLOCK;
+    __MALLOC_UNLOCK;
     return mem;
 }
 
diff --git a/libc/stdlib/malloc-standard/free.c b/libc/stdlib/malloc-standard/free.c
index e0c6ef061..4d24697be 100644
--- a/libc/stdlib/malloc-standard/free.c
+++ b/libc/stdlib/malloc-standard/free.c
@@ -282,7 +282,7 @@ void free(void* mem)
     if (mem == NULL)
 	return;
 
-    LOCK;
+    __MALLOC_LOCK;
     av = get_malloc_state();
     p = mem2chunk(mem);
     size = chunksize(p);
@@ -406,6 +406,6 @@ void free(void* mem)
 	av->mmapped_mem -= (size + offset);
 	munmap((char*)p - offset, size + offset);
     }
-    UNLOCK;
+    __MALLOC_UNLOCK;
 }
 
diff --git a/libc/stdlib/malloc-standard/mallinfo.c b/libc/stdlib/malloc-standard/mallinfo.c
index 4f274ed32..18331010a 100644
--- a/libc/stdlib/malloc-standard/mallinfo.c
+++ b/libc/stdlib/malloc-standard/mallinfo.c
@@ -32,7 +32,7 @@ struct mallinfo mallinfo(void)
     int nblocks;
     int nfastblocks;
 
-    LOCK;
+    __MALLOC_LOCK;
     av = get_malloc_state();
     /* Ensure initialization */
     if (av->top == 0)  {
@@ -77,7 +77,7 @@ struct mallinfo mallinfo(void)
     mi.fsmblks = fastavail;
     mi.keepcost = chunksize(av->top);
     mi.usmblks = av->max_total_mem;
-    UNLOCK;
+    __MALLOC_UNLOCK;
     return mi;
 }
 libc_hidden_def(mallinfo)
@@ -91,19 +91,36 @@ void malloc_stats(FILE *file)
     }
 
     mi = mallinfo();
-    fprintf(file, "total bytes allocated             = %10u\n", (unsigned int)(mi.arena + mi.hblkhd));
-    fprintf(file, "total bytes in use bytes          = %10u\n", (unsigned int)(mi.uordblks + mi.hblkhd));
-    fprintf(file, "total non-mmapped bytes allocated = %10d\n", mi.arena);
-    fprintf(file, "number of mmapped regions         = %10d\n", mi.hblks);
-    fprintf(file, "total allocated mmap space        = %10d\n", mi.hblkhd);
-    fprintf(file, "total allocated sbrk space        = %10d\n", mi.uordblks);
+    fprintf(file,
+	    "total bytes allocated             = %10u\n"
+	    "total bytes in use bytes          = %10u\n"
+	    "total non-mmapped bytes allocated = %10d\n"
+	    "number of mmapped regions         = %10d\n"
+	    "total allocated mmap space        = %10d\n"
+	    "total allocated sbrk space        = %10d\n"
 #if 0
-    fprintf(file, "number of free chunks             = %10d\n", mi.ordblks);
-    fprintf(file, "number of fastbin blocks          = %10d\n", mi.smblks);
-    fprintf(file, "space in freed fastbin blocks     = %10d\n", mi.fsmblks);
+	    "number of free chunks             = %10d\n"
+	    "number of fastbin blocks          = %10d\n"
+	    "space in freed fastbin blocks     = %10d\n"
 #endif
-    fprintf(file, "maximum total allocated space     = %10d\n", mi.usmblks);
-    fprintf(file, "total free space                  = %10d\n", mi.fordblks);
-    fprintf(file, "memory releasable via malloc_trim = %10d\n", mi.keepcost);
+	    "maximum total allocated space     = %10d\n"
+	    "total free space                  = %10d\n"
+	    "memory releasable via malloc_trim = %10d\n",
+
+	    (unsigned int)(mi.arena + mi.hblkhd),
+	    (unsigned int)(mi.uordblks + mi.hblkhd),
+	    mi.arena,
+	    mi.hblks,
+	    mi.hblkhd,
+	    mi.uordblks,
+#if 0
+	    mi.ordblks,
+	    mi.smblks,
+	    mi.fsmblks,
+#endif
+	    mi.usmblks,
+	    mi.fordblks,
+	    mi.keepcost
+	   );
 }
 
diff --git a/libc/stdlib/malloc-standard/malloc.c b/libc/stdlib/malloc-standard/malloc.c
index 7a87ffd36..8c06fdf1f 100644
--- a/libc/stdlib/malloc-standard/malloc.c
+++ b/libc/stdlib/malloc-standard/malloc.c
@@ -17,9 +17,7 @@
 #include "malloc.h"
 
 
-#ifdef __UCLIBC_HAS_THREADS__
-pthread_mutex_t __malloc_lock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
-#endif
+__UCLIBC_MUTEX_INIT(__malloc_lock, PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP);
 
 /*
    There is exactly one instance of this struct in this malloc.
@@ -825,12 +823,13 @@ void* malloc(size_t bytes)
     mchunkptr       fwd;              /* misc temp for linking */
     mchunkptr       bck;              /* misc temp for linking */
     void *          sysmem;
+    void *          retval;
 
 #if !defined(__MALLOC_GLIBC_COMPAT__)
     if (!bytes) return NULL;
 #endif
 
-    LOCK;
+    __MALLOC_LOCK;
     av = get_malloc_state();
     /*
        Convert request size to internal form by adding (sizeof(size_t)) bytes
@@ -861,8 +860,8 @@ void* malloc(size_t bytes)
 	if ( (victim = *fb) != 0) {
 	    *fb = victim->fd;
 	    check_remalloced_chunk(victim, nb);
-	    UNLOCK;
-	    return chunk2mem(victim);
+	    retval = chunk2mem(victim);
+	    goto DONE;
 	}
     }
 
@@ -885,8 +884,8 @@ void* malloc(size_t bytes)
 	    bck->fd = bin;
 
 	    check_malloced_chunk(victim, nb);
-	    UNLOCK;
-	    return chunk2mem(victim);
+	    retval = chunk2mem(victim);
+	    goto DONE;
 	}
     }
 
@@ -902,7 +901,7 @@ void* malloc(size_t bytes)
 
     else {
 	idx = __malloc_largebin_index(nb);
-	if (have_fastchunks(av)) 
+	if (have_fastchunks(av))
 	    __malloc_consolidate(av);
     }
 
@@ -942,8 +941,8 @@ void* malloc(size_t bytes)
 	    set_foot(remainder, remainder_size);
 
 	    check_malloced_chunk(victim, nb);
-	    UNLOCK;
-	    return chunk2mem(victim);
+	    retval = chunk2mem(victim);
+	    goto DONE;
 	}
 
 	/* remove from unsorted list */
@@ -955,8 +954,8 @@ void* malloc(size_t bytes)
 	if (size == nb) {
 	    set_inuse_bit_at_offset(victim, size);
 	    check_malloced_chunk(victim, nb);
-	    UNLOCK;
-	    return chunk2mem(victim);
+	    retval = chunk2mem(victim);
+	    goto DONE;
 	}
 
 	/* place chunk in bin */
@@ -1019,8 +1018,8 @@ void* malloc(size_t bytes)
 		if (remainder_size < MINSIZE)  {
 		    set_inuse_bit_at_offset(victim, size);
 		    check_malloced_chunk(victim, nb);
-		    UNLOCK;
-		    return chunk2mem(victim);
+		    retval = chunk2mem(victim);
+		    goto DONE;
 		}
 		/* Split */
 		else {
@@ -1031,8 +1030,8 @@ void* malloc(size_t bytes)
 		    set_head(remainder, remainder_size | PREV_INUSE);
 		    set_foot(remainder, remainder_size);
 		    check_malloced_chunk(victim, nb);
-		    UNLOCK;
-		    return chunk2mem(victim);
+		    retval = chunk2mem(victim);
+		    goto DONE;
 		}
 	    }
 	}
@@ -1100,8 +1099,8 @@ void* malloc(size_t bytes)
 	    if (remainder_size < MINSIZE) {
 		set_inuse_bit_at_offset(victim, size);
 		check_malloced_chunk(victim, nb);
-		UNLOCK;
-		return chunk2mem(victim);
+		retval = chunk2mem(victim);
+		goto DONE;
 	    }
 
 	    /* Split */
@@ -1118,8 +1117,8 @@ void* malloc(size_t bytes)
 		set_head(remainder, remainder_size | PREV_INUSE);
 		set_foot(remainder, remainder_size);
 		check_malloced_chunk(victim, nb);
-		UNLOCK;
-		return chunk2mem(victim);
+		retval = chunk2mem(victim);
+		goto DONE;
 	    }
 	}
     }
@@ -1151,13 +1150,15 @@ use_top:
 	set_head(remainder, remainder_size | PREV_INUSE);
 
 	check_malloced_chunk(victim, nb);
-	UNLOCK;
-	return chunk2mem(victim);
+	retval = chunk2mem(victim);
+	goto DONE;
     }
 
     /* If no space in top, relay to handle system-dependent cases */
     sysmem = __malloc_alloc(nb, av);
-    UNLOCK;
-    return sysmem;
+    retval = sysmem;
+DONE:
+    __MALLOC_UNLOCK;
+    return retval;
 }
 
diff --git a/libc/stdlib/malloc-standard/malloc.h b/libc/stdlib/malloc-standard/malloc.h
index 556aef4d7..389f1f7d4 100644
--- a/libc/stdlib/malloc-standard/malloc.h
+++ b/libc/stdlib/malloc-standard/malloc.h
@@ -22,18 +22,17 @@
 #include <malloc.h>
 #include <stdlib.h>
 #include <sys/mman.h>
+#include <bits/uClibc_mutex.h>
 
 libc_hidden_proto(mmap)
 libc_hidden_proto(sysconf)
 libc_hidden_proto(sbrk)
 libc_hidden_proto(abort)
 
-#ifdef __UCLIBC_HAS_THREADS__
-# include <pthread.h>
-extern pthread_mutex_t __malloc_lock;
-#endif
-#define LOCK	__pthread_mutex_lock(&__malloc_lock)
-#define UNLOCK	__pthread_mutex_unlock(&__malloc_lock)
+
+__UCLIBC_MUTEX_EXTERN(__malloc_lock);
+#define __MALLOC_LOCK		__UCLIBC_MUTEX_LOCK(__malloc_lock)
+#define __MALLOC_UNLOCK		__UCLIBC_MUTEX_UNLOCK(__malloc_lock)
 
 
 
diff --git a/libc/stdlib/malloc-standard/mallopt.c b/libc/stdlib/malloc-standard/mallopt.c
index e28792099..053242dbe 100644
--- a/libc/stdlib/malloc-standard/mallopt.c
+++ b/libc/stdlib/malloc-standard/mallopt.c
@@ -25,7 +25,7 @@ int mallopt(int param_number, int value)
 
     ret = 0;
 
-    LOCK;
+    __MALLOC_LOCK;
     av = get_malloc_state();
     /* Ensure initialization/consolidation */
     __malloc_consolidate(av);
@@ -58,7 +58,7 @@ int mallopt(int param_number, int value)
 	    ret = 1;
 	    break;
     }
-    UNLOCK;
+    __MALLOC_UNLOCK;
     return ret;
 }
 
diff --git a/libc/stdlib/malloc-standard/memalign.c b/libc/stdlib/malloc-standard/memalign.c
index 27502893d..7e0674be5 100644
--- a/libc/stdlib/malloc-standard/memalign.c
+++ b/libc/stdlib/malloc-standard/memalign.c
@@ -35,6 +35,7 @@ void* memalign(size_t alignment, size_t bytes)
     mchunkptr       remainder;      /* spare room at end to split off */
     unsigned long    remainder_size; /* its size */
     size_t size;
+    void *retval;
 
     /* If need less alignment than we give anyway, just relay to malloc */
 
@@ -51,7 +52,7 @@ void* memalign(size_t alignment, size_t bytes)
 	alignment = a;
     }
 
-    LOCK;
+    __MALLOC_LOCK;
     checked_request2size(bytes, nb);
 
     /* Strategy: find a spot within that chunk that meets the alignment
@@ -63,8 +64,8 @@ void* memalign(size_t alignment, size_t bytes)
     m  = (char*)(malloc(nb + alignment + MINSIZE));
 
     if (m == 0) {
-	UNLOCK;
-	return 0; /* propagate failure */
+	retval = 0; /* propagate failure */
+	goto DONE;
     }
 
     p = mem2chunk(m);
@@ -92,8 +93,8 @@ void* memalign(size_t alignment, size_t bytes)
 	if (chunk_is_mmapped(p)) {
 	    newp->prev_size = p->prev_size + leadsize;
 	    set_head(newp, newsize|IS_MMAPPED);
-	    UNLOCK;
-	    return chunk2mem(newp);
+	    retval = chunk2mem(newp);
+	    goto DONE;
 	}
 
 	/* Otherwise, give back leader, use the rest */
@@ -120,7 +121,10 @@ void* memalign(size_t alignment, size_t bytes)
     }
 
     check_inuse_chunk(p);
-    UNLOCK;
-    return chunk2mem(p);
+    retval = chunk2mem(p);
+
+ DONE:
+    __MALLOC_UNLOCK;
+	return retval;
 }
 
diff --git a/libc/stdlib/malloc-standard/realloc.c b/libc/stdlib/malloc-standard/realloc.c
index f25d6d989..ef436da0d 100644
--- a/libc/stdlib/malloc-standard/realloc.c
+++ b/libc/stdlib/malloc-standard/realloc.c
@@ -46,6 +46,7 @@ void* realloc(void* oldmem, size_t bytes)
     size_t* s;               /* copy source */
     size_t* d;               /* copy destination */
 
+    void *retval;
 
     /* Check for special cases.  */
     if (! oldmem)
@@ -55,7 +56,7 @@ void* realloc(void* oldmem, size_t bytes)
 	return NULL;
     }
 
-    LOCK;
+    __MALLOC_LOCK;
     av = get_malloc_state();
     checked_request2size(bytes, nb);
 
@@ -82,8 +83,8 @@ void* realloc(void* oldmem, size_t bytes)
 		set_head_size(oldp, nb);
 		av->top = chunk_at_offset(oldp, nb);
 		set_head(av->top, (newsize - nb) | PREV_INUSE);
-		UNLOCK;
-		return chunk2mem(oldp);
+		retval = chunk2mem(oldp);
+		goto DONE;
 	    }
 
 	    /* Try to expand forward into next chunk;  split off remainder below */
@@ -99,8 +100,8 @@ void* realloc(void* oldmem, size_t bytes)
 	    else {
 		newmem = malloc(nb - MALLOC_ALIGN_MASK);
 		if (newmem == 0) {
-		    UNLOCK;
-		    return 0; /* propagate failure */
+		    retval = 0; /* propagate failure */
+		    goto DONE;
 		}
 
 		newp = mem2chunk(newmem);
@@ -149,8 +150,8 @@ void* realloc(void* oldmem, size_t bytes)
 
 		    free(oldmem);
 		    check_inuse_chunk(newp);
-		    UNLOCK;
-		    return chunk2mem(newp);
+		    retval = chunk2mem(newp);
+		    goto DONE;
 		}
 	    }
 	}
@@ -175,8 +176,8 @@ void* realloc(void* oldmem, size_t bytes)
 	}
 
 	check_inuse_chunk(newp);
-	UNLOCK;
-	return chunk2mem(newp);
+	retval = chunk2mem(newp);
+	goto DONE;
     }
 
     /*
@@ -194,8 +195,8 @@ void* realloc(void* oldmem, size_t bytes)
 
 	/* don't need to remap if still within same page */
 	if (oldsize == newsize - offset) {
-	    UNLOCK;
-	    return oldmem;
+	    retval = oldmem;
+	    goto DONE;
 	}
 
 	cp = (char*)mremap((char*)oldp - offset, oldsize + offset, newsize, 1);
@@ -216,8 +217,8 @@ void* realloc(void* oldmem, size_t bytes)
 	    if (sum > (unsigned long)(av->max_total_mem))
 		av->max_total_mem = sum;
 
-	    UNLOCK;
-	    return chunk2mem(newp);
+	    retval = chunk2mem(newp);
+	    goto DONE;
 	}
 
 	/* Note the extra (sizeof(size_t)) overhead. */
@@ -231,8 +232,11 @@ void* realloc(void* oldmem, size_t bytes)
 		free(oldmem);
 	    }
 	}
-	UNLOCK;
-	return newmem;
+	retval = newmem;
     }
+
+ DONE:
+    __MALLOC_UNLOCK;
+    return retval;
 }
 
diff --git a/libc/stdlib/random.c b/libc/stdlib/random.c
index 9f1977ee3..3eb8aed8a 100644
--- a/libc/stdlib/random.c
+++ b/libc/stdlib/random.c
@@ -32,13 +32,12 @@ libc_hidden_proto(srandom_r)
 libc_hidden_proto(setstate_r)
 libc_hidden_proto(initstate_r)
 
-#ifdef __UCLIBC_HAS_THREADS__
-# include <pthread.h>
 /* POSIX.1c requires that there is mutual exclusion for the `rand' and
    `srand' functions to prevent concurrent calls from modifying common
    data.  */
-static pthread_mutex_t lock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
-#endif
+#include <bits/uClibc_mutex.h>
+__UCLIBC_MUTEX_STATIC(mylock, PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP);
+
 
 /* An improved random number generation package.  In addition to the standard
    rand()/srand() like interface, this package also has a special state info
@@ -186,9 +185,9 @@ static struct random_data unsafe_state =
    for default usage relies on values produced by this routine.  */
 void srandom (unsigned int x)
 {
-    __pthread_mutex_lock(&lock);
+    __UCLIBC_MUTEX_LOCK(mylock);
     srandom_r (x, &unsafe_state);
-    __pthread_mutex_unlock(&lock);
+    __UCLIBC_MUTEX_UNLOCK(mylock);
 }
 strong_alias(srandom,srand)
 
@@ -207,10 +206,10 @@ char * initstate (unsigned int seed, char *arg_state, size_t n)
 {
     int32_t *ostate;
 
-    __pthread_mutex_lock(&lock);
+    __UCLIBC_MUTEX_LOCK(mylock);
     ostate = &unsafe_state.state[-1];
     initstate_r (seed, arg_state, n, &unsafe_state);
-    __pthread_mutex_unlock(&lock);
+    __UCLIBC_MUTEX_UNLOCK(mylock);
     return (char *) ostate;
 }
 
@@ -226,11 +225,11 @@ char * setstate (char *arg_state)
 {
     int32_t *ostate;
 
-    __pthread_mutex_lock(&lock);
+    __UCLIBC_MUTEX_LOCK(mylock);
     ostate = &unsafe_state.state[-1];
     if (setstate_r (arg_state, &unsafe_state) < 0)
 	ostate = NULL;
-    __pthread_mutex_unlock(&lock);
+    __UCLIBC_MUTEX_UNLOCK(mylock);
     return (char *) ostate;
 }
 
@@ -250,9 +249,9 @@ long int random (void)
 {
   int32_t retval;
 
-  __pthread_mutex_lock(&lock);
+  __UCLIBC_MUTEX_LOCK(mylock);
   random_r (&unsafe_state, &retval);
-  __pthread_mutex_unlock(&lock);
+  __UCLIBC_MUTEX_UNLOCK(mylock);
   return retval;
 }
 libc_hidden_def(random)
diff --git a/libc/stdlib/setenv.c b/libc/stdlib/setenv.c
index fa61675fb..896cf7fc2 100644
--- a/libc/stdlib/setenv.c
+++ b/libc/stdlib/setenv.c
@@ -14,10 +14,10 @@
    You should have received a copy of the GNU Lesser General Public
    License along with the GNU C Library; if not, write to the Free
    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-   02111-1307 USA.  
-   
+   02111-1307 USA.
+
    modified for uClibc by Erik Andersen <andersen@codepoet.org>
-   */
+*/
 
 #include <features.h>
 #include <errno.h>
@@ -32,12 +32,8 @@ libc_hidden_proto(strncmp)
 libc_hidden_proto(strndup)
 libc_hidden_proto(unsetenv)
 
-#ifdef __UCLIBC_HAS_THREADS__
-# include <pthread.h>
-static pthread_mutex_t mylock = PTHREAD_MUTEX_INITIALIZER;
-#endif
-#define LOCK	__pthread_mutex_lock(&mylock)
-#define UNLOCK	__pthread_mutex_unlock(&mylock)
+#include <bits/uClibc_mutex.h>
+__UCLIBC_MUTEX_STATIC(mylock, PTHREAD_MUTEX_INITIALIZER);
 
 
 /* If this variable is not a null pointer we allocated the current
@@ -51,17 +47,18 @@ static char **last_environ;
    must be used directly.  This is all complicated by the fact that we try
    to reuse values once generated for a `setenv' call since we can never
    free the strings.  */
-int __add_to_environ (const char *name, const char *value, 
+int __add_to_environ (const char *name, const char *value,
 	const char *combined, int replace) attribute_hidden;
-int __add_to_environ (const char *name, const char *value, 
-	const char *combined, int replace)
+int __add_to_environ (const char *name, const char *value,
+					  const char *combined, int replace)
 {
     register char **ep;
     register size_t size;
     const size_t namelen = strlen (name);
     const size_t vallen = value != NULL ? strlen (value) + 1 : 0;
+    int rv = -1;
 
-    LOCK;
+    __UCLIBC_MUTEX_LOCK(mylock);
 
     /* We have to get the pointer now that we have the lock and not earlier
        since another thread might have created a new environment.  */
@@ -69,72 +66,71 @@ int __add_to_environ (const char *name, const char *value,
 
     size = 0;
     if (ep != NULL) {
-	for (; *ep != NULL; ++ep) {
-	    if (!strncmp (*ep, name, namelen) && (*ep)[namelen] == '=')
-		break;
-	    else
-		++size;
-	}
+		for (; *ep != NULL; ++ep) {
+			if (!strncmp (*ep, name, namelen) && (*ep)[namelen] == '=')
+				break;
+			else
+				++size;
+		}
     }
 
     if (ep == NULL || *ep == NULL) {
-	char **new_environ;
-
-	/* We allocated this space; we can extend it.  */
-	new_environ = (char **) realloc (last_environ,
-		(size + 2) * sizeof (char *));
-	if (new_environ == NULL) {
-	    UNLOCK;
-	    return -1;
-	}
-
-	/* If the whole entry is given add it.  */
-	if (combined != NULL) {
-	    /* We must not add the string to the search tree since it belongs
-	       to the user.  */
-	    new_environ[size] = (char *) combined;
-	} else {
-	    /* See whether the value is already known.  */
-	    new_environ[size] = (char *) malloc (namelen + 1 + vallen);
-	    if (new_environ[size] == NULL) {
-		__set_errno (ENOMEM);
-		UNLOCK;
-		return -1;
-	    }
-
-	    memcpy (new_environ[size], name, namelen);
-	    new_environ[size][namelen] = '=';
-	    memcpy (&new_environ[size][namelen + 1], value, vallen);
-	}
-
-	if (__environ != last_environ) {
-	    memcpy ((char *) new_environ, (char *) __environ,
-		    size * sizeof (char *));
-	}
-
-	new_environ[size + 1] = NULL;
-	last_environ = __environ = new_environ;
+		char **new_environ;
+
+		/* We allocated this space; we can extend it.  */
+		new_environ = (char **) realloc (last_environ, (size + 2) * sizeof (char *));
+		if (new_environ == NULL) {
+			goto DONE;
+		}
+
+		/* If the whole entry is given add it.  */
+		if (combined != NULL) {
+			/* We must not add the string to the search tree since it belongs
+			   to the user.  */
+			new_environ[size] = (char *) combined;
+		} else {
+			/* See whether the value is already known.  */
+			new_environ[size] = (char *) malloc (namelen + 1 + vallen);
+			if (new_environ[size] == NULL) {
+				__set_errno (ENOMEM);
+				goto DONE;
+			}
+
+			memcpy (new_environ[size], name, namelen);
+			new_environ[size][namelen] = '=';
+			memcpy (&new_environ[size][namelen + 1], value, vallen);
+		}
+
+		if (__environ != last_environ) {
+			memcpy ((char *) new_environ, (char *) __environ,
+					size * sizeof (char *));
+		}
+
+		new_environ[size + 1] = NULL;
+		last_environ = __environ = new_environ;
     } else if (replace) {
-	char *np;
-
-	/* Use the user string if given.  */
-	if (combined != NULL) {
-	    np = (char *) combined;
-	} else {
-	    np = malloc (namelen + 1 + vallen);
-	    if (np == NULL) {
-		UNLOCK;
-		return -1;
-	    }
-	    memcpy (np, name, namelen);
-	    np[namelen] = '=';
-	    memcpy (&np[namelen + 1], value, vallen);
-	}
-	*ep = np;
+		char *np;
+
+		/* Use the user string if given.  */
+		if (combined != NULL) {
+			np = (char *) combined;
+		} else {
+			np = malloc (namelen + 1 + vallen);
+			if (np == NULL) {
+				goto DONE;
+			}
+			memcpy (np, name, namelen);
+			np[namelen] = '=';
+			memcpy (&np[namelen + 1], value, vallen);
+		}
+		*ep = np;
     }
 
-    UNLOCK;
-    return 0;
+    rv = 0;
+
+ DONE:
+    __UCLIBC_MUTEX_UNLOCK(mylock);
+    return rv;
 }
 
 libc_hidden_proto(setenv)
@@ -151,26 +147,26 @@ int unsetenv (const char *name)
     char **ep;
 
     if (name == NULL || *name == '\0' || strchr (name, '=') != NULL) {
-	__set_errno (EINVAL);
-	return -1;
+		__set_errno (EINVAL);
+		return -1;
     }
 
     len = strlen (name);
-    LOCK;
+    __UCLIBC_MUTEX_LOCK(mylock);
     ep = __environ;
     while (*ep != NULL) {
-	if (!strncmp (*ep, name, len) && (*ep)[len] == '=') {
-	    /* Found it.  Remove this pointer by moving later ones back.  */
-	    char **dp = ep;
-	    do {
-		dp[0] = dp[1];
-	    } while (*dp++);
-	    /* Continue the loop in case NAME appears again.  */
-	} else {
-	    ++ep;
-	}
+		if (!strncmp (*ep, name, len) && (*ep)[len] == '=') {
+			/* Found it.  Remove this pointer by moving later ones back.  */
+			char **dp = ep;
+			do {
+				dp[0] = dp[1];
+			} while (*dp++);
+			/* Continue the loop in case NAME appears again.  */
+		} else {
+			++ep;
+		}
     }
-    UNLOCK;
+    __UCLIBC_MUTEX_UNLOCK(mylock);
     return 0;
 }
 libc_hidden_def(unsetenv)
@@ -180,15 +176,15 @@ libc_hidden_def(unsetenv)
    for Fortran 77) requires this function.  */
 int clearenv (void)
 {
-    LOCK;
+    __UCLIBC_MUTEX_LOCK(mylock);
     if (__environ == last_environ && __environ != NULL) {
-	/* We allocated this environment so we can free it.  */
-	free (__environ);
-	last_environ = NULL;
+		/* We allocated this environment so we can free it.  */
+		free (__environ);
+		last_environ = NULL;
     }
     /* Clear the environment pointer removes the whole environment.  */
     __environ = NULL;
-    UNLOCK;
+    __UCLIBC_MUTEX_UNLOCK(mylock);
     return 0;
 }
 
@@ -199,10 +195,10 @@ int putenv (char *string)
     const char *const name_end = strchr (string, '=');
 
     if (name_end != NULL) {
-	char *name = strndup(string, name_end - string);
-	result = __add_to_environ (name, NULL, string, 1);
-	free(name);
-	return(result);
+		char *name = strndup(string, name_end - string);
+		result = __add_to_environ (name, NULL, string, 1);
+		free(name);
+		return(result);
     }
     unsetenv (string);
     return 0;