diff options
Diffstat (limited to 'libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S')
| -rw-r--r-- | libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S | 334 |
1 files changed, 254 insertions, 80 deletions
diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S index 1e461ad41..b0d04c75b 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002-2006, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -19,39 +19,74 @@ #include <sysdep.h> #include <pthread-errnos.h> +#include <bits/kernel-features.h> +#include <lowlevellock.h> .text -#ifndef LOCK -# ifdef UP -# define LOCK +#ifdef __ASSUME_PRIVATE_FUTEX +# define LOAD_PRIVATE_FUTEX_WAIT(reg) \ + movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg +# define LOAD_PRIVATE_FUTEX_WAKE(reg) \ + movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg +# define LOAD_FUTEX_WAIT(reg) \ + xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg +# define LOAD_FUTEX_WAIT_ABS(reg) \ + xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg +# define LOAD_FUTEX_WAKE(reg) \ + xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg +#else +# if FUTEX_WAIT == 0 +# define LOAD_PRIVATE_FUTEX_WAIT(reg) \ + movl %fs:PRIVATE_FUTEX, reg +# else +# define LOAD_PRIVATE_FUTEX_WAIT(reg) \ + movl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAIT, reg +# endif +# define LOAD_PRIVATE_FUTEX_WAKE(reg) \ + movl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAKE, reg +# if FUTEX_WAIT == 0 +# define LOAD_FUTEX_WAIT(reg) \ + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg # else -# define LOCK lock +# define LOAD_FUTEX_WAIT(reg) \ + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAIT, reg # endif +# define LOAD_FUTEX_WAIT_ABS(reg) \ + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg +# define LOAD_FUTEX_WAKE(reg) \ + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAKE, reg #endif -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 /* For the calculation see asm/vsyscall.h. */ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 - .globl __lll_mutex_lock_wait - .type __lll_mutex_lock_wait,@function - .hidden __lll_mutex_lock_wait + .globl __lll_lock_wait_private + .type __lll_lock_wait_private,@function + .hidden __lll_lock_wait_private .align 16 -__lll_mutex_lock_wait: +__lll_lock_wait_private: + cfi_startproc pushq %r10 + cfi_adjust_cfa_offset(8) pushq %rdx - + cfi_adjust_cfa_offset(8) + cfi_offset(%r10, -16) + cfi_offset(%rdx, -24) xorq %r10, %r10 /* No timeout. */ movl $2, %edx -#if FUTEX_WAIT == 0 - xorl %esi, %esi -#else - movl $FUTEX_WAIT, %esi -#endif + LOAD_PRIVATE_FUTEX_WAIT (%esi) cmpl %edx, %eax /* NB: %edx == 2 */ jne 2f @@ -66,33 +101,144 @@ __lll_mutex_lock_wait: jnz 1b popq %rdx + cfi_adjust_cfa_offset(-8) + cfi_restore(%rdx) popq %r10 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r10) retq - .size __lll_mutex_lock_wait,.-__lll_mutex_lock_wait - + cfi_endproc + .size __lll_lock_wait_private,.-__lll_lock_wait_private #ifdef NOT_IN_libc - .globl __lll_mutex_timedlock_wait - .type __lll_mutex_timedlock_wait,@function - .hidden __lll_mutex_timedlock_wait + .globl __lll_lock_wait + .type __lll_lock_wait,@function + .hidden __lll_lock_wait + .align 16 +__lll_lock_wait: + cfi_startproc + pushq %r10 + cfi_adjust_cfa_offset(8) + pushq %rdx + cfi_adjust_cfa_offset(8) + cfi_offset(%r10, -16) + cfi_offset(%rdx, -24) + xorq %r10, %r10 /* No timeout. */ + movl $2, %edx + LOAD_FUTEX_WAIT (%esi) + + cmpl %edx, %eax /* NB: %edx == 2 */ + jne 2f + +1: movl $SYS_futex, %eax + syscall + +2: movl %edx, %eax + xchgl %eax, (%rdi) /* NB: lock is implied */ + + testl %eax, %eax + jnz 1b + + popq %rdx + cfi_adjust_cfa_offset(-8) + cfi_restore(%rdx) + popq %r10 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r10) + retq + cfi_endproc + .size __lll_lock_wait,.-__lll_lock_wait + + /* %rdi: futex + %rsi: flags + %rdx: timeout + %eax: futex value + */ + .globl __lll_timedlock_wait + .type __lll_timedlock_wait,@function + .hidden __lll_timedlock_wait .align 16 -__lll_mutex_timedlock_wait: +__lll_timedlock_wait: + cfi_startproc +# ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +# endif + + pushq %r9 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r9, 0) + movq %rdx, %r10 + movl $0xffffffff, %r9d + LOAD_FUTEX_WAIT_ABS (%esi) + + movl $2, %edx + cmpl %edx, %eax + jne 2f + +1: movl $SYS_futex, %eax + movl $2, %edx + syscall + +2: xchgl %edx, (%rdi) /* NB: lock is implied */ + + testl %edx, %edx + jz 3f + + cmpl $-ETIMEDOUT, %eax + je 4f + cmpl $-EINVAL, %eax + jne 1b +4: movl %eax, %edx + negl %edx + +3: movl %edx, %eax + popq %r9 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r9) + retq + +# ifndef __ASSUME_FUTEX_CLOCK_REALTIME +.Lreltmo: /* Check for a valid timeout value. */ cmpq $1000000000, 8(%rdx) jae 3f pushq %r8 + cfi_adjust_cfa_offset(8) pushq %r9 + cfi_adjust_cfa_offset(8) pushq %r12 + cfi_adjust_cfa_offset(8) pushq %r13 + cfi_adjust_cfa_offset(8) pushq %r14 + cfi_adjust_cfa_offset(8) + cfi_offset(%r8, -16) + cfi_offset(%r9, -24) + cfi_offset(%r12, -32) + cfi_offset(%r13, -40) + cfi_offset(%r14, -48) + pushq %rsi + cfi_adjust_cfa_offset(8) /* Stack frame for the timespec and timeval structs. */ - subq $16, %rsp + subq $24, %rsp + cfi_adjust_cfa_offset(24) movq %rdi, %r12 movq %rdx, %r13 + movl $2, %edx + xchgl %edx, (%r12) + + testl %edx, %edx + je 6f + 1: /* Get current time. */ movq %rsp, %rdi @@ -114,118 +260,137 @@ __lll_mutex_timedlock_wait: addq $1000000000, %rsi decq %rdi 4: testq %rdi, %rdi - js 5f /* Time is already up. */ + js 2f /* Time is already up. */ - /* Futex call. */ - movq %rdi, (%rsp) /* Store relative timeout. */ + /* Store relative timeout. */ + movq %rdi, (%rsp) movq %rsi, 8(%rsp) - movl $1, %eax + /* Futex call. */ movl $2, %edx - LOCK - cmpxchgl %edx, (%r12) - - testl %eax, %eax - je 8f - + movl $1, %eax movq %rsp, %r10 -#if FUTEX_WAIT == 0 - xorl %esi, %esi -#else - movl $FUTEX_WAIT, %esi -#endif + movl 24(%rsp), %esi + LOAD_FUTEX_WAIT (%esi) movq %r12, %rdi movl $SYS_futex, %eax syscall - movq %rax, %rcx -8: /* NB: %edx == 2 */ - xorl %eax, %eax - LOCK - cmpxchgl %edx, (%rdi) - jnz 7f + /* NB: %edx == 2 */ + xchgl %edx, (%r12) + + testl %edx, %edx + je 6f + + cmpl $-ETIMEDOUT, %eax + jne 1b +2: movl $ETIMEDOUT, %edx -6: addq $16, %rsp +6: addq $32, %rsp + cfi_adjust_cfa_offset(-32) popq %r14 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r14) popq %r13 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) popq %r12 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) popq %r9 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r9) popq %r8 - retq - - /* Check whether the time expired. */ -7: cmpq $-ETIMEDOUT, %rcx - je 5f - - /* Make sure the current holder knows we are going to sleep. */ + cfi_adjust_cfa_offset(-8) + cfi_restore(%r8) movl %edx, %eax - xchgl %eax, (%rdi) - testl %eax, %eax - jz 6b - jmp 1b + retq 3: movl $EINVAL, %eax retq - -5: movl $ETIMEDOUT, %eax - jmp 6b - .size __lll_mutex_timedlock_wait,.-__lll_mutex_timedlock_wait +# endif + cfi_endproc + .size __lll_timedlock_wait,.-__lll_timedlock_wait #endif -#ifdef NOT_IN_libc - .globl lll_unlock_wake_cb - .type lll_unlock_wake_cb,@function - .hidden lll_unlock_wake_cb + .globl __lll_unlock_wake_private + .type __lll_unlock_wake_private,@function + .hidden __lll_unlock_wake_private .align 16 -lll_unlock_wake_cb: +__lll_unlock_wake_private: + cfi_startproc pushq %rsi + cfi_adjust_cfa_offset(8) pushq %rdx + cfi_adjust_cfa_offset(8) + cfi_offset(%rsi, -16) + cfi_offset(%rdx, -24) - LOCK - addl $1, (%rdi) - jng 1f + movl $0, (%rdi) + LOAD_PRIVATE_FUTEX_WAKE (%esi) + movl $1, %edx /* Wake one thread. */ + movl $SYS_futex, %eax + syscall popq %rdx + cfi_adjust_cfa_offset(-8) + cfi_restore(%rdx) popq %rsi + cfi_adjust_cfa_offset(-8) + cfi_restore(%rsi) retq - .size lll_unlock_wake_cb,.-lll_unlock_wake_cb -#endif - + cfi_endproc + .size __lll_unlock_wake_private,.-__lll_unlock_wake_private - .globl __lll_mutex_unlock_wake - .type __lll_mutex_unlock_wake,@function - .hidden __lll_mutex_unlock_wake +#ifdef NOT_IN_libc + .globl __lll_unlock_wake + .type __lll_unlock_wake,@function + .hidden __lll_unlock_wake .align 16 -__lll_mutex_unlock_wake: +__lll_unlock_wake: + cfi_startproc pushq %rsi + cfi_adjust_cfa_offset(8) pushq %rdx + cfi_adjust_cfa_offset(8) + cfi_offset(%rsi, -16) + cfi_offset(%rdx, -24) movl $0, (%rdi) - movl $FUTEX_WAKE, %esi + LOAD_FUTEX_WAKE (%esi) movl $1, %edx /* Wake one thread. */ movl $SYS_futex, %eax syscall popq %rdx + cfi_adjust_cfa_offset(-8) + cfi_restore(%rdx) popq %rsi + cfi_adjust_cfa_offset(-8) + cfi_restore(%rsi) retq - .size __lll_mutex_unlock_wake,.-__lll_mutex_unlock_wake + cfi_endproc + .size __lll_unlock_wake,.-__lll_unlock_wake - -#ifdef NOT_IN_libc .globl __lll_timedwait_tid .type __lll_timedwait_tid,@function .hidden __lll_timedwait_tid .align 16 __lll_timedwait_tid: + cfi_startproc pushq %r12 + cfi_adjust_cfa_offset(8) pushq %r13 + cfi_adjust_cfa_offset(8) + cfi_offset(%r12, -16) + cfi_offset(%r13, -24) movq %rdi, %r12 movq %rsi, %r13 subq $16, %rsp + cfi_adjust_cfa_offset(16) /* Get current time. */ 2: movq %rsp, %rdi @@ -255,6 +420,8 @@ __lll_timedwait_tid: jz 4f movq %rsp, %r10 + /* XXX The kernel so far uses global futex for the wakeup at + all times. */ #if FUTEX_WAIT == 0 xorl %esi, %esi #else @@ -269,14 +436,21 @@ __lll_timedwait_tid: 4: xorl %eax, %eax 8: addq $16, %rsp + cfi_adjust_cfa_offset(-16) popq %r13 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) popq %r12 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) retq + cfi_adjust_cfa_offset(32) 1: cmpq $-ETIMEDOUT, %rax jne 2b 6: movl $ETIMEDOUT, %eax jmp 8b + cfi_endproc .size __lll_timedwait_tid,.-__lll_timedwait_tid #endif |