]>
Commit | Line | Data |
---|---|---|
2011d8fd GKH |
1 | From 59c33fa7791e9948ba467c2b83e307a0d087ab49 Mon Sep 17 00:00:00 2001 |
2 | From: Linus Torvalds <torvalds@linux-foundation.org> | |
3 | Date: Tue, 12 Jan 2010 16:21:09 -0800 | |
4 | Subject: x86-32: clean up rwsem inline asm statements | |
5 | ||
6 | From: Linus Torvalds <torvalds@linux-foundation.org> | |
7 | ||
8 | commit 59c33fa7791e9948ba467c2b83e307a0d087ab49 upstream. | |
9 | ||
10 | This makes gcc use the right register names and instruction operand sizes | |
11 | automatically for the rwsem inline asm statements. | |
12 | ||
13 | So instead of using "(%%eax)" to specify the memory address that is the | |
14 | semaphore, we use "(%1)" or similar. And instead of forcing the operation | |
15 | to always be 32-bit, we use "%z0", taking the size from the actual | |
16 | semaphore data structure itself. | |
17 | ||
18 | This doesn't actually matter on x86-32, but if we want to use the same | |
19 | inline asm for x86-64, we'll need to have the compiler generate the proper | |
20 | 64-bit names for the registers (%rax instead of %eax), and if we want to | |
21 | use a 64-bit counter too (in order to avoid the 15-bit limit on the | |
22 | write counter that limits concurrent users to 32767 threads), we'll need | |
23 | to be able to generate instructions with "q" accesses rather than "l". | |
24 | ||
25 | Since this header currently isn't enabled on x86-64, none of that matters, | |
26 | but we do want to use the xadd version of the semaphores rather than have | |
27 | to take spinlocks to do a rwsem. The mm->mmap_sem can be heavily contended | |
28 | when you have lots of threads all taking page faults, and the fallback | |
29 | rwsem code that uses a spinlock performs abysmally badly in that case. | |
30 | ||
31 | [ hpa: modified the patch to skip size suffixes entirely when they are | |
32 | redundant due to register operands. ] | |
33 | ||
34 | Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> | |
35 | LKML-Reference: <alpine.LFD.2.00.1001121613560.17145@localhost.localdomain> | |
36 | Signed-off-by: H. Peter Anvin <hpa@zytor.com> | |
37 | Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de> | |
38 | ||
39 | --- | |
40 | arch/x86/include/asm/rwsem.h | 30 +++++++++++++++--------------- | |
41 | 1 file changed, 15 insertions(+), 15 deletions(-) | |
42 | ||
43 | --- a/arch/x86/include/asm/rwsem.h | |
44 | +++ b/arch/x86/include/asm/rwsem.h | |
45 | @@ -105,7 +105,7 @@ do { \ | |
46 | static inline void __down_read(struct rw_semaphore *sem) | |
47 | { | |
48 | asm volatile("# beginning down_read\n\t" | |
49 | - LOCK_PREFIX " incl (%%eax)\n\t" | |
50 | + LOCK_PREFIX " inc%z0 (%1)\n\t" | |
51 | /* adds 0x00000001, returns the old value */ | |
52 | " jns 1f\n" | |
53 | " call call_rwsem_down_read_failed\n" | |
54 | @@ -123,12 +123,12 @@ static inline int __down_read_trylock(st | |
55 | { | |
56 | __s32 result, tmp; | |
57 | asm volatile("# beginning __down_read_trylock\n\t" | |
58 | - " movl %0,%1\n\t" | |
59 | + " mov %0,%1\n\t" | |
60 | "1:\n\t" | |
61 | - " movl %1,%2\n\t" | |
62 | - " addl %3,%2\n\t" | |
63 | + " mov %1,%2\n\t" | |
64 | + " add %3,%2\n\t" | |
65 | " jle 2f\n\t" | |
66 | - LOCK_PREFIX " cmpxchgl %2,%0\n\t" | |
67 | + LOCK_PREFIX " cmpxchg %2,%0\n\t" | |
68 | " jnz 1b\n\t" | |
69 | "2:\n\t" | |
70 | "# ending __down_read_trylock\n\t" | |
71 | @@ -147,9 +147,9 @@ static inline void __down_write_nested(s | |
72 | ||
73 | tmp = RWSEM_ACTIVE_WRITE_BIAS; | |
74 | asm volatile("# beginning down_write\n\t" | |
75 | - LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" | |
76 | + LOCK_PREFIX " xadd %1,(%2)\n\t" | |
77 | /* subtract 0x0000ffff, returns the old value */ | |
78 | - " testl %%edx,%%edx\n\t" | |
79 | + " test %1,%1\n\t" | |
80 | /* was the count 0 before? */ | |
81 | " jz 1f\n" | |
82 | " call call_rwsem_down_write_failed\n" | |
83 | @@ -185,7 +185,7 @@ static inline void __up_read(struct rw_s | |
84 | { | |
85 | __s32 tmp = -RWSEM_ACTIVE_READ_BIAS; | |
86 | asm volatile("# beginning __up_read\n\t" | |
87 | - LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" | |
88 | + LOCK_PREFIX " xadd %1,(%2)\n\t" | |
89 | /* subtracts 1, returns the old value */ | |
90 | " jns 1f\n\t" | |
91 | " call call_rwsem_wake\n" | |
92 | @@ -201,18 +201,18 @@ static inline void __up_read(struct rw_s | |
93 | */ | |
94 | static inline void __up_write(struct rw_semaphore *sem) | |
95 | { | |
96 | + unsigned long tmp; | |
97 | asm volatile("# beginning __up_write\n\t" | |
98 | - " movl %2,%%edx\n\t" | |
99 | - LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" | |
100 | + LOCK_PREFIX " xadd %1,(%2)\n\t" | |
101 | /* tries to transition | |
102 | 0xffff0001 -> 0x00000000 */ | |
103 | " jz 1f\n" | |
104 | " call call_rwsem_wake\n" | |
105 | "1:\n\t" | |
106 | "# ending __up_write\n" | |
107 | - : "+m" (sem->count) | |
108 | - : "a" (sem), "i" (-RWSEM_ACTIVE_WRITE_BIAS) | |
109 | - : "memory", "cc", "edx"); | |
110 | + : "+m" (sem->count), "=d" (tmp) | |
111 | + : "a" (sem), "1" (-RWSEM_ACTIVE_WRITE_BIAS) | |
112 | + : "memory", "cc"); | |
113 | } | |
114 | ||
115 | /* | |
116 | @@ -221,7 +221,7 @@ static inline void __up_write(struct rw_ | |
117 | static inline void __downgrade_write(struct rw_semaphore *sem) | |
118 | { | |
119 | asm volatile("# beginning __downgrade_write\n\t" | |
120 | - LOCK_PREFIX " addl %2,(%%eax)\n\t" | |
121 | + LOCK_PREFIX " add%z0 %2,(%1)\n\t" | |
122 | /* transitions 0xZZZZ0001 -> 0xYYYY0001 */ | |
123 | " jns 1f\n\t" | |
124 | " call call_rwsem_downgrade_wake\n" | |
125 | @@ -237,7 +237,7 @@ static inline void __downgrade_write(str | |
126 | */ | |
127 | static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem) | |
128 | { | |
129 | - asm volatile(LOCK_PREFIX "addl %1,%0" | |
130 | + asm volatile(LOCK_PREFIX "add%z0 %1,%0" | |
131 | : "+m" (sem->count) | |
132 | : "ir" (delta)); | |
133 | } |