]>
Commit | Line | Data |
---|---|---|
cc90b958 BS |
1 | From: jbeulich@novell.com |
2 | Subject: properly predict phys<->mach translations | |
3 | Patch-mainline: obsolete | |
4 | ||
5 | Index: head-2008-11-04/include/asm-x86/mach-xen/asm/maddr_32.h | |
6 | =================================================================== | |
7 | --- head-2008-11-04.orig/include/asm-x86/mach-xen/asm/maddr_32.h 2008-11-04 11:54:55.000000000 +0100 | |
8 | +++ head-2008-11-04/include/asm-x86/mach-xen/asm/maddr_32.h 2008-10-17 12:45:23.000000000 +0200 | |
9 | @@ -30,17 +30,19 @@ extern unsigned int machine_to_phys_or | |
10 | ||
11 | static inline unsigned long pfn_to_mfn(unsigned long pfn) | |
12 | { | |
13 | - if (xen_feature(XENFEAT_auto_translated_physmap)) | |
14 | + if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) | |
15 | return pfn; | |
16 | - BUG_ON(max_mapnr && pfn >= max_mapnr); | |
17 | + if (likely(max_mapnr)) | |
18 | + BUG_ON(pfn >= max_mapnr); | |
19 | return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT; | |
20 | } | |
21 | ||
22 | static inline int phys_to_machine_mapping_valid(unsigned long pfn) | |
23 | { | |
24 | - if (xen_feature(XENFEAT_auto_translated_physmap)) | |
25 | + if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) | |
26 | return 1; | |
27 | - BUG_ON(max_mapnr && pfn >= max_mapnr); | |
28 | + if (likely(max_mapnr)) | |
29 | + BUG_ON(pfn >= max_mapnr); | |
30 | return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY); | |
31 | } | |
32 | ||
33 | @@ -48,7 +50,7 @@ static inline unsigned long mfn_to_pfn(u | |
34 | { | |
35 | unsigned long pfn; | |
36 | ||
37 | - if (xen_feature(XENFEAT_auto_translated_physmap)) | |
38 | + if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) | |
39 | return mfn; | |
40 | ||
41 | if (unlikely((mfn >> machine_to_phys_order) != 0)) | |
42 | @@ -95,17 +97,18 @@ static inline unsigned long mfn_to_pfn(u | |
43 | static inline unsigned long mfn_to_local_pfn(unsigned long mfn) | |
44 | { | |
45 | unsigned long pfn = mfn_to_pfn(mfn); | |
46 | - if ((pfn < max_mapnr) | |
47 | - && !xen_feature(XENFEAT_auto_translated_physmap) | |
48 | - && (phys_to_machine_mapping[pfn] != mfn)) | |
49 | + if (likely(pfn < max_mapnr) | |
50 | + && likely(!xen_feature(XENFEAT_auto_translated_physmap)) | |
51 | + && unlikely(phys_to_machine_mapping[pfn] != mfn)) | |
52 | return max_mapnr; /* force !pfn_valid() */ | |
53 | return pfn; | |
54 | } | |
55 | ||
56 | static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn) | |
57 | { | |
58 | - BUG_ON(max_mapnr && pfn >= max_mapnr); | |
59 | - if (xen_feature(XENFEAT_auto_translated_physmap)) { | |
60 | + if (likely(max_mapnr)) | |
61 | + BUG_ON(pfn >= max_mapnr); | |
62 | + if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) { | |
63 | BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY); | |
64 | return; | |
65 | } | |
66 | Index: head-2008-11-04/include/asm-x86/mach-xen/asm/maddr_64.h | |
67 | =================================================================== | |
68 | --- head-2008-11-04.orig/include/asm-x86/mach-xen/asm/maddr_64.h 2008-11-04 11:54:55.000000000 +0100 | |
69 | +++ head-2008-11-04/include/asm-x86/mach-xen/asm/maddr_64.h 2008-10-17 12:46:35.000000000 +0200 | |
70 | @@ -25,17 +25,19 @@ extern unsigned int machine_to_phys_or | |
71 | ||
72 | static inline unsigned long pfn_to_mfn(unsigned long pfn) | |
73 | { | |
74 | - if (xen_feature(XENFEAT_auto_translated_physmap)) | |
75 | + if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) | |
76 | return pfn; | |
77 | - BUG_ON(max_mapnr && pfn >= max_mapnr); | |
78 | + if (likely(max_mapnr)) | |
79 | + BUG_ON(pfn >= max_mapnr); | |
80 | return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT; | |
81 | } | |
82 | ||
83 | static inline int phys_to_machine_mapping_valid(unsigned long pfn) | |
84 | { | |
85 | - if (xen_feature(XENFEAT_auto_translated_physmap)) | |
86 | + if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) | |
87 | return 1; | |
88 | - BUG_ON(max_mapnr && pfn >= max_mapnr); | |
89 | + if (likely(max_mapnr)) | |
90 | + BUG_ON(pfn >= max_mapnr); | |
91 | return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY); | |
92 | } | |
93 | ||
94 | @@ -43,7 +45,7 @@ static inline unsigned long mfn_to_pfn(u | |
95 | { | |
96 | unsigned long pfn; | |
97 | ||
98 | - if (xen_feature(XENFEAT_auto_translated_physmap)) | |
99 | + if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) | |
100 | return mfn; | |
101 | ||
102 | if (unlikely((mfn >> machine_to_phys_order) != 0)) | |
103 | @@ -90,17 +92,18 @@ static inline unsigned long mfn_to_pfn(u | |
104 | static inline unsigned long mfn_to_local_pfn(unsigned long mfn) | |
105 | { | |
106 | unsigned long pfn = mfn_to_pfn(mfn); | |
107 | - if ((pfn < max_mapnr) | |
108 | - && !xen_feature(XENFEAT_auto_translated_physmap) | |
109 | - && (phys_to_machine_mapping[pfn] != mfn)) | |
110 | + if (likely(pfn < max_mapnr) | |
111 | + && likely(!xen_feature(XENFEAT_auto_translated_physmap)) | |
112 | + && unlikely(phys_to_machine_mapping[pfn] != mfn)) | |
113 | return max_mapnr; /* force !pfn_valid() */ | |
114 | return pfn; | |
115 | } | |
116 | ||
117 | static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn) | |
118 | { | |
119 | - BUG_ON(max_mapnr && pfn >= max_mapnr); | |
120 | - if (xen_feature(XENFEAT_auto_translated_physmap)) { | |
121 | + if (likely(max_mapnr)) | |
122 | + BUG_ON(pfn >= max_mapnr); | |
123 | + if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) { | |
124 | BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY); | |
125 | return; | |
126 | } | |
127 | Index: head-2008-11-04/include/asm-x86/mach-xen/asm/page.h | |
128 | =================================================================== | |
129 | --- head-2008-11-04.orig/include/asm-x86/mach-xen/asm/page.h 2008-11-04 11:55:22.000000000 +0100 | |
130 | +++ head-2008-11-04/include/asm-x86/mach-xen/asm/page.h 2008-10-17 12:53:40.000000000 +0200 | |
131 | @@ -102,7 +102,7 @@ static inline void copy_user_page(void * | |
132 | #define __pgd_ma(x) ((pgd_t) { (x) } ) | |
133 | static inline pgd_t xen_make_pgd(pgdval_t val) | |
134 | { | |
135 | - if (val & _PAGE_PRESENT) | |
136 | + if (likely(val & _PAGE_PRESENT)) | |
137 | val = pte_phys_to_machine(val); | |
138 | return (pgd_t) { val }; | |
139 | } | |
140 | @@ -112,10 +112,10 @@ static inline pgdval_t xen_pgd_val(pgd_t | |
141 | { | |
142 | pgdval_t ret = __pgd_val(pgd); | |
143 | #if PAGETABLE_LEVELS == 2 && CONFIG_XEN_COMPAT <= 0x030002 | |
144 | - if (ret) | |
145 | + if (likely(ret)) | |
146 | ret = machine_to_phys(ret) | _PAGE_PRESENT; | |
147 | #else | |
148 | - if (ret & _PAGE_PRESENT) | |
149 | + if (likely(ret & _PAGE_PRESENT)) | |
150 | ret = pte_machine_to_phys(ret); | |
151 | #endif | |
152 | return ret; | |
153 | @@ -128,7 +128,7 @@ typedef struct { pudval_t pud; } pud_t; | |
154 | #define __pud_ma(x) ((pud_t) { (x) } ) | |
155 | static inline pud_t xen_make_pud(pudval_t val) | |
156 | { | |
157 | - if (val & _PAGE_PRESENT) | |
158 | + if (likely(val & _PAGE_PRESENT)) | |
159 | val = pte_phys_to_machine(val); | |
160 | return (pud_t) { val }; | |
161 | } | |
162 | @@ -137,7 +137,7 @@ static inline pud_t xen_make_pud(pudval_ | |
163 | static inline pudval_t xen_pud_val(pud_t pud) | |
164 | { | |
165 | pudval_t ret = __pud_val(pud); | |
166 | - if (ret & _PAGE_PRESENT) | |
167 | + if (likely(ret & _PAGE_PRESENT)) | |
168 | ret = pte_machine_to_phys(ret); | |
169 | return ret; | |
170 | } | |
171 | @@ -156,7 +156,7 @@ typedef struct { pmdval_t pmd; } pmd_t; | |
172 | #define __pmd_ma(x) ((pmd_t) { (x) } ) | |
173 | static inline pmd_t xen_make_pmd(pmdval_t val) | |
174 | { | |
175 | - if (val & _PAGE_PRESENT) | |
176 | + if (likely(val & _PAGE_PRESENT)) | |
177 | val = pte_phys_to_machine(val); | |
178 | return (pmd_t) { val }; | |
179 | } | |
180 | @@ -166,10 +166,10 @@ static inline pmdval_t xen_pmd_val(pmd_t | |
181 | { | |
182 | pmdval_t ret = __pmd_val(pmd); | |
183 | #if CONFIG_XEN_COMPAT <= 0x030002 | |
184 | - if (ret) | |
185 | + if (likely(ret)) | |
186 | ret = pte_machine_to_phys(ret) | _PAGE_PRESENT; | |
187 | #else | |
188 | - if (ret & _PAGE_PRESENT) | |
189 | + if (likely(ret & _PAGE_PRESENT)) | |
190 | ret = pte_machine_to_phys(ret); | |
191 | #endif | |
192 | return ret; | |
193 | @@ -188,7 +188,7 @@ static inline pmdval_t xen_pmd_val(pmd_t | |
194 | #define __pte_ma(x) ((pte_t) { .pte = (x) } ) | |
195 | static inline pte_t xen_make_pte(pteval_t val) | |
196 | { | |
197 | - if ((val & (_PAGE_PRESENT|_PAGE_IO)) == _PAGE_PRESENT) | |
198 | + if (likely((val & (_PAGE_PRESENT|_PAGE_IO)) == _PAGE_PRESENT)) | |
199 | val = pte_phys_to_machine(val); | |
200 | return (pte_t) { .pte = val }; | |
201 | } | |
202 | @@ -197,7 +197,7 @@ static inline pte_t xen_make_pte(pteval_ | |
203 | static inline pteval_t xen_pte_val(pte_t pte) | |
204 | { | |
205 | pteval_t ret = __pte_val(pte); | |
206 | - if ((pte.pte_low & (_PAGE_PRESENT|_PAGE_IO)) == _PAGE_PRESENT) | |
207 | + if (likely((pte.pte_low & (_PAGE_PRESENT|_PAGE_IO)) == _PAGE_PRESENT)) | |
208 | ret = pte_machine_to_phys(ret); | |
209 | return ret; | |
210 | } |