]>
Commit | Line | Data |
---|---|---|
caab277b | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
cfc5180e MZ |
2 | /* |
3 | * FP/SIMD state saving and restoring macros | |
4 | * | |
5 | * Copyright (C) 2012 ARM Ltd. | |
6 | * Author: Catalin Marinas <catalin.marinas@arm.com> | |
cfc5180e MZ |
7 | */ |
8 | ||
9 | .macro fpsimd_save state, tmpnr | |
10 | stp q0, q1, [\state, #16 * 0] | |
11 | stp q2, q3, [\state, #16 * 2] | |
12 | stp q4, q5, [\state, #16 * 4] | |
13 | stp q6, q7, [\state, #16 * 6] | |
14 | stp q8, q9, [\state, #16 * 8] | |
15 | stp q10, q11, [\state, #16 * 10] | |
16 | stp q12, q13, [\state, #16 * 12] | |
17 | stp q14, q15, [\state, #16 * 14] | |
18 | stp q16, q17, [\state, #16 * 16] | |
19 | stp q18, q19, [\state, #16 * 18] | |
20 | stp q20, q21, [\state, #16 * 20] | |
21 | stp q22, q23, [\state, #16 * 22] | |
22 | stp q24, q25, [\state, #16 * 24] | |
23 | stp q26, q27, [\state, #16 * 26] | |
24 | stp q28, q29, [\state, #16 * 28] | |
25 | stp q30, q31, [\state, #16 * 30]! | |
26 | mrs x\tmpnr, fpsr | |
27 | str w\tmpnr, [\state, #16 * 2] | |
28 | mrs x\tmpnr, fpcr | |
29 | str w\tmpnr, [\state, #16 * 2 + 4] | |
30 | .endm | |
31 | ||
5959e257 WD |
32 | .macro fpsimd_restore_fpcr state, tmp |
33 | /* | |
34 | * Writes to fpcr may be self-synchronising, so avoid restoring | |
35 | * the register if it hasn't changed. | |
36 | */ | |
37 | mrs \tmp, fpcr | |
38 | cmp \tmp, \state | |
39 | b.eq 9999f | |
40 | msr fpcr, \state | |
41 | 9999: | |
42 | .endm | |
43 | ||
44 | /* Clobbers \state */ | |
cfc5180e MZ |
45 | .macro fpsimd_restore state, tmpnr |
46 | ldp q0, q1, [\state, #16 * 0] | |
47 | ldp q2, q3, [\state, #16 * 2] | |
48 | ldp q4, q5, [\state, #16 * 4] | |
49 | ldp q6, q7, [\state, #16 * 6] | |
50 | ldp q8, q9, [\state, #16 * 8] | |
51 | ldp q10, q11, [\state, #16 * 10] | |
52 | ldp q12, q13, [\state, #16 * 12] | |
53 | ldp q14, q15, [\state, #16 * 14] | |
54 | ldp q16, q17, [\state, #16 * 16] | |
55 | ldp q18, q19, [\state, #16 * 18] | |
56 | ldp q20, q21, [\state, #16 * 20] | |
57 | ldp q22, q23, [\state, #16 * 22] | |
58 | ldp q24, q25, [\state, #16 * 24] | |
59 | ldp q26, q27, [\state, #16 * 26] | |
60 | ldp q28, q29, [\state, #16 * 28] | |
61 | ldp q30, q31, [\state, #16 * 30]! | |
62 | ldr w\tmpnr, [\state, #16 * 2] | |
63 | msr fpsr, x\tmpnr | |
64 | ldr w\tmpnr, [\state, #16 * 2 + 4] | |
5959e257 | 65 | fpsimd_restore_fpcr x\tmpnr, \state |
cfc5180e | 66 | .endm |
1fc5dce7 DM |
67 | |
68 | /* Sanity-check macros to help avoid encoding garbage instructions */ | |
69 | ||
70 | .macro _check_general_reg nr | |
71 | .if (\nr) < 0 || (\nr) > 30 | |
72 | .error "Bad register number \nr." | |
73 | .endif | |
74 | .endm | |
75 | ||
76 | .macro _sve_check_zreg znr | |
77 | .if (\znr) < 0 || (\znr) > 31 | |
78 | .error "Bad Scalable Vector Extension vector register number \znr." | |
79 | .endif | |
80 | .endm | |
81 | ||
82 | .macro _sve_check_preg pnr | |
83 | .if (\pnr) < 0 || (\pnr) > 15 | |
84 | .error "Bad Scalable Vector Extension predicate register number \pnr." | |
85 | .endif | |
86 | .endm | |
87 | ||
88 | .macro _check_num n, min, max | |
89 | .if (\n) < (\min) || (\n) > (\max) | |
90 | .error "Number \n out of range [\min,\max]" | |
91 | .endif | |
92 | .endm | |
93 | ||
94 | /* SVE instruction encodings for non-SVE-capable assemblers */ | |
95 | ||
96 | /* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */ | |
97 | .macro _sve_str_v nz, nxbase, offset=0 | |
98 | _sve_check_zreg \nz | |
99 | _check_general_reg \nxbase | |
100 | _check_num (\offset), -0x100, 0xff | |
101 | .inst 0xe5804000 \ | |
102 | | (\nz) \ | |
103 | | ((\nxbase) << 5) \ | |
104 | | (((\offset) & 7) << 10) \ | |
105 | | (((\offset) & 0x1f8) << 13) | |
106 | .endm | |
107 | ||
108 | /* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */ | |
109 | .macro _sve_ldr_v nz, nxbase, offset=0 | |
110 | _sve_check_zreg \nz | |
111 | _check_general_reg \nxbase | |
112 | _check_num (\offset), -0x100, 0xff | |
113 | .inst 0x85804000 \ | |
114 | | (\nz) \ | |
115 | | ((\nxbase) << 5) \ | |
116 | | (((\offset) & 7) << 10) \ | |
117 | | (((\offset) & 0x1f8) << 13) | |
118 | .endm | |
119 | ||
120 | /* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */ | |
121 | .macro _sve_str_p np, nxbase, offset=0 | |
122 | _sve_check_preg \np | |
123 | _check_general_reg \nxbase | |
124 | _check_num (\offset), -0x100, 0xff | |
125 | .inst 0xe5800000 \ | |
126 | | (\np) \ | |
127 | | ((\nxbase) << 5) \ | |
128 | | (((\offset) & 7) << 10) \ | |
129 | | (((\offset) & 0x1f8) << 13) | |
130 | .endm | |
131 | ||
132 | /* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */ | |
133 | .macro _sve_ldr_p np, nxbase, offset=0 | |
134 | _sve_check_preg \np | |
135 | _check_general_reg \nxbase | |
136 | _check_num (\offset), -0x100, 0xff | |
137 | .inst 0x85800000 \ | |
138 | | (\np) \ | |
139 | | ((\nxbase) << 5) \ | |
140 | | (((\offset) & 7) << 10) \ | |
141 | | (((\offset) & 0x1f8) << 13) | |
142 | .endm | |
143 | ||
144 | /* RDVL X\nx, #\imm */ | |
145 | .macro _sve_rdvl nx, imm | |
146 | _check_general_reg \nx | |
147 | _check_num (\imm), -0x20, 0x1f | |
148 | .inst 0x04bf5000 \ | |
149 | | (\nx) \ | |
150 | | (((\imm) & 0x3f) << 5) | |
151 | .endm | |
152 | ||
153 | /* RDFFR (unpredicated): RDFFR P\np.B */ | |
154 | .macro _sve_rdffr np | |
155 | _sve_check_preg \np | |
156 | .inst 0x2519f000 \ | |
157 | | (\np) | |
158 | .endm | |
159 | ||
160 | /* WRFFR P\np.B */ | |
161 | .macro _sve_wrffr np | |
162 | _sve_check_preg \np | |
163 | .inst 0x25289000 \ | |
164 | | ((\np) << 5) | |
165 | .endm | |
166 | ||
167 | .macro __for from:req, to:req | |
168 | .if (\from) == (\to) | |
169 | _for__body \from | |
170 | .else | |
171 | __for \from, (\from) + ((\to) - (\from)) / 2 | |
172 | __for (\from) + ((\to) - (\from)) / 2 + 1, \to | |
173 | .endif | |
174 | .endm | |
175 | ||
176 | .macro _for var:req, from:req, to:req, insn:vararg | |
177 | .macro _for__body \var:req | |
178 | \insn | |
179 | .endm | |
180 | ||
181 | __for \from, \to | |
182 | ||
183 | .purgem _for__body | |
184 | .endm | |
185 | ||
186 | .macro sve_save nxbase, xpfpsr, nxtmp | |
187 | _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34 | |
188 | _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16 | |
189 | _sve_rdffr 0 | |
190 | _sve_str_p 0, \nxbase | |
191 | _sve_ldr_p 0, \nxbase, -16 | |
192 | ||
193 | mrs x\nxtmp, fpsr | |
194 | str w\nxtmp, [\xpfpsr] | |
195 | mrs x\nxtmp, fpcr | |
196 | str w\nxtmp, [\xpfpsr, #4] | |
197 | .endm | |
198 | ||
159fd7b8 | 199 | .macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2 |
1fc5dce7 | 200 | mrs_s x\nxtmp, SYS_ZCR_EL1 |
159fd7b8 DM |
201 | bic \xtmp2, x\nxtmp, ZCR_ELx_LEN_MASK |
202 | orr \xtmp2, \xtmp2, \xvqminus1 | |
203 | cmp \xtmp2, x\nxtmp | |
204 | b.eq 921f | |
205 | msr_s SYS_ZCR_EL1, \xtmp2 // self-synchronising | |
206 | 921: | |
1fc5dce7 DM |
207 | _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34 |
208 | _sve_ldr_p 0, \nxbase | |
209 | _sve_wrffr 0 | |
210 | _for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16 | |
211 | ||
212 | ldr w\nxtmp, [\xpfpsr] | |
213 | msr fpsr, x\nxtmp | |
214 | ldr w\nxtmp, [\xpfpsr, #4] | |
215 | msr fpcr, x\nxtmp | |
216 | .endm |