]> git.ipfire.org Git - thirdparty/qemu.git/blame - tcg/tcg-op.c
projects / thirdparty / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'remotes/stsquad/tags/pull-travis-speedup-130318-1'...
[thirdparty/qemu.git] / tcg / tcg-op.c
CommitLineData
951c6300
RH		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
757e725b 25#include "qemu/osdep.h"
33c11879
PB		 26#include "qemu-common.h"
27#include "cpu.h"
63c91552 28#include "exec/exec-all.h"
951c6300
RH		 29#include "tcg.h"
30#include "tcg-op.h"
b32dc337 31#include "tcg-mo.h"
dcdaadb6
LV		 32#include "trace-tcg.h"
33#include "trace/mem.h"
951c6300 34
3a13c3f3
RH		 35/* Reduce the number of ifdefs below. This assumes that all uses of
36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
37 the compiler can eliminate. */
38#if TCG_TARGET_REG_BITS == 64
39extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
40extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
41#define TCGV_LOW TCGV_LOW_link_error
42#define TCGV_HIGH TCGV_HIGH_link_error
43#endif
951c6300 44
b7e8b17a 45void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
951c6300 46{
b7e8b17a 47 TCGOp *op = tcg_emit_op(opc);
75e8b9b7 48 op->args[0] = a1;
951c6300
RH		 49}
50
b7e8b17a 51void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
951c6300 52{
b7e8b17a 53 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 54 op->args[0] = a1;
55 op->args[1] = a2;
951c6300
RH		 56}
57
b7e8b17a 58void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
951c6300 59{
b7e8b17a 60 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 61 op->args[0] = a1;
62 op->args[1] = a2;
63 op->args[2] = a3;
951c6300
RH		 64}
65
b7e8b17a 66void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
951c6300 67{
b7e8b17a 68 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 69 op->args[0] = a1;
70 op->args[1] = a2;
71 op->args[2] = a3;
72 op->args[3] = a4;
951c6300
RH		 73}
74
b7e8b17a
RH		 75void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
76 TCGArg a4, TCGArg a5)
951c6300 77{
b7e8b17a 78 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 79 op->args[0] = a1;
80 op->args[1] = a2;
81 op->args[2] = a3;
82 op->args[3] = a4;
83 op->args[4] = a5;
951c6300
RH		 84}
85
b7e8b17a
RH		 86void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
87 TCGArg a4, TCGArg a5, TCGArg a6)
951c6300 88{
b7e8b17a 89 TCGOp *op = tcg_emit_op(opc);
75e8b9b7
RH		 90 op->args[0] = a1;
91 op->args[1] = a2;
92 op->args[2] = a3;
93 op->args[3] = a4;
94 op->args[4] = a5;
95 op->args[5] = a6;
951c6300
RH		 96}
97
f65e19bc
PK		 98void tcg_gen_mb(TCGBar mb_type)
99{
b1311c4a 100 if (tcg_ctx->tb_cflags & CF_PARALLEL) {
b7e8b17a 101 tcg_gen_op1(INDEX_op_mb, mb_type);
f65e19bc
PK		 102 }
103}
104
951c6300
RH		 105/* 32 bit ops */
106
107void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
108{
109 /* some cases can be optimized here */
110 if (arg2 == 0) {
111 tcg_gen_mov_i32(ret, arg1);
112 } else {
113 TCGv_i32 t0 = tcg_const_i32(arg2);
114 tcg_gen_add_i32(ret, arg1, t0);
115 tcg_temp_free_i32(t0);
116 }
117}
118
119void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
120{
121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
122 /* Don't recurse with tcg_gen_neg_i32. */
123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
124 } else {
125 TCGv_i32 t0 = tcg_const_i32(arg1);
126 tcg_gen_sub_i32(ret, t0, arg2);
127 tcg_temp_free_i32(t0);
128 }
129}
130
131void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
132{
133 /* some cases can be optimized here */
134 if (arg2 == 0) {
135 tcg_gen_mov_i32(ret, arg1);
136 } else {
137 TCGv_i32 t0 = tcg_const_i32(arg2);
138 tcg_gen_sub_i32(ret, arg1, t0);
139 tcg_temp_free_i32(t0);
140 }
141}
142
474b2e8f 143void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300
RH		 144{
145 TCGv_i32 t0;
146 /* Some cases can be optimized here. */
147 switch (arg2) {
148 case 0:
149 tcg_gen_movi_i32(ret, 0);
150 return;
474b2e8f 151 case -1:
951c6300
RH		 152 tcg_gen_mov_i32(ret, arg1);
153 return;
474b2e8f 154 case 0xff:
951c6300
RH		 155 /* Don't recurse with tcg_gen_ext8u_i32. */
156 if (TCG_TARGET_HAS_ext8u_i32) {
157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
158 return;
159 }
160 break;
474b2e8f 161 case 0xffff:
951c6300
RH		 162 if (TCG_TARGET_HAS_ext16u_i32) {
163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
164 return;
165 }
166 break;
167 }
168 t0 = tcg_const_i32(arg2);
169 tcg_gen_and_i32(ret, arg1, t0);
170 tcg_temp_free_i32(t0);
171}
172
173void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
174{
175 /* Some cases can be optimized here. */
176 if (arg2 == -1) {
177 tcg_gen_movi_i32(ret, -1);
178 } else if (arg2 == 0) {
179 tcg_gen_mov_i32(ret, arg1);
180 } else {
181 TCGv_i32 t0 = tcg_const_i32(arg2);
182 tcg_gen_or_i32(ret, arg1, t0);
183 tcg_temp_free_i32(t0);
184 }
185}
186
187void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
188{
189 /* Some cases can be optimized here. */
190 if (arg2 == 0) {
191 tcg_gen_mov_i32(ret, arg1);
192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
193 /* Don't recurse with tcg_gen_not_i32. */
194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
195 } else {
196 TCGv_i32 t0 = tcg_const_i32(arg2);
197 tcg_gen_xor_i32(ret, arg1, t0);
198 tcg_temp_free_i32(t0);
199 }
200}
201
474b2e8f 202void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 203{
474b2e8f 204 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 205 if (arg2 == 0) {
206 tcg_gen_mov_i32(ret, arg1);
207 } else {
208 TCGv_i32 t0 = tcg_const_i32(arg2);
209 tcg_gen_shl_i32(ret, arg1, t0);
210 tcg_temp_free_i32(t0);
211 }
212}
213
474b2e8f 214void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 215{
474b2e8f 216 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 217 if (arg2 == 0) {
218 tcg_gen_mov_i32(ret, arg1);
219 } else {
220 TCGv_i32 t0 = tcg_const_i32(arg2);
221 tcg_gen_shr_i32(ret, arg1, t0);
222 tcg_temp_free_i32(t0);
223 }
224}
225
474b2e8f 226void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
951c6300 227{
474b2e8f 228 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
951c6300
RH		 229 if (arg2 == 0) {
230 tcg_gen_mov_i32(ret, arg1);
231 } else {
232 TCGv_i32 t0 = tcg_const_i32(arg2);
233 tcg_gen_sar_i32(ret, arg1, t0);
234 tcg_temp_free_i32(t0);
235 }
236}
237
42a268c2 238void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
951c6300
RH		 239{
240 if (cond == TCG_COND_ALWAYS) {
42a268c2 241 tcg_gen_br(l);
951c6300 242 } else if (cond != TCG_COND_NEVER) {
42a268c2 243 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
951c6300
RH		 244 }
245}
246
42a268c2 247void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
951c6300 248{
37ed3bf1
RH		 249 if (cond == TCG_COND_ALWAYS) {
250 tcg_gen_br(l);
251 } else if (cond != TCG_COND_NEVER) {
252 TCGv_i32 t0 = tcg_const_i32(arg2);
253 tcg_gen_brcond_i32(cond, arg1, t0, l);
254 tcg_temp_free_i32(t0);
255 }
951c6300
RH		 256}
257
258void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
259 TCGv_i32 arg1, TCGv_i32 arg2)
260{
261 if (cond == TCG_COND_ALWAYS) {
262 tcg_gen_movi_i32(ret, 1);
263 } else if (cond == TCG_COND_NEVER) {
264 tcg_gen_movi_i32(ret, 0);
265 } else {
266 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
267 }
268}
269
270void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
271 TCGv_i32 arg1, int32_t arg2)
272{
273 TCGv_i32 t0 = tcg_const_i32(arg2);
274 tcg_gen_setcond_i32(cond, ret, arg1, t0);
275 tcg_temp_free_i32(t0);
276}
277
278void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
279{
280 TCGv_i32 t0 = tcg_const_i32(arg2);
281 tcg_gen_mul_i32(ret, arg1, t0);
282 tcg_temp_free_i32(t0);
283}
284
285void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
286{
287 if (TCG_TARGET_HAS_div_i32) {
288 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
289 } else if (TCG_TARGET_HAS_div2_i32) {
290 TCGv_i32 t0 = tcg_temp_new_i32();
291 tcg_gen_sari_i32(t0, arg1, 31);
292 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
293 tcg_temp_free_i32(t0);
294 } else {
295 gen_helper_div_i32(ret, arg1, arg2);
296 }
297}
298
299void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
300{
301 if (TCG_TARGET_HAS_rem_i32) {
302 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
303 } else if (TCG_TARGET_HAS_div_i32) {
304 TCGv_i32 t0 = tcg_temp_new_i32();
305 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
306 tcg_gen_mul_i32(t0, t0, arg2);
307 tcg_gen_sub_i32(ret, arg1, t0);
308 tcg_temp_free_i32(t0);
309 } else if (TCG_TARGET_HAS_div2_i32) {
310 TCGv_i32 t0 = tcg_temp_new_i32();
311 tcg_gen_sari_i32(t0, arg1, 31);
312 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
313 tcg_temp_free_i32(t0);
314 } else {
315 gen_helper_rem_i32(ret, arg1, arg2);
316 }
317}
318
319void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
320{
321 if (TCG_TARGET_HAS_div_i32) {
322 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
323 } else if (TCG_TARGET_HAS_div2_i32) {
324 TCGv_i32 t0 = tcg_temp_new_i32();
325 tcg_gen_movi_i32(t0, 0);
326 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
327 tcg_temp_free_i32(t0);
328 } else {
329 gen_helper_divu_i32(ret, arg1, arg2);
330 }
331}
332
333void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
334{
335 if (TCG_TARGET_HAS_rem_i32) {
336 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
337 } else if (TCG_TARGET_HAS_div_i32) {
338 TCGv_i32 t0 = tcg_temp_new_i32();
339 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
340 tcg_gen_mul_i32(t0, t0, arg2);
341 tcg_gen_sub_i32(ret, arg1, t0);
342 tcg_temp_free_i32(t0);
343 } else if (TCG_TARGET_HAS_div2_i32) {
344 TCGv_i32 t0 = tcg_temp_new_i32();
345 tcg_gen_movi_i32(t0, 0);
346 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
347 tcg_temp_free_i32(t0);
348 } else {
349 gen_helper_remu_i32(ret, arg1, arg2);
350 }
351}
352
353void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
354{
355 if (TCG_TARGET_HAS_andc_i32) {
356 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
357 } else {
358 TCGv_i32 t0 = tcg_temp_new_i32();
359 tcg_gen_not_i32(t0, arg2);
360 tcg_gen_and_i32(ret, arg1, t0);
361 tcg_temp_free_i32(t0);
362 }
363}
364
365void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
366{
367 if (TCG_TARGET_HAS_eqv_i32) {
368 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
369 } else {
370 tcg_gen_xor_i32(ret, arg1, arg2);
371 tcg_gen_not_i32(ret, ret);
372 }
373}
374
375void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
376{
377 if (TCG_TARGET_HAS_nand_i32) {
378 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
379 } else {
380 tcg_gen_and_i32(ret, arg1, arg2);
381 tcg_gen_not_i32(ret, ret);
382 }
383}
384
385void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
386{
387 if (TCG_TARGET_HAS_nor_i32) {
388 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
389 } else {
390 tcg_gen_or_i32(ret, arg1, arg2);
391 tcg_gen_not_i32(ret, ret);
392 }
393}
394
395void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
396{
397 if (TCG_TARGET_HAS_orc_i32) {
398 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
399 } else {
400 TCGv_i32 t0 = tcg_temp_new_i32();
401 tcg_gen_not_i32(t0, arg2);
402 tcg_gen_or_i32(ret, arg1, t0);
403 tcg_temp_free_i32(t0);
404 }
405}
406
0e28d006
RH		 407void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
408{
409 if (TCG_TARGET_HAS_clz_i32) {
410 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
411 } else if (TCG_TARGET_HAS_clz_i64) {
412 TCGv_i64 t1 = tcg_temp_new_i64();
413 TCGv_i64 t2 = tcg_temp_new_i64();
414 tcg_gen_extu_i32_i64(t1, arg1);
415 tcg_gen_extu_i32_i64(t2, arg2);
416 tcg_gen_addi_i64(t2, t2, 32);
417 tcg_gen_clz_i64(t1, t1, t2);
418 tcg_gen_extrl_i64_i32(ret, t1);
419 tcg_temp_free_i64(t1);
420 tcg_temp_free_i64(t2);
421 tcg_gen_subi_i32(ret, ret, 32);
422 } else {
423 gen_helper_clz_i32(ret, arg1, arg2);
424 }
425}
426
427void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
428{
429 TCGv_i32 t = tcg_const_i32(arg2);
430 tcg_gen_clz_i32(ret, arg1, t);
431 tcg_temp_free_i32(t);
432}
433
434void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
435{
436 if (TCG_TARGET_HAS_ctz_i32) {
437 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
438 } else if (TCG_TARGET_HAS_ctz_i64) {
439 TCGv_i64 t1 = tcg_temp_new_i64();
440 TCGv_i64 t2 = tcg_temp_new_i64();
441 tcg_gen_extu_i32_i64(t1, arg1);
442 tcg_gen_extu_i32_i64(t2, arg2);
443 tcg_gen_ctz_i64(t1, t1, t2);
444 tcg_gen_extrl_i64_i32(ret, t1);
445 tcg_temp_free_i64(t1);
446 tcg_temp_free_i64(t2);
14e99210
RH		 447 } else if (TCG_TARGET_HAS_ctpop_i32
448 || TCG_TARGET_HAS_ctpop_i64
449 || TCG_TARGET_HAS_clz_i32
450 || TCG_TARGET_HAS_clz_i64) {
451 TCGv_i32 z, t = tcg_temp_new_i32();
452
453 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
454 tcg_gen_subi_i32(t, arg1, 1);
455 tcg_gen_andc_i32(t, t, arg1);
456 tcg_gen_ctpop_i32(t, t);
457 } else {
458 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
459 tcg_gen_neg_i32(t, arg1);
460 tcg_gen_and_i32(t, t, arg1);
461 tcg_gen_clzi_i32(t, t, 32);
462 tcg_gen_xori_i32(t, t, 31);
463 }
464 z = tcg_const_i32(0);
465 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
466 tcg_temp_free_i32(t);
467 tcg_temp_free_i32(z);
0e28d006
RH		 468 } else {
469 gen_helper_ctz_i32(ret, arg1, arg2);
470 }
471}
472
473void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
474{
14e99210
RH		 475 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
476 /* This equivalence has the advantage of not requiring a fixup. */
477 TCGv_i32 t = tcg_temp_new_i32();
478 tcg_gen_subi_i32(t, arg1, 1);
479 tcg_gen_andc_i32(t, t, arg1);
480 tcg_gen_ctpop_i32(ret, t);
481 tcg_temp_free_i32(t);
482 } else {
483 TCGv_i32 t = tcg_const_i32(arg2);
484 tcg_gen_ctz_i32(ret, arg1, t);
485 tcg_temp_free_i32(t);
486 }
0e28d006
RH		 487}
488
086920c2
RH		 489void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
490{
491 if (TCG_TARGET_HAS_clz_i32) {
492 TCGv_i32 t = tcg_temp_new_i32();
493 tcg_gen_sari_i32(t, arg, 31);
494 tcg_gen_xor_i32(t, t, arg);
495 tcg_gen_clzi_i32(t, t, 32);
496 tcg_gen_subi_i32(ret, t, 1);
497 tcg_temp_free_i32(t);
498 } else {
499 gen_helper_clrsb_i32(ret, arg);
500 }
501}
502
a768e4e9
RH		 503void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
504{
505 if (TCG_TARGET_HAS_ctpop_i32) {
506 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
507 } else if (TCG_TARGET_HAS_ctpop_i64) {
508 TCGv_i64 t = tcg_temp_new_i64();
509 tcg_gen_extu_i32_i64(t, arg1);
510 tcg_gen_ctpop_i64(t, t);
511 tcg_gen_extrl_i64_i32(ret, t);
512 tcg_temp_free_i64(t);
513 } else {
514 gen_helper_ctpop_i32(ret, arg1);
515 }
516}
517
951c6300
RH		 518void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
519{
520 if (TCG_TARGET_HAS_rot_i32) {
521 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
522 } else {
523 TCGv_i32 t0, t1;
524
525 t0 = tcg_temp_new_i32();
526 t1 = tcg_temp_new_i32();
527 tcg_gen_shl_i32(t0, arg1, arg2);
528 tcg_gen_subfi_i32(t1, 32, arg2);
529 tcg_gen_shr_i32(t1, arg1, t1);
530 tcg_gen_or_i32(ret, t0, t1);
531 tcg_temp_free_i32(t0);
532 tcg_temp_free_i32(t1);
533 }
534}
535
536void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
537{
538 tcg_debug_assert(arg2 < 32);
539 /* some cases can be optimized here */
540 if (arg2 == 0) {
541 tcg_gen_mov_i32(ret, arg1);
542 } else if (TCG_TARGET_HAS_rot_i32) {
543 TCGv_i32 t0 = tcg_const_i32(arg2);
544 tcg_gen_rotl_i32(ret, arg1, t0);
545 tcg_temp_free_i32(t0);
546 } else {
547 TCGv_i32 t0, t1;
548 t0 = tcg_temp_new_i32();
549 t1 = tcg_temp_new_i32();
550 tcg_gen_shli_i32(t0, arg1, arg2);
551 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
552 tcg_gen_or_i32(ret, t0, t1);
553 tcg_temp_free_i32(t0);
554 tcg_temp_free_i32(t1);
555 }
556}
557
558void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
559{
560 if (TCG_TARGET_HAS_rot_i32) {
561 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
562 } else {
563 TCGv_i32 t0, t1;
564
565 t0 = tcg_temp_new_i32();
566 t1 = tcg_temp_new_i32();
567 tcg_gen_shr_i32(t0, arg1, arg2);
568 tcg_gen_subfi_i32(t1, 32, arg2);
569 tcg_gen_shl_i32(t1, arg1, t1);
570 tcg_gen_or_i32(ret, t0, t1);
571 tcg_temp_free_i32(t0);
572 tcg_temp_free_i32(t1);
573 }
574}
575
576void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
577{
578 tcg_debug_assert(arg2 < 32);
579 /* some cases can be optimized here */
580 if (arg2 == 0) {
581 tcg_gen_mov_i32(ret, arg1);
582 } else {
583 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
584 }
585}
586
587void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
588 unsigned int ofs, unsigned int len)
589{
590 uint32_t mask;
591 TCGv_i32 t1;
592
593 tcg_debug_assert(ofs < 32);
0d0d309d 594 tcg_debug_assert(len > 0);
951c6300
RH		 595 tcg_debug_assert(len <= 32);
596 tcg_debug_assert(ofs + len <= 32);
597
0d0d309d 598 if (len == 32) {
951c6300
RH		 599 tcg_gen_mov_i32(ret, arg2);
600 return;
601 }
602 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
603 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
604 return;
605 }
606
607 mask = (1u << len) - 1;
608 t1 = tcg_temp_new_i32();
609
610 if (ofs + len < 32) {
611 tcg_gen_andi_i32(t1, arg2, mask);
612 tcg_gen_shli_i32(t1, t1, ofs);
613 } else {
614 tcg_gen_shli_i32(t1, arg2, ofs);
615 }
616 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
617 tcg_gen_or_i32(ret, ret, t1);
618
619 tcg_temp_free_i32(t1);
620}
621
07cc68d5
RH		 622void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
623 unsigned int ofs, unsigned int len)
624{
625 tcg_debug_assert(ofs < 32);
626 tcg_debug_assert(len > 0);
627 tcg_debug_assert(len <= 32);
628 tcg_debug_assert(ofs + len <= 32);
629
630 if (ofs + len == 32) {
631 tcg_gen_shli_i32(ret, arg, ofs);
632 } else if (ofs == 0) {
633 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
634 } else if (TCG_TARGET_HAS_deposit_i32
635 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
636 TCGv_i32 zero = tcg_const_i32(0);
637 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
638 tcg_temp_free_i32(zero);
639 } else {
640 /* To help two-operand hosts we prefer to zero-extend first,
641 which allows ARG to stay live. */
642 switch (len) {
643 case 16:
644 if (TCG_TARGET_HAS_ext16u_i32) {
645 tcg_gen_ext16u_i32(ret, arg);
646 tcg_gen_shli_i32(ret, ret, ofs);
647 return;
648 }
649 break;
650 case 8:
651 if (TCG_TARGET_HAS_ext8u_i32) {
652 tcg_gen_ext8u_i32(ret, arg);
653 tcg_gen_shli_i32(ret, ret, ofs);
654 return;
655 }
656 break;
657 }
658 /* Otherwise prefer zero-extension over AND for code size. */
659 switch (ofs + len) {
660 case 16:
661 if (TCG_TARGET_HAS_ext16u_i32) {
662 tcg_gen_shli_i32(ret, arg, ofs);
663 tcg_gen_ext16u_i32(ret, ret);
664 return;
665 }
666 break;
667 case 8:
668 if (TCG_TARGET_HAS_ext8u_i32) {
669 tcg_gen_shli_i32(ret, arg, ofs);
670 tcg_gen_ext8u_i32(ret, ret);
671 return;
672 }
673 break;
674 }
675 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
676 tcg_gen_shli_i32(ret, ret, ofs);
677 }
678}
679
7ec8bab3
RH		 680void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
681 unsigned int ofs, unsigned int len)
682{
683 tcg_debug_assert(ofs < 32);
684 tcg_debug_assert(len > 0);
685 tcg_debug_assert(len <= 32);
686 tcg_debug_assert(ofs + len <= 32);
687
688 /* Canonicalize certain special cases, even if extract is supported. */
689 if (ofs + len == 32) {
690 tcg_gen_shri_i32(ret, arg, 32 - len);
691 return;
692 }
693 if (ofs == 0) {
694 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
695 return;
696 }
697
698 if (TCG_TARGET_HAS_extract_i32
699 && TCG_TARGET_extract_i32_valid(ofs, len)) {
700 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
701 return;
702 }
703
704 /* Assume that zero-extension, if available, is cheaper than a shift. */
705 switch (ofs + len) {
706 case 16:
707 if (TCG_TARGET_HAS_ext16u_i32) {
708 tcg_gen_ext16u_i32(ret, arg);
709 tcg_gen_shri_i32(ret, ret, ofs);
710 return;
711 }
712 break;
713 case 8:
714 if (TCG_TARGET_HAS_ext8u_i32) {
715 tcg_gen_ext8u_i32(ret, arg);
716 tcg_gen_shri_i32(ret, ret, ofs);
717 return;
718 }
719 break;
720 }
721
722 /* ??? Ideally we'd know what values are available for immediate AND.
723 Assume that 8 bits are available, plus the special case of 16,
724 so that we get ext8u, ext16u. */
725 switch (len) {
726 case 1 ... 8: case 16:
727 tcg_gen_shri_i32(ret, arg, ofs);
728 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
729 break;
730 default:
731 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
732 tcg_gen_shri_i32(ret, ret, 32 - len);
733 break;
734 }
735}
736
737void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
738 unsigned int ofs, unsigned int len)
739{
740 tcg_debug_assert(ofs < 32);
741 tcg_debug_assert(len > 0);
742 tcg_debug_assert(len <= 32);
743 tcg_debug_assert(ofs + len <= 32);
744
745 /* Canonicalize certain special cases, even if extract is supported. */
746 if (ofs + len == 32) {
747 tcg_gen_sari_i32(ret, arg, 32 - len);
748 return;
749 }
750 if (ofs == 0) {
751 switch (len) {
752 case 16:
753 tcg_gen_ext16s_i32(ret, arg);
754 return;
755 case 8:
756 tcg_gen_ext8s_i32(ret, arg);
757 return;
758 }
759 }
760
761 if (TCG_TARGET_HAS_sextract_i32
762 && TCG_TARGET_extract_i32_valid(ofs, len)) {
763 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
764 return;
765 }
766
767 /* Assume that sign-extension, if available, is cheaper than a shift. */
768 switch (ofs + len) {
769 case 16:
770 if (TCG_TARGET_HAS_ext16s_i32) {
771 tcg_gen_ext16s_i32(ret, arg);
772 tcg_gen_sari_i32(ret, ret, ofs);
773 return;
774 }
775 break;
776 case 8:
777 if (TCG_TARGET_HAS_ext8s_i32) {
778 tcg_gen_ext8s_i32(ret, arg);
779 tcg_gen_sari_i32(ret, ret, ofs);
780 return;
781 }
782 break;
783 }
784 switch (len) {
785 case 16:
786 if (TCG_TARGET_HAS_ext16s_i32) {
787 tcg_gen_shri_i32(ret, arg, ofs);
788 tcg_gen_ext16s_i32(ret, ret);
789 return;
790 }
791 break;
792 case 8:
793 if (TCG_TARGET_HAS_ext8s_i32) {
794 tcg_gen_shri_i32(ret, arg, ofs);
795 tcg_gen_ext8s_i32(ret, ret);
796 return;
797 }
798 break;
799 }
800
801 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
802 tcg_gen_sari_i32(ret, ret, 32 - len);
803}
804
951c6300
RH		 805void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
806 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
807{
37ed3bf1
RH		 808 if (cond == TCG_COND_ALWAYS) {
809 tcg_gen_mov_i32(ret, v1);
810 } else if (cond == TCG_COND_NEVER) {
811 tcg_gen_mov_i32(ret, v2);
812 } else if (TCG_TARGET_HAS_movcond_i32) {
951c6300
RH		 813 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
814 } else {
815 TCGv_i32 t0 = tcg_temp_new_i32();
816 TCGv_i32 t1 = tcg_temp_new_i32();
817 tcg_gen_setcond_i32(cond, t0, c1, c2);
818 tcg_gen_neg_i32(t0, t0);
819 tcg_gen_and_i32(t1, v1, t0);
820 tcg_gen_andc_i32(ret, v2, t0);
821 tcg_gen_or_i32(ret, ret, t1);
822 tcg_temp_free_i32(t0);
823 tcg_temp_free_i32(t1);
824 }
825}
826
827void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
828 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
829{
830 if (TCG_TARGET_HAS_add2_i32) {
831 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
951c6300
RH		 832 } else {
833 TCGv_i64 t0 = tcg_temp_new_i64();
834 TCGv_i64 t1 = tcg_temp_new_i64();
835 tcg_gen_concat_i32_i64(t0, al, ah);
836 tcg_gen_concat_i32_i64(t1, bl, bh);
837 tcg_gen_add_i64(t0, t0, t1);
838 tcg_gen_extr_i64_i32(rl, rh, t0);
839 tcg_temp_free_i64(t0);
840 tcg_temp_free_i64(t1);
841 }
842}
843
844void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
845 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
846{
847 if (TCG_TARGET_HAS_sub2_i32) {
848 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
951c6300
RH		 849 } else {
850 TCGv_i64 t0 = tcg_temp_new_i64();
851 TCGv_i64 t1 = tcg_temp_new_i64();
852 tcg_gen_concat_i32_i64(t0, al, ah);
853 tcg_gen_concat_i32_i64(t1, bl, bh);
854 tcg_gen_sub_i64(t0, t0, t1);
855 tcg_gen_extr_i64_i32(rl, rh, t0);
856 tcg_temp_free_i64(t0);
857 tcg_temp_free_i64(t1);
858 }
859}
860
861void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
862{
863 if (TCG_TARGET_HAS_mulu2_i32) {
864 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
951c6300
RH		 865 } else if (TCG_TARGET_HAS_muluh_i32) {
866 TCGv_i32 t = tcg_temp_new_i32();
867 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
868 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
869 tcg_gen_mov_i32(rl, t);
870 tcg_temp_free_i32(t);
871 } else {
872 TCGv_i64 t0 = tcg_temp_new_i64();
873 TCGv_i64 t1 = tcg_temp_new_i64();
874 tcg_gen_extu_i32_i64(t0, arg1);
875 tcg_gen_extu_i32_i64(t1, arg2);
876 tcg_gen_mul_i64(t0, t0, t1);
877 tcg_gen_extr_i64_i32(rl, rh, t0);
878 tcg_temp_free_i64(t0);
879 tcg_temp_free_i64(t1);
880 }
881}
882
883void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
884{
885 if (TCG_TARGET_HAS_muls2_i32) {
886 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
951c6300
RH		 887 } else if (TCG_TARGET_HAS_mulsh_i32) {
888 TCGv_i32 t = tcg_temp_new_i32();
889 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
890 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
891 tcg_gen_mov_i32(rl, t);
892 tcg_temp_free_i32(t);
893 } else if (TCG_TARGET_REG_BITS == 32) {
894 TCGv_i32 t0 = tcg_temp_new_i32();
895 TCGv_i32 t1 = tcg_temp_new_i32();
896 TCGv_i32 t2 = tcg_temp_new_i32();
897 TCGv_i32 t3 = tcg_temp_new_i32();
898 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
899 /* Adjust for negative inputs. */
900 tcg_gen_sari_i32(t2, arg1, 31);
901 tcg_gen_sari_i32(t3, arg2, 31);
902 tcg_gen_and_i32(t2, t2, arg2);
903 tcg_gen_and_i32(t3, t3, arg1);
904 tcg_gen_sub_i32(rh, t1, t2);
905 tcg_gen_sub_i32(rh, rh, t3);
906 tcg_gen_mov_i32(rl, t0);
907 tcg_temp_free_i32(t0);
908 tcg_temp_free_i32(t1);
909 tcg_temp_free_i32(t2);
910 tcg_temp_free_i32(t3);
911 } else {
912 TCGv_i64 t0 = tcg_temp_new_i64();
913 TCGv_i64 t1 = tcg_temp_new_i64();
914 tcg_gen_ext_i32_i64(t0, arg1);
915 tcg_gen_ext_i32_i64(t1, arg2);
916 tcg_gen_mul_i64(t0, t0, t1);
917 tcg_gen_extr_i64_i32(rl, rh, t0);
918 tcg_temp_free_i64(t0);
919 tcg_temp_free_i64(t1);
920 }
921}
922
5087abfb
RH		 923void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
924{
925 if (TCG_TARGET_REG_BITS == 32) {
926 TCGv_i32 t0 = tcg_temp_new_i32();
927 TCGv_i32 t1 = tcg_temp_new_i32();
928 TCGv_i32 t2 = tcg_temp_new_i32();
929 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
930 /* Adjust for negative input for the signed arg1. */
931 tcg_gen_sari_i32(t2, arg1, 31);
932 tcg_gen_and_i32(t2, t2, arg2);
933 tcg_gen_sub_i32(rh, t1, t2);
934 tcg_gen_mov_i32(rl, t0);
935 tcg_temp_free_i32(t0);
936 tcg_temp_free_i32(t1);
937 tcg_temp_free_i32(t2);
938 } else {
939 TCGv_i64 t0 = tcg_temp_new_i64();
940 TCGv_i64 t1 = tcg_temp_new_i64();
941 tcg_gen_ext_i32_i64(t0, arg1);
942 tcg_gen_extu_i32_i64(t1, arg2);
943 tcg_gen_mul_i64(t0, t0, t1);
944 tcg_gen_extr_i64_i32(rl, rh, t0);
945 tcg_temp_free_i64(t0);
946 tcg_temp_free_i64(t1);
947 }
948}
949
951c6300
RH		 950void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
951{
952 if (TCG_TARGET_HAS_ext8s_i32) {
953 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
954 } else {
955 tcg_gen_shli_i32(ret, arg, 24);
956 tcg_gen_sari_i32(ret, ret, 24);
957 }
958}
959
960void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
961{
962 if (TCG_TARGET_HAS_ext16s_i32) {
963 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
964 } else {
965 tcg_gen_shli_i32(ret, arg, 16);
966 tcg_gen_sari_i32(ret, ret, 16);
967 }
968}
969
970void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
971{
972 if (TCG_TARGET_HAS_ext8u_i32) {
973 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
974 } else {
975 tcg_gen_andi_i32(ret, arg, 0xffu);
976 }
977}
978
979void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
980{
981 if (TCG_TARGET_HAS_ext16u_i32) {
982 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
983 } else {
984 tcg_gen_andi_i32(ret, arg, 0xffffu);
985 }
986}
987
988/* Note: we assume the two high bytes are set to zero */
989void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
990{
991 if (TCG_TARGET_HAS_bswap16_i32) {
992 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
993 } else {
994 TCGv_i32 t0 = tcg_temp_new_i32();
995
996 tcg_gen_ext8u_i32(t0, arg);
997 tcg_gen_shli_i32(t0, t0, 8);
998 tcg_gen_shri_i32(ret, arg, 8);
999 tcg_gen_or_i32(ret, ret, t0);
1000 tcg_temp_free_i32(t0);
1001 }
1002}
1003
1004void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1005{
1006 if (TCG_TARGET_HAS_bswap32_i32) {
1007 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1008 } else {
1009 TCGv_i32 t0, t1;
1010 t0 = tcg_temp_new_i32();
1011 t1 = tcg_temp_new_i32();
1012
1013 tcg_gen_shli_i32(t0, arg, 24);
1014
1015 tcg_gen_andi_i32(t1, arg, 0x0000ff00);
1016 tcg_gen_shli_i32(t1, t1, 8);
1017 tcg_gen_or_i32(t0, t0, t1);
1018
1019 tcg_gen_shri_i32(t1, arg, 8);
1020 tcg_gen_andi_i32(t1, t1, 0x0000ff00);
1021 tcg_gen_or_i32(t0, t0, t1);
1022
1023 tcg_gen_shri_i32(t1, arg, 24);
1024 tcg_gen_or_i32(ret, t0, t1);
1025 tcg_temp_free_i32(t0);
1026 tcg_temp_free_i32(t1);
1027 }
1028}
1029
1030/* 64-bit ops */
1031
1032#if TCG_TARGET_REG_BITS == 32
1033/* These are all inline for TCG_TARGET_REG_BITS == 64. */
1034
1035void tcg_gen_discard_i64(TCGv_i64 arg)
1036{
1037 tcg_gen_discard_i32(TCGV_LOW(arg));
1038 tcg_gen_discard_i32(TCGV_HIGH(arg));
1039}
1040
1041void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1042{
1043 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1044 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1045}
1046
1047void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1048{
1049 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1050 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1051}
1052
1053void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1054{
1055 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1056 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1057}
1058
1059void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1060{
1061 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
3ff91d7e 1062 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
951c6300
RH		 1063}
1064
1065void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1066{
1067 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1068 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1069}
1070
1071void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1072{
1073 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1074 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1075}
1076
1077void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1078{
1079 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1080 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1081}
1082
1083void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1084{
1085 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1086 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1087}
1088
1089void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1090{
1091 /* Since arg2 and ret have different types,
1092 they cannot be the same temporary */
cf811fff 1093#ifdef HOST_WORDS_BIGENDIAN
951c6300
RH		 1094 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1095 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1096#else
1097 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1098 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1099#endif
1100}
1101
1102void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1103{
cf811fff 1104#ifdef HOST_WORDS_BIGENDIAN
951c6300
RH		 1105 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1106 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1107#else
1108 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1109 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1110#endif
1111}
1112
1113void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1114{
1115 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1116 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1117}
1118
1119void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1120{
1121 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1122 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1123}
1124
1125void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1126{
1127 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1128 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1129}
1130
1131void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1132{
1133 gen_helper_shl_i64(ret, arg1, arg2);
1134}
1135
1136void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1137{
1138 gen_helper_shr_i64(ret, arg1, arg2);
1139}
1140
1141void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1142{
1143 gen_helper_sar_i64(ret, arg1, arg2);
1144}
1145
1146void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1147{
1148 TCGv_i64 t0;
1149 TCGv_i32 t1;
1150
1151 t0 = tcg_temp_new_i64();
1152 t1 = tcg_temp_new_i32();
1153
1154 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1155 TCGV_LOW(arg1), TCGV_LOW(arg2));
1156
1157 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1158 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1159 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1160 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1161
1162 tcg_gen_mov_i64(ret, t0);
1163 tcg_temp_free_i64(t0);
1164 tcg_temp_free_i32(t1);
1165}
1166#endif /* TCG_TARGET_REG_SIZE == 32 */
1167
1168void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1169{
1170 /* some cases can be optimized here */
1171 if (arg2 == 0) {
1172 tcg_gen_mov_i64(ret, arg1);
1173 } else {
1174 TCGv_i64 t0 = tcg_const_i64(arg2);
1175 tcg_gen_add_i64(ret, arg1, t0);
1176 tcg_temp_free_i64(t0);
1177 }
1178}
1179
1180void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1181{
1182 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1183 /* Don't recurse with tcg_gen_neg_i64. */
1184 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1185 } else {
1186 TCGv_i64 t0 = tcg_const_i64(arg1);
1187 tcg_gen_sub_i64(ret, t0, arg2);
1188 tcg_temp_free_i64(t0);
1189 }
1190}
1191
1192void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1193{
1194 /* some cases can be optimized here */
1195 if (arg2 == 0) {
1196 tcg_gen_mov_i64(ret, arg1);
1197 } else {
1198 TCGv_i64 t0 = tcg_const_i64(arg2);
1199 tcg_gen_sub_i64(ret, arg1, t0);
1200 tcg_temp_free_i64(t0);
1201 }
1202}
1203
474b2e8f 1204void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1205{
951c6300 1206 TCGv_i64 t0;
3a13c3f3
RH		 1207
1208 if (TCG_TARGET_REG_BITS == 32) {
1209 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1210 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1211 return;
1212 }
1213
951c6300
RH		 1214 /* Some cases can be optimized here. */
1215 switch (arg2) {
1216 case 0:
1217 tcg_gen_movi_i64(ret, 0);
1218 return;
474b2e8f 1219 case -1:
951c6300
RH		 1220 tcg_gen_mov_i64(ret, arg1);
1221 return;
474b2e8f 1222 case 0xff:
951c6300
RH		 1223 /* Don't recurse with tcg_gen_ext8u_i64. */
1224 if (TCG_TARGET_HAS_ext8u_i64) {
1225 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1226 return;
1227 }
1228 break;
474b2e8f 1229 case 0xffff:
951c6300
RH		 1230 if (TCG_TARGET_HAS_ext16u_i64) {
1231 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1232 return;
1233 }
1234 break;
474b2e8f 1235 case 0xffffffffu:
951c6300
RH		 1236 if (TCG_TARGET_HAS_ext32u_i64) {
1237 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1238 return;
1239 }
1240 break;
1241 }
1242 t0 = tcg_const_i64(arg2);
1243 tcg_gen_and_i64(ret, arg1, t0);
1244 tcg_temp_free_i64(t0);
951c6300
RH		 1245}
1246
1247void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1248{
3a13c3f3
RH		 1249 if (TCG_TARGET_REG_BITS == 32) {
1250 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1251 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1252 return;
1253 }
951c6300
RH		 1254 /* Some cases can be optimized here. */
1255 if (arg2 == -1) {
1256 tcg_gen_movi_i64(ret, -1);
1257 } else if (arg2 == 0) {
1258 tcg_gen_mov_i64(ret, arg1);
1259 } else {
1260 TCGv_i64 t0 = tcg_const_i64(arg2);
1261 tcg_gen_or_i64(ret, arg1, t0);
1262 tcg_temp_free_i64(t0);
1263 }
951c6300
RH		 1264}
1265
1266void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1267{
3a13c3f3
RH		 1268 if (TCG_TARGET_REG_BITS == 32) {
1269 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1270 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1271 return;
1272 }
951c6300
RH		 1273 /* Some cases can be optimized here. */
1274 if (arg2 == 0) {
1275 tcg_gen_mov_i64(ret, arg1);
1276 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1277 /* Don't recurse with tcg_gen_not_i64. */
1278 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1279 } else {
1280 TCGv_i64 t0 = tcg_const_i64(arg2);
1281 tcg_gen_xor_i64(ret, arg1, t0);
1282 tcg_temp_free_i64(t0);
1283 }
951c6300
RH		 1284}
1285
951c6300
RH		 1286static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1287 unsigned c, bool right, bool arith)
1288{
1289 tcg_debug_assert(c < 64);
1290 if (c == 0) {
1291 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1292 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1293 } else if (c >= 32) {
1294 c -= 32;
1295 if (right) {
1296 if (arith) {
1297 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1298 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1299 } else {
1300 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1301 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1302 }
1303 } else {
1304 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1305 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1306 }
1307 } else {
1308 TCGv_i32 t0, t1;
1309
1310 t0 = tcg_temp_new_i32();
1311 t1 = tcg_temp_new_i32();
1312 if (right) {
1313 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
1314 if (arith) {
1315 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
1316 } else {
1317 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
1318 }
1319 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1320 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
1321 tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
1322 } else {
1323 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1324 /* Note: ret can be the same as arg1, so we use t1 */
1325 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
1326 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1327 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
1328 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1329 }
1330 tcg_temp_free_i32(t0);
1331 tcg_temp_free_i32(t1);
1332 }
1333}
1334
474b2e8f 1335void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1336{
474b2e8f 1337 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1338 if (TCG_TARGET_REG_BITS == 32) {
1339 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1340 } else if (arg2 == 0) {
951c6300
RH		 1341 tcg_gen_mov_i64(ret, arg1);
1342 } else {
1343 TCGv_i64 t0 = tcg_const_i64(arg2);
1344 tcg_gen_shl_i64(ret, arg1, t0);
1345 tcg_temp_free_i64(t0);
1346 }
1347}
1348
474b2e8f 1349void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1350{
474b2e8f 1351 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1352 if (TCG_TARGET_REG_BITS == 32) {
1353 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1354 } else if (arg2 == 0) {
951c6300
RH		 1355 tcg_gen_mov_i64(ret, arg1);
1356 } else {
1357 TCGv_i64 t0 = tcg_const_i64(arg2);
1358 tcg_gen_shr_i64(ret, arg1, t0);
1359 tcg_temp_free_i64(t0);
1360 }
1361}
1362
474b2e8f 1363void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
951c6300 1364{
474b2e8f 1365 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
3a13c3f3
RH		 1366 if (TCG_TARGET_REG_BITS == 32) {
1367 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1368 } else if (arg2 == 0) {
951c6300
RH		 1369 tcg_gen_mov_i64(ret, arg1);
1370 } else {
1371 TCGv_i64 t0 = tcg_const_i64(arg2);
1372 tcg_gen_sar_i64(ret, arg1, t0);
1373 tcg_temp_free_i64(t0);
1374 }
1375}
951c6300 1376
42a268c2 1377void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
951c6300
RH		 1378{
1379 if (cond == TCG_COND_ALWAYS) {
42a268c2 1380 tcg_gen_br(l);
951c6300 1381 } else if (cond != TCG_COND_NEVER) {
3a13c3f3
RH		 1382 if (TCG_TARGET_REG_BITS == 32) {
1383 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1384 TCGV_HIGH(arg1), TCGV_LOW(arg2),
42a268c2 1385 TCGV_HIGH(arg2), cond, label_arg(l));
3a13c3f3 1386 } else {
42a268c2
RH		 1387 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1388 label_arg(l));
3a13c3f3 1389 }
951c6300
RH		 1390 }
1391}
1392
42a268c2 1393void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
951c6300
RH		 1394{
1395 if (cond == TCG_COND_ALWAYS) {
42a268c2 1396 tcg_gen_br(l);
951c6300
RH		 1397 } else if (cond != TCG_COND_NEVER) {
1398 TCGv_i64 t0 = tcg_const_i64(arg2);
42a268c2 1399 tcg_gen_brcond_i64(cond, arg1, t0, l);
951c6300
RH		 1400 tcg_temp_free_i64(t0);
1401 }
1402}
1403
1404void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1405 TCGv_i64 arg1, TCGv_i64 arg2)
1406{
1407 if (cond == TCG_COND_ALWAYS) {
1408 tcg_gen_movi_i64(ret, 1);
1409 } else if (cond == TCG_COND_NEVER) {
1410 tcg_gen_movi_i64(ret, 0);
1411 } else {
3a13c3f3
RH		 1412 if (TCG_TARGET_REG_BITS == 32) {
1413 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1414 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1415 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1416 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1417 } else {
1418 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1419 }
951c6300
RH		 1420 }
1421}
1422
1423void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1424 TCGv_i64 arg1, int64_t arg2)
1425{
1426 TCGv_i64 t0 = tcg_const_i64(arg2);
1427 tcg_gen_setcond_i64(cond, ret, arg1, t0);
1428 tcg_temp_free_i64(t0);
1429}
1430
1431void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1432{
1433 TCGv_i64 t0 = tcg_const_i64(arg2);
1434 tcg_gen_mul_i64(ret, arg1, t0);
1435 tcg_temp_free_i64(t0);
1436}
1437
1438void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1439{
1440 if (TCG_TARGET_HAS_div_i64) {
1441 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1442 } else if (TCG_TARGET_HAS_div2_i64) {
1443 TCGv_i64 t0 = tcg_temp_new_i64();
1444 tcg_gen_sari_i64(t0, arg1, 63);
1445 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1446 tcg_temp_free_i64(t0);
1447 } else {
1448 gen_helper_div_i64(ret, arg1, arg2);
1449 }
1450}
1451
1452void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1453{
1454 if (TCG_TARGET_HAS_rem_i64) {
1455 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1456 } else if (TCG_TARGET_HAS_div_i64) {
1457 TCGv_i64 t0 = tcg_temp_new_i64();
1458 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1459 tcg_gen_mul_i64(t0, t0, arg2);
1460 tcg_gen_sub_i64(ret, arg1, t0);
1461 tcg_temp_free_i64(t0);
1462 } else if (TCG_TARGET_HAS_div2_i64) {
1463 TCGv_i64 t0 = tcg_temp_new_i64();
1464 tcg_gen_sari_i64(t0, arg1, 63);
1465 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1466 tcg_temp_free_i64(t0);
1467 } else {
1468 gen_helper_rem_i64(ret, arg1, arg2);
1469 }
1470}
1471
1472void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1473{
1474 if (TCG_TARGET_HAS_div_i64) {
1475 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1476 } else if (TCG_TARGET_HAS_div2_i64) {
1477 TCGv_i64 t0 = tcg_temp_new_i64();
1478 tcg_gen_movi_i64(t0, 0);
1479 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1480 tcg_temp_free_i64(t0);
1481 } else {
1482 gen_helper_divu_i64(ret, arg1, arg2);
1483 }
1484}
1485
1486void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1487{
1488 if (TCG_TARGET_HAS_rem_i64) {
1489 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1490 } else if (TCG_TARGET_HAS_div_i64) {
1491 TCGv_i64 t0 = tcg_temp_new_i64();
1492 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1493 tcg_gen_mul_i64(t0, t0, arg2);
1494 tcg_gen_sub_i64(ret, arg1, t0);
1495 tcg_temp_free_i64(t0);
1496 } else if (TCG_TARGET_HAS_div2_i64) {
1497 TCGv_i64 t0 = tcg_temp_new_i64();
1498 tcg_gen_movi_i64(t0, 0);
1499 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1500 tcg_temp_free_i64(t0);
1501 } else {
1502 gen_helper_remu_i64(ret, arg1, arg2);
1503 }
1504}
1505
1506void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1507{
3a13c3f3
RH		 1508 if (TCG_TARGET_REG_BITS == 32) {
1509 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1510 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1511 } else if (TCG_TARGET_HAS_ext8s_i64) {
951c6300
RH		 1512 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1513 } else {
1514 tcg_gen_shli_i64(ret, arg, 56);
1515 tcg_gen_sari_i64(ret, ret, 56);
1516 }
951c6300
RH		 1517}
1518
1519void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1520{
3a13c3f3
RH		 1521 if (TCG_TARGET_REG_BITS == 32) {
1522 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1523 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1524 } else if (TCG_TARGET_HAS_ext16s_i64) {
951c6300
RH		 1525 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1526 } else {
1527 tcg_gen_shli_i64(ret, arg, 48);
1528 tcg_gen_sari_i64(ret, ret, 48);
1529 }
951c6300
RH		 1530}
1531
1532void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1533{
3a13c3f3
RH		 1534 if (TCG_TARGET_REG_BITS == 32) {
1535 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1536 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1537 } else if (TCG_TARGET_HAS_ext32s_i64) {
951c6300
RH		 1538 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1539 } else {
1540 tcg_gen_shli_i64(ret, arg, 32);
1541 tcg_gen_sari_i64(ret, ret, 32);
1542 }
951c6300
RH		 1543}
1544
1545void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1546{
3a13c3f3
RH		 1547 if (TCG_TARGET_REG_BITS == 32) {
1548 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1549 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1550 } else if (TCG_TARGET_HAS_ext8u_i64) {
951c6300
RH		 1551 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1552 } else {
1553 tcg_gen_andi_i64(ret, arg, 0xffu);
1554 }
951c6300
RH		 1555}
1556
1557void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1558{
3a13c3f3
RH		 1559 if (TCG_TARGET_REG_BITS == 32) {
1560 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1561 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1562 } else if (TCG_TARGET_HAS_ext16u_i64) {
951c6300
RH		 1563 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1564 } else {
1565 tcg_gen_andi_i64(ret, arg, 0xffffu);
1566 }
951c6300
RH		 1567}
1568
1569void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1570{
3a13c3f3
RH		 1571 if (TCG_TARGET_REG_BITS == 32) {
1572 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1573 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1574 } else if (TCG_TARGET_HAS_ext32u_i64) {
951c6300
RH		 1575 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1576 } else {
1577 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1578 }
951c6300
RH		 1579}
1580
1581/* Note: we assume the six high bytes are set to zero */
1582void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1583{
3a13c3f3
RH		 1584 if (TCG_TARGET_REG_BITS == 32) {
1585 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1586 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1587 } else if (TCG_TARGET_HAS_bswap16_i64) {
951c6300
RH		 1588 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1589 } else {
1590 TCGv_i64 t0 = tcg_temp_new_i64();
1591
1592 tcg_gen_ext8u_i64(t0, arg);
1593 tcg_gen_shli_i64(t0, t0, 8);
1594 tcg_gen_shri_i64(ret, arg, 8);
1595 tcg_gen_or_i64(ret, ret, t0);
1596 tcg_temp_free_i64(t0);
1597 }
951c6300
RH		 1598}
1599
1600/* Note: we assume the four high bytes are set to zero */
1601void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1602{
3a13c3f3
RH		 1603 if (TCG_TARGET_REG_BITS == 32) {
1604 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1605 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1606 } else if (TCG_TARGET_HAS_bswap32_i64) {
951c6300
RH		 1607 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1608 } else {
1609 TCGv_i64 t0, t1;
1610 t0 = tcg_temp_new_i64();
1611 t1 = tcg_temp_new_i64();
1612
1613 tcg_gen_shli_i64(t0, arg, 24);
1614 tcg_gen_ext32u_i64(t0, t0);
1615
1616 tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1617 tcg_gen_shli_i64(t1, t1, 8);
1618 tcg_gen_or_i64(t0, t0, t1);
1619
1620 tcg_gen_shri_i64(t1, arg, 8);
1621 tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1622 tcg_gen_or_i64(t0, t0, t1);
1623
1624 tcg_gen_shri_i64(t1, arg, 24);
1625 tcg_gen_or_i64(ret, t0, t1);
1626 tcg_temp_free_i64(t0);
1627 tcg_temp_free_i64(t1);
1628 }
951c6300
RH		 1629}
1630
1631void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1632{
3a13c3f3
RH		 1633 if (TCG_TARGET_REG_BITS == 32) {
1634 TCGv_i32 t0, t1;
1635 t0 = tcg_temp_new_i32();
1636 t1 = tcg_temp_new_i32();
951c6300 1637
3a13c3f3
RH		 1638 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1639 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1640 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1641 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1642 tcg_temp_free_i32(t0);
1643 tcg_temp_free_i32(t1);
1644 } else if (TCG_TARGET_HAS_bswap64_i64) {
951c6300
RH		 1645 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1646 } else {
1647 TCGv_i64 t0 = tcg_temp_new_i64();
1648 TCGv_i64 t1 = tcg_temp_new_i64();
1649
1650 tcg_gen_shli_i64(t0, arg, 56);
1651
1652 tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1653 tcg_gen_shli_i64(t1, t1, 40);
1654 tcg_gen_or_i64(t0, t0, t1);
1655
1656 tcg_gen_andi_i64(t1, arg, 0x00ff0000);
1657 tcg_gen_shli_i64(t1, t1, 24);
1658 tcg_gen_or_i64(t0, t0, t1);
1659
1660 tcg_gen_andi_i64(t1, arg, 0xff000000);
1661 tcg_gen_shli_i64(t1, t1, 8);
1662 tcg_gen_or_i64(t0, t0, t1);
1663
1664 tcg_gen_shri_i64(t1, arg, 8);
1665 tcg_gen_andi_i64(t1, t1, 0xff000000);
1666 tcg_gen_or_i64(t0, t0, t1);
1667
1668 tcg_gen_shri_i64(t1, arg, 24);
1669 tcg_gen_andi_i64(t1, t1, 0x00ff0000);
1670 tcg_gen_or_i64(t0, t0, t1);
1671
1672 tcg_gen_shri_i64(t1, arg, 40);
1673 tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1674 tcg_gen_or_i64(t0, t0, t1);
1675
1676 tcg_gen_shri_i64(t1, arg, 56);
1677 tcg_gen_or_i64(ret, t0, t1);
1678 tcg_temp_free_i64(t0);
1679 tcg_temp_free_i64(t1);
1680 }
951c6300
RH		 1681}
1682
1683void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1684{
3a13c3f3
RH		 1685 if (TCG_TARGET_REG_BITS == 32) {
1686 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1687 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1688 } else if (TCG_TARGET_HAS_not_i64) {
951c6300
RH		 1689 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1690 } else {
1691 tcg_gen_xori_i64(ret, arg, -1);
1692 }
951c6300
RH		 1693}
1694
1695void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1696{
3a13c3f3
RH		 1697 if (TCG_TARGET_REG_BITS == 32) {
1698 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1699 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1700 } else if (TCG_TARGET_HAS_andc_i64) {
951c6300
RH		 1701 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1702 } else {
1703 TCGv_i64 t0 = tcg_temp_new_i64();
1704 tcg_gen_not_i64(t0, arg2);
1705 tcg_gen_and_i64(ret, arg1, t0);
1706 tcg_temp_free_i64(t0);
1707 }
951c6300
RH		 1708}
1709
1710void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1711{
3a13c3f3
RH		 1712 if (TCG_TARGET_REG_BITS == 32) {
1713 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1714 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1715 } else if (TCG_TARGET_HAS_eqv_i64) {
951c6300
RH		 1716 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1717 } else {
1718 tcg_gen_xor_i64(ret, arg1, arg2);
1719 tcg_gen_not_i64(ret, ret);
1720 }
951c6300
RH		 1721}
1722
1723void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1724{
3a13c3f3
RH		 1725 if (TCG_TARGET_REG_BITS == 32) {
1726 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1727 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1728 } else if (TCG_TARGET_HAS_nand_i64) {
951c6300
RH		 1729 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1730 } else {
1731 tcg_gen_and_i64(ret, arg1, arg2);
1732 tcg_gen_not_i64(ret, ret);
1733 }
951c6300
RH		 1734}
1735
1736void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1737{
3a13c3f3
RH		 1738 if (TCG_TARGET_REG_BITS == 32) {
1739 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1740 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1741 } else if (TCG_TARGET_HAS_nor_i64) {
951c6300
RH		 1742 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1743 } else {
1744 tcg_gen_or_i64(ret, arg1, arg2);
1745 tcg_gen_not_i64(ret, ret);
1746 }
951c6300
RH		 1747}
1748
1749void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1750{
3a13c3f3
RH		 1751 if (TCG_TARGET_REG_BITS == 32) {
1752 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1753 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1754 } else if (TCG_TARGET_HAS_orc_i64) {
951c6300
RH		 1755 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1756 } else {
1757 TCGv_i64 t0 = tcg_temp_new_i64();
1758 tcg_gen_not_i64(t0, arg2);
1759 tcg_gen_or_i64(ret, arg1, t0);
1760 tcg_temp_free_i64(t0);
1761 }
951c6300
RH		 1762}
1763
0e28d006
RH		 1764void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1765{
1766 if (TCG_TARGET_HAS_clz_i64) {
1767 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1768 } else {
1769 gen_helper_clz_i64(ret, arg1, arg2);
1770 }
1771}
1772
1773void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1774{
1775 if (TCG_TARGET_REG_BITS == 32
1776 && TCG_TARGET_HAS_clz_i32
1777 && arg2 <= 0xffffffffu) {
1778 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1779 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1780 tcg_gen_addi_i32(t, t, 32);
1781 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1782 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1783 tcg_temp_free_i32(t);
1784 } else {
1785 TCGv_i64 t = tcg_const_i64(arg2);
1786 tcg_gen_clz_i64(ret, arg1, t);
1787 tcg_temp_free_i64(t);
1788 }
1789}
1790
1791void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1792{
1793 if (TCG_TARGET_HAS_ctz_i64) {
1794 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
14e99210
RH		 1795 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1796 TCGv_i64 z, t = tcg_temp_new_i64();
1797
1798 if (TCG_TARGET_HAS_ctpop_i64) {
1799 tcg_gen_subi_i64(t, arg1, 1);
1800 tcg_gen_andc_i64(t, t, arg1);
1801 tcg_gen_ctpop_i64(t, t);
1802 } else {
1803 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1804 tcg_gen_neg_i64(t, arg1);
1805 tcg_gen_and_i64(t, t, arg1);
1806 tcg_gen_clzi_i64(t, t, 64);
1807 tcg_gen_xori_i64(t, t, 63);
1808 }
1809 z = tcg_const_i64(0);
1810 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1811 tcg_temp_free_i64(t);
1812 tcg_temp_free_i64(z);
0e28d006
RH		 1813 } else {
1814 gen_helper_ctz_i64(ret, arg1, arg2);
1815 }
1816}
1817
1818void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1819{
1820 if (TCG_TARGET_REG_BITS == 32
1821 && TCG_TARGET_HAS_ctz_i32
1822 && arg2 <= 0xffffffffu) {
1823 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1824 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1825 tcg_gen_addi_i32(t32, t32, 32);
1826 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1827 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1828 tcg_temp_free_i32(t32);
14e99210
RH		 1829 } else if (!TCG_TARGET_HAS_ctz_i64
1830 && TCG_TARGET_HAS_ctpop_i64
1831 && arg2 == 64) {
1832 /* This equivalence has the advantage of not requiring a fixup. */
1833 TCGv_i64 t = tcg_temp_new_i64();
1834 tcg_gen_subi_i64(t, arg1, 1);
1835 tcg_gen_andc_i64(t, t, arg1);
1836 tcg_gen_ctpop_i64(ret, t);
1837 tcg_temp_free_i64(t);
0e28d006
RH		 1838 } else {
1839 TCGv_i64 t64 = tcg_const_i64(arg2);
1840 tcg_gen_ctz_i64(ret, arg1, t64);
1841 tcg_temp_free_i64(t64);
1842 }
1843}
1844
086920c2
RH		 1845void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1846{
1847 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1848 TCGv_i64 t = tcg_temp_new_i64();
1849 tcg_gen_sari_i64(t, arg, 63);
1850 tcg_gen_xor_i64(t, t, arg);
1851 tcg_gen_clzi_i64(t, t, 64);
1852 tcg_gen_subi_i64(ret, t, 1);
1853 tcg_temp_free_i64(t);
1854 } else {
1855 gen_helper_clrsb_i64(ret, arg);
1856 }
1857}
1858
a768e4e9
RH		 1859void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1860{
1861 if (TCG_TARGET_HAS_ctpop_i64) {
1862 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1863 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1864 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1865 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1866 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1867 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1868 } else {
1869 gen_helper_ctpop_i64(ret, arg1);
1870 }
1871}
1872
951c6300
RH		 1873void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1874{
1875 if (TCG_TARGET_HAS_rot_i64) {
1876 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1877 } else {
1878 TCGv_i64 t0, t1;
1879 t0 = tcg_temp_new_i64();
1880 t1 = tcg_temp_new_i64();
1881 tcg_gen_shl_i64(t0, arg1, arg2);
1882 tcg_gen_subfi_i64(t1, 64, arg2);
1883 tcg_gen_shr_i64(t1, arg1, t1);
1884 tcg_gen_or_i64(ret, t0, t1);
1885 tcg_temp_free_i64(t0);
1886 tcg_temp_free_i64(t1);
1887 }
1888}
1889
1890void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1891{
1892 tcg_debug_assert(arg2 < 64);
1893 /* some cases can be optimized here */
1894 if (arg2 == 0) {
1895 tcg_gen_mov_i64(ret, arg1);
1896 } else if (TCG_TARGET_HAS_rot_i64) {
1897 TCGv_i64 t0 = tcg_const_i64(arg2);
1898 tcg_gen_rotl_i64(ret, arg1, t0);
1899 tcg_temp_free_i64(t0);
1900 } else {
1901 TCGv_i64 t0, t1;
1902 t0 = tcg_temp_new_i64();
1903 t1 = tcg_temp_new_i64();
1904 tcg_gen_shli_i64(t0, arg1, arg2);
1905 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1906 tcg_gen_or_i64(ret, t0, t1);
1907 tcg_temp_free_i64(t0);
1908 tcg_temp_free_i64(t1);
1909 }
1910}
1911
1912void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1913{
1914 if (TCG_TARGET_HAS_rot_i64) {
1915 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1916 } else {
1917 TCGv_i64 t0, t1;
1918 t0 = tcg_temp_new_i64();
1919 t1 = tcg_temp_new_i64();
1920 tcg_gen_shr_i64(t0, arg1, arg2);
1921 tcg_gen_subfi_i64(t1, 64, arg2);
1922 tcg_gen_shl_i64(t1, arg1, t1);
1923 tcg_gen_or_i64(ret, t0, t1);
1924 tcg_temp_free_i64(t0);
1925 tcg_temp_free_i64(t1);
1926 }
1927}
1928
1929void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1930{
1931 tcg_debug_assert(arg2 < 64);
1932 /* some cases can be optimized here */
1933 if (arg2 == 0) {
1934 tcg_gen_mov_i64(ret, arg1);
1935 } else {
1936 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
1937 }
1938}
1939
1940void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
1941 unsigned int ofs, unsigned int len)
1942{
1943 uint64_t mask;
1944 TCGv_i64 t1;
1945
1946 tcg_debug_assert(ofs < 64);
0d0d309d 1947 tcg_debug_assert(len > 0);
951c6300
RH		 1948 tcg_debug_assert(len <= 64);
1949 tcg_debug_assert(ofs + len <= 64);
1950
0d0d309d 1951 if (len == 64) {
951c6300
RH		 1952 tcg_gen_mov_i64(ret, arg2);
1953 return;
1954 }
1955 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
1956 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
1957 return;
1958 }
1959
3a13c3f3
RH		 1960 if (TCG_TARGET_REG_BITS == 32) {
1961 if (ofs >= 32) {
1962 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
1963 TCGV_LOW(arg2), ofs - 32, len);
1964 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1965 return;
1966 }
1967 if (ofs + len <= 32) {
1968 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
1969 TCGV_LOW(arg2), ofs, len);
1970 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1971 return;
1972 }
951c6300 1973 }
951c6300
RH		 1974
1975 mask = (1ull << len) - 1;
1976 t1 = tcg_temp_new_i64();
1977
1978 if (ofs + len < 64) {
1979 tcg_gen_andi_i64(t1, arg2, mask);
1980 tcg_gen_shli_i64(t1, t1, ofs);
1981 } else {
1982 tcg_gen_shli_i64(t1, arg2, ofs);
1983 }
1984 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
1985 tcg_gen_or_i64(ret, ret, t1);
1986
1987 tcg_temp_free_i64(t1);
1988}
1989
07cc68d5
RH		 1990void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
1991 unsigned int ofs, unsigned int len)
1992{
1993 tcg_debug_assert(ofs < 64);
1994 tcg_debug_assert(len > 0);
1995 tcg_debug_assert(len <= 64);
1996 tcg_debug_assert(ofs + len <= 64);
1997
1998 if (ofs + len == 64) {
1999 tcg_gen_shli_i64(ret, arg, ofs);
2000 } else if (ofs == 0) {
2001 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2002 } else if (TCG_TARGET_HAS_deposit_i64
2003 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2004 TCGv_i64 zero = tcg_const_i64(0);
2005 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2006 tcg_temp_free_i64(zero);
2007 } else {
2008 if (TCG_TARGET_REG_BITS == 32) {
2009 if (ofs >= 32) {
2010 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2011 ofs - 32, len);
2012 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2013 return;
2014 }
2015 if (ofs + len <= 32) {
2016 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2017 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2018 return;
2019 }
2020 }
2021 /* To help two-operand hosts we prefer to zero-extend first,
2022 which allows ARG to stay live. */
2023 switch (len) {
2024 case 32:
2025 if (TCG_TARGET_HAS_ext32u_i64) {
2026 tcg_gen_ext32u_i64(ret, arg);
2027 tcg_gen_shli_i64(ret, ret, ofs);
2028 return;
2029 }
2030 break;
2031 case 16:
2032 if (TCG_TARGET_HAS_ext16u_i64) {
2033 tcg_gen_ext16u_i64(ret, arg);
2034 tcg_gen_shli_i64(ret, ret, ofs);
2035 return;
2036 }
2037 break;
2038 case 8:
2039 if (TCG_TARGET_HAS_ext8u_i64) {
2040 tcg_gen_ext8u_i64(ret, arg);
2041 tcg_gen_shli_i64(ret, ret, ofs);
2042 return;
2043 }
2044 break;
2045 }
2046 /* Otherwise prefer zero-extension over AND for code size. */
2047 switch (ofs + len) {
2048 case 32:
2049 if (TCG_TARGET_HAS_ext32u_i64) {
2050 tcg_gen_shli_i64(ret, arg, ofs);
2051 tcg_gen_ext32u_i64(ret, ret);
2052 return;
2053 }
2054 break;
2055 case 16:
2056 if (TCG_TARGET_HAS_ext16u_i64) {
2057 tcg_gen_shli_i64(ret, arg, ofs);
2058 tcg_gen_ext16u_i64(ret, ret);
2059 return;
2060 }
2061 break;
2062 case 8:
2063 if (TCG_TARGET_HAS_ext8u_i64) {
2064 tcg_gen_shli_i64(ret, arg, ofs);
2065 tcg_gen_ext8u_i64(ret, ret);
2066 return;
2067 }
2068 break;
2069 }
2070 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2071 tcg_gen_shli_i64(ret, ret, ofs);
2072 }
2073}
2074
7ec8bab3
RH		 2075void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2076 unsigned int ofs, unsigned int len)
2077{
2078 tcg_debug_assert(ofs < 64);
2079 tcg_debug_assert(len > 0);
2080 tcg_debug_assert(len <= 64);
2081 tcg_debug_assert(ofs + len <= 64);
2082
2083 /* Canonicalize certain special cases, even if extract is supported. */
2084 if (ofs + len == 64) {
2085 tcg_gen_shri_i64(ret, arg, 64 - len);
2086 return;
2087 }
2088 if (ofs == 0) {
2089 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2090 return;
2091 }
2092
2093 if (TCG_TARGET_REG_BITS == 32) {
2094 /* Look for a 32-bit extract within one of the two words. */
2095 if (ofs >= 32) {
2096 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2097 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2098 return;
2099 }
2100 if (ofs + len <= 32) {
2101 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2102 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2103 return;
2104 }
2105 /* The field is split across two words. One double-word
2106 shift is better than two double-word shifts. */
2107 goto do_shift_and;
2108 }
2109
2110 if (TCG_TARGET_HAS_extract_i64
2111 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2112 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2113 return;
2114 }
2115
2116 /* Assume that zero-extension, if available, is cheaper than a shift. */
2117 switch (ofs + len) {
2118 case 32:
2119 if (TCG_TARGET_HAS_ext32u_i64) {
2120 tcg_gen_ext32u_i64(ret, arg);
2121 tcg_gen_shri_i64(ret, ret, ofs);
2122 return;
2123 }
2124 break;
2125 case 16:
2126 if (TCG_TARGET_HAS_ext16u_i64) {
2127 tcg_gen_ext16u_i64(ret, arg);
2128 tcg_gen_shri_i64(ret, ret, ofs);
2129 return;
2130 }
2131 break;
2132 case 8:
2133 if (TCG_TARGET_HAS_ext8u_i64) {
2134 tcg_gen_ext8u_i64(ret, arg);
2135 tcg_gen_shri_i64(ret, ret, ofs);
2136 return;
2137 }
2138 break;
2139 }
2140
2141 /* ??? Ideally we'd know what values are available for immediate AND.
2142 Assume that 8 bits are available, plus the special cases of 16 and 32,
2143 so that we get ext8u, ext16u, and ext32u. */
2144 switch (len) {
2145 case 1 ... 8: case 16: case 32:
2146 do_shift_and:
2147 tcg_gen_shri_i64(ret, arg, ofs);
2148 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2149 break;
2150 default:
2151 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2152 tcg_gen_shri_i64(ret, ret, 64 - len);
2153 break;
2154 }
2155}
2156
2157void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2158 unsigned int ofs, unsigned int len)
2159{
2160 tcg_debug_assert(ofs < 64);
2161 tcg_debug_assert(len > 0);
2162 tcg_debug_assert(len <= 64);
2163 tcg_debug_assert(ofs + len <= 64);
2164
2165 /* Canonicalize certain special cases, even if sextract is supported. */
2166 if (ofs + len == 64) {
2167 tcg_gen_sari_i64(ret, arg, 64 - len);
2168 return;
2169 }
2170 if (ofs == 0) {
2171 switch (len) {
2172 case 32:
2173 tcg_gen_ext32s_i64(ret, arg);
2174 return;
2175 case 16:
2176 tcg_gen_ext16s_i64(ret, arg);
2177 return;
2178 case 8:
2179 tcg_gen_ext8s_i64(ret, arg);
2180 return;
2181 }
2182 }
2183
2184 if (TCG_TARGET_REG_BITS == 32) {
2185 /* Look for a 32-bit extract within one of the two words. */
2186 if (ofs >= 32) {
2187 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2188 } else if (ofs + len <= 32) {
2189 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2190 } else if (ofs == 0) {
2191 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2192 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2193 return;
2194 } else if (len > 32) {
2195 TCGv_i32 t = tcg_temp_new_i32();
2196 /* Extract the bits for the high word normally. */
2197 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2198 /* Shift the field down for the low part. */
2199 tcg_gen_shri_i64(ret, arg, ofs);
2200 /* Overwrite the shift into the high part. */
2201 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2202 tcg_temp_free_i32(t);
2203 return;
2204 } else {
2205 /* Shift the field down for the low part, such that the
2206 field sits at the MSB. */
2207 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2208 /* Shift the field down from the MSB, sign extending. */
2209 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2210 }
2211 /* Sign-extend the field from 32 bits. */
2212 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2213 return;
2214 }
2215
2216 if (TCG_TARGET_HAS_sextract_i64
2217 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2218 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2219 return;
2220 }
2221
2222 /* Assume that sign-extension, if available, is cheaper than a shift. */
2223 switch (ofs + len) {
2224 case 32:
2225 if (TCG_TARGET_HAS_ext32s_i64) {
2226 tcg_gen_ext32s_i64(ret, arg);
2227 tcg_gen_sari_i64(ret, ret, ofs);
2228 return;
2229 }
2230 break;
2231 case 16:
2232 if (TCG_TARGET_HAS_ext16s_i64) {
2233 tcg_gen_ext16s_i64(ret, arg);
2234 tcg_gen_sari_i64(ret, ret, ofs);
2235 return;
2236 }
2237 break;
2238 case 8:
2239 if (TCG_TARGET_HAS_ext8s_i64) {
2240 tcg_gen_ext8s_i64(ret, arg);
2241 tcg_gen_sari_i64(ret, ret, ofs);
2242 return;
2243 }
2244 break;
2245 }
2246 switch (len) {
2247 case 32:
2248 if (TCG_TARGET_HAS_ext32s_i64) {
2249 tcg_gen_shri_i64(ret, arg, ofs);
2250 tcg_gen_ext32s_i64(ret, ret);
2251 return;
2252 }
2253 break;
2254 case 16:
2255 if (TCG_TARGET_HAS_ext16s_i64) {
2256 tcg_gen_shri_i64(ret, arg, ofs);
2257 tcg_gen_ext16s_i64(ret, ret);
2258 return;
2259 }
2260 break;
2261 case 8:
2262 if (TCG_TARGET_HAS_ext8s_i64) {
2263 tcg_gen_shri_i64(ret, arg, ofs);
2264 tcg_gen_ext8s_i64(ret, ret);
2265 return;
2266 }
2267 break;
2268 }
2269 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2270 tcg_gen_sari_i64(ret, ret, 64 - len);
2271}
2272
951c6300
RH		 2273void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2274 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2275{
37ed3bf1
RH		 2276 if (cond == TCG_COND_ALWAYS) {
2277 tcg_gen_mov_i64(ret, v1);
2278 } else if (cond == TCG_COND_NEVER) {
2279 tcg_gen_mov_i64(ret, v2);
2280 } else if (TCG_TARGET_REG_BITS == 32) {
3a13c3f3
RH		 2281 TCGv_i32 t0 = tcg_temp_new_i32();
2282 TCGv_i32 t1 = tcg_temp_new_i32();
2283 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2284 TCGV_LOW(c1), TCGV_HIGH(c1),
2285 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2286
2287 if (TCG_TARGET_HAS_movcond_i32) {
2288 tcg_gen_movi_i32(t1, 0);
2289 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2290 TCGV_LOW(v1), TCGV_LOW(v2));
2291 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2292 TCGV_HIGH(v1), TCGV_HIGH(v2));
2293 } else {
2294 tcg_gen_neg_i32(t0, t0);
951c6300 2295
3a13c3f3
RH		 2296 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2297 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2298 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
951c6300 2299
3a13c3f3
RH		 2300 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2301 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2302 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2303 }
2304 tcg_temp_free_i32(t0);
2305 tcg_temp_free_i32(t1);
2306 } else if (TCG_TARGET_HAS_movcond_i64) {
951c6300
RH		 2307 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2308 } else {
2309 TCGv_i64 t0 = tcg_temp_new_i64();
2310 TCGv_i64 t1 = tcg_temp_new_i64();
2311 tcg_gen_setcond_i64(cond, t0, c1, c2);
2312 tcg_gen_neg_i64(t0, t0);
2313 tcg_gen_and_i64(t1, v1, t0);
2314 tcg_gen_andc_i64(ret, v2, t0);
2315 tcg_gen_or_i64(ret, ret, t1);
2316 tcg_temp_free_i64(t0);
2317 tcg_temp_free_i64(t1);
2318 }
951c6300
RH		 2319}
2320
2321void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2322 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2323{
2324 if (TCG_TARGET_HAS_add2_i64) {
2325 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
951c6300
RH		 2326 } else {
2327 TCGv_i64 t0 = tcg_temp_new_i64();
2328 TCGv_i64 t1 = tcg_temp_new_i64();
2329 tcg_gen_add_i64(t0, al, bl);
2330 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2331 tcg_gen_add_i64(rh, ah, bh);
2332 tcg_gen_add_i64(rh, rh, t1);
2333 tcg_gen_mov_i64(rl, t0);
2334 tcg_temp_free_i64(t0);
2335 tcg_temp_free_i64(t1);
2336 }
2337}
2338
2339void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2340 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2341{
2342 if (TCG_TARGET_HAS_sub2_i64) {
2343 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
951c6300
RH		 2344 } else {
2345 TCGv_i64 t0 = tcg_temp_new_i64();
2346 TCGv_i64 t1 = tcg_temp_new_i64();
2347 tcg_gen_sub_i64(t0, al, bl);
2348 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2349 tcg_gen_sub_i64(rh, ah, bh);
2350 tcg_gen_sub_i64(rh, rh, t1);
2351 tcg_gen_mov_i64(rl, t0);
2352 tcg_temp_free_i64(t0);
2353 tcg_temp_free_i64(t1);
2354 }
2355}
2356
2357void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2358{
2359 if (TCG_TARGET_HAS_mulu2_i64) {
2360 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
951c6300
RH		 2361 } else if (TCG_TARGET_HAS_muluh_i64) {
2362 TCGv_i64 t = tcg_temp_new_i64();
2363 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2364 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2365 tcg_gen_mov_i64(rl, t);
2366 tcg_temp_free_i64(t);
2367 } else {
2368 TCGv_i64 t0 = tcg_temp_new_i64();
2369 tcg_gen_mul_i64(t0, arg1, arg2);
2370 gen_helper_muluh_i64(rh, arg1, arg2);
2371 tcg_gen_mov_i64(rl, t0);
2372 tcg_temp_free_i64(t0);
2373 }
2374}
2375
2376void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2377{
2378 if (TCG_TARGET_HAS_muls2_i64) {
2379 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
951c6300
RH		 2380 } else if (TCG_TARGET_HAS_mulsh_i64) {
2381 TCGv_i64 t = tcg_temp_new_i64();
2382 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2383 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2384 tcg_gen_mov_i64(rl, t);
2385 tcg_temp_free_i64(t);
2386 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2387 TCGv_i64 t0 = tcg_temp_new_i64();
2388 TCGv_i64 t1 = tcg_temp_new_i64();
2389 TCGv_i64 t2 = tcg_temp_new_i64();
2390 TCGv_i64 t3 = tcg_temp_new_i64();
2391 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2392 /* Adjust for negative inputs. */
2393 tcg_gen_sari_i64(t2, arg1, 63);
2394 tcg_gen_sari_i64(t3, arg2, 63);
2395 tcg_gen_and_i64(t2, t2, arg2);
2396 tcg_gen_and_i64(t3, t3, arg1);
2397 tcg_gen_sub_i64(rh, t1, t2);
2398 tcg_gen_sub_i64(rh, rh, t3);
2399 tcg_gen_mov_i64(rl, t0);
2400 tcg_temp_free_i64(t0);
2401 tcg_temp_free_i64(t1);
2402 tcg_temp_free_i64(t2);
2403 tcg_temp_free_i64(t3);
2404 } else {
2405 TCGv_i64 t0 = tcg_temp_new_i64();
2406 tcg_gen_mul_i64(t0, arg1, arg2);
2407 gen_helper_mulsh_i64(rh, arg1, arg2);
2408 tcg_gen_mov_i64(rl, t0);
2409 tcg_temp_free_i64(t0);
2410 }
2411}
2412
5087abfb
RH		 2413void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2414{
2415 TCGv_i64 t0 = tcg_temp_new_i64();
2416 TCGv_i64 t1 = tcg_temp_new_i64();
2417 TCGv_i64 t2 = tcg_temp_new_i64();
2418 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2419 /* Adjust for negative input for the signed arg1. */
2420 tcg_gen_sari_i64(t2, arg1, 63);
2421 tcg_gen_and_i64(t2, t2, arg2);
2422 tcg_gen_sub_i64(rh, t1, t2);
2423 tcg_gen_mov_i64(rl, t0);
2424 tcg_temp_free_i64(t0);
2425 tcg_temp_free_i64(t1);
2426 tcg_temp_free_i64(t2);
2427}
2428
951c6300
RH		 2429/* Size changing operations. */
2430
609ad705 2431void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
951c6300 2432{
3a13c3f3 2433 if (TCG_TARGET_REG_BITS == 32) {
609ad705
RH		 2434 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2435 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
b7e8b17a 2436 tcg_gen_op2(INDEX_op_extrl_i64_i32,
ae8b75dc 2437 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
609ad705 2438 } else {
dc41aa7d 2439 tcg_gen_mov_i32(ret, (TCGv_i32)arg);
609ad705
RH		 2440 }
2441}
2442
2443void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2444{
2445 if (TCG_TARGET_REG_BITS == 32) {
2446 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2447 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
b7e8b17a 2448 tcg_gen_op2(INDEX_op_extrh_i64_i32,
ae8b75dc 2449 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
951c6300
RH		 2450 } else {
2451 TCGv_i64 t = tcg_temp_new_i64();
609ad705 2452 tcg_gen_shri_i64(t, arg, 32);
dc41aa7d 2453 tcg_gen_mov_i32(ret, (TCGv_i32)t);
951c6300
RH		 2454 tcg_temp_free_i64(t);
2455 }
951c6300
RH		 2456}
2457
2458void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2459{
3a13c3f3
RH		 2460 if (TCG_TARGET_REG_BITS == 32) {
2461 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2462 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2463 } else {
b7e8b17a 2464 tcg_gen_op2(INDEX_op_extu_i32_i64,
ae8b75dc 2465 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3a13c3f3 2466 }
951c6300
RH		 2467}
2468
2469void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2470{
3a13c3f3
RH		 2471 if (TCG_TARGET_REG_BITS == 32) {
2472 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2473 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2474 } else {
b7e8b17a 2475 tcg_gen_op2(INDEX_op_ext_i32_i64,
ae8b75dc 2476 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3a13c3f3 2477 }
951c6300
RH		 2478}
2479
2480void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2481{
3a13c3f3
RH		 2482 TCGv_i64 tmp;
2483
2484 if (TCG_TARGET_REG_BITS == 32) {
2485 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2486 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2487 return;
2488 }
2489
2490 tmp = tcg_temp_new_i64();
951c6300
RH		 2491 /* These extensions are only needed for type correctness.
2492 We may be able to do better given target specific information. */
2493 tcg_gen_extu_i32_i64(tmp, high);
2494 tcg_gen_extu_i32_i64(dest, low);
2495 /* If deposit is available, use it. Otherwise use the extra
2496 knowledge that we have of the zero-extensions above. */
2497 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2498 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2499 } else {
2500 tcg_gen_shli_i64(tmp, tmp, 32);
2501 tcg_gen_or_i64(dest, dest, tmp);
2502 }
2503 tcg_temp_free_i64(tmp);
951c6300
RH		 2504}
2505
2506void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2507{
3a13c3f3
RH		 2508 if (TCG_TARGET_REG_BITS == 32) {
2509 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2510 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2511 } else {
609ad705
RH		 2512 tcg_gen_extrl_i64_i32(lo, arg);
2513 tcg_gen_extrh_i64_i32(hi, arg);
3a13c3f3 2514 }
951c6300
RH		 2515}
2516
2517void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2518{
2519 tcg_gen_ext32u_i64(lo, arg);
2520 tcg_gen_shri_i64(hi, arg, 32);
2521}
2522
2523/* QEMU specific operations. */
2524
2525void tcg_gen_goto_tb(unsigned idx)
2526{
2527 /* We only support two chained exits. */
2528 tcg_debug_assert(idx <= 1);
2529#ifdef CONFIG_DEBUG_TCG
2530 /* Verify that we havn't seen this numbered exit before. */
b1311c4a
EC		 2531 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2532 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
951c6300
RH		 2533#endif
2534 tcg_gen_op1i(INDEX_op_goto_tb, idx);
2535}
2536
7f11636d 2537void tcg_gen_lookup_and_goto_ptr(void)
cedbcb01
EC		 2538{
2539 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2540 TCGv_ptr ptr = tcg_temp_new_ptr();
1c2adb95 2541 gen_helper_lookup_tb_ptr(ptr, cpu_env);
ae8b75dc 2542 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
cedbcb01
EC		 2543 tcg_temp_free_ptr(ptr);
2544 } else {
2545 tcg_gen_exit_tb(0);
2546 }
2547}
2548
951c6300
RH		 2549static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
2550{
1f00b27f
SS		 2551 /* Trigger the asserts within as early as possible. */
2552 (void)get_alignment_bits(op);
2553
951c6300
RH		 2554 switch (op & MO_SIZE) {
2555 case MO_8:
2556 op &= ~MO_BSWAP;
2557 break;
2558 case MO_16:
2559 break;
2560 case MO_32:
2561 if (!is64) {
2562 op &= ~MO_SIGN;
2563 }
2564 break;
2565 case MO_64:
2566 if (!is64) {
2567 tcg_abort();
2568 }
2569 break;
2570 }
2571 if (st) {
2572 op &= ~MO_SIGN;
2573 }
2574 return op;
2575}
2576
c45cb8bb
RH		 2577static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2578 TCGMemOp memop, TCGArg idx)
951c6300 2579{
59227d5d 2580 TCGMemOpIdx oi = make_memop_idx(memop, idx);
c45cb8bb 2581#if TARGET_LONG_BITS == 32
59227d5d 2582 tcg_gen_op3i_i32(opc, val, addr, oi);
c45cb8bb 2583#else
3a13c3f3 2584 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2585 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
3a13c3f3 2586 } else {
ae8b75dc 2587 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
3a13c3f3 2588 }
c45cb8bb 2589#endif
951c6300
RH		 2590}
2591
c45cb8bb
RH		 2592static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2593 TCGMemOp memop, TCGArg idx)
2594{
59227d5d 2595 TCGMemOpIdx oi = make_memop_idx(memop, idx);
951c6300 2596#if TARGET_LONG_BITS == 32
c45cb8bb 2597 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2598 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
c45cb8bb 2599 } else {
ae8b75dc 2600 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
c45cb8bb 2601 }
951c6300 2602#else
c45cb8bb 2603 if (TCG_TARGET_REG_BITS == 32) {
59227d5d
RH		 2604 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2605 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
c45cb8bb 2606 } else {
59227d5d 2607 tcg_gen_op3i_i64(opc, val, addr, oi);
c45cb8bb 2608 }
951c6300 2609#endif
c45cb8bb 2610}
951c6300 2611
b32dc337
PK		 2612static void tcg_gen_req_mo(TCGBar type)
2613{
2614#ifdef TCG_GUEST_DEFAULT_MO
2615 type &= TCG_GUEST_DEFAULT_MO;
2616#endif
2617 type &= ~TCG_TARGET_DEFAULT_MO;
2618 if (type) {
2619 tcg_gen_mb(type | TCG_BAR_SC);
2620 }
2621}
2622
951c6300
RH		 2623void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2624{
b32dc337 2625 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
951c6300 2626 memop = tcg_canonicalize_memop(memop, 0, 0);
1c2adb95 2627 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2628 addr, trace_mem_get_info(memop, 0));
c45cb8bb 2629 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
951c6300
RH		 2630}
2631
2632void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2633{
b32dc337 2634 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
951c6300 2635 memop = tcg_canonicalize_memop(memop, 0, 1);
1c2adb95 2636 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2637 addr, trace_mem_get_info(memop, 1));
c45cb8bb 2638 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
951c6300
RH		 2639}
2640
2641void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2642{
b32dc337 2643 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
3a13c3f3 2644 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 2645 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2646 if (memop & MO_SIGN) {
2647 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2648 } else {
2649 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2650 }
2651 return;
2652 }
951c6300 2653
c45cb8bb 2654 memop = tcg_canonicalize_memop(memop, 1, 0);
1c2adb95 2655 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2656 addr, trace_mem_get_info(memop, 0));
c45cb8bb 2657 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
951c6300
RH		 2658}
2659
2660void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2661{
b32dc337 2662 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
3a13c3f3 2663 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 2664 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2665 return;
2666 }
951c6300 2667
c45cb8bb 2668 memop = tcg_canonicalize_memop(memop, 1, 1);
1c2adb95 2669 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
dcdaadb6 2670 addr, trace_mem_get_info(memop, 1));
c45cb8bb 2671 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
951c6300 2672}
c482cb11
RH		 2673
2674static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
2675{
2676 switch (opc & MO_SSIZE) {
2677 case MO_SB:
2678 tcg_gen_ext8s_i32(ret, val);
2679 break;
2680 case MO_UB:
2681 tcg_gen_ext8u_i32(ret, val);
2682 break;
2683 case MO_SW:
2684 tcg_gen_ext16s_i32(ret, val);
2685 break;
2686 case MO_UW:
2687 tcg_gen_ext16u_i32(ret, val);
2688 break;
2689 default:
2690 tcg_gen_mov_i32(ret, val);
2691 break;
2692 }
2693}
2694
2695static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
2696{
2697 switch (opc & MO_SSIZE) {
2698 case MO_SB:
2699 tcg_gen_ext8s_i64(ret, val);
2700 break;
2701 case MO_UB:
2702 tcg_gen_ext8u_i64(ret, val);
2703 break;
2704 case MO_SW:
2705 tcg_gen_ext16s_i64(ret, val);
2706 break;
2707 case MO_UW:
2708 tcg_gen_ext16u_i64(ret, val);
2709 break;
2710 case MO_SL:
2711 tcg_gen_ext32s_i64(ret, val);
2712 break;
2713 case MO_UL:
2714 tcg_gen_ext32u_i64(ret, val);
2715 break;
2716 default:
2717 tcg_gen_mov_i64(ret, val);
2718 break;
2719 }
2720}
2721
2722#ifdef CONFIG_SOFTMMU
2723typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
2724 TCGv_i32, TCGv_i32, TCGv_i32);
2725typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
2726 TCGv_i64, TCGv_i64, TCGv_i32);
2727typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
2728 TCGv_i32, TCGv_i32);
2729typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
2730 TCGv_i64, TCGv_i32);
2731#else
2732typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
2733typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
2734typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
2735typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
2736#endif
2737
df79b996
RH		 2738#ifdef CONFIG_ATOMIC64
2739# define WITH_ATOMIC64(X) X,
2740#else
2741# define WITH_ATOMIC64(X)
2742#endif
2743
c482cb11
RH		 2744static void * const table_cmpxchg[16] = {
2745 [MO_8] = gen_helper_atomic_cmpxchgb,
2746 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
2747 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
2748 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
2749 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
df79b996
RH		 2750 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
2751 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
c482cb11
RH		 2752};
2753
2754void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
2755 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
2756{
2757 memop = tcg_canonicalize_memop(memop, 0, 0);
2758
b1311c4a 2759 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
c482cb11
RH		 2760 TCGv_i32 t1 = tcg_temp_new_i32();
2761 TCGv_i32 t2 = tcg_temp_new_i32();
2762
2763 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
2764
2765 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2766 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
2767 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2768 tcg_temp_free_i32(t2);
2769
2770 if (memop & MO_SIGN) {
2771 tcg_gen_ext_i32(retv, t1, memop);
2772 } else {
2773 tcg_gen_mov_i32(retv, t1);
2774 }
2775 tcg_temp_free_i32(t1);
2776 } else {
2777 gen_atomic_cx_i32 gen;
2778
2779 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2780 tcg_debug_assert(gen != NULL);
2781
2782#ifdef CONFIG_SOFTMMU
2783 {
2784 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
1c2adb95 2785 gen(retv, cpu_env, addr, cmpv, newv, oi);
c482cb11
RH		 2786 tcg_temp_free_i32(oi);
2787 }
2788#else
1c2adb95 2789 gen(retv, cpu_env, addr, cmpv, newv);
c482cb11
RH		 2790#endif
2791
2792 if (memop & MO_SIGN) {
2793 tcg_gen_ext_i32(retv, retv, memop);
2794 }
2795 }
2796}
2797
2798void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
2799 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
2800{
2801 memop = tcg_canonicalize_memop(memop, 1, 0);
2802
b1311c4a 2803 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
c482cb11
RH		 2804 TCGv_i64 t1 = tcg_temp_new_i64();
2805 TCGv_i64 t2 = tcg_temp_new_i64();
2806
2807 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
2808
2809 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2810 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
2811 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2812 tcg_temp_free_i64(t2);
2813
2814 if (memop & MO_SIGN) {
2815 tcg_gen_ext_i64(retv, t1, memop);
2816 } else {
2817 tcg_gen_mov_i64(retv, t1);
2818 }
2819 tcg_temp_free_i64(t1);
2820 } else if ((memop & MO_SIZE) == MO_64) {
df79b996 2821#ifdef CONFIG_ATOMIC64
c482cb11
RH		 2822 gen_atomic_cx_i64 gen;
2823
2824 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2825 tcg_debug_assert(gen != NULL);
2826
2827#ifdef CONFIG_SOFTMMU
2828 {
2829 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
1c2adb95 2830 gen(retv, cpu_env, addr, cmpv, newv, oi);
c482cb11
RH		 2831 tcg_temp_free_i32(oi);
2832 }
2833#else
1c2adb95 2834 gen(retv, cpu_env, addr, cmpv, newv);
c482cb11 2835#endif
df79b996 2836#else
1c2adb95 2837 gen_helper_exit_atomic(cpu_env);
79b1af90
RH		 2838 /* Produce a result, so that we have a well-formed opcode stream
2839 with respect to uses of the result in the (dead) code following. */
2840 tcg_gen_movi_i64(retv, 0);
df79b996 2841#endif /* CONFIG_ATOMIC64 */
c482cb11
RH		 2842 } else {
2843 TCGv_i32 c32 = tcg_temp_new_i32();
2844 TCGv_i32 n32 = tcg_temp_new_i32();
2845 TCGv_i32 r32 = tcg_temp_new_i32();
2846
2847 tcg_gen_extrl_i64_i32(c32, cmpv);
2848 tcg_gen_extrl_i64_i32(n32, newv);
2849 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
2850 tcg_temp_free_i32(c32);
2851 tcg_temp_free_i32(n32);
2852
2853 tcg_gen_extu_i32_i64(retv, r32);
2854 tcg_temp_free_i32(r32);
2855
2856 if (memop & MO_SIGN) {
2857 tcg_gen_ext_i64(retv, retv, memop);
2858 }
2859 }
2860}
2861
2862static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2863 TCGArg idx, TCGMemOp memop, bool new_val,
2864 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
2865{
2866 TCGv_i32 t1 = tcg_temp_new_i32();
2867 TCGv_i32 t2 = tcg_temp_new_i32();
2868
2869 memop = tcg_canonicalize_memop(memop, 0, 0);
2870
2871 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2872 gen(t2, t1, val);
2873 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2874
2875 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
2876 tcg_temp_free_i32(t1);
2877 tcg_temp_free_i32(t2);
2878}
2879
2880static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2881 TCGArg idx, TCGMemOp memop, void * const table[])
2882{
2883 gen_atomic_op_i32 gen;
2884
2885 memop = tcg_canonicalize_memop(memop, 0, 0);
2886
2887 gen = table[memop & (MO_SIZE | MO_BSWAP)];
2888 tcg_debug_assert(gen != NULL);
2889
2890#ifdef CONFIG_SOFTMMU
2891 {
2892 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
1c2adb95 2893 gen(ret, cpu_env, addr, val, oi);
c482cb11
RH		 2894 tcg_temp_free_i32(oi);
2895 }
2896#else
1c2adb95 2897 gen(ret, cpu_env, addr, val);
c482cb11
RH		 2898#endif
2899
2900 if (memop & MO_SIGN) {
2901 tcg_gen_ext_i32(ret, ret, memop);
2902 }
2903}
2904
2905static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2906 TCGArg idx, TCGMemOp memop, bool new_val,
2907 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
2908{
2909 TCGv_i64 t1 = tcg_temp_new_i64();
2910 TCGv_i64 t2 = tcg_temp_new_i64();
2911
2912 memop = tcg_canonicalize_memop(memop, 1, 0);
2913
2914 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2915 gen(t2, t1, val);
2916 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2917
2918 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
2919 tcg_temp_free_i64(t1);
2920 tcg_temp_free_i64(t2);
2921}
2922
2923static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2924 TCGArg idx, TCGMemOp memop, void * const table[])
2925{
2926 memop = tcg_canonicalize_memop(memop, 1, 0);
2927
2928 if ((memop & MO_SIZE) == MO_64) {
df79b996 2929#ifdef CONFIG_ATOMIC64
c482cb11
RH		 2930 gen_atomic_op_i64 gen;
2931
2932 gen = table[memop & (MO_SIZE | MO_BSWAP)];
2933 tcg_debug_assert(gen != NULL);
2934
2935#ifdef CONFIG_SOFTMMU
2936 {
2937 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
1c2adb95 2938 gen(ret, cpu_env, addr, val, oi);
c482cb11
RH		 2939 tcg_temp_free_i32(oi);
2940 }
2941#else
1c2adb95 2942 gen(ret, cpu_env, addr, val);
c482cb11 2943#endif
df79b996 2944#else
1c2adb95 2945 gen_helper_exit_atomic(cpu_env);
79b1af90
RH		 2946 /* Produce a result, so that we have a well-formed opcode stream
2947 with respect to uses of the result in the (dead) code following. */
2948 tcg_gen_movi_i64(ret, 0);
df79b996 2949#endif /* CONFIG_ATOMIC64 */
c482cb11
RH		 2950 } else {
2951 TCGv_i32 v32 = tcg_temp_new_i32();
2952 TCGv_i32 r32 = tcg_temp_new_i32();
2953
2954 tcg_gen_extrl_i64_i32(v32, val);
2955 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
2956 tcg_temp_free_i32(v32);
2957
2958 tcg_gen_extu_i32_i64(ret, r32);
2959 tcg_temp_free_i32(r32);
2960
2961 if (memop & MO_SIGN) {
2962 tcg_gen_ext_i64(ret, ret, memop);
2963 }
2964 }
2965}
2966
2967#define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
2968static void * const table_##NAME[16] = { \
2969 [MO_8] = gen_helper_atomic_##NAME##b, \
2970 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
2971 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
2972 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
2973 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
df79b996
RH		 2974 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
2975 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
c482cb11
RH		 2976}; \
2977void tcg_gen_atomic_##NAME##_i32 \
2978 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
2979{ \
b1311c4a 2980 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
c482cb11
RH		 2981 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
2982 } else { \
2983 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
2984 tcg_gen_##OP##_i32); \
2985 } \
2986} \
2987void tcg_gen_atomic_##NAME##_i64 \
2988 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
2989{ \
b1311c4a 2990 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
c482cb11
RH		 2991 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
2992 } else { \
2993 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
2994 tcg_gen_##OP##_i64); \
2995 } \
2996}
2997
2998GEN_ATOMIC_HELPER(fetch_add, add, 0)
2999GEN_ATOMIC_HELPER(fetch_and, and, 0)
3000GEN_ATOMIC_HELPER(fetch_or, or, 0)
3001GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3002
3003GEN_ATOMIC_HELPER(add_fetch, add, 1)
3004GEN_ATOMIC_HELPER(and_fetch, and, 1)
3005GEN_ATOMIC_HELPER(or_fetch, or, 1)
3006GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3007
3008static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3009{
3010 tcg_gen_mov_i32(r, b);
3011}
3012
3013static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3014{
3015 tcg_gen_mov_i64(r, b);
3016}
3017
3018GEN_ATOMIC_HELPER(xchg, mov2, 0)
3019
3020#undef GEN_ATOMIC_HELPER
Mirror of https://git.qemu.org/git/qemu.git