]> git.ipfire.org Git - thirdparty/qemu.git/blame - tcg/ppc/tcg-target.inc.c
projects / thirdparty / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg/ppc: Add support for vector add/subtract
[thirdparty/qemu.git] / tcg / ppc / tcg-target.inc.c
CommitLineData
810260a8 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
53c89efd
RH		 25#include "elf.h"
26#include "tcg-pool.inc.c"
27
ffcfbece
RH		 28#if defined _CALL_DARWIN || defined __APPLE__
29#define TCG_TARGET_CALL_DARWIN
30#endif
7f25c469
RH		 31#ifdef _CALL_SYSV
32# define TCG_TARGET_CALL_ALIGN_ARGS 1
33#endif
ffcfbece 34
dfca1778
RH		 35/* For some memory operations, we need a scratch that isn't R0. For the AIX
36 calling convention, we can re-use the TOC register since we'll be reloading
37 it at every call. Otherwise R12 will do nicely as neither a call-saved
38 register nor a parameter register. */
39#ifdef _CALL_AIX
40# define TCG_REG_TMP1 TCG_REG_R2
41#else
42# define TCG_REG_TMP1 TCG_REG_R12
43#endif
44
42281ec6
RH		 45#define TCG_VEC_TMP1 TCG_REG_V0
46#define TCG_VEC_TMP2 TCG_REG_V1
47
5964fca8
RH		 48#define TCG_REG_TB TCG_REG_R31
49#define USE_REG_TB (TCG_TARGET_REG_BITS == 64)
a84ac4cb 50
de3d636d
RH		 51/* Shorthand for size of a pointer. Avoid promotion to unsigned. */
52#define SZP ((int)sizeof(void *))
53
4c3831a0
RH		 54/* Shorthand for size of a register. */
55#define SZR (TCG_TARGET_REG_BITS / 8)
56
3d582c61
RH		 57#define TCG_CT_CONST_S16 0x100
58#define TCG_CT_CONST_U16 0x200
59#define TCG_CT_CONST_S32 0x400
60#define TCG_CT_CONST_U32 0x800
61#define TCG_CT_CONST_ZERO 0x1000
6c858762 62#define TCG_CT_CONST_MONE 0x2000
d0b07481 63#define TCG_CT_CONST_WSZ 0x4000
fe6f943f 64
e083c4a2 65static tcg_insn_unit *tb_ret_addr;
810260a8 66
7d9dae0a 67TCGPowerISA have_isa;
63922f46 68static bool have_isel;
4b06c216 69bool have_altivec;
49d9870a 70
4cbea598 71#ifndef CONFIG_SOFTMMU
f6548c0a 72#define TCG_GUEST_BASE_REG 30
f6548c0a 73#endif
74
8d8fdbae 75#ifdef CONFIG_DEBUG_TCG
42281ec6
RH		 76static const char tcg_target_reg_names[TCG_TARGET_NB_REGS][4] = {
77 "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7",
78 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
79 "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
80 "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
81 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
82 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
83 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
84 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",
810260a8 85};
d4a9eb1f 86#endif
810260a8 87
88static const int tcg_target_reg_alloc_order[] = {
5e1702b0 89 TCG_REG_R14, /* call saved registers */
810260a8 90 TCG_REG_R15,
91 TCG_REG_R16,
92 TCG_REG_R17,
93 TCG_REG_R18,
94 TCG_REG_R19,
95 TCG_REG_R20,
96 TCG_REG_R21,
97 TCG_REG_R22,
98 TCG_REG_R23,
5e1702b0
RH		 99 TCG_REG_R24,
100 TCG_REG_R25,
101 TCG_REG_R26,
102 TCG_REG_R27,
810260a8 103 TCG_REG_R28,
104 TCG_REG_R29,
105 TCG_REG_R30,
106 TCG_REG_R31,
5e1702b0
RH		 107 TCG_REG_R12, /* call clobbered, non-arguments */
108 TCG_REG_R11,
dfca1778
RH		 109 TCG_REG_R2,
110 TCG_REG_R13,
5e1702b0 111 TCG_REG_R10, /* call clobbered, arguments */
810260a8 112 TCG_REG_R9,
5e1702b0
RH		 113 TCG_REG_R8,
114 TCG_REG_R7,
115 TCG_REG_R6,
116 TCG_REG_R5,
117 TCG_REG_R4,
118 TCG_REG_R3,
42281ec6
RH		 119
120 /* V0 and V1 reserved as temporaries; V20 - V31 are call-saved */
121 TCG_REG_V2, /* call clobbered, vectors */
122 TCG_REG_V3,
123 TCG_REG_V4,
124 TCG_REG_V5,
125 TCG_REG_V6,
126 TCG_REG_V7,
127 TCG_REG_V8,
128 TCG_REG_V9,
129 TCG_REG_V10,
130 TCG_REG_V11,
131 TCG_REG_V12,
132 TCG_REG_V13,
133 TCG_REG_V14,
134 TCG_REG_V15,
135 TCG_REG_V16,
136 TCG_REG_V17,
137 TCG_REG_V18,
138 TCG_REG_V19,
810260a8 139};
140
141static const int tcg_target_call_iarg_regs[] = {
142 TCG_REG_R3,
143 TCG_REG_R4,
144 TCG_REG_R5,
145 TCG_REG_R6,
146 TCG_REG_R7,
147 TCG_REG_R8,
148 TCG_REG_R9,
149 TCG_REG_R10
150};
151
be9c4183 152static const int tcg_target_call_oarg_regs[] = {
dfca1778
RH		 153 TCG_REG_R3,
154 TCG_REG_R4
810260a8 155};
156
157static const int tcg_target_callee_save_regs[] = {
dfca1778 158#ifdef TCG_TARGET_CALL_DARWIN
5d7ff5bb
AF		 159 TCG_REG_R11,
160#endif
810260a8 161 TCG_REG_R14,
162 TCG_REG_R15,
163 TCG_REG_R16,
164 TCG_REG_R17,
165 TCG_REG_R18,
166 TCG_REG_R19,
167 TCG_REG_R20,
168 TCG_REG_R21,
169 TCG_REG_R22,
170 TCG_REG_R23,
095271d4 171 TCG_REG_R24,
172 TCG_REG_R25,
173 TCG_REG_R26,
cea5f9a2 174 TCG_REG_R27, /* currently used for the global env */
810260a8 175 TCG_REG_R28,
176 TCG_REG_R29,
177 TCG_REG_R30,
178 TCG_REG_R31
179};
180
b0940da0
RH		 181static inline bool in_range_b(tcg_target_long target)
182{
183 return target == sextract64(target, 0, 26);
184}
185
e083c4a2 186static uint32_t reloc_pc24_val(tcg_insn_unit *pc, tcg_insn_unit *target)
810260a8 187{
e083c4a2 188 ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
eabb7b91 189 tcg_debug_assert(in_range_b(disp));
810260a8 190 return disp & 0x3fffffc;
191}
192
d5132903 193static bool reloc_pc24(tcg_insn_unit *pc, tcg_insn_unit *target)
810260a8 194{
d5132903
RH		 195 ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
196 if (in_range_b(disp)) {
197 *pc = (*pc & ~0x3fffffc) | (disp & 0x3fffffc);
198 return true;
199 }
200 return false;
810260a8 201}
202
e083c4a2 203static uint16_t reloc_pc14_val(tcg_insn_unit *pc, tcg_insn_unit *target)
810260a8 204{
e083c4a2 205 ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
eabb7b91 206 tcg_debug_assert(disp == (int16_t) disp);
810260a8 207 return disp & 0xfffc;
208}
209
d5132903 210static bool reloc_pc14(tcg_insn_unit *pc, tcg_insn_unit *target)
810260a8 211{
d5132903
RH		 212 ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
213 if (disp == (int16_t) disp) {
214 *pc = (*pc & ~0xfffc) | (disp & 0xfffc);
215 return true;
216 }
217 return false;
810260a8 218}
219
810260a8 220/* parse target specific constraints */
069ea736
RH		 221static const char *target_parse_constraint(TCGArgConstraint *ct,
222 const char *ct_str, TCGType type)
810260a8 223{
069ea736 224 switch (*ct_str++) {
810260a8 225 case 'A': case 'B': case 'C': case 'D':
226 ct->ct |= TCG_CT_REG;
541dd4ce 227 tcg_regset_set_reg(ct->u.regs, 3 + ct_str[0] - 'A');
810260a8 228 break;
229 case 'r':
230 ct->ct |= TCG_CT_REG;
f46934df 231 ct->u.regs = 0xffffffff;
810260a8 232 break;
6ef14d7e
RH		 233 case 'v':
234 ct->ct |= TCG_CT_REG;
235 ct->u.regs = 0xffffffff00000000ull;
236 break;
810260a8 237 case 'L': /* qemu_ld constraint */
238 ct->ct |= TCG_CT_REG;
f46934df 239 ct->u.regs = 0xffffffff;
541dd4ce 240 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
735ee40d 241#ifdef CONFIG_SOFTMMU
541dd4ce
RH		 242 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
243 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
735ee40d 244#endif
810260a8 245 break;
c070355d 246 case 'S': /* qemu_st constraint */
810260a8 247 ct->ct |= TCG_CT_REG;
f46934df 248 ct->u.regs = 0xffffffff;
541dd4ce 249 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
735ee40d 250#ifdef CONFIG_SOFTMMU
541dd4ce
RH		 251 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
252 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
253 tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
735ee40d 254#endif
810260a8 255 break;
3d582c61
RH		 256 case 'I':
257 ct->ct |= TCG_CT_CONST_S16;
258 break;
259 case 'J':
260 ct->ct |= TCG_CT_CONST_U16;
261 break;
6c858762
RH		 262 case 'M':
263 ct->ct |= TCG_CT_CONST_MONE;
264 break;
3d582c61
RH		 265 case 'T':
266 ct->ct |= TCG_CT_CONST_S32;
267 break;
268 case 'U':
fe6f943f 269 ct->ct |= TCG_CT_CONST_U32;
270 break;
d0b07481
RH		 271 case 'W':
272 ct->ct |= TCG_CT_CONST_WSZ;
273 break;
3d582c61
RH		 274 case 'Z':
275 ct->ct |= TCG_CT_CONST_ZERO;
276 break;
810260a8 277 default:
069ea736 278 return NULL;
810260a8 279 }
069ea736 280 return ct_str;
810260a8 281}
282
283/* test if a constant matches the constraint */
f6c6afc1 284static int tcg_target_const_match(tcg_target_long val, TCGType type,
541dd4ce 285 const TCGArgConstraint *arg_ct)
810260a8 286{
3d582c61
RH		 287 int ct = arg_ct->ct;
288 if (ct & TCG_CT_CONST) {
289 return 1;
1194dcba
RH		 290 }
291
292 /* The only 32-bit constraint we use aside from
293 TCG_CT_CONST is TCG_CT_CONST_S16. */
294 if (type == TCG_TYPE_I32) {
295 val = (int32_t)val;
296 }
297
298 if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) {
3d582c61
RH		 299 return 1;
300 } else if ((ct & TCG_CT_CONST_U16) && val == (uint16_t)val) {
810260a8 301 return 1;
3d582c61 302 } else if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
fe6f943f 303 return 1;
3d582c61
RH		 304 } else if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
305 return 1;
306 } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
307 return 1;
6c858762
RH		 308 } else if ((ct & TCG_CT_CONST_MONE) && val == -1) {
309 return 1;
d0b07481
RH		 310 } else if ((ct & TCG_CT_CONST_WSZ)
311 && val == (type == TCG_TYPE_I32 ? 32 : 64)) {
312 return 1;
3d582c61 313 }
810260a8 314 return 0;
315}
316
317#define OPCD(opc) ((opc)<<26)
318#define XO19(opc) (OPCD(19)|((opc)<<1))
8a94cfb0
AB		 319#define MD30(opc) (OPCD(30)|((opc)<<2))
320#define MDS30(opc) (OPCD(30)|((opc)<<1))
810260a8 321#define XO31(opc) (OPCD(31)|((opc)<<1))
322#define XO58(opc) (OPCD(58)|(opc))
323#define XO62(opc) (OPCD(62)|(opc))
1838905e 324#define VX4(opc) (OPCD(4)|(opc))
810260a8 325
326#define B OPCD( 18)
327#define BC OPCD( 16)
328#define LBZ OPCD( 34)
329#define LHZ OPCD( 40)
330#define LHA OPCD( 42)
331#define LWZ OPCD( 32)
644f591a 332#define LWZUX XO31( 55)
810260a8 333#define STB OPCD( 38)
334#define STH OPCD( 44)
335#define STW OPCD( 36)
336
337#define STD XO62( 0)
338#define STDU XO62( 1)
339#define STDX XO31(149)
340
341#define LD XO58( 0)
342#define LDX XO31( 21)
343#define LDU XO58( 1)
644f591a 344#define LDUX XO31( 53)
301f6d90 345#define LWA XO58( 2)
810260a8 346#define LWAX XO31(341)
347
1cd62ae9 348#define ADDIC OPCD( 12)
810260a8 349#define ADDI OPCD( 14)
350#define ADDIS OPCD( 15)
351#define ORI OPCD( 24)
352#define ORIS OPCD( 25)
353#define XORI OPCD( 26)
354#define XORIS OPCD( 27)
355#define ANDI OPCD( 28)
356#define ANDIS OPCD( 29)
357#define MULLI OPCD( 7)
358#define CMPLI OPCD( 10)
359#define CMPI OPCD( 11)
148bdd23 360#define SUBFIC OPCD( 8)
810260a8 361
362#define LWZU OPCD( 33)
363#define STWU OPCD( 37)
364
313d91c7 365#define RLWIMI OPCD( 20)
810260a8 366#define RLWINM OPCD( 21)
313d91c7 367#define RLWNM OPCD( 23)
810260a8 368
8a94cfb0
AB		 369#define RLDICL MD30( 0)
370#define RLDICR MD30( 1)
371#define RLDIMI MD30( 3)
372#define RLDCL MDS30( 8)
810260a8 373
374#define BCLR XO19( 16)
375#define BCCTR XO19(528)
376#define CRAND XO19(257)
377#define CRANDC XO19(129)
378#define CRNAND XO19(225)
379#define CROR XO19(449)
1cd62ae9 380#define CRNOR XO19( 33)
810260a8 381
382#define EXTSB XO31(954)
383#define EXTSH XO31(922)
384#define EXTSW XO31(986)
385#define ADD XO31(266)
386#define ADDE XO31(138)
6c858762
RH		 387#define ADDME XO31(234)
388#define ADDZE XO31(202)
810260a8 389#define ADDC XO31( 10)
390#define AND XO31( 28)
391#define SUBF XO31( 40)
392#define SUBFC XO31( 8)
393#define SUBFE XO31(136)
6c858762
RH		 394#define SUBFME XO31(232)
395#define SUBFZE XO31(200)
810260a8 396#define OR XO31(444)
397#define XOR XO31(316)
398#define MULLW XO31(235)
8fa391a0 399#define MULHW XO31( 75)
810260a8 400#define MULHWU XO31( 11)
401#define DIVW XO31(491)
402#define DIVWU XO31(459)
403#define CMP XO31( 0)
404#define CMPL XO31( 32)
405#define LHBRX XO31(790)
406#define LWBRX XO31(534)
49d9870a 407#define LDBRX XO31(532)
810260a8 408#define STHBRX XO31(918)
409#define STWBRX XO31(662)
49d9870a 410#define STDBRX XO31(660)
810260a8 411#define MFSPR XO31(339)
412#define MTSPR XO31(467)
413#define SRAWI XO31(824)
414#define NEG XO31(104)
1cd62ae9 415#define MFCR XO31( 19)
6995a4a0 416#define MFOCRF (MFCR | (1u << 20))
157f2662 417#define NOR XO31(124)
1cd62ae9 418#define CNTLZW XO31( 26)
419#define CNTLZD XO31( 58)
d0b07481
RH		 420#define CNTTZW XO31(538)
421#define CNTTZD XO31(570)
33e75fb9
RH		 422#define CNTPOPW XO31(378)
423#define CNTPOPD XO31(506)
ce1010d6
RH		 424#define ANDC XO31( 60)
425#define ORC XO31(412)
426#define EQV XO31(284)
427#define NAND XO31(476)
70fac59a 428#define ISEL XO31( 15)
810260a8 429
430#define MULLD XO31(233)
431#define MULHD XO31( 73)
432#define MULHDU XO31( 9)
433#define DIVD XO31(489)
434#define DIVDU XO31(457)
435
436#define LBZX XO31( 87)
4f4a67ae 437#define LHZX XO31(279)
810260a8 438#define LHAX XO31(343)
439#define LWZX XO31( 23)
440#define STBX XO31(215)
441#define STHX XO31(407)
442#define STWX XO31(151)
443
7b4af5ee
PK		 444#define EIEIO XO31(854)
445#define HWSYNC XO31(598)
446#define LWSYNC (HWSYNC | (1u << 21))
447
541dd4ce 448#define SPR(a, b) ((((a)<<5)|(b))<<11)
810260a8 449#define LR SPR(8, 0)
450#define CTR SPR(9, 0)
451
452#define SLW XO31( 24)
453#define SRW XO31(536)
454#define SRAW XO31(792)
455
456#define SLD XO31( 27)
457#define SRD XO31(539)
458#define SRAD XO31(794)
fe6f943f 459#define SRADI XO31(413<<1)
810260a8 460
810260a8 461#define TW XO31( 4)
541dd4ce 462#define TRAP (TW | TO(31))
810260a8 463
a84ac4cb
RH		 464#define NOP ORI /* ori 0,0,0 */
465
6ef14d7e
RH		 466#define LVX XO31(103)
467#define LVEBX XO31(7)
468#define LVEHX XO31(39)
469#define LVEWX XO31(71)
470
471#define STVX XO31(231)
472#define STVEWX XO31(199)
473
d6750811
RH		 474#define VADDUBM VX4(0)
475#define VADDUHM VX4(64)
476#define VADDUWM VX4(128)
477
478#define VSUBUBM VX4(1024)
479#define VSUBUHM VX4(1088)
480#define VSUBUWM VX4(1152)
481
e2382972
RH		 482#define VMAXSB VX4(258)
483#define VMAXSH VX4(322)
484#define VMAXSW VX4(386)
485#define VMAXUB VX4(2)
486#define VMAXUH VX4(66)
487#define VMAXUW VX4(130)
488#define VMINSB VX4(770)
489#define VMINSH VX4(834)
490#define VMINSW VX4(898)
491#define VMINUB VX4(514)
492#define VMINUH VX4(578)
493#define VMINUW VX4(642)
494
6ef14d7e
RH		 495#define VCMPEQUB VX4(6)
496#define VCMPEQUH VX4(70)
497#define VCMPEQUW VX4(134)
498#define VCMPGTSB VX4(774)
499#define VCMPGTSH VX4(838)
500#define VCMPGTSW VX4(902)
501#define VCMPGTUB VX4(518)
502#define VCMPGTUH VX4(582)
503#define VCMPGTUW VX4(646)
504
505#define VAND VX4(1028)
506#define VANDC VX4(1092)
507#define VNOR VX4(1284)
508#define VOR VX4(1156)
509#define VXOR VX4(1220)
510
511#define VSPLTB VX4(524)
512#define VSPLTH VX4(588)
513#define VSPLTW VX4(652)
514#define VSPLTISB VX4(780)
515#define VSPLTISH VX4(844)
516#define VSPLTISW VX4(908)
517
518#define VSLDOI VX4(44)
519
810260a8 520#define RT(r) ((r)<<21)
521#define RS(r) ((r)<<21)
522#define RA(r) ((r)<<16)
523#define RB(r) ((r)<<11)
524#define TO(t) ((t)<<21)
525#define SH(s) ((s)<<11)
526#define MB(b) ((b)<<6)
527#define ME(e) ((e)<<1)
528#define BO(o) ((o)<<21)
529#define MB64(b) ((b)<<5)
6995a4a0 530#define FXM(b) (1 << (19 - (b)))
810260a8 531
b82f769c
RH		 532#define VRT(r) (((r) & 31) << 21)
533#define VRA(r) (((r) & 31) << 16)
534#define VRB(r) (((r) & 31) << 11)
535#define VRC(r) (((r) & 31) << 6)
536
810260a8 537#define LK 1
538
2fd8eddc
RH		 539#define TAB(t, a, b) (RT(t) | RA(a) | RB(b))
540#define SAB(s, a, b) (RS(s) | RA(a) | RB(b))
541#define TAI(s, a, i) (RT(s) | RA(a) | ((i) & 0xffff))
542#define SAI(s, a, i) (RS(s) | RA(a) | ((i) & 0xffff))
810260a8 543
544#define BF(n) ((n)<<23)
545#define BI(n, c) (((c)+((n)*4))<<16)
546#define BT(n, c) (((c)+((n)*4))<<21)
547#define BA(n, c) (((c)+((n)*4))<<16)
548#define BB(n, c) (((c)+((n)*4))<<11)
70fac59a 549#define BC_(n, c) (((c)+((n)*4))<<6)
810260a8 550
541dd4ce
RH		 551#define BO_COND_TRUE BO(12)
552#define BO_COND_FALSE BO( 4)
553#define BO_ALWAYS BO(20)
810260a8 554
555enum {
556 CR_LT,
557 CR_GT,
558 CR_EQ,
559 CR_SO
560};
561
0aed257f 562static const uint32_t tcg_to_bc[] = {
541dd4ce
RH		 563 [TCG_COND_EQ] = BC | BI(7, CR_EQ) | BO_COND_TRUE,
564 [TCG_COND_NE] = BC | BI(7, CR_EQ) | BO_COND_FALSE,
565 [TCG_COND_LT] = BC | BI(7, CR_LT) | BO_COND_TRUE,
566 [TCG_COND_GE] = BC | BI(7, CR_LT) | BO_COND_FALSE,
567 [TCG_COND_LE] = BC | BI(7, CR_GT) | BO_COND_FALSE,
568 [TCG_COND_GT] = BC | BI(7, CR_GT) | BO_COND_TRUE,
569 [TCG_COND_LTU] = BC | BI(7, CR_LT) | BO_COND_TRUE,
570 [TCG_COND_GEU] = BC | BI(7, CR_LT) | BO_COND_FALSE,
571 [TCG_COND_LEU] = BC | BI(7, CR_GT) | BO_COND_FALSE,
572 [TCG_COND_GTU] = BC | BI(7, CR_GT) | BO_COND_TRUE,
810260a8 573};
574
70fac59a
RH		 575/* The low bit here is set if the RA and RB fields must be inverted. */
576static const uint32_t tcg_to_isel[] = {
577 [TCG_COND_EQ] = ISEL | BC_(7, CR_EQ),
578 [TCG_COND_NE] = ISEL | BC_(7, CR_EQ) | 1,
579 [TCG_COND_LT] = ISEL | BC_(7, CR_LT),
580 [TCG_COND_GE] = ISEL | BC_(7, CR_LT) | 1,
581 [TCG_COND_LE] = ISEL | BC_(7, CR_GT) | 1,
582 [TCG_COND_GT] = ISEL | BC_(7, CR_GT),
583 [TCG_COND_LTU] = ISEL | BC_(7, CR_LT),
584 [TCG_COND_GEU] = ISEL | BC_(7, CR_LT) | 1,
585 [TCG_COND_LEU] = ISEL | BC_(7, CR_GT) | 1,
586 [TCG_COND_GTU] = ISEL | BC_(7, CR_GT),
587};
588
6ac17786 589static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
030ffe39
RH		 590 intptr_t value, intptr_t addend)
591{
592 tcg_insn_unit *target;
6ef14d7e
RH		 593 int16_t lo;
594 int32_t hi;
030ffe39
RH		 595
596 value += addend;
597 target = (tcg_insn_unit *)value;
598
599 switch (type) {
600 case R_PPC_REL14:
d5132903 601 return reloc_pc14(code_ptr, target);
030ffe39 602 case R_PPC_REL24:
d5132903 603 return reloc_pc24(code_ptr, target);
030ffe39 604 case R_PPC_ADDR16:
a7cdaf71
RH		 605 /*
606 * We are (slightly) abusing this relocation type. In particular,
607 * assert that the low 2 bits are zero, and do not modify them.
608 * That way we can use this with LD et al that have opcode bits
609 * in the low 2 bits of the insn.
610 */
611 if ((value & 3) || value != (int16_t)value) {
612 return false;
030ffe39 613 }
a7cdaf71 614 *code_ptr = (*code_ptr & ~0xfffc) | (value & 0xfffc);
030ffe39 615 break;
6ef14d7e
RH		 616 case R_PPC_ADDR32:
617 /*
618 * We are abusing this relocation type. Again, this points to
619 * a pair of insns, lis + load. This is an absolute address
620 * relocation for PPC32 so the lis cannot be removed.
621 */
622 lo = value;
623 hi = value - lo;
624 if (hi + lo != value) {
625 return false;
626 }
627 code_ptr[0] = deposit32(code_ptr[0], 0, 16, hi >> 16);
628 code_ptr[1] = deposit32(code_ptr[1], 0, 16, lo);
629 break;
030ffe39
RH		 630 default:
631 g_assert_not_reached();
632 }
6ac17786 633 return true;
030ffe39
RH		 634}
635
a84ac4cb
RH		 636static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
637 TCGReg base, tcg_target_long offset);
638
78113e83 639static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
810260a8 640{
6ef14d7e
RH		 641 if (ret == arg) {
642 return true;
643 }
644 switch (type) {
645 case TCG_TYPE_I64:
646 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
647 /* fallthru */
648 case TCG_TYPE_I32:
649 if (ret < TCG_REG_V0 && arg < TCG_REG_V0) {
650 tcg_out32(s, OR | SAB(arg, ret, arg));
651 break;
652 } else if (ret < TCG_REG_V0 || arg < TCG_REG_V0) {
653 /* Altivec does not support vector/integer moves. */
654 return false;
655 }
656 /* fallthru */
657 case TCG_TYPE_V64:
658 case TCG_TYPE_V128:
659 tcg_debug_assert(ret >= TCG_REG_V0 && arg >= TCG_REG_V0);
660 tcg_out32(s, VOR | VRT(ret) | VRA(arg) | VRB(arg));
661 break;
662 default:
663 g_assert_not_reached();
f8b84129 664 }
78113e83 665 return true;
810260a8 666}
667
aceac8d6
RH		 668static inline void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs,
669 int sh, int mb)
810260a8 670{
eabb7b91 671 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
541dd4ce
RH		 672 sh = SH(sh & 0x1f) | (((sh >> 5) & 1) << 1);
673 mb = MB64((mb >> 5) | ((mb << 1) & 0x3f));
674 tcg_out32(s, op | RA(ra) | RS(rs) | sh | mb);
810260a8 675}
676
9e555b73
RH		 677static inline void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs,
678 int sh, int mb, int me)
679{
680 tcg_out32(s, op | RA(ra) | RS(rs) | SH(sh) | MB(mb) | ME(me));
681}
682
6e5e0602
RH		 683static inline void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
684{
685 tcg_out_rld(s, RLDICL, dst, src, 0, 32);
686}
687
a757e1ee
RH		 688static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
689{
690 tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
691}
692
0a9564b9
RH		 693static inline void tcg_out_shli64(TCGContext *s, TCGReg dst, TCGReg src, int c)
694{
695 tcg_out_rld(s, RLDICR, dst, src, c, 63 - c);
696}
697
a757e1ee
RH		 698static inline void tcg_out_shri32(TCGContext *s, TCGReg dst, TCGReg src, int c)
699{
700 tcg_out_rlw(s, RLWINM, dst, src, 32 - c, c, 31);
701}
702
5e916c28
RH		 703static inline void tcg_out_shri64(TCGContext *s, TCGReg dst, TCGReg src, int c)
704{
705 tcg_out_rld(s, RLDICL, dst, src, 64 - c, c);
706}
707
77bfc7c0
RH		 708/* Emit a move into ret of arg, if it can be done in one insn. */
709static bool tcg_out_movi_one(TCGContext *s, TCGReg ret, tcg_target_long arg)
710{
711 if (arg == (int16_t)arg) {
712 tcg_out32(s, ADDI | TAI(ret, 0, arg));
713 return true;
714 }
715 if (arg == (int32_t)arg && (arg & 0xffff) == 0) {
716 tcg_out32(s, ADDIS | TAI(ret, 0, arg >> 16));
717 return true;
718 }
719 return false;
720}
721
5964fca8
RH		 722static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
723 tcg_target_long arg, bool in_prologue)
810260a8 724{
5964fca8 725 intptr_t tb_diff;
77bfc7c0
RH		 726 tcg_target_long tmp;
727 int shift;
5964fca8
RH		 728
729 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
730
731 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
732 arg = (int32_t)arg;
733 }
734
735 /* Load 16-bit immediates with one insn. */
77bfc7c0 736 if (tcg_out_movi_one(s, ret, arg)) {
5964fca8
RH		 737 return;
738 }
739
740 /* Load addresses within the TB with one insn. */
741 tb_diff = arg - (intptr_t)s->code_gen_ptr;
742 if (!in_prologue && USE_REG_TB && tb_diff == (int16_t)tb_diff) {
743 tcg_out32(s, ADDI | TAI(ret, TCG_REG_TB, tb_diff));
744 return;
745 }
746
77bfc7c0
RH		 747 /* Load 32-bit immediates with two insns. Note that we've already
748 eliminated bare ADDIS, so we know both insns are required. */
5964fca8 749 if (TCG_TARGET_REG_BITS == 32 || arg == (int32_t)arg) {
2fd8eddc 750 tcg_out32(s, ADDIS | TAI(ret, 0, arg >> 16));
77bfc7c0 751 tcg_out32(s, ORI | SAI(ret, ret, arg));
5964fca8 752 return;
810260a8 753 }
5964fca8 754 if (arg == (uint32_t)arg && !(arg & 0x8000)) {
421233a1
RH		 755 tcg_out32(s, ADDI | TAI(ret, 0, arg));
756 tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16));
5964fca8
RH		 757 return;
758 }
a84ac4cb 759
77bfc7c0
RH		 760 /* Load masked 16-bit value. */
761 if (arg > 0 && (arg & 0x8000)) {
762 tmp = arg | 0x7fff;
763 if ((tmp & (tmp + 1)) == 0) {
764 int mb = clz64(tmp + 1) + 1;
765 tcg_out32(s, ADDI | TAI(ret, 0, arg));
766 tcg_out_rld(s, RLDICL, ret, ret, 0, mb);
767 return;
768 }
769 }
770
771 /* Load common masks with 2 insns. */
772 shift = ctz64(arg);
773 tmp = arg >> shift;
774 if (tmp == (int16_t)tmp) {
775 tcg_out32(s, ADDI | TAI(ret, 0, tmp));
776 tcg_out_shli64(s, ret, ret, shift);
777 return;
778 }
779 shift = clz64(arg);
780 if (tcg_out_movi_one(s, ret, arg << shift)) {
781 tcg_out_shri64(s, ret, ret, shift);
782 return;
783 }
784
5964fca8
RH		 785 /* Load addresses within 2GB of TB with 2 (or rarely 3) insns. */
786 if (!in_prologue && USE_REG_TB && tb_diff == (int32_t)tb_diff) {
787 tcg_out_mem_long(s, ADDI, ADD, ret, TCG_REG_TB, tb_diff);
788 return;
789 }
a84ac4cb 790
53c89efd
RH		 791 /* Use the constant pool, if possible. */
792 if (!in_prologue && USE_REG_TB) {
793 new_pool_label(s, arg, R_PPC_ADDR16, s->code_ptr,
794 -(intptr_t)s->code_gen_ptr);
a7cdaf71 795 tcg_out32(s, LD | TAI(ret, TCG_REG_TB, 0));
53c89efd
RH		 796 return;
797 }
798
77bfc7c0
RH		 799 tmp = arg >> 31 >> 1;
800 tcg_out_movi(s, TCG_TYPE_I32, ret, tmp);
801 if (tmp) {
5964fca8 802 tcg_out_shli64(s, ret, ret, 32);
810260a8 803 }
5964fca8
RH		 804 if (arg & 0xffff0000) {
805 tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16));
806 }
807 if (arg & 0xffff) {
808 tcg_out32(s, ORI | SAI(ret, ret, arg));
809 }
810}
811
4b06c216
RH		 812static void tcg_out_dupi_vec(TCGContext *s, TCGType type, TCGReg ret,
813 tcg_target_long val)
5964fca8 814{
6ef14d7e
RH		 815 uint32_t load_insn;
816 int rel, low;
817 intptr_t add;
818
819 low = (int8_t)val;
820 if (low >= -16 && low < 16) {
821 if (val == (tcg_target_long)dup_const(MO_8, low)) {
822 tcg_out32(s, VSPLTISB | VRT(ret) | ((val & 31) << 16));
823 return;
824 }
825 if (val == (tcg_target_long)dup_const(MO_16, low)) {
826 tcg_out32(s, VSPLTISH | VRT(ret) | ((val & 31) << 16));
827 return;
828 }
829 if (val == (tcg_target_long)dup_const(MO_32, low)) {
830 tcg_out32(s, VSPLTISW | VRT(ret) | ((val & 31) << 16));
831 return;
832 }
833 }
834
835 /*
836 * Otherwise we must load the value from the constant pool.
837 */
838 if (USE_REG_TB) {
839 rel = R_PPC_ADDR16;
840 add = -(intptr_t)s->code_gen_ptr;
841 } else {
842 rel = R_PPC_ADDR32;
843 add = 0;
844 }
845
846 load_insn = LVX | VRT(ret) | RB(TCG_REG_TMP1);
847 if (TCG_TARGET_REG_BITS == 64) {
848 new_pool_l2(s, rel, s->code_ptr, add, val, val);
849 } else {
850 new_pool_l4(s, rel, s->code_ptr, add, val, val, val, val);
851 }
852
853 if (USE_REG_TB) {
854 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, 0, 0));
855 load_insn |= RA(TCG_REG_TB);
856 } else {
857 tcg_out32(s, ADDIS | TAI(TCG_REG_TMP1, 0, 0));
858 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, TCG_REG_TMP1, 0));
859 }
860 tcg_out32(s, load_insn);
4b06c216
RH		 861}
862
863static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret,
864 tcg_target_long arg)
865{
866 switch (type) {
867 case TCG_TYPE_I32:
868 case TCG_TYPE_I64:
869 tcg_debug_assert(ret < TCG_REG_V0);
870 tcg_out_movi_int(s, type, ret, arg, false);
871 break;
872
873 case TCG_TYPE_V64:
874 case TCG_TYPE_V128:
875 tcg_debug_assert(ret >= TCG_REG_V0);
876 tcg_out_dupi_vec(s, type, ret, arg);
877 break;
878
879 default:
880 g_assert_not_reached();
881 }
810260a8 882}
883
637af30c 884static bool mask_operand(uint32_t c, int *mb, int *me)
a9249dff
RH		 885{
886 uint32_t lsb, test;
887
888 /* Accept a bit pattern like:
889 0....01....1
890 1....10....0
891 0..01..10..0
892 Keep track of the transitions. */
893 if (c == 0 || c == -1) {
894 return false;
895 }
896 test = c;
897 lsb = test & -test;
898 test += lsb;
899 if (test & (test - 1)) {
900 return false;
901 }
902
903 *me = clz32(lsb);
904 *mb = test ? clz32(test & -test) + 1 : 0;
905 return true;
906}
907
637af30c
RH		 908static bool mask64_operand(uint64_t c, int *mb, int *me)
909{
910 uint64_t lsb;
911
912 if (c == 0) {
913 return false;
914 }
915
916 lsb = c & -c;
917 /* Accept 1..10..0. */
918 if (c == -lsb) {
919 *mb = 0;
920 *me = clz64(lsb);
921 return true;
922 }
923 /* Accept 0..01..1. */
924 if (lsb == 1 && (c & (c + 1)) == 0) {
925 *mb = clz64(c + 1) + 1;
926 *me = 63;
927 return true;
928 }
929 return false;
930}
931
a9249dff
RH		 932static void tcg_out_andi32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
933{
934 int mb, me;
935
1e1df962
RH		 936 if (mask_operand(c, &mb, &me)) {
937 tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
938 } else if ((c & 0xffff) == c) {
a9249dff
RH		 939 tcg_out32(s, ANDI | SAI(src, dst, c));
940 return;
941 } else if ((c & 0xffff0000) == c) {
942 tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
943 return;
a9249dff 944 } else {
8327a470
RH		 945 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R0, c);
946 tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
a9249dff
RH		 947 }
948}
949
637af30c
RH		 950static void tcg_out_andi64(TCGContext *s, TCGReg dst, TCGReg src, uint64_t c)
951{
952 int mb, me;
953
eabb7b91 954 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
1e1df962 955 if (mask64_operand(c, &mb, &me)) {
637af30c
RH		 956 if (mb == 0) {
957 tcg_out_rld(s, RLDICR, dst, src, 0, me);
958 } else {
959 tcg_out_rld(s, RLDICL, dst, src, 0, mb);
960 }
1e1df962
RH		 961 } else if ((c & 0xffff) == c) {
962 tcg_out32(s, ANDI | SAI(src, dst, c));
963 return;
964 } else if ((c & 0xffff0000) == c) {
965 tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
966 return;
637af30c 967 } else {
8327a470
RH		 968 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, c);
969 tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
637af30c
RH		 970 }
971}
972
dce74c57
RH		 973static void tcg_out_zori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c,
974 int op_lo, int op_hi)
975{
976 if (c >> 16) {
977 tcg_out32(s, op_hi | SAI(src, dst, c >> 16));
978 src = dst;
979 }
980 if (c & 0xffff) {
981 tcg_out32(s, op_lo | SAI(src, dst, c));
982 src = dst;
983 }
984}
985
986static void tcg_out_ori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
987{
988 tcg_out_zori32(s, dst, src, c, ORI, ORIS);
989}
990
991static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
992{
993 tcg_out_zori32(s, dst, src, c, XORI, XORIS);
994}
995
e083c4a2 996static void tcg_out_b(TCGContext *s, int mask, tcg_insn_unit *target)
5d7ff5bb 997{
e083c4a2 998 ptrdiff_t disp = tcg_pcrel_diff(s, target);
b0940da0 999 if (in_range_b(disp)) {
541dd4ce
RH		 1000 tcg_out32(s, B | (disp & 0x3fffffc) | mask);
1001 } else {
de3d636d 1002 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, (uintptr_t)target);
8327a470 1003 tcg_out32(s, MTSPR | RS(TCG_REG_R0) | CTR);
541dd4ce 1004 tcg_out32(s, BCCTR | BO_ALWAYS | mask);
5d7ff5bb
AF		 1005 }
1006}
1007
b18d5d2b
RH		 1008static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
1009 TCGReg base, tcg_target_long offset)
810260a8 1010{
b18d5d2b 1011 tcg_target_long orig = offset, l0, l1, extra = 0, align = 0;
de7761a3 1012 bool is_store = false;
dfca1778 1013 TCGReg rs = TCG_REG_TMP1;
b18d5d2b 1014
b18d5d2b
RH		 1015 switch (opi) {
1016 case LD: case LWA:
1017 align = 3;
1018 /* FALLTHRU */
1019 default:
6ef14d7e 1020 if (rt > TCG_REG_R0 && rt < TCG_REG_V0) {
b18d5d2b 1021 rs = rt;
de7761a3 1022 break;
b18d5d2b
RH		 1023 }
1024 break;
1025 case STD:
1026 align = 3;
de7761a3 1027 /* FALLTHRU */
b18d5d2b 1028 case STB: case STH: case STW:
de7761a3 1029 is_store = true;
b18d5d2b 1030 break;
810260a8 1031 }
810260a8 1032
b18d5d2b 1033 /* For unaligned, or very large offsets, use the indexed form. */
6ef14d7e 1034 if (offset & align || offset != (int32_t)offset || opi == 0) {
d4cba13b
RH		 1035 if (rs == base) {
1036 rs = TCG_REG_R0;
1037 }
1038 tcg_debug_assert(!is_store || rs != rt);
de7761a3 1039 tcg_out_movi(s, TCG_TYPE_PTR, rs, orig);
6ef14d7e 1040 tcg_out32(s, opx | TAB(rt & 31, base, rs));
b18d5d2b
RH		 1041 return;
1042 }
1043
1044 l0 = (int16_t)offset;
1045 offset = (offset - l0) >> 16;
1046 l1 = (int16_t)offset;
1047
1048 if (l1 < 0 && orig >= 0) {
1049 extra = 0x4000;
1050 l1 = (int16_t)(offset - 0x4000);
1051 }
1052 if (l1) {
1053 tcg_out32(s, ADDIS | TAI(rs, base, l1));
1054 base = rs;
1055 }
1056 if (extra) {
1057 tcg_out32(s, ADDIS | TAI(rs, base, extra));
1058 base = rs;
1059 }
1060 if (opi != ADDI || base != rt || l0 != 0) {
6ef14d7e 1061 tcg_out32(s, opi | TAI(rt & 31, base, l0));
828808f5 1062 }
1063}
1064
6ef14d7e
RH		 1065static void tcg_out_vsldoi(TCGContext *s, TCGReg ret,
1066 TCGReg va, TCGReg vb, int shb)
d604f1a9 1067{
6ef14d7e
RH		 1068 tcg_out32(s, VSLDOI | VRT(ret) | VRA(va) | VRB(vb) | (shb << 6));
1069}
810260a8 1070
6ef14d7e
RH		 1071static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
1072 TCGReg base, intptr_t offset)
1073{
1074 int shift;
1075
1076 switch (type) {
1077 case TCG_TYPE_I32:
1078 if (ret < TCG_REG_V0) {
1079 tcg_out_mem_long(s, LWZ, LWZX, ret, base, offset);
1080 break;
1081 }
1082 tcg_debug_assert((offset & 3) == 0);
1083 tcg_out_mem_long(s, 0, LVEWX, ret, base, offset);
1084 shift = (offset - 4) & 0xc;
1085 if (shift) {
1086 tcg_out_vsldoi(s, ret, ret, ret, shift);
1087 }
1088 break;
1089 case TCG_TYPE_I64:
1090 if (ret < TCG_REG_V0) {
1091 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
1092 tcg_out_mem_long(s, LD, LDX, ret, base, offset);
1093 break;
1094 }
1095 /* fallthru */
1096 case TCG_TYPE_V64:
1097 tcg_debug_assert(ret >= TCG_REG_V0);
1098 tcg_debug_assert((offset & 7) == 0);
1099 tcg_out_mem_long(s, 0, LVX, ret, base, offset & -16);
1100 if (offset & 8) {
1101 tcg_out_vsldoi(s, ret, ret, ret, 8);
1102 }
1103 break;
1104 case TCG_TYPE_V128:
1105 tcg_debug_assert(ret >= TCG_REG_V0);
1106 tcg_debug_assert((offset & 15) == 0);
1107 tcg_out_mem_long(s, 0, LVX, ret, base, offset);
1108 break;
1109 default:
1110 g_assert_not_reached();
d604f1a9 1111 }
d604f1a9 1112}
fedee3e7 1113
6ef14d7e
RH		 1114static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
1115 TCGReg base, intptr_t offset)
810260a8 1116{
6ef14d7e 1117 int shift;
fedee3e7 1118
6ef14d7e
RH		 1119 switch (type) {
1120 case TCG_TYPE_I32:
1121 if (arg < TCG_REG_V0) {
1122 tcg_out_mem_long(s, STW, STWX, arg, base, offset);
1123 break;
1124 }
1125 tcg_debug_assert((offset & 3) == 0);
1126 shift = (offset - 4) & 0xc;
1127 if (shift) {
1128 tcg_out_vsldoi(s, TCG_VEC_TMP1, arg, arg, shift);
1129 arg = TCG_VEC_TMP1;
1130 }
1131 tcg_out_mem_long(s, 0, STVEWX, arg, base, offset);
1132 break;
1133 case TCG_TYPE_I64:
1134 if (arg < TCG_REG_V0) {
1135 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
1136 tcg_out_mem_long(s, STD, STDX, arg, base, offset);
1137 break;
1138 }
1139 /* fallthru */
1140 case TCG_TYPE_V64:
1141 tcg_debug_assert(arg >= TCG_REG_V0);
1142 tcg_debug_assert((offset & 7) == 0);
1143 if (offset & 8) {
1144 tcg_out_vsldoi(s, TCG_VEC_TMP1, arg, arg, 8);
1145 arg = TCG_VEC_TMP1;
1146 }
1147 tcg_out_mem_long(s, 0, STVEWX, arg, base, offset);
1148 tcg_out_mem_long(s, 0, STVEWX, arg, base, offset + 4);
1149 break;
1150 case TCG_TYPE_V128:
1151 tcg_debug_assert(arg >= TCG_REG_V0);
1152 tcg_out_mem_long(s, 0, STVX, arg, base, offset);
1153 break;
1154 default:
1155 g_assert_not_reached();
fedee3e7 1156 }
d604f1a9 1157}
810260a8 1158
59d7c14e
RH		 1159static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1160 TCGReg base, intptr_t ofs)
1161{
1162 return false;
1163}
1164
d604f1a9
RH		 1165static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
1166 int const_arg2, int cr, TCGType type)
1167{
1168 int imm;
1169 uint32_t op;
810260a8 1170
abcf61c4
RH		 1171 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
1172
d604f1a9
RH		 1173 /* Simplify the comparisons below wrt CMPI. */
1174 if (type == TCG_TYPE_I32) {
1175 arg2 = (int32_t)arg2;
4a40e231 1176 }
fedee3e7 1177
d604f1a9
RH		 1178 switch (cond) {
1179 case TCG_COND_EQ:
1180 case TCG_COND_NE:
1181 if (const_arg2) {
1182 if ((int16_t) arg2 == arg2) {
1183 op = CMPI;
1184 imm = 1;
1185 break;
1186 } else if ((uint16_t) arg2 == arg2) {
1187 op = CMPLI;
1188 imm = 1;
1189 break;
1190 }
1191 }
1192 op = CMPL;
1193 imm = 0;
1194 break;
fedee3e7 1195
d604f1a9
RH		 1196 case TCG_COND_LT:
1197 case TCG_COND_GE:
1198 case TCG_COND_LE:
1199 case TCG_COND_GT:
1200 if (const_arg2) {
1201 if ((int16_t) arg2 == arg2) {
1202 op = CMPI;
1203 imm = 1;
1204 break;
1205 }
1206 }
1207 op = CMP;
1208 imm = 0;
1209 break;
fedee3e7 1210
d604f1a9
RH		 1211 case TCG_COND_LTU:
1212 case TCG_COND_GEU:
1213 case TCG_COND_LEU:
1214 case TCG_COND_GTU:
1215 if (const_arg2) {
1216 if ((uint16_t) arg2 == arg2) {
1217 op = CMPLI;
1218 imm = 1;
1219 break;
1220 }
1221 }
1222 op = CMPL;
1223 imm = 0;
1224 break;
fedee3e7 1225
d604f1a9
RH		 1226 default:
1227 tcg_abort();
fedee3e7 1228 }
d604f1a9 1229 op |= BF(cr) | ((type == TCG_TYPE_I64) << 21);
fedee3e7 1230
d604f1a9
RH		 1231 if (imm) {
1232 tcg_out32(s, op | RA(arg1) | (arg2 & 0xffff));
1233 } else {
1234 if (const_arg2) {
1235 tcg_out_movi(s, type, TCG_REG_R0, arg2);
1236 arg2 = TCG_REG_R0;
1237 }
1238 tcg_out32(s, op | RA(arg1) | RB(arg2));
1239 }
810260a8 1240}
1241
d604f1a9
RH		 1242static void tcg_out_setcond_eq0(TCGContext *s, TCGType type,
1243 TCGReg dst, TCGReg src)
7f12d649 1244{
a757e1ee
RH		 1245 if (type == TCG_TYPE_I32) {
1246 tcg_out32(s, CNTLZW | RS(src) | RA(dst));
1247 tcg_out_shri32(s, dst, dst, 5);
1248 } else {
1249 tcg_out32(s, CNTLZD | RS(src) | RA(dst));
1250 tcg_out_shri64(s, dst, dst, 6);
1251 }
7f12d649
RH		 1252}
1253
d604f1a9 1254static void tcg_out_setcond_ne0(TCGContext *s, TCGReg dst, TCGReg src)
7f12d649 1255{
d604f1a9
RH		 1256 /* X != 0 implies X + -1 generates a carry. Extra addition
1257 trickery means: R = X-1 + ~X + C = X-1 + (-X+1) + C = C. */
1258 if (dst != src) {
1259 tcg_out32(s, ADDIC | TAI(dst, src, -1));
1260 tcg_out32(s, SUBFE | TAB(dst, dst, src));
7f12d649 1261 } else {
d604f1a9
RH		 1262 tcg_out32(s, ADDIC | TAI(TCG_REG_R0, src, -1));
1263 tcg_out32(s, SUBFE | TAB(dst, TCG_REG_R0, src));
7f12d649 1264 }
d604f1a9 1265}
7f12d649 1266
d604f1a9
RH		 1267static TCGReg tcg_gen_setcond_xor(TCGContext *s, TCGReg arg1, TCGArg arg2,
1268 bool const_arg2)
1269{
1270 if (const_arg2) {
1271 if ((uint32_t)arg2 == arg2) {
1272 tcg_out_xori32(s, TCG_REG_R0, arg1, arg2);
1273 } else {
1274 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, arg2);
1275 tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, TCG_REG_R0));
1276 }
1277 } else {
1278 tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, arg2));
1279 }
1280 return TCG_REG_R0;
7f12d649
RH		 1281}
1282
d604f1a9
RH		 1283static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
1284 TCGArg arg0, TCGArg arg1, TCGArg arg2,
1285 int const_arg2)
7f12d649 1286{
d604f1a9 1287 int crop, sh;
7f12d649 1288
eabb7b91 1289 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
a757e1ee 1290
d604f1a9
RH		 1291 /* Ignore high bits of a potential constant arg2. */
1292 if (type == TCG_TYPE_I32) {
1293 arg2 = (uint32_t)arg2;
1294 }
7f12d649 1295
d604f1a9
RH		 1296 /* Handle common and trivial cases before handling anything else. */
1297 if (arg2 == 0) {
1298 switch (cond) {
1299 case TCG_COND_EQ:
1300 tcg_out_setcond_eq0(s, type, arg0, arg1);
1301 return;
1302 case TCG_COND_NE:
a757e1ee 1303 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
d604f1a9
RH		 1304 tcg_out_ext32u(s, TCG_REG_R0, arg1);
1305 arg1 = TCG_REG_R0;
1306 }
1307 tcg_out_setcond_ne0(s, arg0, arg1);
1308 return;
1309 case TCG_COND_GE:
1310 tcg_out32(s, NOR | SAB(arg1, arg0, arg1));
1311 arg1 = arg0;
1312 /* FALLTHRU */
1313 case TCG_COND_LT:
1314 /* Extract the sign bit. */
a757e1ee
RH		 1315 if (type == TCG_TYPE_I32) {
1316 tcg_out_shri32(s, arg0, arg1, 31);
1317 } else {
1318 tcg_out_shri64(s, arg0, arg1, 63);
1319 }
d604f1a9
RH		 1320 return;
1321 default:
1322 break;
1323 }
1324 }
7f12d649 1325
d604f1a9
RH		 1326 /* If we have ISEL, we can implement everything with 3 or 4 insns.
1327 All other cases below are also at least 3 insns, so speed up the
1328 code generator by not considering them and always using ISEL. */
63922f46 1329 if (have_isel) {
d604f1a9 1330 int isel, tab;
7f12d649 1331
d604f1a9 1332 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
7f12d649 1333
d604f1a9 1334 isel = tcg_to_isel[cond];
7f12d649 1335
d604f1a9
RH		 1336 tcg_out_movi(s, type, arg0, 1);
1337 if (isel & 1) {
1338 /* arg0 = (bc ? 0 : 1) */
1339 tab = TAB(arg0, 0, arg0);
1340 isel &= ~1;
1341 } else {
1342 /* arg0 = (bc ? 1 : 0) */
1343 tcg_out_movi(s, type, TCG_REG_R0, 0);
1344 tab = TAB(arg0, arg0, TCG_REG_R0);
1345 }
1346 tcg_out32(s, isel | tab);
1347 return;
1348 }
49d9870a 1349
d604f1a9
RH		 1350 switch (cond) {
1351 case TCG_COND_EQ:
1352 arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
1353 tcg_out_setcond_eq0(s, type, arg0, arg1);
1354 return;
810260a8 1355
d604f1a9
RH		 1356 case TCG_COND_NE:
1357 arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
1358 /* Discard the high bits only once, rather than both inputs. */
a757e1ee 1359 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
d604f1a9
RH		 1360 tcg_out_ext32u(s, TCG_REG_R0, arg1);
1361 arg1 = TCG_REG_R0;
1362 }
1363 tcg_out_setcond_ne0(s, arg0, arg1);
1364 return;
810260a8 1365
d604f1a9
RH		 1366 case TCG_COND_GT:
1367 case TCG_COND_GTU:
1368 sh = 30;
1369 crop = 0;
1370 goto crtest;
810260a8 1371
d604f1a9
RH		 1372 case TCG_COND_LT:
1373 case TCG_COND_LTU:
1374 sh = 29;
1375 crop = 0;
1376 goto crtest;
810260a8 1377
d604f1a9
RH		 1378 case TCG_COND_GE:
1379 case TCG_COND_GEU:
1380 sh = 31;
1381 crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_LT) | BB(7, CR_LT);
1382 goto crtest;
810260a8 1383
d604f1a9
RH		 1384 case TCG_COND_LE:
1385 case TCG_COND_LEU:
1386 sh = 31;
1387 crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_GT) | BB(7, CR_GT);
1388 crtest:
1389 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
1390 if (crop) {
1391 tcg_out32(s, crop);
1392 }
1393 tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7));
1394 tcg_out_rlw(s, RLWINM, arg0, TCG_REG_R0, sh, 31, 31);
1395 break;
1396
1397 default:
1398 tcg_abort();
1399 }
810260a8 1400}
1401
bec16311 1402static void tcg_out_bc(TCGContext *s, int bc, TCGLabel *l)
810260a8 1403{
d604f1a9 1404 if (l->has_value) {
f9c7246f 1405 bc |= reloc_pc14_val(s->code_ptr, l->u.value_ptr);
49d9870a 1406 } else {
bec16311 1407 tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, l, 0);
810260a8 1408 }
f9c7246f 1409 tcg_out32(s, bc);
810260a8 1410}
1411
d604f1a9
RH		 1412static void tcg_out_brcond(TCGContext *s, TCGCond cond,
1413 TCGArg arg1, TCGArg arg2, int const_arg2,
bec16311 1414 TCGLabel *l, TCGType type)
810260a8 1415{
d604f1a9 1416 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
bec16311 1417 tcg_out_bc(s, tcg_to_bc[cond], l);
d604f1a9 1418}
fa94c3be 1419
d604f1a9
RH		 1420static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond,
1421 TCGArg dest, TCGArg c1, TCGArg c2, TCGArg v1,
1422 TCGArg v2, bool const_c2)
1423{
1424 /* If for some reason both inputs are zero, don't produce bad code. */
1425 if (v1 == 0 && v2 == 0) {
1426 tcg_out_movi(s, type, dest, 0);
1427 return;
b9e946c7 1428 }
f6548c0a 1429
d604f1a9 1430 tcg_out_cmp(s, cond, c1, c2, const_c2, 7, type);
a69abbe0 1431
63922f46 1432 if (have_isel) {
d604f1a9 1433 int isel = tcg_to_isel[cond];
810260a8 1434
d604f1a9
RH		 1435 /* Swap the V operands if the operation indicates inversion. */
1436 if (isel & 1) {
1437 int t = v1;
1438 v1 = v2;
1439 v2 = t;
1440 isel &= ~1;
1441 }
1442 /* V1 == 0 is handled by isel; V2 == 0 must be handled by hand. */
1443 if (v2 == 0) {
1444 tcg_out_movi(s, type, TCG_REG_R0, 0);
1445 }
1446 tcg_out32(s, isel | TAB(dest, v1, v2));
1447 } else {
1448 if (dest == v2) {
1449 cond = tcg_invert_cond(cond);
1450 v2 = v1;
1451 } else if (dest != v1) {
1452 if (v1 == 0) {
1453 tcg_out_movi(s, type, dest, 0);
1454 } else {
1455 tcg_out_mov(s, type, dest, v1);
1456 }
1457 }
1458 /* Branch forward over one insn */
1459 tcg_out32(s, tcg_to_bc[cond] | 8);
1460 if (v2 == 0) {
1461 tcg_out_movi(s, type, dest, 0);
1462 } else {
1463 tcg_out_mov(s, type, dest, v2);
1464 }
29b69198 1465 }
810260a8 1466}
1467
d0b07481
RH		 1468static void tcg_out_cntxz(TCGContext *s, TCGType type, uint32_t opc,
1469 TCGArg a0, TCGArg a1, TCGArg a2, bool const_a2)
1470{
1471 if (const_a2 && a2 == (type == TCG_TYPE_I32 ? 32 : 64)) {
1472 tcg_out32(s, opc | RA(a0) | RS(a1));
1473 } else {
1474 tcg_out_cmp(s, TCG_COND_EQ, a1, 0, 1, 7, type);
1475 /* Note that the only other valid constant for a2 is 0. */
63922f46 1476 if (have_isel) {
d0b07481
RH		 1477 tcg_out32(s, opc | RA(TCG_REG_R0) | RS(a1));
1478 tcg_out32(s, tcg_to_isel[TCG_COND_EQ] | TAB(a0, a2, TCG_REG_R0));
1479 } else if (!const_a2 && a0 == a2) {
1480 tcg_out32(s, tcg_to_bc[TCG_COND_EQ] | 8);
1481 tcg_out32(s, opc | RA(a0) | RS(a1));
1482 } else {
1483 tcg_out32(s, opc | RA(a0) | RS(a1));
1484 tcg_out32(s, tcg_to_bc[TCG_COND_NE] | 8);
1485 if (const_a2) {
1486 tcg_out_movi(s, type, a0, 0);
1487 } else {
1488 tcg_out_mov(s, type, a0, a2);
1489 }
1490 }
1491 }
1492}
1493
abcf61c4
RH		 1494static void tcg_out_cmp2(TCGContext *s, const TCGArg *args,
1495 const int *const_args)
1496{
1497 static const struct { uint8_t bit1, bit2; } bits[] = {
1498 [TCG_COND_LT ] = { CR_LT, CR_LT },
1499 [TCG_COND_LE ] = { CR_LT, CR_GT },
1500 [TCG_COND_GT ] = { CR_GT, CR_GT },
1501 [TCG_COND_GE ] = { CR_GT, CR_LT },
1502 [TCG_COND_LTU] = { CR_LT, CR_LT },
1503 [TCG_COND_LEU] = { CR_LT, CR_GT },
1504 [TCG_COND_GTU] = { CR_GT, CR_GT },
1505 [TCG_COND_GEU] = { CR_GT, CR_LT },
1506 };
1507
1508 TCGCond cond = args[4], cond2;
1509 TCGArg al, ah, bl, bh;
1510 int blconst, bhconst;
1511 int op, bit1, bit2;
1512
1513 al = args[0];
1514 ah = args[1];
1515 bl = args[2];
1516 bh = args[3];
1517 blconst = const_args[2];
1518 bhconst = const_args[3];
1519
1520 switch (cond) {
1521 case TCG_COND_EQ:
1522 op = CRAND;
1523 goto do_equality;
1524 case TCG_COND_NE:
1525 op = CRNAND;
1526 do_equality:
1527 tcg_out_cmp(s, cond, al, bl, blconst, 6, TCG_TYPE_I32);
1528 tcg_out_cmp(s, cond, ah, bh, bhconst, 7, TCG_TYPE_I32);
1529 tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
1530 break;
1531
1532 case TCG_COND_LT:
1533 case TCG_COND_LE:
1534 case TCG_COND_GT:
1535 case TCG_COND_GE:
1536 case TCG_COND_LTU:
1537 case TCG_COND_LEU:
1538 case TCG_COND_GTU:
1539 case TCG_COND_GEU:
1540 bit1 = bits[cond].bit1;
1541 bit2 = bits[cond].bit2;
1542 op = (bit1 != bit2 ? CRANDC : CRAND);
1543 cond2 = tcg_unsigned_cond(cond);
1544
1545 tcg_out_cmp(s, cond, ah, bh, bhconst, 6, TCG_TYPE_I32);
1546 tcg_out_cmp(s, cond2, al, bl, blconst, 7, TCG_TYPE_I32);
1547 tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, bit2));
1548 tcg_out32(s, CROR | BT(7, CR_EQ) | BA(6, bit1) | BB(7, CR_EQ));
1549 break;
1550
1551 default:
1552 tcg_abort();
1553 }
1554}
1555
1556static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
1557 const int *const_args)
1558{
1559 tcg_out_cmp2(s, args + 1, const_args + 1);
1560 tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7));
1561 tcg_out_rlw(s, RLWINM, args[0], TCG_REG_R0, 31, 31, 31);
1562}
1563
1564static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args,
1565 const int *const_args)
1566{
1567 tcg_out_cmp2(s, args, const_args);
bec16311 1568 tcg_out_bc(s, BC | BI(7, CR_EQ) | BO_COND_TRUE, arg_label(args[5]));
abcf61c4
RH		 1569}
1570
7b4af5ee
PK		 1571static void tcg_out_mb(TCGContext *s, TCGArg a0)
1572{
1573 uint32_t insn = HWSYNC;
1574 a0 &= TCG_MO_ALL;
1575 if (a0 == TCG_MO_LD_LD) {
1576 insn = LWSYNC;
1577 } else if (a0 == TCG_MO_ST_ST) {
1578 insn = EIEIO;
1579 }
1580 tcg_out32(s, insn);
1581}
1582
a8583393
RH		 1583void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
1584 uintptr_t addr)
810260a8 1585{
5964fca8
RH		 1586 if (TCG_TARGET_REG_BITS == 64) {
1587 tcg_insn_unit i1, i2;
1588 intptr_t tb_diff = addr - tc_ptr;
1589 intptr_t br_diff = addr - (jmp_addr + 4);
1590 uint64_t pair;
1591
1592 /* This does not exercise the range of the branch, but we do
1593 still need to be able to load the new value of TCG_REG_TB.
1594 But this does still happen quite often. */
1595 if (tb_diff == (int16_t)tb_diff) {
1596 i1 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, tb_diff);
1597 i2 = B | (br_diff & 0x3fffffc);
1598 } else {
1599 intptr_t lo = (int16_t)tb_diff;
1600 intptr_t hi = (int32_t)(tb_diff - lo);
1601 assert(tb_diff == hi + lo);
1602 i1 = ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, hi >> 16);
1603 i2 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, lo);
1604 }
5bfd75a3 1605#ifdef HOST_WORDS_BIGENDIAN
5964fca8 1606 pair = (uint64_t)i1 << 32 | i2;
5bfd75a3 1607#else
5964fca8 1608 pair = (uint64_t)i2 << 32 | i1;
5bfd75a3
RH		 1609#endif
1610
ba026602
PMD		 1611 /* As per the enclosing if, this is ppc64. Avoid the _Static_assert
1612 within atomic_set that would fail to build a ppc32 host. */
1613 atomic_set__nocheck((uint64_t *)jmp_addr, pair);
5964fca8
RH		 1614 flush_icache_range(jmp_addr, jmp_addr + 8);
1615 } else {
1616 intptr_t diff = addr - jmp_addr;
1617 tcg_debug_assert(in_range_b(diff));
1618 atomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc));
1619 flush_icache_range(jmp_addr, jmp_addr + 4);
1620 }
399f1648 1621}
810260a8 1622
d604f1a9 1623static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
810260a8 1624{
eaf7d1cf 1625#ifdef _CALL_AIX
d604f1a9
RH		 1626 /* Look through the descriptor. If the branch is in range, and we
1627 don't have to spend too much effort on building the toc. */
1628 void *tgt = ((void **)target)[0];
1629 uintptr_t toc = ((uintptr_t *)target)[1];
1630 intptr_t diff = tcg_pcrel_diff(s, tgt);
b18d5d2b 1631
d604f1a9 1632 if (in_range_b(diff) && toc == (uint32_t)toc) {
dfca1778 1633 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, toc);
d604f1a9 1634 tcg_out_b(s, LK, tgt);
541dd4ce 1635 } else {
d604f1a9
RH		 1636 /* Fold the low bits of the constant into the addresses below. */
1637 intptr_t arg = (intptr_t)target;
1638 int ofs = (int16_t)arg;
1639
1640 if (ofs + 8 < 0x8000) {
1641 arg -= ofs;
1642 } else {
1643 ofs = 0;
1644 }
dfca1778
RH		 1645 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, arg);
1646 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_TMP1, ofs);
d604f1a9 1647 tcg_out32(s, MTSPR | RA(TCG_REG_R0) | CTR);
dfca1778 1648 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_REG_TMP1, ofs + SZP);
d604f1a9 1649 tcg_out32(s, BCCTR | BO_ALWAYS | LK);
541dd4ce 1650 }
77e58d0d
UW		 1651#elif defined(_CALL_ELF) && _CALL_ELF == 2
1652 intptr_t diff;
1653
1654 /* In the ELFv2 ABI, we have to set up r12 to contain the destination
1655 address, which the callee uses to compute its TOC address. */
1656 /* FIXME: when the branch is in range, we could avoid r12 load if we
1657 knew that the destination uses the same TOC, and what its local
1658 entry point offset is. */
1659 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R12, (intptr_t)target);
1660
1661 diff = tcg_pcrel_diff(s, target);
1662 if (in_range_b(diff)) {
1663 tcg_out_b(s, LK, target);
1664 } else {
1665 tcg_out32(s, MTSPR | RS(TCG_REG_R12) | CTR);
1666 tcg_out32(s, BCCTR | BO_ALWAYS | LK);
1667 }
eaf7d1cf
RH		 1668#else
1669 tcg_out_b(s, LK, target);
d604f1a9 1670#endif
810260a8 1671}
1672
d604f1a9
RH		 1673static const uint32_t qemu_ldx_opc[16] = {
1674 [MO_UB] = LBZX,
1675 [MO_UW] = LHZX,
1676 [MO_UL] = LWZX,
1677 [MO_Q] = LDX,
1678 [MO_SW] = LHAX,
1679 [MO_SL] = LWAX,
1680 [MO_BSWAP | MO_UB] = LBZX,
1681 [MO_BSWAP | MO_UW] = LHBRX,
1682 [MO_BSWAP | MO_UL] = LWBRX,
1683 [MO_BSWAP | MO_Q] = LDBRX,
1684};
810260a8 1685
d604f1a9
RH		 1686static const uint32_t qemu_stx_opc[16] = {
1687 [MO_UB] = STBX,
1688 [MO_UW] = STHX,
1689 [MO_UL] = STWX,
1690 [MO_Q] = STDX,
1691 [MO_BSWAP | MO_UB] = STBX,
1692 [MO_BSWAP | MO_UW] = STHBRX,
1693 [MO_BSWAP | MO_UL] = STWBRX,
1694 [MO_BSWAP | MO_Q] = STDBRX,
1695};
991041a4 1696
d604f1a9
RH		 1697static const uint32_t qemu_exts_opc[4] = {
1698 EXTSB, EXTSH, EXTSW, 0
1699};
810260a8 1700
d604f1a9 1701#if defined (CONFIG_SOFTMMU)
659ef5cb
RH		 1702#include "tcg-ldst.inc.c"
1703
d604f1a9
RH		 1704/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
1705 * int mmu_idx, uintptr_t ra)
1706 */
1707static void * const qemu_ld_helpers[16] = {
1708 [MO_UB] = helper_ret_ldub_mmu,
1709 [MO_LEUW] = helper_le_lduw_mmu,
1710 [MO_LEUL] = helper_le_ldul_mmu,
1711 [MO_LEQ] = helper_le_ldq_mmu,
1712 [MO_BEUW] = helper_be_lduw_mmu,
1713 [MO_BEUL] = helper_be_ldul_mmu,
1714 [MO_BEQ] = helper_be_ldq_mmu,
1715};
810260a8 1716
d604f1a9
RH		 1717/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
1718 * uintxx_t val, int mmu_idx, uintptr_t ra)
1719 */
1720static void * const qemu_st_helpers[16] = {
1721 [MO_UB] = helper_ret_stb_mmu,
1722 [MO_LEUW] = helper_le_stw_mmu,
1723 [MO_LEUL] = helper_le_stl_mmu,
1724 [MO_LEQ] = helper_le_stq_mmu,
1725 [MO_BEUW] = helper_be_stw_mmu,
1726 [MO_BEUL] = helper_be_stl_mmu,
1727 [MO_BEQ] = helper_be_stq_mmu,
1728};
810260a8 1729
269bd5d8
RH		 1730/* We expect to use a 16-bit negative offset from ENV. */
1731QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
1732QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -32768);
1733
d604f1a9
RH		 1734/* Perform the TLB load and compare. Places the result of the comparison
1735 in CR7, loads the addend of the TLB into R3, and returns the register
1736 containing the guest address (zero-extended into R4). Clobbers R0 and R2. */
1737
14776ab5 1738static TCGReg tcg_out_tlb_read(TCGContext *s, MemOp opc,
7f25c469 1739 TCGReg addrlo, TCGReg addrhi,
d604f1a9
RH		 1740 int mem_index, bool is_read)
1741{
1742 int cmp_off
1743 = (is_read
644f591a
RH		 1744 ? offsetof(CPUTLBEntry, addr_read)
1745 : offsetof(CPUTLBEntry, addr_write));
269bd5d8
RH		 1746 int fast_off = TLB_MASK_TABLE_OFS(mem_index);
1747 int mask_off = fast_off + offsetof(CPUTLBDescFast, mask);
1748 int table_off = fast_off + offsetof(CPUTLBDescFast, table);
85aa8081
RH		 1749 unsigned s_bits = opc & MO_SIZE;
1750 unsigned a_bits = get_alignment_bits(opc);
d604f1a9 1751
644f591a 1752 /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */
269bd5d8
RH		 1753 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_AREG0, mask_off);
1754 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R4, TCG_AREG0, table_off);
644f591a
RH		 1755
1756 /* Extract the page index, shifted into place for tlb index. */
1757 if (TCG_TARGET_REG_BITS == 32) {
1758 tcg_out_shri32(s, TCG_REG_TMP1, addrlo,
1759 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
4c314da6 1760 } else {
644f591a
RH		 1761 tcg_out_shri64(s, TCG_REG_TMP1, addrlo,
1762 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
810260a8 1763 }
644f591a 1764 tcg_out32(s, AND | SAB(TCG_REG_R3, TCG_REG_R3, TCG_REG_TMP1));
810260a8 1765
644f591a
RH		 1766 /* Load the TLB comparator. */
1767 if (cmp_off == 0 && TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) {
1768 uint32_t lxu = (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32
1769 ? LWZUX : LDUX);
1770 tcg_out32(s, lxu | TAB(TCG_REG_TMP1, TCG_REG_R3, TCG_REG_R4));
7f25c469 1771 } else {
644f591a
RH		 1772 tcg_out32(s, ADD | TAB(TCG_REG_R3, TCG_REG_R3, TCG_REG_R4));
1773 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
1774 tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_TMP1, TCG_REG_R3, cmp_off + 4);
1775 tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_R4, TCG_REG_R3, cmp_off);
1776 } else {
1777 tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP1, TCG_REG_R3, cmp_off);
1778 }
7f25c469 1779 }
d604f1a9
RH		 1780
1781 /* Load the TLB addend for use on the fast path. Do this asap
1782 to minimize any load use delay. */
644f591a
RH		 1783 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_REG_R3,
1784 offsetof(CPUTLBEntry, addend));
d604f1a9 1785
68d45bb6 1786 /* Clear the non-page, non-alignment bits from the address */
85aa8081
RH		 1787 if (TCG_TARGET_REG_BITS == 32) {
1788 /* We don't support unaligned accesses on 32-bits.
1789 * Preserve the bottom bits and thus trigger a comparison
1790 * failure on unaligned accesses.
68d45bb6 1791 */
85aa8081 1792 if (a_bits < s_bits) {
1f00b27f
SS		 1793 a_bits = s_bits;
1794 }
7f25c469 1795 tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0,
1f00b27f 1796 (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS);
85aa8081
RH		 1797 } else {
1798 TCGReg t = addrlo;
1799
1800 /* If the access is unaligned, we need to make sure we fail if we
1801 * cross a page boundary. The trick is to add the access size-1
1802 * to the address before masking the low bits. That will make the
1803 * address overflow to the next page if we cross a page boundary,
1804 * which will then force a mismatch of the TLB compare.
1805 */
1806 if (a_bits < s_bits) {
1807 unsigned a_mask = (1 << a_bits) - 1;
1808 unsigned s_mask = (1 << s_bits) - 1;
1809 tcg_out32(s, ADDI | TAI(TCG_REG_R0, t, s_mask - a_mask));
1810 t = TCG_REG_R0;
1811 }
1812
1813 /* Mask the address for the requested alignment. */
1814 if (TARGET_LONG_BITS == 32) {
1815 tcg_out_rlw(s, RLWINM, TCG_REG_R0, t, 0,
1816 (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS);
644f591a
RH		 1817 /* Zero-extend the address for use in the final address. */
1818 tcg_out_ext32u(s, TCG_REG_R4, addrlo);
1819 addrlo = TCG_REG_R4;
85aa8081
RH		 1820 } else if (a_bits == 0) {
1821 tcg_out_rld(s, RLDICR, TCG_REG_R0, t, 0, 63 - TARGET_PAGE_BITS);
1822 } else {
1823 tcg_out_rld(s, RLDICL, TCG_REG_R0, t,
1f00b27f 1824 64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - a_bits);
68d45bb6 1825 tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
68d45bb6 1826 }
70fac59a 1827 }
d604f1a9 1828
7f25c469 1829 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
dfca1778
RH		 1830 tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1,
1831 0, 7, TCG_TYPE_I32);
7f25c469
RH		 1832 tcg_out_cmp(s, TCG_COND_EQ, addrhi, TCG_REG_R4, 0, 6, TCG_TYPE_I32);
1833 tcg_out32(s, CRAND | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
1834 } else {
dfca1778
RH		 1835 tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1,
1836 0, 7, TCG_TYPE_TL);
7f25c469 1837 }
d604f1a9 1838
7f25c469 1839 return addrlo;
70fac59a 1840}
1cd62ae9 1841
d604f1a9
RH		 1842/* Record the context of a call to the out of line helper code for the slow
1843 path for a load or store, so that we can later generate the correct
1844 helper code. */
3972ef6f 1845static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
7f25c469
RH		 1846 TCGReg datalo_reg, TCGReg datahi_reg,
1847 TCGReg addrlo_reg, TCGReg addrhi_reg,
3972ef6f 1848 tcg_insn_unit *raddr, tcg_insn_unit *lptr)
70fac59a 1849{
d604f1a9
RH		 1850 TCGLabelQemuLdst *label = new_ldst_label(s);
1851
1852 label->is_ld = is_ld;
3972ef6f 1853 label->oi = oi;
7f25c469
RH		 1854 label->datalo_reg = datalo_reg;
1855 label->datahi_reg = datahi_reg;
1856 label->addrlo_reg = addrlo_reg;
1857 label->addrhi_reg = addrhi_reg;
d604f1a9 1858 label->raddr = raddr;
7f25c469 1859 label->label_ptr[0] = lptr;
70fac59a 1860}
1cd62ae9 1861
aeee05f5 1862static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
70fac59a 1863{
3972ef6f 1864 TCGMemOpIdx oi = lb->oi;
14776ab5 1865 MemOp opc = get_memop(oi);
7f25c469 1866 TCGReg hi, lo, arg = TCG_REG_R3;
70fac59a 1867
aeee05f5
RH		 1868 if (!reloc_pc14(lb->label_ptr[0], s->code_ptr)) {
1869 return false;
1870 }
70fac59a 1871
7f25c469 1872 tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0);
1cd62ae9 1873
7f25c469
RH		 1874 lo = lb->addrlo_reg;
1875 hi = lb->addrhi_reg;
1876 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
1877#ifdef TCG_TARGET_CALL_ALIGN_ARGS
1878 arg |= 1;
1879#endif
1880 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
1881 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
1882 } else {
1883 /* If the address needed to be zero-extended, we'll have already
1884 placed it in R4. The only remaining case is 64-bit guest. */
1885 tcg_out_mov(s, TCG_TYPE_TL, arg++, lo);
1886 }
752c1fdb 1887
3972ef6f 1888 tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
7f25c469 1889 tcg_out32(s, MFSPR | RT(arg) | LR);
70fac59a 1890
2b7ec66f 1891 tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
70fac59a 1892
7f25c469
RH		 1893 lo = lb->datalo_reg;
1894 hi = lb->datahi_reg;
1895 if (TCG_TARGET_REG_BITS == 32 && (opc & MO_SIZE) == MO_64) {
1896 tcg_out_mov(s, TCG_TYPE_I32, lo, TCG_REG_R4);
1897 tcg_out_mov(s, TCG_TYPE_I32, hi, TCG_REG_R3);
1898 } else if (opc & MO_SIGN) {
d604f1a9 1899 uint32_t insn = qemu_exts_opc[opc & MO_SIZE];
7f25c469 1900 tcg_out32(s, insn | RA(lo) | RS(TCG_REG_R3));
d604f1a9 1901 } else {
7f25c469 1902 tcg_out_mov(s, TCG_TYPE_REG, lo, TCG_REG_R3);
70fac59a
RH		 1903 }
1904
d604f1a9 1905 tcg_out_b(s, 0, lb->raddr);
aeee05f5 1906 return true;
d604f1a9 1907}
70fac59a 1908
aeee05f5 1909static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
d604f1a9 1910{
3972ef6f 1911 TCGMemOpIdx oi = lb->oi;
14776ab5
TN		 1912 MemOp opc = get_memop(oi);
1913 MemOp s_bits = opc & MO_SIZE;
7f25c469 1914 TCGReg hi, lo, arg = TCG_REG_R3;
1cd62ae9 1915
aeee05f5
RH		 1916 if (!reloc_pc14(lb->label_ptr[0], s->code_ptr)) {
1917 return false;
1918 }
1cd62ae9 1919
7f25c469
RH		 1920 tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0);
1921
1922 lo = lb->addrlo_reg;
1923 hi = lb->addrhi_reg;
1924 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
1925#ifdef TCG_TARGET_CALL_ALIGN_ARGS
1926 arg |= 1;
1927#endif
1928 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
1929 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
1930 } else {
1931 /* If the address needed to be zero-extended, we'll have already
1932 placed it in R4. The only remaining case is 64-bit guest. */
1933 tcg_out_mov(s, TCG_TYPE_TL, arg++, lo);
1934 }
1cd62ae9 1935
7f25c469
RH		 1936 lo = lb->datalo_reg;
1937 hi = lb->datahi_reg;
1938 if (TCG_TARGET_REG_BITS == 32) {
1939 switch (s_bits) {
1940 case MO_64:
1941#ifdef TCG_TARGET_CALL_ALIGN_ARGS
1942 arg |= 1;
1943#endif
1944 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
1945 /* FALLTHRU */
1946 case MO_32:
1947 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
1948 break;
1949 default:
1950 tcg_out_rlw(s, RLWINM, arg++, lo, 0, 32 - (8 << s_bits), 31);
1951 break;
1952 }
1953 } else {
1954 if (s_bits == MO_64) {
1955 tcg_out_mov(s, TCG_TYPE_I64, arg++, lo);
1956 } else {
1957 tcg_out_rld(s, RLDICL, arg++, lo, 0, 64 - (8 << s_bits));
1958 }
1959 }
1cd62ae9 1960
3972ef6f 1961 tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
7f25c469 1962 tcg_out32(s, MFSPR | RT(arg) | LR);
1cd62ae9 1963
2b7ec66f 1964 tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
d604f1a9
RH		 1965
1966 tcg_out_b(s, 0, lb->raddr);
aeee05f5 1967 return true;
1cd62ae9 1968}
d604f1a9 1969#endif /* SOFTMMU */
1cd62ae9 1970
7f25c469 1971static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
810260a8 1972{
7f25c469
RH		 1973 TCGReg datalo, datahi, addrlo, rbase;
1974 TCGReg addrhi __attribute__((unused));
59227d5d 1975 TCGMemOpIdx oi;
14776ab5 1976 MemOp opc, s_bits;
d604f1a9 1977#ifdef CONFIG_SOFTMMU
7f25c469 1978 int mem_index;
d604f1a9
RH		 1979 tcg_insn_unit *label_ptr;
1980#endif
810260a8 1981
7f25c469
RH		 1982 datalo = *args++;
1983 datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0);
1984 addrlo = *args++;
1985 addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0);
59227d5d
RH		 1986 oi = *args++;
1987 opc = get_memop(oi);
7f25c469
RH		 1988 s_bits = opc & MO_SIZE;
1989
d604f1a9 1990#ifdef CONFIG_SOFTMMU
59227d5d 1991 mem_index = get_mmuidx(oi);
68d45bb6 1992 addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, true);
d604f1a9
RH		 1993
1994 /* Load a pointer into the current opcode w/conditional branch-link. */
1995 label_ptr = s->code_ptr;
f9c7246f 1996 tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
d604f1a9
RH		 1997
1998 rbase = TCG_REG_R3;
1999#else /* !CONFIG_SOFTMMU */
b76f21a7 2000 rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
7f25c469 2001 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
dfca1778
RH		 2002 tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
2003 addrlo = TCG_REG_TMP1;
d604f1a9
RH		 2004 }
2005#endif
2006
7f25c469
RH		 2007 if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
2008 if (opc & MO_BSWAP) {
2009 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
2010 tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
2011 tcg_out32(s, LWBRX | TAB(datahi, rbase, TCG_REG_R0));
2012 } else if (rbase != 0) {
2013 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
2014 tcg_out32(s, LWZX | TAB(datahi, rbase, addrlo));
2015 tcg_out32(s, LWZX | TAB(datalo, rbase, TCG_REG_R0));
2016 } else if (addrlo == datahi) {
2017 tcg_out32(s, LWZ | TAI(datalo, addrlo, 4));
2018 tcg_out32(s, LWZ | TAI(datahi, addrlo, 0));
2019 } else {
2020 tcg_out32(s, LWZ | TAI(datahi, addrlo, 0));
2021 tcg_out32(s, LWZ | TAI(datalo, addrlo, 4));
2022 }
541dd4ce 2023 } else {
2b7ec66f 2024 uint32_t insn = qemu_ldx_opc[opc & (MO_BSWAP | MO_SSIZE)];
4e33fe01 2025 if (!have_isa_2_06 && insn == LDBRX) {
7f25c469
RH		 2026 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
2027 tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
2028 tcg_out32(s, LWBRX | TAB(TCG_REG_R0, rbase, TCG_REG_R0));
2029 tcg_out_rld(s, RLDIMI, datalo, TCG_REG_R0, 32, 0);
2030 } else if (insn) {
2031 tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
2032 } else {
2033 insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)];
2034 tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
2035 insn = qemu_exts_opc[s_bits];
2036 tcg_out32(s, insn | RA(datalo) | RS(datalo));
2037 }
810260a8 2038 }
810260a8 2039
d604f1a9 2040#ifdef CONFIG_SOFTMMU
3972ef6f
RH		 2041 add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
2042 s->code_ptr, label_ptr);
d604f1a9 2043#endif
810260a8 2044}
2045
7f25c469 2046static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
027ffea9 2047{
7f25c469
RH		 2048 TCGReg datalo, datahi, addrlo, rbase;
2049 TCGReg addrhi __attribute__((unused));
59227d5d 2050 TCGMemOpIdx oi;
14776ab5 2051 MemOp opc, s_bits;
d604f1a9 2052#ifdef CONFIG_SOFTMMU
7f25c469 2053 int mem_index;
d604f1a9
RH		 2054 tcg_insn_unit *label_ptr;
2055#endif
027ffea9 2056
7f25c469
RH		 2057 datalo = *args++;
2058 datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0);
2059 addrlo = *args++;
2060 addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0);
59227d5d
RH		 2061 oi = *args++;
2062 opc = get_memop(oi);
7f25c469
RH		 2063 s_bits = opc & MO_SIZE;
2064
d604f1a9 2065#ifdef CONFIG_SOFTMMU
59227d5d 2066 mem_index = get_mmuidx(oi);
68d45bb6 2067 addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, false);
027ffea9 2068
d604f1a9
RH		 2069 /* Load a pointer into the current opcode w/conditional branch-link. */
2070 label_ptr = s->code_ptr;
f9c7246f 2071 tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
027ffea9 2072
d604f1a9
RH		 2073 rbase = TCG_REG_R3;
2074#else /* !CONFIG_SOFTMMU */
b76f21a7 2075 rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
7f25c469 2076 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
dfca1778
RH		 2077 tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
2078 addrlo = TCG_REG_TMP1;
d604f1a9
RH		 2079 }
2080#endif
2081
7f25c469
RH		 2082 if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
2083 if (opc & MO_BSWAP) {
2084 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
2085 tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
2086 tcg_out32(s, STWBRX | SAB(datahi, rbase, TCG_REG_R0));
2087 } else if (rbase != 0) {
2088 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
2089 tcg_out32(s, STWX | SAB(datahi, rbase, addrlo));
2090 tcg_out32(s, STWX | SAB(datalo, rbase, TCG_REG_R0));
2091 } else {
2092 tcg_out32(s, STW | TAI(datahi, addrlo, 0));
2093 tcg_out32(s, STW | TAI(datalo, addrlo, 4));
2094 }
027ffea9 2095 } else {
2b7ec66f 2096 uint32_t insn = qemu_stx_opc[opc & (MO_BSWAP | MO_SIZE)];
4e33fe01 2097 if (!have_isa_2_06 && insn == STDBRX) {
7f25c469 2098 tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
dfca1778 2099 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, addrlo, 4));
7f25c469 2100 tcg_out_shri64(s, TCG_REG_R0, datalo, 32);
dfca1778 2101 tcg_out32(s, STWBRX | SAB(TCG_REG_R0, rbase, TCG_REG_TMP1));
7f25c469
RH		 2102 } else {
2103 tcg_out32(s, insn | SAB(datalo, rbase, addrlo));
2104 }
027ffea9 2105 }
d604f1a9
RH		 2106
2107#ifdef CONFIG_SOFTMMU
3972ef6f
RH		 2108 add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
2109 s->code_ptr, label_ptr);
d604f1a9 2110#endif
027ffea9
RH		 2111}
2112
53c89efd
RH		 2113static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
2114{
2115 int i;
2116 for (i = 0; i < count; ++i) {
2117 p[i] = NOP;
2118 }
2119}
2120
a921fddc
RH		 2121/* Parameters for function call generation, used in tcg.c. */
2122#define TCG_TARGET_STACK_ALIGN 16
a921fddc
RH		 2123#define TCG_TARGET_EXTEND_ARGS 1
2124
802ca56e
RH		 2125#ifdef _CALL_AIX
2126# define LINK_AREA_SIZE (6 * SZR)
2127# define LR_OFFSET (1 * SZR)
2128# define TCG_TARGET_CALL_STACK_OFFSET (LINK_AREA_SIZE + 8 * SZR)
1045fc04
PM		 2129#elif defined(TCG_TARGET_CALL_DARWIN)
2130# define LINK_AREA_SIZE (6 * SZR)
2131# define LR_OFFSET (2 * SZR)
ffcfbece
RH		 2132#elif TCG_TARGET_REG_BITS == 64
2133# if defined(_CALL_ELF) && _CALL_ELF == 2
2134# define LINK_AREA_SIZE (4 * SZR)
2135# define LR_OFFSET (1 * SZR)
2136# endif
2137#else /* TCG_TARGET_REG_BITS == 32 */
2138# if defined(_CALL_SYSV)
ffcfbece
RH		 2139# define LINK_AREA_SIZE (2 * SZR)
2140# define LR_OFFSET (1 * SZR)
ffcfbece
RH		 2141# endif
2142#endif
2143#ifndef LR_OFFSET
2144# error "Unhandled abi"
2145#endif
2146#ifndef TCG_TARGET_CALL_STACK_OFFSET
a2a98f80 2147# define TCG_TARGET_CALL_STACK_OFFSET LINK_AREA_SIZE
802ca56e
RH		 2148#endif
2149
2150#define CPU_TEMP_BUF_SIZE (CPU_TEMP_BUF_NLONGS * (int)sizeof(long))
2151#define REG_SAVE_SIZE ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * SZR)
d604f1a9 2152
802ca56e
RH		 2153#define FRAME_SIZE ((TCG_TARGET_CALL_STACK_OFFSET \
2154 + TCG_STATIC_CALL_ARGS_SIZE \
2155 + CPU_TEMP_BUF_SIZE \
2156 + REG_SAVE_SIZE \
2157 + TCG_TARGET_STACK_ALIGN - 1) \
2158 & -TCG_TARGET_STACK_ALIGN)
2159
2160#define REG_SAVE_BOT (FRAME_SIZE - REG_SAVE_SIZE)
d604f1a9
RH		 2161
2162static void tcg_target_qemu_prologue(TCGContext *s)
810260a8 2163{
d604f1a9 2164 int i;
810260a8 2165
802ca56e 2166#ifdef _CALL_AIX
a84ac4cb
RH		 2167 void **desc = (void **)s->code_ptr;
2168 desc[0] = desc + 2; /* entry point */
2169 desc[1] = 0; /* environment pointer */
2170 s->code_ptr = (void *)(desc + 2); /* skip over descriptor */
d604f1a9
RH		 2171#endif
2172
a84ac4cb
RH		 2173 tcg_set_frame(s, TCG_REG_CALL_STACK, REG_SAVE_BOT - CPU_TEMP_BUF_SIZE,
2174 CPU_TEMP_BUF_SIZE);
2175
d604f1a9
RH		 2176 /* Prologue */
2177 tcg_out32(s, MFSPR | RT(TCG_REG_R0) | LR);
ffcfbece
RH		 2178 tcg_out32(s, (SZR == 8 ? STDU : STWU)
2179 | SAI(TCG_REG_R1, TCG_REG_R1, -FRAME_SIZE));
802ca56e 2180
d604f1a9 2181 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
4c3831a0
RH		 2182 tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
2183 TCG_REG_R1, REG_SAVE_BOT + i * SZR);
d604f1a9 2184 }
802ca56e 2185 tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
d604f1a9 2186
4cbea598 2187#ifndef CONFIG_SOFTMMU
b76f21a7 2188 if (guest_base) {
5964fca8 2189 tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base, true);
d604f1a9
RH		 2190 tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
2191 }
2192#endif
2193
2194 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
2195 tcg_out32(s, MTSPR | RS(tcg_target_call_iarg_regs[1]) | CTR);
5964fca8
RH		 2196 if (USE_REG_TB) {
2197 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, tcg_target_call_iarg_regs[1]);
a84ac4cb 2198 }
5964fca8 2199 tcg_out32(s, BCCTR | BO_ALWAYS);
d604f1a9
RH		 2200
2201 /* Epilogue */
5964fca8 2202 s->code_gen_epilogue = tb_ret_addr = s->code_ptr;
d604f1a9 2203
802ca56e 2204 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
d604f1a9 2205 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
4c3831a0
RH		 2206 tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
2207 TCG_REG_R1, REG_SAVE_BOT + i * SZR);
d604f1a9 2208 }
d604f1a9
RH		 2209 tcg_out32(s, MTSPR | RS(TCG_REG_R0) | LR);
2210 tcg_out32(s, ADDI | TAI(TCG_REG_R1, TCG_REG_R1, FRAME_SIZE));
2211 tcg_out32(s, BCLR | BO_ALWAYS);
810260a8 2212}
2213
541dd4ce
RH		 2214static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
2215 const int *const_args)
810260a8 2216{
ee924fa6 2217 TCGArg a0, a1, a2;
e46b9681 2218 int c;
2219
810260a8 2220 switch (opc) {
2221 case INDEX_op_exit_tb:
de3d636d 2222 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
e083c4a2 2223 tcg_out_b(s, 0, tb_ret_addr);
810260a8 2224 break;
2225 case INDEX_op_goto_tb:
5964fca8
RH		 2226 if (s->tb_jmp_insn_offset) {
2227 /* Direct jump. */
2228 if (TCG_TARGET_REG_BITS == 64) {
2229 /* Ensure the next insns are 8-byte aligned. */
2230 if ((uintptr_t)s->code_ptr & 7) {
2231 tcg_out32(s, NOP);
2232 }
2233 s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
2234 tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
2235 tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
2236 } else {
2237 s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
2238 tcg_out32(s, B);
2239 s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
2240 break;
2241 }
2242 } else {
2243 /* Indirect jump. */
2244 tcg_debug_assert(s->tb_jmp_insn_offset == NULL);
2245 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, 0,
2246 (intptr_t)(s->tb_jmp_insn_offset + args[0]));
2247 }
2248 tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
5bfd75a3 2249 tcg_out32(s, BCCTR | BO_ALWAYS);
9f754620 2250 set_jmp_reset_offset(s, args[0]);
5964fca8
RH		 2251 if (USE_REG_TB) {
2252 /* For the unlinked case, need to reset TCG_REG_TB. */
9f754620 2253 c = -tcg_current_code_size(s);
5964fca8
RH		 2254 assert(c == (int16_t)c);
2255 tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, c));
2256 }
810260a8 2257 break;
0c240785
RH		 2258 case INDEX_op_goto_ptr:
2259 tcg_out32(s, MTSPR | RS(args[0]) | CTR);
5964fca8
RH		 2260 if (USE_REG_TB) {
2261 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, args[0]);
2262 }
2263 tcg_out32(s, ADDI | TAI(TCG_REG_R3, 0, 0));
0c240785
RH		 2264 tcg_out32(s, BCCTR | BO_ALWAYS);
2265 break;
810260a8 2266 case INDEX_op_br:
2267 {
bec16311 2268 TCGLabel *l = arg_label(args[0]);
f9c7246f 2269 uint32_t insn = B;
810260a8 2270
2271 if (l->has_value) {
f9c7246f 2272 insn |= reloc_pc24_val(s->code_ptr, l->u.value_ptr);
541dd4ce 2273 } else {
bec16311 2274 tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, l, 0);
810260a8 2275 }
f9c7246f 2276 tcg_out32(s, insn);
810260a8 2277 }
2278 break;
810260a8 2279 case INDEX_op_ld8u_i32:
2280 case INDEX_op_ld8u_i64:
b18d5d2b 2281 tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
810260a8 2282 break;
2283 case INDEX_op_ld8s_i32:
2284 case INDEX_op_ld8s_i64:
b18d5d2b 2285 tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
541dd4ce 2286 tcg_out32(s, EXTSB | RS(args[0]) | RA(args[0]));
810260a8 2287 break;
2288 case INDEX_op_ld16u_i32:
2289 case INDEX_op_ld16u_i64:
b18d5d2b 2290 tcg_out_mem_long(s, LHZ, LHZX, args[0], args[1], args[2]);
810260a8 2291 break;
2292 case INDEX_op_ld16s_i32:
2293 case INDEX_op_ld16s_i64:
b18d5d2b 2294 tcg_out_mem_long(s, LHA, LHAX, args[0], args[1], args[2]);
810260a8 2295 break;
2296 case INDEX_op_ld_i32:
2297 case INDEX_op_ld32u_i64:
b18d5d2b 2298 tcg_out_mem_long(s, LWZ, LWZX, args[0], args[1], args[2]);
810260a8 2299 break;
2300 case INDEX_op_ld32s_i64:
b18d5d2b 2301 tcg_out_mem_long(s, LWA, LWAX, args[0], args[1], args[2]);
810260a8 2302 break;
2303 case INDEX_op_ld_i64:
b18d5d2b 2304 tcg_out_mem_long(s, LD, LDX, args[0], args[1], args[2]);
810260a8 2305 break;
2306 case INDEX_op_st8_i32:
2307 case INDEX_op_st8_i64:
b18d5d2b 2308 tcg_out_mem_long(s, STB, STBX, args[0], args[1], args[2]);
810260a8 2309 break;
2310 case INDEX_op_st16_i32:
2311 case INDEX_op_st16_i64:
b18d5d2b 2312 tcg_out_mem_long(s, STH, STHX, args[0], args[1], args[2]);
810260a8 2313 break;
2314 case INDEX_op_st_i32:
2315 case INDEX_op_st32_i64:
b18d5d2b 2316 tcg_out_mem_long(s, STW, STWX, args[0], args[1], args[2]);
810260a8 2317 break;
2318 case INDEX_op_st_i64:
b18d5d2b 2319 tcg_out_mem_long(s, STD, STDX, args[0], args[1], args[2]);
810260a8 2320 break;
2321
2322 case INDEX_op_add_i32:
ee924fa6
RH		 2323 a0 = args[0], a1 = args[1], a2 = args[2];
2324 if (const_args[2]) {
ee924fa6 2325 do_addi_32:
b18d5d2b 2326 tcg_out_mem_long(s, ADDI, ADD, a0, a1, (int32_t)a2);
ee924fa6
RH		 2327 } else {
2328 tcg_out32(s, ADD | TAB(a0, a1, a2));
2329 }
810260a8 2330 break;
2331 case INDEX_op_sub_i32:
ee924fa6 2332 a0 = args[0], a1 = args[1], a2 = args[2];
148bdd23
RH		 2333 if (const_args[1]) {
2334 if (const_args[2]) {
2335 tcg_out_movi(s, TCG_TYPE_I32, a0, a1 - a2);
2336 } else {
2337 tcg_out32(s, SUBFIC | TAI(a0, a2, a1));
2338 }
2339 } else if (const_args[2]) {
ee924fa6
RH		 2340 a2 = -a2;
2341 goto do_addi_32;
2342 } else {
2343 tcg_out32(s, SUBF | TAB(a0, a2, a1));
2344 }
810260a8 2345 break;
2346
2347 case INDEX_op_and_i32:
37251b98 2348 a0 = args[0], a1 = args[1], a2 = args[2];
a9249dff 2349 if (const_args[2]) {
37251b98 2350 tcg_out_andi32(s, a0, a1, a2);
a9249dff 2351 } else {
37251b98 2352 tcg_out32(s, AND | SAB(a1, a0, a2));
a9249dff
RH		 2353 }
2354 break;
2355 case INDEX_op_and_i64:
37251b98 2356 a0 = args[0], a1 = args[1], a2 = args[2];
810260a8 2357 if (const_args[2]) {
37251b98 2358 tcg_out_andi64(s, a0, a1, a2);
637af30c 2359 } else {
37251b98 2360 tcg_out32(s, AND | SAB(a1, a0, a2));
810260a8 2361 }
810260a8 2362 break;
fe6f943f 2363 case INDEX_op_or_i64:
810260a8 2364 case INDEX_op_or_i32:
dce74c57 2365 a0 = args[0], a1 = args[1], a2 = args[2];
810260a8 2366 if (const_args[2]) {
dce74c57
RH		 2367 tcg_out_ori32(s, a0, a1, a2);
2368 } else {
2369 tcg_out32(s, OR | SAB(a1, a0, a2));
810260a8 2370 }
810260a8 2371 break;
fe6f943f 2372 case INDEX_op_xor_i64:
810260a8 2373 case INDEX_op_xor_i32:
dce74c57 2374 a0 = args[0], a1 = args[1], a2 = args[2];
810260a8 2375 if (const_args[2]) {
dce74c57
RH		 2376 tcg_out_xori32(s, a0, a1, a2);
2377 } else {
2378 tcg_out32(s, XOR | SAB(a1, a0, a2));
810260a8 2379 }
810260a8 2380 break;
ce1010d6 2381 case INDEX_op_andc_i32:
37251b98
RH		 2382 a0 = args[0], a1 = args[1], a2 = args[2];
2383 if (const_args[2]) {
2384 tcg_out_andi32(s, a0, a1, ~a2);
2385 } else {
2386 tcg_out32(s, ANDC | SAB(a1, a0, a2));
2387 }
2388 break;
ce1010d6 2389 case INDEX_op_andc_i64:
37251b98
RH		 2390 a0 = args[0], a1 = args[1], a2 = args[2];
2391 if (const_args[2]) {
2392 tcg_out_andi64(s, a0, a1, ~a2);
2393 } else {
2394 tcg_out32(s, ANDC | SAB(a1, a0, a2));
2395 }
ce1010d6
RH		 2396 break;
2397 case INDEX_op_orc_i32:
37251b98
RH		 2398 if (const_args[2]) {
2399 tcg_out_ori32(s, args[0], args[1], ~args[2]);
2400 break;
2401 }
2402 /* FALLTHRU */
ce1010d6
RH		 2403 case INDEX_op_orc_i64:
2404 tcg_out32(s, ORC | SAB(args[1], args[0], args[2]));
2405 break;
2406 case INDEX_op_eqv_i32:
37251b98
RH		 2407 if (const_args[2]) {
2408 tcg_out_xori32(s, args[0], args[1], ~args[2]);
2409 break;
2410 }
2411 /* FALLTHRU */
ce1010d6
RH		 2412 case INDEX_op_eqv_i64:
2413 tcg_out32(s, EQV | SAB(args[1], args[0], args[2]));
2414 break;
2415 case INDEX_op_nand_i32:
2416 case INDEX_op_nand_i64:
2417 tcg_out32(s, NAND | SAB(args[1], args[0], args[2]));
2418 break;
2419 case INDEX_op_nor_i32:
2420 case INDEX_op_nor_i64:
2421 tcg_out32(s, NOR | SAB(args[1], args[0], args[2]));
2422 break;
810260a8 2423
d0b07481
RH		 2424 case INDEX_op_clz_i32:
2425 tcg_out_cntxz(s, TCG_TYPE_I32, CNTLZW, args[0], args[1],
2426 args[2], const_args[2]);
2427 break;
2428 case INDEX_op_ctz_i32:
2429 tcg_out_cntxz(s, TCG_TYPE_I32, CNTTZW, args[0], args[1],
2430 args[2], const_args[2]);
2431 break;
33e75fb9
RH		 2432 case INDEX_op_ctpop_i32:
2433 tcg_out32(s, CNTPOPW | SAB(args[1], args[0], 0));
2434 break;
d0b07481
RH		 2435
2436 case INDEX_op_clz_i64:
2437 tcg_out_cntxz(s, TCG_TYPE_I64, CNTLZD, args[0], args[1],
2438 args[2], const_args[2]);
2439 break;
2440 case INDEX_op_ctz_i64:
2441 tcg_out_cntxz(s, TCG_TYPE_I64, CNTTZD, args[0], args[1],
2442 args[2], const_args[2]);
2443 break;
33e75fb9
RH		 2444 case INDEX_op_ctpop_i64:
2445 tcg_out32(s, CNTPOPD | SAB(args[1], args[0], 0));
2446 break;
d0b07481 2447
810260a8 2448 case INDEX_op_mul_i32:
ef809300 2449 a0 = args[0], a1 = args[1], a2 = args[2];
810260a8 2450 if (const_args[2]) {
ef809300
RH		 2451 tcg_out32(s, MULLI | TAI(a0, a1, a2));
2452 } else {
2453 tcg_out32(s, MULLW | TAB(a0, a1, a2));
810260a8 2454 }
810260a8 2455 break;
2456
2457 case INDEX_op_div_i32:
541dd4ce 2458 tcg_out32(s, DIVW | TAB(args[0], args[1], args[2]));
810260a8 2459 break;
2460
2461 case INDEX_op_divu_i32:
541dd4ce 2462 tcg_out32(s, DIVWU | TAB(args[0], args[1], args[2]));
810260a8 2463 break;
2464
810260a8 2465 case INDEX_op_shl_i32:
2466 if (const_args[2]) {
a757e1ee 2467 tcg_out_shli32(s, args[0], args[1], args[2]);
9e555b73 2468 } else {
541dd4ce 2469 tcg_out32(s, SLW | SAB(args[1], args[0], args[2]));
9e555b73 2470 }
810260a8 2471 break;
2472 case INDEX_op_shr_i32:
2473 if (const_args[2]) {
a757e1ee 2474 tcg_out_shri32(s, args[0], args[1], args[2]);
9e555b73 2475 } else {
541dd4ce 2476 tcg_out32(s, SRW | SAB(args[1], args[0], args[2]));
9e555b73 2477 }
810260a8 2478 break;
2479 case INDEX_op_sar_i32:
541dd4ce
RH		 2480 if (const_args[2]) {
2481 tcg_out32(s, SRAWI | RS(args[1]) | RA(args[0]) | SH(args[2]));
2482 } else {
2483 tcg_out32(s, SRAW | SAB(args[1], args[0], args[2]));
2484 }
810260a8 2485 break;
313d91c7
RH		 2486 case INDEX_op_rotl_i32:
2487 if (const_args[2]) {
2488 tcg_out_rlw(s, RLWINM, args[0], args[1], args[2], 0, 31);
2489 } else {
2490 tcg_out32(s, RLWNM | SAB(args[1], args[0], args[2])
2491 | MB(0) | ME(31));
2492 }
2493 break;
2494 case INDEX_op_rotr_i32:
2495 if (const_args[2]) {
2496 tcg_out_rlw(s, RLWINM, args[0], args[1], 32 - args[2], 0, 31);
2497 } else {
8327a470
RH		 2498 tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 32));
2499 tcg_out32(s, RLWNM | SAB(args[1], args[0], TCG_REG_R0)
313d91c7
RH		 2500 | MB(0) | ME(31));
2501 }
2502 break;
810260a8 2503
2504 case INDEX_op_brcond_i32:
4c314da6 2505 tcg_out_brcond(s, args[2], args[0], args[1], const_args[1],
bec16311 2506 arg_label(args[3]), TCG_TYPE_I32);
e924bbec 2507 break;
810260a8 2508 case INDEX_op_brcond_i64:
4c314da6 2509 tcg_out_brcond(s, args[2], args[0], args[1], const_args[1],
bec16311 2510 arg_label(args[3]), TCG_TYPE_I64);
810260a8 2511 break;
abcf61c4
RH		 2512 case INDEX_op_brcond2_i32:
2513 tcg_out_brcond2(s, args, const_args);
2514 break;
810260a8 2515
2516 case INDEX_op_neg_i32:
810260a8 2517 case INDEX_op_neg_i64:
541dd4ce 2518 tcg_out32(s, NEG | RT(args[0]) | RA(args[1]));
810260a8 2519 break;
2520
157f2662 2521 case INDEX_op_not_i32:
2522 case INDEX_op_not_i64:
541dd4ce 2523 tcg_out32(s, NOR | SAB(args[1], args[0], args[1]));
157f2662 2524 break;
2525
810260a8 2526 case INDEX_op_add_i64:
ee924fa6
RH		 2527 a0 = args[0], a1 = args[1], a2 = args[2];
2528 if (const_args[2]) {
ee924fa6 2529 do_addi_64:
b18d5d2b 2530 tcg_out_mem_long(s, ADDI, ADD, a0, a1, a2);
ee924fa6
RH		 2531 } else {
2532 tcg_out32(s, ADD | TAB(a0, a1, a2));
2533 }
810260a8 2534 break;
2535 case INDEX_op_sub_i64:
ee924fa6 2536 a0 = args[0], a1 = args[1], a2 = args[2];
148bdd23
RH		 2537 if (const_args[1]) {
2538 if (const_args[2]) {
2539 tcg_out_movi(s, TCG_TYPE_I64, a0, a1 - a2);
2540 } else {
2541 tcg_out32(s, SUBFIC | TAI(a0, a2, a1));
2542 }
2543 } else if (const_args[2]) {
ee924fa6
RH		 2544 a2 = -a2;
2545 goto do_addi_64;
2546 } else {
2547 tcg_out32(s, SUBF | TAB(a0, a2, a1));
2548 }
810260a8 2549 break;
2550
2551 case INDEX_op_shl_i64:
541dd4ce 2552 if (const_args[2]) {
0a9564b9 2553 tcg_out_shli64(s, args[0], args[1], args[2]);
541dd4ce
RH		 2554 } else {
2555 tcg_out32(s, SLD | SAB(args[1], args[0], args[2]));
2556 }
810260a8 2557 break;
2558 case INDEX_op_shr_i64:
541dd4ce 2559 if (const_args[2]) {
5e916c28 2560 tcg_out_shri64(s, args[0], args[1], args[2]);
541dd4ce
RH		 2561 } else {
2562 tcg_out32(s, SRD | SAB(args[1], args[0], args[2]));
2563 }
810260a8 2564 break;
2565 case INDEX_op_sar_i64:
fe6f943f 2566 if (const_args[2]) {
541dd4ce
RH		 2567 int sh = SH(args[2] & 0x1f) | (((args[2] >> 5) & 1) << 1);
2568 tcg_out32(s, SRADI | RA(args[0]) | RS(args[1]) | sh);
2569 } else {
2570 tcg_out32(s, SRAD | SAB(args[1], args[0], args[2]));
fe6f943f 2571 }
810260a8 2572 break;
313d91c7
RH		 2573 case INDEX_op_rotl_i64:
2574 if (const_args[2]) {
2575 tcg_out_rld(s, RLDICL, args[0], args[1], args[2], 0);
2576 } else {
2577 tcg_out32(s, RLDCL | SAB(args[1], args[0], args[2]) | MB64(0));
2578 }
2579 break;
2580 case INDEX_op_rotr_i64:
2581 if (const_args[2]) {
2582 tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 0);
2583 } else {
8327a470
RH		 2584 tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 64));
2585 tcg_out32(s, RLDCL | SAB(args[1], args[0], TCG_REG_R0) | MB64(0));
313d91c7
RH		 2586 }
2587 break;
810260a8 2588
2589 case INDEX_op_mul_i64:
ef809300
RH		 2590 a0 = args[0], a1 = args[1], a2 = args[2];
2591 if (const_args[2]) {
2592 tcg_out32(s, MULLI | TAI(a0, a1, a2));
2593 } else {
2594 tcg_out32(s, MULLD | TAB(a0, a1, a2));
2595 }
810260a8 2596 break;
2597 case INDEX_op_div_i64:
541dd4ce 2598 tcg_out32(s, DIVD | TAB(args[0], args[1], args[2]));
810260a8 2599 break;
2600 case INDEX_op_divu_i64:
541dd4ce 2601 tcg_out32(s, DIVDU | TAB(args[0], args[1], args[2]));
810260a8 2602 break;
810260a8 2603
1768ec06 2604 case INDEX_op_qemu_ld_i32:
7f25c469
RH		 2605 tcg_out_qemu_ld(s, args, false);
2606 break;
1768ec06 2607 case INDEX_op_qemu_ld_i64:
7f25c469 2608 tcg_out_qemu_ld(s, args, true);
810260a8 2609 break;
1768ec06 2610 case INDEX_op_qemu_st_i32:
7f25c469
RH		 2611 tcg_out_qemu_st(s, args, false);
2612 break;
1768ec06 2613 case INDEX_op_qemu_st_i64:
7f25c469 2614 tcg_out_qemu_st(s, args, true);
810260a8 2615 break;
2616
e46b9681 2617 case INDEX_op_ext8s_i32:
2618 case INDEX_op_ext8s_i64:
2619 c = EXTSB;
2620 goto gen_ext;
2621 case INDEX_op_ext16s_i32:
2622 case INDEX_op_ext16s_i64:
2623 c = EXTSH;
2624 goto gen_ext;
4f2331e5 2625 case INDEX_op_ext_i32_i64:
e46b9681 2626 case INDEX_op_ext32s_i64:
2627 c = EXTSW;
2628 goto gen_ext;
2629 gen_ext:
541dd4ce 2630 tcg_out32(s, c | RS(args[1]) | RA(args[0]));
e46b9681 2631 break;
4f2331e5
AJ		 2632 case INDEX_op_extu_i32_i64:
2633 tcg_out_ext32u(s, args[0], args[1]);
2634 break;
e46b9681 2635
1cd62ae9 2636 case INDEX_op_setcond_i32:
541dd4ce
RH		 2637 tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
2638 const_args[2]);
1cd62ae9 2639 break;
2640 case INDEX_op_setcond_i64:
541dd4ce
RH		 2641 tcg_out_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], args[2],
2642 const_args[2]);
1cd62ae9 2643 break;
abcf61c4
RH		 2644 case INDEX_op_setcond2_i32:
2645 tcg_out_setcond2(s, args, const_args);
2646 break;
1cd62ae9 2647
5d221582
RH		 2648 case INDEX_op_bswap16_i32:
2649 case INDEX_op_bswap16_i64:
2650 a0 = args[0], a1 = args[1];
2651 /* a1 = abcd */
2652 if (a0 != a1) {
2653 /* a0 = (a1 r<< 24) & 0xff # 000c */
2654 tcg_out_rlw(s, RLWINM, a0, a1, 24, 24, 31);
2655 /* a0 = (a0 & ~0xff00) | (a1 r<< 8) & 0xff00 # 00dc */
2656 tcg_out_rlw(s, RLWIMI, a0, a1, 8, 16, 23);
2657 } else {
2658 /* r0 = (a1 r<< 8) & 0xff00 # 00d0 */
2659 tcg_out_rlw(s, RLWINM, TCG_REG_R0, a1, 8, 16, 23);
2660 /* a0 = (a1 r<< 24) & 0xff # 000c */
2661 tcg_out_rlw(s, RLWINM, a0, a1, 24, 24, 31);
2662 /* a0 = a0 | r0 # 00dc */
2663 tcg_out32(s, OR | SAB(TCG_REG_R0, a0, a0));
2664 }
2665 break;
2666
2667 case INDEX_op_bswap32_i32:
2668 case INDEX_op_bswap32_i64:
2669 /* Stolen from gcc's builtin_bswap32 */
2670 a1 = args[1];
2671 a0 = args[0] == a1 ? TCG_REG_R0 : args[0];
2672
2673 /* a1 = args[1] # abcd */
2674 /* a0 = rotate_left (a1, 8) # bcda */
2675 tcg_out_rlw(s, RLWINM, a0, a1, 8, 0, 31);
2676 /* a0 = (a0 & ~0xff000000) | ((a1 r<< 24) & 0xff000000) # dcda */
2677 tcg_out_rlw(s, RLWIMI, a0, a1, 24, 0, 7);
2678 /* a0 = (a0 & ~0x0000ff00) | ((a1 r<< 24) & 0x0000ff00) # dcba */
2679 tcg_out_rlw(s, RLWIMI, a0, a1, 24, 16, 23);
2680
2681 if (a0 == TCG_REG_R0) {
de3d636d 2682 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
5d221582
RH		 2683 }
2684 break;
2685
68aebd45 2686 case INDEX_op_bswap64_i64:
8327a470 2687 a0 = args[0], a1 = args[1], a2 = TCG_REG_R0;
68aebd45 2688 if (a0 == a1) {
8327a470 2689 a0 = TCG_REG_R0;
68aebd45
RH		 2690 a2 = a1;
2691 }
2692
2693 /* a1 = # abcd efgh */
2694 /* a0 = rl32(a1, 8) # 0000 fghe */
2695 tcg_out_rlw(s, RLWINM, a0, a1, 8, 0, 31);
2696 /* a0 = dep(a0, rl32(a1, 24), 0xff000000) # 0000 hghe */
2697 tcg_out_rlw(s, RLWIMI, a0, a1, 24, 0, 7);
2698 /* a0 = dep(a0, rl32(a1, 24), 0x0000ff00) # 0000 hgfe */
2699 tcg_out_rlw(s, RLWIMI, a0, a1, 24, 16, 23);
2700
2701 /* a0 = rl64(a0, 32) # hgfe 0000 */
2702 /* a2 = rl64(a1, 32) # efgh abcd */
2703 tcg_out_rld(s, RLDICL, a0, a0, 32, 0);
2704 tcg_out_rld(s, RLDICL, a2, a1, 32, 0);
2705
2706 /* a0 = dep(a0, rl32(a2, 8), 0xffffffff) # hgfe bcda */
2707 tcg_out_rlw(s, RLWIMI, a0, a2, 8, 0, 31);
2708 /* a0 = dep(a0, rl32(a2, 24), 0xff000000) # hgfe dcda */
2709 tcg_out_rlw(s, RLWIMI, a0, a2, 24, 0, 7);
2710 /* a0 = dep(a0, rl32(a2, 24), 0x0000ff00) # hgfe dcba */
2711 tcg_out_rlw(s, RLWIMI, a0, a2, 24, 16, 23);
2712
2713 if (a0 == 0) {
de3d636d 2714 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
68aebd45
RH		 2715 }
2716 break;
2717
33de9ed2 2718 case INDEX_op_deposit_i32:
39dc85b9
RH		 2719 if (const_args[2]) {
2720 uint32_t mask = ((2u << (args[4] - 1)) - 1) << args[3];
2721 tcg_out_andi32(s, args[0], args[0], ~mask);
2722 } else {
2723 tcg_out_rlw(s, RLWIMI, args[0], args[2], args[3],
2724 32 - args[3] - args[4], 31 - args[3]);
2725 }
33de9ed2
RH		 2726 break;
2727 case INDEX_op_deposit_i64:
39dc85b9
RH		 2728 if (const_args[2]) {
2729 uint64_t mask = ((2ull << (args[4] - 1)) - 1) << args[3];
2730 tcg_out_andi64(s, args[0], args[0], ~mask);
2731 } else {
2732 tcg_out_rld(s, RLDIMI, args[0], args[2], args[3],
2733 64 - args[3] - args[4]);
2734 }
33de9ed2
RH		 2735 break;
2736
c05021c3
RH		 2737 case INDEX_op_extract_i32:
2738 tcg_out_rlw(s, RLWINM, args[0], args[1],
2739 32 - args[2], 32 - args[3], 31);
2740 break;
2741 case INDEX_op_extract_i64:
2742 tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 64 - args[3]);
2743 break;
2744
027ffea9
RH		 2745 case INDEX_op_movcond_i32:
2746 tcg_out_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1], args[2],
2747 args[3], args[4], const_args[2]);
2748 break;
2749 case INDEX_op_movcond_i64:
2750 tcg_out_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1], args[2],
2751 args[3], args[4], const_args[2]);
2752 break;
2753
796f1a68 2754#if TCG_TARGET_REG_BITS == 64
6c858762 2755 case INDEX_op_add2_i64:
796f1a68
RH		 2756#else
2757 case INDEX_op_add2_i32:
2758#endif
6c858762
RH		 2759 /* Note that the CA bit is defined based on the word size of the
2760 environment. So in 64-bit mode it's always carry-out of bit 63.
2761 The fallback code using deposit works just as well for 32-bit. */
2762 a0 = args[0], a1 = args[1];
84247357 2763 if (a0 == args[3] || (!const_args[5] && a0 == args[5])) {
6c858762
RH		 2764 a0 = TCG_REG_R0;
2765 }
84247357
AB		 2766 if (const_args[4]) {
2767 tcg_out32(s, ADDIC | TAI(a0, args[2], args[4]));
6c858762 2768 } else {
84247357 2769 tcg_out32(s, ADDC | TAB(a0, args[2], args[4]));
6c858762
RH		 2770 }
2771 if (const_args[5]) {
84247357 2772 tcg_out32(s, (args[5] ? ADDME : ADDZE) | RT(a1) | RA(args[3]));
6c858762 2773 } else {
84247357 2774 tcg_out32(s, ADDE | TAB(a1, args[3], args[5]));
6c858762
RH		 2775 }
2776 if (a0 != args[0]) {
de3d636d 2777 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
6c858762
RH		 2778 }
2779 break;
2780
796f1a68 2781#if TCG_TARGET_REG_BITS == 64
6c858762 2782 case INDEX_op_sub2_i64:
796f1a68
RH		 2783#else
2784 case INDEX_op_sub2_i32:
2785#endif
6c858762 2786 a0 = args[0], a1 = args[1];
b31284ce 2787 if (a0 == args[5] || (!const_args[3] && a0 == args[3])) {
6c858762
RH		 2788 a0 = TCG_REG_R0;
2789 }
2790 if (const_args[2]) {
b31284ce 2791 tcg_out32(s, SUBFIC | TAI(a0, args[4], args[2]));
6c858762 2792 } else {
b31284ce 2793 tcg_out32(s, SUBFC | TAB(a0, args[4], args[2]));
6c858762 2794 }
b31284ce
RH		 2795 if (const_args[3]) {
2796 tcg_out32(s, (args[3] ? SUBFME : SUBFZE) | RT(a1) | RA(args[5]));
6c858762 2797 } else {
b31284ce 2798 tcg_out32(s, SUBFE | TAB(a1, args[5], args[3]));
6c858762
RH		 2799 }
2800 if (a0 != args[0]) {
de3d636d 2801 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
6c858762
RH		 2802 }
2803 break;
2804
abcf61c4
RH		 2805 case INDEX_op_muluh_i32:
2806 tcg_out32(s, MULHWU | TAB(args[0], args[1], args[2]));
2807 break;
8fa391a0
RH		 2808 case INDEX_op_mulsh_i32:
2809 tcg_out32(s, MULHW | TAB(args[0], args[1], args[2]));
2810 break;
32f5717f
RH		 2811 case INDEX_op_muluh_i64:
2812 tcg_out32(s, MULHDU | TAB(args[0], args[1], args[2]));
2813 break;
2814 case INDEX_op_mulsh_i64:
2815 tcg_out32(s, MULHD | TAB(args[0], args[1], args[2]));
6645c147
RH		 2816 break;
2817
7b4af5ee
PK		 2818 case INDEX_op_mb:
2819 tcg_out_mb(s, args[0]);
2820 break;
2821
96d0ee7f
RH		 2822 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
2823 case INDEX_op_mov_i64:
2824 case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */
2825 case INDEX_op_movi_i64:
2826 case INDEX_op_call: /* Always emitted via tcg_out_call. */
810260a8 2827 default:
541dd4ce 2828 tcg_abort();
810260a8 2829 }
2830}
2831
4b06c216
RH		 2832int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
2833{
6ef14d7e
RH		 2834 switch (opc) {
2835 case INDEX_op_and_vec:
2836 case INDEX_op_or_vec:
2837 case INDEX_op_xor_vec:
2838 case INDEX_op_andc_vec:
2839 case INDEX_op_not_vec:
2840 return 1;
d6750811
RH		 2841 case INDEX_op_add_vec:
2842 case INDEX_op_sub_vec:
e2382972
RH		 2843 case INDEX_op_smax_vec:
2844 case INDEX_op_smin_vec:
2845 case INDEX_op_umax_vec:
2846 case INDEX_op_umin_vec:
2847 return vece <= MO_32;
6ef14d7e
RH		 2848 case INDEX_op_cmp_vec:
2849 return vece <= MO_32 ? -1 : 0;
2850 default:
2851 return 0;
2852 }
4b06c216
RH		 2853}
2854
2855static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
2856 TCGReg dst, TCGReg src)
2857{
6ef14d7e
RH		 2858 tcg_debug_assert(dst >= TCG_REG_V0);
2859 tcg_debug_assert(src >= TCG_REG_V0);
2860
2861 /*
2862 * Recall we use (or emulate) VSX integer loads, so the integer is
2863 * right justified within the left (zero-index) double-word.
2864 */
2865 switch (vece) {
2866 case MO_8:
2867 tcg_out32(s, VSPLTB | VRT(dst) | VRB(src) | (7 << 16));
2868 break;
2869 case MO_16:
2870 tcg_out32(s, VSPLTH | VRT(dst) | VRB(src) | (3 << 16));
2871 break;
2872 case MO_32:
2873 tcg_out32(s, VSPLTW | VRT(dst) | VRB(src) | (1 << 16));
2874 break;
2875 case MO_64:
2876 tcg_out_vsldoi(s, TCG_VEC_TMP1, src, src, 8);
2877 tcg_out_vsldoi(s, dst, TCG_VEC_TMP1, src, 8);
2878 break;
2879 default:
2880 g_assert_not_reached();
2881 }
2882 return true;
4b06c216
RH		 2883}
2884
2885static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
2886 TCGReg out, TCGReg base, intptr_t offset)
2887{
6ef14d7e
RH		 2888 int elt;
2889
2890 tcg_debug_assert(out >= TCG_REG_V0);
2891 switch (vece) {
2892 case MO_8:
2893 tcg_out_mem_long(s, 0, LVEBX, out, base, offset);
2894 elt = extract32(offset, 0, 4);
2895#ifndef HOST_WORDS_BIGENDIAN
2896 elt ^= 15;
2897#endif
2898 tcg_out32(s, VSPLTB | VRT(out) | VRB(out) | (elt << 16));
2899 break;
2900 case MO_16:
2901 tcg_debug_assert((offset & 1) == 0);
2902 tcg_out_mem_long(s, 0, LVEHX, out, base, offset);
2903 elt = extract32(offset, 1, 3);
2904#ifndef HOST_WORDS_BIGENDIAN
2905 elt ^= 7;
2906#endif
2907 tcg_out32(s, VSPLTH | VRT(out) | VRB(out) | (elt << 16));
2908 break;
2909 case MO_32:
2910 tcg_debug_assert((offset & 3) == 0);
2911 tcg_out_mem_long(s, 0, LVEWX, out, base, offset);
2912 elt = extract32(offset, 2, 2);
2913#ifndef HOST_WORDS_BIGENDIAN
2914 elt ^= 3;
2915#endif
2916 tcg_out32(s, VSPLTW | VRT(out) | VRB(out) | (elt << 16));
2917 break;
2918 case MO_64:
2919 tcg_debug_assert((offset & 7) == 0);
2920 tcg_out_mem_long(s, 0, LVX, out, base, offset & -16);
2921 tcg_out_vsldoi(s, TCG_VEC_TMP1, out, out, 8);
2922 elt = extract32(offset, 3, 1);
2923#ifndef HOST_WORDS_BIGENDIAN
2924 elt = !elt;
2925#endif
2926 if (elt) {
2927 tcg_out_vsldoi(s, out, out, TCG_VEC_TMP1, 8);
2928 } else {
2929 tcg_out_vsldoi(s, out, TCG_VEC_TMP1, out, 8);
2930 }
2931 break;
2932 default:
2933 g_assert_not_reached();
2934 }
2935 return true;
4b06c216
RH		 2936}
2937
2938static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
2939 unsigned vecl, unsigned vece,
2940 const TCGArg *args, const int *const_args)
2941{
6ef14d7e 2942 static const uint32_t
d6750811
RH		 2943 add_op[4] = { VADDUBM, VADDUHM, VADDUWM, 0 },
2944 sub_op[4] = { VSUBUBM, VSUBUHM, VSUBUWM, 0 },
6ef14d7e
RH		 2945 eq_op[4] = { VCMPEQUB, VCMPEQUH, VCMPEQUW, 0 },
2946 gts_op[4] = { VCMPGTSB, VCMPGTSH, VCMPGTSW, 0 },
e2382972
RH		 2947 gtu_op[4] = { VCMPGTUB, VCMPGTUH, VCMPGTUW, 0 },
2948 umin_op[4] = { VMINUB, VMINUH, VMINUW, 0 },
2949 smin_op[4] = { VMINSB, VMINSH, VMINSW, 0 },
2950 umax_op[4] = { VMAXUB, VMAXUH, VMAXUW, 0 },
2951 smax_op[4] = { VMAXSB, VMAXSH, VMAXSW, 0 };
6ef14d7e
RH		 2952
2953 TCGType type = vecl + TCG_TYPE_V64;
2954 TCGArg a0 = args[0], a1 = args[1], a2 = args[2];
2955 uint32_t insn;
2956
2957 switch (opc) {
2958 case INDEX_op_ld_vec:
2959 tcg_out_ld(s, type, a0, a1, a2);
2960 return;
2961 case INDEX_op_st_vec:
2962 tcg_out_st(s, type, a0, a1, a2);
2963 return;
2964 case INDEX_op_dupm_vec:
2965 tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
2966 return;
2967
d6750811
RH		 2968 case INDEX_op_add_vec:
2969 insn = add_op[vece];
2970 break;
2971 case INDEX_op_sub_vec:
2972 insn = sub_op[vece];
2973 break;
e2382972
RH		 2974 case INDEX_op_smin_vec:
2975 insn = smin_op[vece];
2976 break;
2977 case INDEX_op_umin_vec:
2978 insn = umin_op[vece];
2979 break;
2980 case INDEX_op_smax_vec:
2981 insn = smax_op[vece];
2982 break;
2983 case INDEX_op_umax_vec:
2984 insn = umax_op[vece];
2985 break;
6ef14d7e
RH		 2986 case INDEX_op_and_vec:
2987 insn = VAND;
2988 break;
2989 case INDEX_op_or_vec:
2990 insn = VOR;
2991 break;
2992 case INDEX_op_xor_vec:
2993 insn = VXOR;
2994 break;
2995 case INDEX_op_andc_vec:
2996 insn = VANDC;
2997 break;
2998 case INDEX_op_not_vec:
2999 insn = VNOR;
3000 a2 = a1;
3001 break;
3002
3003 case INDEX_op_cmp_vec:
3004 switch (args[3]) {
3005 case TCG_COND_EQ:
3006 insn = eq_op[vece];
3007 break;
3008 case TCG_COND_GT:
3009 insn = gts_op[vece];
3010 break;
3011 case TCG_COND_GTU:
3012 insn = gtu_op[vece];
3013 break;
3014 default:
3015 g_assert_not_reached();
3016 }
3017 break;
3018
3019 case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
3020 case INDEX_op_dupi_vec: /* Always emitted via tcg_out_movi. */
3021 case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
3022 default:
3023 g_assert_not_reached();
3024 }
3025
3026 tcg_debug_assert(insn != 0);
3027 tcg_out32(s, insn | VRT(a0) | VRA(a1) | VRB(a2));
3028}
3029
3030static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0,
3031 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
3032{
3033 bool need_swap = false, need_inv = false;
3034
3035 tcg_debug_assert(vece <= MO_32);
3036
3037 switch (cond) {
3038 case TCG_COND_EQ:
3039 case TCG_COND_GT:
3040 case TCG_COND_GTU:
3041 break;
3042 case TCG_COND_NE:
3043 case TCG_COND_LE:
3044 case TCG_COND_LEU:
3045 need_inv = true;
3046 break;
3047 case TCG_COND_LT:
3048 case TCG_COND_LTU:
3049 need_swap = true;
3050 break;
3051 case TCG_COND_GE:
3052 case TCG_COND_GEU:
3053 need_swap = need_inv = true;
3054 break;
3055 default:
3056 g_assert_not_reached();
3057 }
3058
3059 if (need_inv) {
3060 cond = tcg_invert_cond(cond);
3061 }
3062 if (need_swap) {
3063 TCGv_vec t1;
3064 t1 = v1, v1 = v2, v2 = t1;
3065 cond = tcg_swap_cond(cond);
3066 }
3067
3068 vec_gen_4(INDEX_op_cmp_vec, type, vece, tcgv_vec_arg(v0),
3069 tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond);
3070
3071 if (need_inv) {
3072 tcg_gen_not_vec(vece, v0, v0);
3073 }
4b06c216
RH		 3074}
3075
3076void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
3077 TCGArg a0, ...)
3078{
6ef14d7e
RH		 3079 va_list va;
3080 TCGv_vec v0, v1, v2;
3081
3082 va_start(va, a0);
3083 v0 = temp_tcgv_vec(arg_temp(a0));
3084 v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3085 v2 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3086
3087 switch (opc) {
3088 case INDEX_op_cmp_vec:
3089 expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg));
3090 break;
3091 default:
3092 g_assert_not_reached();
3093 }
3094 va_end(va);
4b06c216
RH		 3095}
3096
6cb3658a
RH		 3097static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op)
3098{
3099 static const TCGTargetOpDef r = { .args_ct_str = { "r" } };
3100 static const TCGTargetOpDef r_r = { .args_ct_str = { "r", "r" } };
3101 static const TCGTargetOpDef r_L = { .args_ct_str = { "r", "L" } };
3102 static const TCGTargetOpDef S_S = { .args_ct_str = { "S", "S" } };
3103 static const TCGTargetOpDef r_ri = { .args_ct_str = { "r", "ri" } };
3104 static const TCGTargetOpDef r_r_r = { .args_ct_str = { "r", "r", "r" } };
3105 static const TCGTargetOpDef r_L_L = { .args_ct_str = { "r", "L", "L" } };
3106 static const TCGTargetOpDef L_L_L = { .args_ct_str = { "L", "L", "L" } };
3107 static const TCGTargetOpDef S_S_S = { .args_ct_str = { "S", "S", "S" } };
3108 static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } };
3109 static const TCGTargetOpDef r_r_rI = { .args_ct_str = { "r", "r", "rI" } };
3110 static const TCGTargetOpDef r_r_rT = { .args_ct_str = { "r", "r", "rT" } };
3111 static const TCGTargetOpDef r_r_rU = { .args_ct_str = { "r", "r", "rU" } };
3112 static const TCGTargetOpDef r_rI_ri
3113 = { .args_ct_str = { "r", "rI", "ri" } };
3114 static const TCGTargetOpDef r_rI_rT
3115 = { .args_ct_str = { "r", "rI", "rT" } };
3116 static const TCGTargetOpDef r_r_rZW
3117 = { .args_ct_str = { "r", "r", "rZW" } };
3118 static const TCGTargetOpDef L_L_L_L
3119 = { .args_ct_str = { "L", "L", "L", "L" } };
3120 static const TCGTargetOpDef S_S_S_S
3121 = { .args_ct_str = { "S", "S", "S", "S" } };
3122 static const TCGTargetOpDef movc
3123 = { .args_ct_str = { "r", "r", "ri", "rZ", "rZ" } };
3124 static const TCGTargetOpDef dep
3125 = { .args_ct_str = { "r", "0", "rZ" } };
3126 static const TCGTargetOpDef br2
3127 = { .args_ct_str = { "r", "r", "ri", "ri" } };
3128 static const TCGTargetOpDef setc2
3129 = { .args_ct_str = { "r", "r", "r", "ri", "ri" } };
3130 static const TCGTargetOpDef add2
3131 = { .args_ct_str = { "r", "r", "r", "r", "rI", "rZM" } };
3132 static const TCGTargetOpDef sub2
3133 = { .args_ct_str = { "r", "r", "rI", "rZM", "r", "r" } };
6ef14d7e
RH		 3134 static const TCGTargetOpDef v_r = { .args_ct_str = { "v", "r" } };
3135 static const TCGTargetOpDef v_v = { .args_ct_str = { "v", "v" } };
3136 static const TCGTargetOpDef v_v_v = { .args_ct_str = { "v", "v", "v" } };
6cb3658a
RH		 3137
3138 switch (op) {
3139 case INDEX_op_goto_ptr:
3140 return &r;
796f1a68 3141
6cb3658a
RH		 3142 case INDEX_op_ld8u_i32:
3143 case INDEX_op_ld8s_i32:
3144 case INDEX_op_ld16u_i32:
3145 case INDEX_op_ld16s_i32:
3146 case INDEX_op_ld_i32:
3147 case INDEX_op_st8_i32:
3148 case INDEX_op_st16_i32:
3149 case INDEX_op_st_i32:
3150 case INDEX_op_ctpop_i32:
3151 case INDEX_op_neg_i32:
3152 case INDEX_op_not_i32:
3153 case INDEX_op_ext8s_i32:
3154 case INDEX_op_ext16s_i32:
3155 case INDEX_op_bswap16_i32:
3156 case INDEX_op_bswap32_i32:
3157 case INDEX_op_extract_i32:
3158 case INDEX_op_ld8u_i64:
3159 case INDEX_op_ld8s_i64:
3160 case INDEX_op_ld16u_i64:
3161 case INDEX_op_ld16s_i64:
3162 case INDEX_op_ld32u_i64:
3163 case INDEX_op_ld32s_i64:
3164 case INDEX_op_ld_i64:
3165 case INDEX_op_st8_i64:
3166 case INDEX_op_st16_i64:
3167 case INDEX_op_st32_i64:
3168 case INDEX_op_st_i64:
3169 case INDEX_op_ctpop_i64:
3170 case INDEX_op_neg_i64:
3171 case INDEX_op_not_i64:
3172 case INDEX_op_ext8s_i64:
3173 case INDEX_op_ext16s_i64:
3174 case INDEX_op_ext32s_i64:
3175 case INDEX_op_ext_i32_i64:
3176 case INDEX_op_extu_i32_i64:
3177 case INDEX_op_bswap16_i64:
3178 case INDEX_op_bswap32_i64:
3179 case INDEX_op_bswap64_i64:
3180 case INDEX_op_extract_i64:
3181 return &r_r;
abcf61c4 3182
6cb3658a
RH		 3183 case INDEX_op_add_i32:
3184 case INDEX_op_and_i32:
3185 case INDEX_op_or_i32:
3186 case INDEX_op_xor_i32:
3187 case INDEX_op_andc_i32:
3188 case INDEX_op_orc_i32:
3189 case INDEX_op_eqv_i32:
3190 case INDEX_op_shl_i32:
3191 case INDEX_op_shr_i32:
3192 case INDEX_op_sar_i32:
3193 case INDEX_op_rotl_i32:
3194 case INDEX_op_rotr_i32:
3195 case INDEX_op_setcond_i32:
3196 case INDEX_op_and_i64:
3197 case INDEX_op_andc_i64:
3198 case INDEX_op_shl_i64:
3199 case INDEX_op_shr_i64:
3200 case INDEX_op_sar_i64:
3201 case INDEX_op_rotl_i64:
3202 case INDEX_op_rotr_i64:
3203 case INDEX_op_setcond_i64:
3204 return &r_r_ri;
3205 case INDEX_op_mul_i32:
3206 case INDEX_op_mul_i64:
3207 return &r_r_rI;
3208 case INDEX_op_div_i32:
3209 case INDEX_op_divu_i32:
3210 case INDEX_op_nand_i32:
3211 case INDEX_op_nor_i32:
3212 case INDEX_op_muluh_i32:
3213 case INDEX_op_mulsh_i32:
3214 case INDEX_op_orc_i64:
3215 case INDEX_op_eqv_i64:
3216 case INDEX_op_nand_i64:
3217 case INDEX_op_nor_i64:
3218 case INDEX_op_div_i64:
3219 case INDEX_op_divu_i64:
3220 case INDEX_op_mulsh_i64:
3221 case INDEX_op_muluh_i64:
3222 return &r_r_r;
3223 case INDEX_op_sub_i32:
3224 return &r_rI_ri;
3225 case INDEX_op_add_i64:
3226 return &r_r_rT;
3227 case INDEX_op_or_i64:
3228 case INDEX_op_xor_i64:
3229 return &r_r_rU;
3230 case INDEX_op_sub_i64:
3231 return &r_rI_rT;
3232 case INDEX_op_clz_i32:
3233 case INDEX_op_ctz_i32:
3234 case INDEX_op_clz_i64:
3235 case INDEX_op_ctz_i64:
3236 return &r_r_rZW;
796f1a68 3237
6cb3658a
RH		 3238 case INDEX_op_brcond_i32:
3239 case INDEX_op_brcond_i64:
3240 return &r_ri;
6c858762 3241
6cb3658a
RH		 3242 case INDEX_op_movcond_i32:
3243 case INDEX_op_movcond_i64:
3244 return &movc;
3245 case INDEX_op_deposit_i32:
3246 case INDEX_op_deposit_i64:
3247 return &dep;
3248 case INDEX_op_brcond2_i32:
3249 return &br2;
3250 case INDEX_op_setcond2_i32:
3251 return &setc2;
3252 case INDEX_op_add2_i64:
3253 case INDEX_op_add2_i32:
3254 return &add2;
3255 case INDEX_op_sub2_i64:
3256 case INDEX_op_sub2_i32:
3257 return &sub2;
810260a8 3258
6cb3658a
RH		 3259 case INDEX_op_qemu_ld_i32:
3260 return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32
3261 ? &r_L : &r_L_L);
3262 case INDEX_op_qemu_st_i32:
3263 return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32
3264 ? &S_S : &S_S_S);
3265 case INDEX_op_qemu_ld_i64:
3266 return (TCG_TARGET_REG_BITS == 64 ? &r_L
3267 : TARGET_LONG_BITS == 32 ? &L_L_L : &L_L_L_L);
3268 case INDEX_op_qemu_st_i64:
3269 return (TCG_TARGET_REG_BITS == 64 ? &S_S
3270 : TARGET_LONG_BITS == 32 ? &S_S_S : &S_S_S_S);
f69d277e 3271
d6750811
RH		 3272 case INDEX_op_add_vec:
3273 case INDEX_op_sub_vec:
6ef14d7e
RH		 3274 case INDEX_op_and_vec:
3275 case INDEX_op_or_vec:
3276 case INDEX_op_xor_vec:
3277 case INDEX_op_andc_vec:
3278 case INDEX_op_orc_vec:
3279 case INDEX_op_cmp_vec:
e2382972
RH		 3280 case INDEX_op_smax_vec:
3281 case INDEX_op_smin_vec:
3282 case INDEX_op_umax_vec:
3283 case INDEX_op_umin_vec:
6ef14d7e
RH		 3284 return &v_v_v;
3285 case INDEX_op_not_vec:
3286 case INDEX_op_dup_vec:
3287 return &v_v;
3288 case INDEX_op_ld_vec:
3289 case INDEX_op_st_vec:
3290 case INDEX_op_dupm_vec:
3291 return &v_r;
3292
6cb3658a
RH		 3293 default:
3294 return NULL;
f69d277e 3295 }
f69d277e
RH		 3296}
3297
541dd4ce 3298static void tcg_target_init(TCGContext *s)
810260a8 3299{
cd629de1 3300 unsigned long hwcap = qemu_getauxval(AT_HWCAP);
d0b07481
RH		 3301 unsigned long hwcap2 = qemu_getauxval(AT_HWCAP2);
3302
7d9dae0a 3303 have_isa = tcg_isa_base;
1e6e9aca 3304 if (hwcap & PPC_FEATURE_ARCH_2_06) {
7d9dae0a 3305 have_isa = tcg_isa_2_06;
1e6e9aca 3306 }
d0b07481
RH		 3307#ifdef PPC_FEATURE2_ARCH_3_00
3308 if (hwcap2 & PPC_FEATURE2_ARCH_3_00) {
7d9dae0a 3309 have_isa = tcg_isa_3_00;
d0b07481
RH		 3310 }
3311#endif
1e6e9aca 3312
63922f46
RH		 3313#ifdef PPC_FEATURE2_HAS_ISEL
3314 /* Prefer explicit instruction from the kernel. */
3315 have_isel = (hwcap2 & PPC_FEATURE2_HAS_ISEL) != 0;
3316#else
3317 /* Fall back to knowing Power7 (2.06) has ISEL. */
3318 have_isel = have_isa_2_06;
3319#endif
3320
f46934df
RH		 3321 tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff;
3322 tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff;
4b06c216
RH		 3323 if (have_altivec) {
3324 tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
3325 tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull;
3326 }
f46934df
RH		 3327
3328 tcg_target_call_clobber_regs = 0;
3329 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0);
3330 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2);
3331 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3);
3332 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4);
3333 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5);
3334 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6);
3335 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R7);
3336 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
3337 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
3338 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
3339 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
3340 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R12);
810260a8 3341
42281ec6
RH		 3342 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V0);
3343 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V1);
3344 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V2);
3345 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V3);
3346 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V4);
3347 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V5);
3348 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V6);
3349 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V7);
3350 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V8);
3351 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V9);
3352 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V10);
3353 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V11);
3354 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V12);
3355 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V13);
3356 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V14);
3357 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V15);
3358 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V16);
3359 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V17);
3360 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V18);
3361 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V19);
3362
ccb1bb66 3363 s->reserved_regs = 0;
5e1702b0
RH		 3364 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0); /* tcg temp */
3365 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1); /* stack pointer */
dfca1778
RH		 3366#if defined(_CALL_SYSV)
3367 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R2); /* toc pointer */
5d7ff5bb 3368#endif
dfca1778 3369#if defined(_CALL_SYSV) || TCG_TARGET_REG_BITS == 64
5e1702b0 3370 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R13); /* thread pointer */
dfca1778
RH		 3371#endif
3372 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1); /* mem temp */
42281ec6
RH		 3373 tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP1);
3374 tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP2);
5964fca8
RH		 3375 if (USE_REG_TB) {
3376 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB); /* tb->tc_ptr */
a84ac4cb 3377 }
810260a8 3378}
fa94c3be 3379
ffcfbece 3380#ifdef __ELF__
fa94c3be
RH		 3381typedef struct {
3382 DebugFrameCIE cie;
3383 DebugFrameFDEHeader fde;
3384 uint8_t fde_def_cfa[4];
3385 uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2 + 3];
3386} DebugFrame;
3387
3388/* We're expecting a 2 byte uleb128 encoded value. */
3389QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
3390
ffcfbece
RH		 3391#if TCG_TARGET_REG_BITS == 64
3392# define ELF_HOST_MACHINE EM_PPC64
3393#else
3394# define ELF_HOST_MACHINE EM_PPC
3395#endif
fa94c3be
RH		 3396
3397static DebugFrame debug_frame = {
3398 .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
3399 .cie.id = -1,
3400 .cie.version = 1,
3401 .cie.code_align = 1,
802ca56e 3402 .cie.data_align = (-SZR & 0x7f), /* sleb128 -SZR */
fa94c3be
RH		 3403 .cie.return_column = 65,
3404
3405 /* Total FDE size does not include the "len" member. */
3406 .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
3407
3408 .fde_def_cfa = {
802ca56e 3409 12, TCG_REG_R1, /* DW_CFA_def_cfa r1, ... */
fa94c3be
RH		 3410 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
3411 (FRAME_SIZE >> 7)
3412 },
3413 .fde_reg_ofs = {
802ca56e
RH		 3414 /* DW_CFA_offset_extended_sf, lr, LR_OFFSET */
3415 0x11, 65, (LR_OFFSET / -SZR) & 0x7f,
fa94c3be
RH		 3416 }
3417};
3418
3419void tcg_register_jit(void *buf, size_t buf_size)
3420{
3421 uint8_t *p = &debug_frame.fde_reg_ofs[3];
3422 int i;
3423
3424 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i, p += 2) {
3425 p[0] = 0x80 + tcg_target_callee_save_regs[i];
802ca56e 3426 p[1] = (FRAME_SIZE - (REG_SAVE_BOT + i * SZR)) / SZR;
fa94c3be
RH		 3427 }
3428
802ca56e 3429 debug_frame.fde.func_start = (uintptr_t)buf;
fa94c3be
RH		 3430 debug_frame.fde.func_len = buf_size;
3431
3432 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
3433}
ffcfbece 3434#endif /* __ELF__ */
224f9fd4 3435
224f9fd4
RH		 3436void flush_icache_range(uintptr_t start, uintptr_t stop)
3437{
3438 uintptr_t p, start1, stop1;
b255b2c8
EC		 3439 size_t dsize = qemu_dcache_linesize;
3440 size_t isize = qemu_icache_linesize;
224f9fd4
RH		 3441
3442 start1 = start & ~(dsize - 1);
3443 stop1 = (stop + dsize - 1) & ~(dsize - 1);
3444 for (p = start1; p < stop1; p += dsize) {
3445 asm volatile ("dcbst 0,%0" : : "r"(p) : "memory");
3446 }
3447 asm volatile ("sync" : : : "memory");
3448
3449 start &= start & ~(isize - 1);
3450 stop1 = (stop + isize - 1) & ~(isize - 1);
3451 for (p = start1; p < stop1; p += isize) {
3452 asm volatile ("icbi 0,%0" : : "r"(p) : "memory");
3453 }
3454 asm volatile ("sync" : : : "memory");
3455 asm volatile ("isync" : : : "memory");
3456}
Mirror of https://git.qemu.org/git/qemu.git