2 * Common CPU TLB handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
22 #include "hw/core/tcg-cpu-ops.h"
23 #include "exec/exec-all.h"
24 #include "exec/memory.h"
25 #include "exec/cpu_ldst.h"
26 #include "exec/cputlb.h"
27 #include "exec/tb-flush.h"
28 #include "exec/memory-internal.h"
29 #include "exec/ram_addr.h"
31 #include "qemu/error-report.h"
33 #include "exec/helper-proto-common.h"
34 #include "qemu/atomic.h"
35 #include "qemu/atomic128.h"
36 #include "exec/translate-all.h"
39 #include "internal-common.h"
40 #include "internal-target.h"
42 #include "qemu/plugin-memory.h"
44 #include "tcg/tcg-ldst.h"
45 #include "tcg/oversized-guest.h"
47 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
48 /* #define DEBUG_TLB */
49 /* #define DEBUG_TLB_LOG */
52 # define DEBUG_TLB_GATE 1
54 # define DEBUG_TLB_LOG_GATE 1
56 # define DEBUG_TLB_LOG_GATE 0
59 # define DEBUG_TLB_GATE 0
60 # define DEBUG_TLB_LOG_GATE 0
63 #define tlb_debug(fmt, ...) do { \
64 if (DEBUG_TLB_LOG_GATE) { \
65 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
67 } else if (DEBUG_TLB_GATE) { \
68 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
72 #define assert_cpu_is_self(cpu) do { \
73 if (DEBUG_TLB_GATE) { \
74 g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \
78 /* run_on_cpu_data.target_ptr should always be big enough for a
79 * vaddr even on 32 bit builds
81 QEMU_BUILD_BUG_ON(sizeof(vaddr
) > sizeof(run_on_cpu_data
));
83 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
85 QEMU_BUILD_BUG_ON(NB_MMU_MODES
> 16);
86 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
88 static inline size_t tlb_n_entries(CPUTLBDescFast
*fast
)
90 return (fast
->mask
>> CPU_TLB_ENTRY_BITS
) + 1;
93 static inline size_t sizeof_tlb(CPUTLBDescFast
*fast
)
95 return fast
->mask
+ (1 << CPU_TLB_ENTRY_BITS
);
98 static void tlb_window_reset(CPUTLBDesc
*desc
, int64_t ns
,
101 desc
->window_begin_ns
= ns
;
102 desc
->window_max_entries
= max_entries
;
105 static void tb_jmp_cache_clear_page(CPUState
*cpu
, vaddr page_addr
)
107 CPUJumpCache
*jc
= cpu
->tb_jmp_cache
;
114 i0
= tb_jmp_cache_hash_page(page_addr
);
115 for (i
= 0; i
< TB_JMP_PAGE_SIZE
; i
++) {
116 qatomic_set(&jc
->array
[i0
+ i
].tb
, NULL
);
121 * tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary
122 * @desc: The CPUTLBDesc portion of the TLB
123 * @fast: The CPUTLBDescFast portion of the same TLB
125 * Called with tlb_lock_held.
127 * We have two main constraints when resizing a TLB: (1) we only resize it
128 * on a TLB flush (otherwise we'd have to take a perf hit by either rehashing
129 * the array or unnecessarily flushing it), which means we do not control how
130 * frequently the resizing can occur; (2) we don't have access to the guest's
131 * future scheduling decisions, and therefore have to decide the magnitude of
132 * the resize based on past observations.
134 * In general, a memory-hungry process can benefit greatly from an appropriately
135 * sized TLB, since a guest TLB miss is very expensive. This doesn't mean that
136 * we just have to make the TLB as large as possible; while an oversized TLB
137 * results in minimal TLB miss rates, it also takes longer to be flushed
138 * (flushes can be _very_ frequent), and the reduced locality can also hurt
141 * To achieve near-optimal performance for all kinds of workloads, we:
143 * 1. Aggressively increase the size of the TLB when the use rate of the
144 * TLB being flushed is high, since it is likely that in the near future this
145 * memory-hungry process will execute again, and its memory hungriness will
146 * probably be similar.
148 * 2. Slowly reduce the size of the TLB as the use rate declines over a
149 * reasonably large time window. The rationale is that if in such a time window
150 * we have not observed a high TLB use rate, it is likely that we won't observe
151 * it in the near future. In that case, once a time window expires we downsize
152 * the TLB to match the maximum use rate observed in the window.
154 * 3. Try to keep the maximum use rate in a time window in the 30-70% range,
155 * since in that range performance is likely near-optimal. Recall that the TLB
156 * is direct mapped, so we want the use rate to be low (or at least not too
157 * high), since otherwise we are likely to have a significant amount of
160 static void tlb_mmu_resize_locked(CPUTLBDesc
*desc
, CPUTLBDescFast
*fast
,
163 size_t old_size
= tlb_n_entries(fast
);
165 size_t new_size
= old_size
;
166 int64_t window_len_ms
= 100;
167 int64_t window_len_ns
= window_len_ms
* 1000 * 1000;
168 bool window_expired
= now
> desc
->window_begin_ns
+ window_len_ns
;
170 if (desc
->n_used_entries
> desc
->window_max_entries
) {
171 desc
->window_max_entries
= desc
->n_used_entries
;
173 rate
= desc
->window_max_entries
* 100 / old_size
;
176 new_size
= MIN(old_size
<< 1, 1 << CPU_TLB_DYN_MAX_BITS
);
177 } else if (rate
< 30 && window_expired
) {
178 size_t ceil
= pow2ceil(desc
->window_max_entries
);
179 size_t expected_rate
= desc
->window_max_entries
* 100 / ceil
;
182 * Avoid undersizing when the max number of entries seen is just below
183 * a pow2. For instance, if max_entries == 1025, the expected use rate
184 * would be 1025/2048==50%. However, if max_entries == 1023, we'd get
185 * 1023/1024==99.9% use rate, so we'd likely end up doubling the size
186 * later. Thus, make sure that the expected use rate remains below 70%.
187 * (and since we double the size, that means the lowest rate we'd
188 * expect to get is 35%, which is still in the 30-70% range where
189 * we consider that the size is appropriate.)
191 if (expected_rate
> 70) {
194 new_size
= MAX(ceil
, 1 << CPU_TLB_DYN_MIN_BITS
);
197 if (new_size
== old_size
) {
198 if (window_expired
) {
199 tlb_window_reset(desc
, now
, desc
->n_used_entries
);
205 g_free(desc
->fulltlb
);
207 tlb_window_reset(desc
, now
, 0);
208 /* desc->n_used_entries is cleared by the caller */
209 fast
->mask
= (new_size
- 1) << CPU_TLB_ENTRY_BITS
;
210 fast
->table
= g_try_new(CPUTLBEntry
, new_size
);
211 desc
->fulltlb
= g_try_new(CPUTLBEntryFull
, new_size
);
214 * If the allocations fail, try smaller sizes. We just freed some
215 * memory, so going back to half of new_size has a good chance of working.
216 * Increased memory pressure elsewhere in the system might cause the
217 * allocations to fail though, so we progressively reduce the allocation
218 * size, aborting if we cannot even allocate the smallest TLB we support.
220 while (fast
->table
== NULL
|| desc
->fulltlb
== NULL
) {
221 if (new_size
== (1 << CPU_TLB_DYN_MIN_BITS
)) {
222 error_report("%s: %s", __func__
, strerror(errno
));
225 new_size
= MAX(new_size
>> 1, 1 << CPU_TLB_DYN_MIN_BITS
);
226 fast
->mask
= (new_size
- 1) << CPU_TLB_ENTRY_BITS
;
229 g_free(desc
->fulltlb
);
230 fast
->table
= g_try_new(CPUTLBEntry
, new_size
);
231 desc
->fulltlb
= g_try_new(CPUTLBEntryFull
, new_size
);
235 static void tlb_mmu_flush_locked(CPUTLBDesc
*desc
, CPUTLBDescFast
*fast
)
237 desc
->n_used_entries
= 0;
238 desc
->large_page_addr
= -1;
239 desc
->large_page_mask
= -1;
241 memset(fast
->table
, -1, sizeof_tlb(fast
));
242 memset(desc
->vtable
, -1, sizeof(desc
->vtable
));
245 static void tlb_flush_one_mmuidx_locked(CPUState
*cpu
, int mmu_idx
,
248 CPUTLBDesc
*desc
= &cpu
->neg
.tlb
.d
[mmu_idx
];
249 CPUTLBDescFast
*fast
= &cpu
->neg
.tlb
.f
[mmu_idx
];
251 tlb_mmu_resize_locked(desc
, fast
, now
);
252 tlb_mmu_flush_locked(desc
, fast
);
255 static void tlb_mmu_init(CPUTLBDesc
*desc
, CPUTLBDescFast
*fast
, int64_t now
)
257 size_t n_entries
= 1 << CPU_TLB_DYN_DEFAULT_BITS
;
259 tlb_window_reset(desc
, now
, 0);
260 desc
->n_used_entries
= 0;
261 fast
->mask
= (n_entries
- 1) << CPU_TLB_ENTRY_BITS
;
262 fast
->table
= g_new(CPUTLBEntry
, n_entries
);
263 desc
->fulltlb
= g_new(CPUTLBEntryFull
, n_entries
);
264 tlb_mmu_flush_locked(desc
, fast
);
267 static inline void tlb_n_used_entries_inc(CPUState
*cpu
, uintptr_t mmu_idx
)
269 cpu
->neg
.tlb
.d
[mmu_idx
].n_used_entries
++;
272 static inline void tlb_n_used_entries_dec(CPUState
*cpu
, uintptr_t mmu_idx
)
274 cpu
->neg
.tlb
.d
[mmu_idx
].n_used_entries
--;
277 void tlb_init(CPUState
*cpu
)
279 int64_t now
= get_clock_realtime();
282 qemu_spin_init(&cpu
->neg
.tlb
.c
.lock
);
284 /* All tlbs are initialized flushed. */
285 cpu
->neg
.tlb
.c
.dirty
= 0;
287 for (i
= 0; i
< NB_MMU_MODES
; i
++) {
288 tlb_mmu_init(&cpu
->neg
.tlb
.d
[i
], &cpu
->neg
.tlb
.f
[i
], now
);
292 void tlb_destroy(CPUState
*cpu
)
296 qemu_spin_destroy(&cpu
->neg
.tlb
.c
.lock
);
297 for (i
= 0; i
< NB_MMU_MODES
; i
++) {
298 CPUTLBDesc
*desc
= &cpu
->neg
.tlb
.d
[i
];
299 CPUTLBDescFast
*fast
= &cpu
->neg
.tlb
.f
[i
];
302 g_free(desc
->fulltlb
);
306 /* flush_all_helper: run fn across all cpus
308 * If the wait flag is set then the src cpu's helper will be queued as
309 * "safe" work and the loop exited creating a synchronisation point
310 * where all queued work will be finished before execution starts
313 static void flush_all_helper(CPUState
*src
, run_on_cpu_func fn
,
320 async_run_on_cpu(cpu
, fn
, d
);
325 static void tlb_flush_by_mmuidx_async_work(CPUState
*cpu
, run_on_cpu_data data
)
327 uint16_t asked
= data
.host_int
;
328 uint16_t all_dirty
, work
, to_clean
;
329 int64_t now
= get_clock_realtime();
331 assert_cpu_is_self(cpu
);
333 tlb_debug("mmu_idx:0x%04" PRIx16
"\n", asked
);
335 qemu_spin_lock(&cpu
->neg
.tlb
.c
.lock
);
337 all_dirty
= cpu
->neg
.tlb
.c
.dirty
;
338 to_clean
= asked
& all_dirty
;
339 all_dirty
&= ~to_clean
;
340 cpu
->neg
.tlb
.c
.dirty
= all_dirty
;
342 for (work
= to_clean
; work
!= 0; work
&= work
- 1) {
343 int mmu_idx
= ctz32(work
);
344 tlb_flush_one_mmuidx_locked(cpu
, mmu_idx
, now
);
347 qemu_spin_unlock(&cpu
->neg
.tlb
.c
.lock
);
349 tcg_flush_jmp_cache(cpu
);
351 if (to_clean
== ALL_MMUIDX_BITS
) {
352 qatomic_set(&cpu
->neg
.tlb
.c
.full_flush_count
,
353 cpu
->neg
.tlb
.c
.full_flush_count
+ 1);
355 qatomic_set(&cpu
->neg
.tlb
.c
.part_flush_count
,
356 cpu
->neg
.tlb
.c
.part_flush_count
+ ctpop16(to_clean
));
357 if (to_clean
!= asked
) {
358 qatomic_set(&cpu
->neg
.tlb
.c
.elide_flush_count
,
359 cpu
->neg
.tlb
.c
.elide_flush_count
+
360 ctpop16(asked
& ~to_clean
));
365 void tlb_flush_by_mmuidx(CPUState
*cpu
, uint16_t idxmap
)
367 tlb_debug("mmu_idx: 0x%" PRIx16
"\n", idxmap
);
369 if (cpu
->created
&& !qemu_cpu_is_self(cpu
)) {
370 async_run_on_cpu(cpu
, tlb_flush_by_mmuidx_async_work
,
371 RUN_ON_CPU_HOST_INT(idxmap
));
373 tlb_flush_by_mmuidx_async_work(cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
377 void tlb_flush(CPUState
*cpu
)
379 tlb_flush_by_mmuidx(cpu
, ALL_MMUIDX_BITS
);
382 void tlb_flush_by_mmuidx_all_cpus(CPUState
*src_cpu
, uint16_t idxmap
)
384 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
386 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
388 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
389 fn(src_cpu
, RUN_ON_CPU_HOST_INT(idxmap
));
392 void tlb_flush_all_cpus(CPUState
*src_cpu
)
394 tlb_flush_by_mmuidx_all_cpus(src_cpu
, ALL_MMUIDX_BITS
);
397 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
, uint16_t idxmap
)
399 const run_on_cpu_func fn
= tlb_flush_by_mmuidx_async_work
;
401 tlb_debug("mmu_idx: 0x%"PRIx16
"\n", idxmap
);
403 flush_all_helper(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
404 async_safe_run_on_cpu(src_cpu
, fn
, RUN_ON_CPU_HOST_INT(idxmap
));
407 void tlb_flush_all_cpus_synced(CPUState
*src_cpu
)
409 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu
, ALL_MMUIDX_BITS
);
412 static bool tlb_hit_page_mask_anyprot(CPUTLBEntry
*tlb_entry
,
413 vaddr page
, vaddr mask
)
416 mask
&= TARGET_PAGE_MASK
| TLB_INVALID_MASK
;
418 return (page
== (tlb_entry
->addr_read
& mask
) ||
419 page
== (tlb_addr_write(tlb_entry
) & mask
) ||
420 page
== (tlb_entry
->addr_code
& mask
));
423 static inline bool tlb_hit_page_anyprot(CPUTLBEntry
*tlb_entry
, vaddr page
)
425 return tlb_hit_page_mask_anyprot(tlb_entry
, page
, -1);
429 * tlb_entry_is_empty - return true if the entry is not in use
430 * @te: pointer to CPUTLBEntry
432 static inline bool tlb_entry_is_empty(const CPUTLBEntry
*te
)
434 return te
->addr_read
== -1 && te
->addr_write
== -1 && te
->addr_code
== -1;
437 /* Called with tlb_c.lock held */
438 static bool tlb_flush_entry_mask_locked(CPUTLBEntry
*tlb_entry
,
442 if (tlb_hit_page_mask_anyprot(tlb_entry
, page
, mask
)) {
443 memset(tlb_entry
, -1, sizeof(*tlb_entry
));
449 static inline bool tlb_flush_entry_locked(CPUTLBEntry
*tlb_entry
, vaddr page
)
451 return tlb_flush_entry_mask_locked(tlb_entry
, page
, -1);
454 /* Called with tlb_c.lock held */
455 static void tlb_flush_vtlb_page_mask_locked(CPUState
*cpu
, int mmu_idx
,
459 CPUTLBDesc
*d
= &cpu
->neg
.tlb
.d
[mmu_idx
];
462 assert_cpu_is_self(cpu
);
463 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
464 if (tlb_flush_entry_mask_locked(&d
->vtable
[k
], page
, mask
)) {
465 tlb_n_used_entries_dec(cpu
, mmu_idx
);
470 static inline void tlb_flush_vtlb_page_locked(CPUState
*cpu
, int mmu_idx
,
473 tlb_flush_vtlb_page_mask_locked(cpu
, mmu_idx
, page
, -1);
476 static void tlb_flush_page_locked(CPUState
*cpu
, int midx
, vaddr page
)
478 vaddr lp_addr
= cpu
->neg
.tlb
.d
[midx
].large_page_addr
;
479 vaddr lp_mask
= cpu
->neg
.tlb
.d
[midx
].large_page_mask
;
481 /* Check if we need to flush due to large pages. */
482 if ((page
& lp_mask
) == lp_addr
) {
483 tlb_debug("forcing full flush midx %d (%016"
484 VADDR_PRIx
"/%016" VADDR_PRIx
")\n",
485 midx
, lp_addr
, lp_mask
);
486 tlb_flush_one_mmuidx_locked(cpu
, midx
, get_clock_realtime());
488 if (tlb_flush_entry_locked(tlb_entry(cpu
, midx
, page
), page
)) {
489 tlb_n_used_entries_dec(cpu
, midx
);
491 tlb_flush_vtlb_page_locked(cpu
, midx
, page
);
496 * tlb_flush_page_by_mmuidx_async_0:
497 * @cpu: cpu on which to flush
498 * @addr: page of virtual address to flush
499 * @idxmap: set of mmu_idx to flush
501 * Helper for tlb_flush_page_by_mmuidx and friends, flush one page
502 * at @addr from the tlbs indicated by @idxmap from @cpu.
504 static void tlb_flush_page_by_mmuidx_async_0(CPUState
*cpu
,
510 assert_cpu_is_self(cpu
);
512 tlb_debug("page addr: %016" VADDR_PRIx
" mmu_map:0x%x\n", addr
, idxmap
);
514 qemu_spin_lock(&cpu
->neg
.tlb
.c
.lock
);
515 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
516 if ((idxmap
>> mmu_idx
) & 1) {
517 tlb_flush_page_locked(cpu
, mmu_idx
, addr
);
520 qemu_spin_unlock(&cpu
->neg
.tlb
.c
.lock
);
523 * Discard jump cache entries for any tb which might potentially
524 * overlap the flushed page, which includes the previous.
526 tb_jmp_cache_clear_page(cpu
, addr
- TARGET_PAGE_SIZE
);
527 tb_jmp_cache_clear_page(cpu
, addr
);
531 * tlb_flush_page_by_mmuidx_async_1:
532 * @cpu: cpu on which to flush
533 * @data: encoded addr + idxmap
535 * Helper for tlb_flush_page_by_mmuidx and friends, called through
536 * async_run_on_cpu. The idxmap parameter is encoded in the page
537 * offset of the target_ptr field. This limits the set of mmu_idx
538 * that can be passed via this method.
540 static void tlb_flush_page_by_mmuidx_async_1(CPUState
*cpu
,
541 run_on_cpu_data data
)
543 vaddr addr_and_idxmap
= data
.target_ptr
;
544 vaddr addr
= addr_and_idxmap
& TARGET_PAGE_MASK
;
545 uint16_t idxmap
= addr_and_idxmap
& ~TARGET_PAGE_MASK
;
547 tlb_flush_page_by_mmuidx_async_0(cpu
, addr
, idxmap
);
553 } TLBFlushPageByMMUIdxData
;
556 * tlb_flush_page_by_mmuidx_async_2:
557 * @cpu: cpu on which to flush
558 * @data: allocated addr + idxmap
560 * Helper for tlb_flush_page_by_mmuidx and friends, called through
561 * async_run_on_cpu. The addr+idxmap parameters are stored in a
562 * TLBFlushPageByMMUIdxData structure that has been allocated
563 * specifically for this helper. Free the structure when done.
565 static void tlb_flush_page_by_mmuidx_async_2(CPUState
*cpu
,
566 run_on_cpu_data data
)
568 TLBFlushPageByMMUIdxData
*d
= data
.host_ptr
;
570 tlb_flush_page_by_mmuidx_async_0(cpu
, d
->addr
, d
->idxmap
);
574 void tlb_flush_page_by_mmuidx(CPUState
*cpu
, vaddr addr
, uint16_t idxmap
)
576 tlb_debug("addr: %016" VADDR_PRIx
" mmu_idx:%" PRIx16
"\n", addr
, idxmap
);
578 /* This should already be page aligned */
579 addr
&= TARGET_PAGE_MASK
;
581 if (qemu_cpu_is_self(cpu
)) {
582 tlb_flush_page_by_mmuidx_async_0(cpu
, addr
, idxmap
);
583 } else if (idxmap
< TARGET_PAGE_SIZE
) {
585 * Most targets have only a few mmu_idx. In the case where
586 * we can stuff idxmap into the low TARGET_PAGE_BITS, avoid
587 * allocating memory for this operation.
589 async_run_on_cpu(cpu
, tlb_flush_page_by_mmuidx_async_1
,
590 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
592 TLBFlushPageByMMUIdxData
*d
= g_new(TLBFlushPageByMMUIdxData
, 1);
594 /* Otherwise allocate a structure, freed by the worker. */
597 async_run_on_cpu(cpu
, tlb_flush_page_by_mmuidx_async_2
,
598 RUN_ON_CPU_HOST_PTR(d
));
602 void tlb_flush_page(CPUState
*cpu
, vaddr addr
)
604 tlb_flush_page_by_mmuidx(cpu
, addr
, ALL_MMUIDX_BITS
);
607 void tlb_flush_page_by_mmuidx_all_cpus(CPUState
*src_cpu
, vaddr addr
,
610 tlb_debug("addr: %016" VADDR_PRIx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
612 /* This should already be page aligned */
613 addr
&= TARGET_PAGE_MASK
;
616 * Allocate memory to hold addr+idxmap only when needed.
617 * See tlb_flush_page_by_mmuidx for details.
619 if (idxmap
< TARGET_PAGE_SIZE
) {
620 flush_all_helper(src_cpu
, tlb_flush_page_by_mmuidx_async_1
,
621 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
625 /* Allocate a separate data block for each destination cpu. */
626 CPU_FOREACH(dst_cpu
) {
627 if (dst_cpu
!= src_cpu
) {
628 TLBFlushPageByMMUIdxData
*d
629 = g_new(TLBFlushPageByMMUIdxData
, 1);
633 async_run_on_cpu(dst_cpu
, tlb_flush_page_by_mmuidx_async_2
,
634 RUN_ON_CPU_HOST_PTR(d
));
639 tlb_flush_page_by_mmuidx_async_0(src_cpu
, addr
, idxmap
);
642 void tlb_flush_page_all_cpus(CPUState
*src
, vaddr addr
)
644 tlb_flush_page_by_mmuidx_all_cpus(src
, addr
, ALL_MMUIDX_BITS
);
647 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
651 tlb_debug("addr: %016" VADDR_PRIx
" mmu_idx:%"PRIx16
"\n", addr
, idxmap
);
653 /* This should already be page aligned */
654 addr
&= TARGET_PAGE_MASK
;
657 * Allocate memory to hold addr+idxmap only when needed.
658 * See tlb_flush_page_by_mmuidx for details.
660 if (idxmap
< TARGET_PAGE_SIZE
) {
661 flush_all_helper(src_cpu
, tlb_flush_page_by_mmuidx_async_1
,
662 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
663 async_safe_run_on_cpu(src_cpu
, tlb_flush_page_by_mmuidx_async_1
,
664 RUN_ON_CPU_TARGET_PTR(addr
| idxmap
));
667 TLBFlushPageByMMUIdxData
*d
;
669 /* Allocate a separate data block for each destination cpu. */
670 CPU_FOREACH(dst_cpu
) {
671 if (dst_cpu
!= src_cpu
) {
672 d
= g_new(TLBFlushPageByMMUIdxData
, 1);
675 async_run_on_cpu(dst_cpu
, tlb_flush_page_by_mmuidx_async_2
,
676 RUN_ON_CPU_HOST_PTR(d
));
680 d
= g_new(TLBFlushPageByMMUIdxData
, 1);
683 async_safe_run_on_cpu(src_cpu
, tlb_flush_page_by_mmuidx_async_2
,
684 RUN_ON_CPU_HOST_PTR(d
));
688 void tlb_flush_page_all_cpus_synced(CPUState
*src
, vaddr addr
)
690 tlb_flush_page_by_mmuidx_all_cpus_synced(src
, addr
, ALL_MMUIDX_BITS
);
693 static void tlb_flush_range_locked(CPUState
*cpu
, int midx
,
694 vaddr addr
, vaddr len
,
697 CPUTLBDesc
*d
= &cpu
->neg
.tlb
.d
[midx
];
698 CPUTLBDescFast
*f
= &cpu
->neg
.tlb
.f
[midx
];
699 vaddr mask
= MAKE_64BIT_MASK(0, bits
);
702 * If @bits is smaller than the tlb size, there may be multiple entries
703 * within the TLB; otherwise all addresses that match under @mask hit
704 * the same TLB entry.
705 * TODO: Perhaps allow bits to be a few bits less than the size.
706 * For now, just flush the entire TLB.
708 * If @len is larger than the tlb size, then it will take longer to
709 * test all of the entries in the TLB than it will to flush it all.
711 if (mask
< f
->mask
|| len
> f
->mask
) {
712 tlb_debug("forcing full flush midx %d ("
713 "%016" VADDR_PRIx
"/%016" VADDR_PRIx
"+%016" VADDR_PRIx
")\n",
714 midx
, addr
, mask
, len
);
715 tlb_flush_one_mmuidx_locked(cpu
, midx
, get_clock_realtime());
720 * Check if we need to flush due to large pages.
721 * Because large_page_mask contains all 1's from the msb,
722 * we only need to test the end of the range.
724 if (((addr
+ len
- 1) & d
->large_page_mask
) == d
->large_page_addr
) {
725 tlb_debug("forcing full flush midx %d ("
726 "%016" VADDR_PRIx
"/%016" VADDR_PRIx
")\n",
727 midx
, d
->large_page_addr
, d
->large_page_mask
);
728 tlb_flush_one_mmuidx_locked(cpu
, midx
, get_clock_realtime());
732 for (vaddr i
= 0; i
< len
; i
+= TARGET_PAGE_SIZE
) {
733 vaddr page
= addr
+ i
;
734 CPUTLBEntry
*entry
= tlb_entry(cpu
, midx
, page
);
736 if (tlb_flush_entry_mask_locked(entry
, page
, mask
)) {
737 tlb_n_used_entries_dec(cpu
, midx
);
739 tlb_flush_vtlb_page_mask_locked(cpu
, midx
, page
, mask
);
750 static void tlb_flush_range_by_mmuidx_async_0(CPUState
*cpu
,
755 assert_cpu_is_self(cpu
);
757 tlb_debug("range: %016" VADDR_PRIx
"/%u+%016" VADDR_PRIx
" mmu_map:0x%x\n",
758 d
.addr
, d
.bits
, d
.len
, d
.idxmap
);
760 qemu_spin_lock(&cpu
->neg
.tlb
.c
.lock
);
761 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
762 if ((d
.idxmap
>> mmu_idx
) & 1) {
763 tlb_flush_range_locked(cpu
, mmu_idx
, d
.addr
, d
.len
, d
.bits
);
766 qemu_spin_unlock(&cpu
->neg
.tlb
.c
.lock
);
769 * If the length is larger than the jump cache size, then it will take
770 * longer to clear each entry individually than it will to clear it all.
772 if (d
.len
>= (TARGET_PAGE_SIZE
* TB_JMP_CACHE_SIZE
)) {
773 tcg_flush_jmp_cache(cpu
);
778 * Discard jump cache entries for any tb which might potentially
779 * overlap the flushed pages, which includes the previous.
781 d
.addr
-= TARGET_PAGE_SIZE
;
782 for (vaddr i
= 0, n
= d
.len
/ TARGET_PAGE_SIZE
+ 1; i
< n
; i
++) {
783 tb_jmp_cache_clear_page(cpu
, d
.addr
);
784 d
.addr
+= TARGET_PAGE_SIZE
;
788 static void tlb_flush_range_by_mmuidx_async_1(CPUState
*cpu
,
789 run_on_cpu_data data
)
791 TLBFlushRangeData
*d
= data
.host_ptr
;
792 tlb_flush_range_by_mmuidx_async_0(cpu
, *d
);
796 void tlb_flush_range_by_mmuidx(CPUState
*cpu
, vaddr addr
,
797 vaddr len
, uint16_t idxmap
,
803 * If all bits are significant, and len is small,
804 * this devolves to tlb_flush_page.
806 if (bits
>= TARGET_LONG_BITS
&& len
<= TARGET_PAGE_SIZE
) {
807 tlb_flush_page_by_mmuidx(cpu
, addr
, idxmap
);
810 /* If no page bits are significant, this devolves to tlb_flush. */
811 if (bits
< TARGET_PAGE_BITS
) {
812 tlb_flush_by_mmuidx(cpu
, idxmap
);
816 /* This should already be page aligned */
817 d
.addr
= addr
& TARGET_PAGE_MASK
;
822 if (qemu_cpu_is_self(cpu
)) {
823 tlb_flush_range_by_mmuidx_async_0(cpu
, d
);
825 /* Otherwise allocate a structure, freed by the worker. */
826 TLBFlushRangeData
*p
= g_memdup(&d
, sizeof(d
));
827 async_run_on_cpu(cpu
, tlb_flush_range_by_mmuidx_async_1
,
828 RUN_ON_CPU_HOST_PTR(p
));
832 void tlb_flush_page_bits_by_mmuidx(CPUState
*cpu
, vaddr addr
,
833 uint16_t idxmap
, unsigned bits
)
835 tlb_flush_range_by_mmuidx(cpu
, addr
, TARGET_PAGE_SIZE
, idxmap
, bits
);
838 void tlb_flush_range_by_mmuidx_all_cpus(CPUState
*src_cpu
,
839 vaddr addr
, vaddr len
,
840 uint16_t idxmap
, unsigned bits
)
846 * If all bits are significant, and len is small,
847 * this devolves to tlb_flush_page.
849 if (bits
>= TARGET_LONG_BITS
&& len
<= TARGET_PAGE_SIZE
) {
850 tlb_flush_page_by_mmuidx_all_cpus(src_cpu
, addr
, idxmap
);
853 /* If no page bits are significant, this devolves to tlb_flush. */
854 if (bits
< TARGET_PAGE_BITS
) {
855 tlb_flush_by_mmuidx_all_cpus(src_cpu
, idxmap
);
859 /* This should already be page aligned */
860 d
.addr
= addr
& TARGET_PAGE_MASK
;
865 /* Allocate a separate data block for each destination cpu. */
866 CPU_FOREACH(dst_cpu
) {
867 if (dst_cpu
!= src_cpu
) {
868 TLBFlushRangeData
*p
= g_memdup(&d
, sizeof(d
));
869 async_run_on_cpu(dst_cpu
,
870 tlb_flush_range_by_mmuidx_async_1
,
871 RUN_ON_CPU_HOST_PTR(p
));
875 tlb_flush_range_by_mmuidx_async_0(src_cpu
, d
);
878 void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState
*src_cpu
,
879 vaddr addr
, uint16_t idxmap
,
882 tlb_flush_range_by_mmuidx_all_cpus(src_cpu
, addr
, TARGET_PAGE_SIZE
,
886 void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
892 TLBFlushRangeData d
, *p
;
896 * If all bits are significant, and len is small,
897 * this devolves to tlb_flush_page.
899 if (bits
>= TARGET_LONG_BITS
&& len
<= TARGET_PAGE_SIZE
) {
900 tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu
, addr
, idxmap
);
903 /* If no page bits are significant, this devolves to tlb_flush. */
904 if (bits
< TARGET_PAGE_BITS
) {
905 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu
, idxmap
);
909 /* This should already be page aligned */
910 d
.addr
= addr
& TARGET_PAGE_MASK
;
915 /* Allocate a separate data block for each destination cpu. */
916 CPU_FOREACH(dst_cpu
) {
917 if (dst_cpu
!= src_cpu
) {
918 p
= g_memdup(&d
, sizeof(d
));
919 async_run_on_cpu(dst_cpu
, tlb_flush_range_by_mmuidx_async_1
,
920 RUN_ON_CPU_HOST_PTR(p
));
924 p
= g_memdup(&d
, sizeof(d
));
925 async_safe_run_on_cpu(src_cpu
, tlb_flush_range_by_mmuidx_async_1
,
926 RUN_ON_CPU_HOST_PTR(p
));
929 void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState
*src_cpu
,
934 tlb_flush_range_by_mmuidx_all_cpus_synced(src_cpu
, addr
, TARGET_PAGE_SIZE
,
938 /* update the TLBs so that writes to code in the virtual page 'addr'
940 void tlb_protect_code(ram_addr_t ram_addr
)
942 cpu_physical_memory_test_and_clear_dirty(ram_addr
& TARGET_PAGE_MASK
,
947 /* update the TLB so that writes in physical page 'phys_addr' are no longer
948 tested for self modifying code */
949 void tlb_unprotect_code(ram_addr_t ram_addr
)
951 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
);
956 * Dirty write flag handling
958 * When the TCG code writes to a location it looks up the address in
959 * the TLB and uses that data to compute the final address. If any of
960 * the lower bits of the address are set then the slow path is forced.
961 * There are a number of reasons to do this but for normal RAM the
962 * most usual is detecting writes to code regions which may invalidate
965 * Other vCPUs might be reading their TLBs during guest execution, so we update
966 * te->addr_write with qatomic_set. We don't need to worry about this for
967 * oversized guests as MTTCG is disabled for them.
969 * Called with tlb_c.lock held.
971 static void tlb_reset_dirty_range_locked(CPUTLBEntry
*tlb_entry
,
972 uintptr_t start
, uintptr_t length
)
974 uintptr_t addr
= tlb_entry
->addr_write
;
976 if ((addr
& (TLB_INVALID_MASK
| TLB_MMIO
|
977 TLB_DISCARD_WRITE
| TLB_NOTDIRTY
)) == 0) {
978 addr
&= TARGET_PAGE_MASK
;
979 addr
+= tlb_entry
->addend
;
980 if ((addr
- start
) < length
) {
981 #if TARGET_LONG_BITS == 32
982 uint32_t *ptr_write
= (uint32_t *)&tlb_entry
->addr_write
;
983 ptr_write
+= HOST_BIG_ENDIAN
;
984 qatomic_set(ptr_write
, *ptr_write
| TLB_NOTDIRTY
);
985 #elif TCG_OVERSIZED_GUEST
986 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
988 qatomic_set(&tlb_entry
->addr_write
,
989 tlb_entry
->addr_write
| TLB_NOTDIRTY
);
996 * Called with tlb_c.lock held.
997 * Called only from the vCPU context, i.e. the TLB's owner thread.
999 static inline void copy_tlb_helper_locked(CPUTLBEntry
*d
, const CPUTLBEntry
*s
)
1004 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
1006 * We must take tlb_c.lock to avoid racing with another vCPU update. The only
1007 * thing actually updated is the target TLB entry ->addr_write flags.
1009 void tlb_reset_dirty(CPUState
*cpu
, ram_addr_t start1
, ram_addr_t length
)
1013 qemu_spin_lock(&cpu
->neg
.tlb
.c
.lock
);
1014 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1016 unsigned int n
= tlb_n_entries(&cpu
->neg
.tlb
.f
[mmu_idx
]);
1018 for (i
= 0; i
< n
; i
++) {
1019 tlb_reset_dirty_range_locked(&cpu
->neg
.tlb
.f
[mmu_idx
].table
[i
],
1023 for (i
= 0; i
< CPU_VTLB_SIZE
; i
++) {
1024 tlb_reset_dirty_range_locked(&cpu
->neg
.tlb
.d
[mmu_idx
].vtable
[i
],
1028 qemu_spin_unlock(&cpu
->neg
.tlb
.c
.lock
);
1031 /* Called with tlb_c.lock held */
1032 static inline void tlb_set_dirty1_locked(CPUTLBEntry
*tlb_entry
,
1035 if (tlb_entry
->addr_write
== (addr
| TLB_NOTDIRTY
)) {
1036 tlb_entry
->addr_write
= addr
;
1040 /* update the TLB corresponding to virtual page vaddr
1041 so that it is no longer dirty */
1042 static void tlb_set_dirty(CPUState
*cpu
, vaddr addr
)
1046 assert_cpu_is_self(cpu
);
1048 addr
&= TARGET_PAGE_MASK
;
1049 qemu_spin_lock(&cpu
->neg
.tlb
.c
.lock
);
1050 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1051 tlb_set_dirty1_locked(tlb_entry(cpu
, mmu_idx
, addr
), addr
);
1054 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1056 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
1057 tlb_set_dirty1_locked(&cpu
->neg
.tlb
.d
[mmu_idx
].vtable
[k
], addr
);
1060 qemu_spin_unlock(&cpu
->neg
.tlb
.c
.lock
);
1063 /* Our TLB does not support large pages, so remember the area covered by
1064 large pages and trigger a full TLB flush if these are invalidated. */
1065 static void tlb_add_large_page(CPUState
*cpu
, int mmu_idx
,
1066 vaddr addr
, uint64_t size
)
1068 vaddr lp_addr
= cpu
->neg
.tlb
.d
[mmu_idx
].large_page_addr
;
1069 vaddr lp_mask
= ~(size
- 1);
1071 if (lp_addr
== (vaddr
)-1) {
1072 /* No previous large page. */
1075 /* Extend the existing region to include the new page.
1076 This is a compromise between unnecessary flushes and
1077 the cost of maintaining a full variable size TLB. */
1078 lp_mask
&= cpu
->neg
.tlb
.d
[mmu_idx
].large_page_mask
;
1079 while (((lp_addr
^ addr
) & lp_mask
) != 0) {
1083 cpu
->neg
.tlb
.d
[mmu_idx
].large_page_addr
= lp_addr
& lp_mask
;
1084 cpu
->neg
.tlb
.d
[mmu_idx
].large_page_mask
= lp_mask
;
1087 static inline void tlb_set_compare(CPUTLBEntryFull
*full
, CPUTLBEntry
*ent
,
1088 vaddr address
, int flags
,
1089 MMUAccessType access_type
, bool enable
)
1092 address
|= flags
& TLB_FLAGS_MASK
;
1093 flags
&= TLB_SLOW_FLAGS_MASK
;
1095 address
|= TLB_FORCE_SLOW
;
1101 ent
->addr_idx
[access_type
] = address
;
1102 full
->slow_flags
[access_type
] = flags
;
1106 * Add a new TLB entry. At most one entry for a given virtual address
1107 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
1108 * supplied size is only used by tlb_flush_page.
1110 * Called from TCG-generated code, which is under an RCU read-side
1113 void tlb_set_page_full(CPUState
*cpu
, int mmu_idx
,
1114 vaddr addr
, CPUTLBEntryFull
*full
)
1116 CPUTLB
*tlb
= &cpu
->neg
.tlb
;
1117 CPUTLBDesc
*desc
= &tlb
->d
[mmu_idx
];
1118 MemoryRegionSection
*section
;
1119 unsigned int index
, read_flags
, write_flags
;
1121 CPUTLBEntry
*te
, tn
;
1122 hwaddr iotlb
, xlat
, sz
, paddr_page
;
1124 int asidx
, wp_flags
, prot
;
1125 bool is_ram
, is_romd
;
1127 assert_cpu_is_self(cpu
);
1129 if (full
->lg_page_size
<= TARGET_PAGE_BITS
) {
1130 sz
= TARGET_PAGE_SIZE
;
1132 sz
= (hwaddr
)1 << full
->lg_page_size
;
1133 tlb_add_large_page(cpu
, mmu_idx
, addr
, sz
);
1135 addr_page
= addr
& TARGET_PAGE_MASK
;
1136 paddr_page
= full
->phys_addr
& TARGET_PAGE_MASK
;
1139 asidx
= cpu_asidx_from_attrs(cpu
, full
->attrs
);
1140 section
= address_space_translate_for_iotlb(cpu
, asidx
, paddr_page
,
1141 &xlat
, &sz
, full
->attrs
, &prot
);
1142 assert(sz
>= TARGET_PAGE_SIZE
);
1144 tlb_debug("vaddr=%016" VADDR_PRIx
" paddr=0x" HWADDR_FMT_plx
1145 " prot=%x idx=%d\n",
1146 addr
, full
->phys_addr
, prot
, mmu_idx
);
1148 read_flags
= full
->tlb_fill_flags
;
1149 if (full
->lg_page_size
< TARGET_PAGE_BITS
) {
1150 /* Repeat the MMU check and TLB fill on every access. */
1151 read_flags
|= TLB_INVALID_MASK
;
1154 is_ram
= memory_region_is_ram(section
->mr
);
1155 is_romd
= memory_region_is_romd(section
->mr
);
1157 if (is_ram
|| is_romd
) {
1158 /* RAM and ROMD both have associated host memory. */
1159 addend
= (uintptr_t)memory_region_get_ram_ptr(section
->mr
) + xlat
;
1161 /* I/O does not; force the host address to NULL. */
1165 write_flags
= read_flags
;
1167 iotlb
= memory_region_get_ram_addr(section
->mr
) + xlat
;
1168 assert(!(iotlb
& ~TARGET_PAGE_MASK
));
1170 * Computing is_clean is expensive; avoid all that unless
1171 * the page is actually writable.
1173 if (prot
& PAGE_WRITE
) {
1174 if (section
->readonly
) {
1175 write_flags
|= TLB_DISCARD_WRITE
;
1176 } else if (cpu_physical_memory_is_clean(iotlb
)) {
1177 write_flags
|= TLB_NOTDIRTY
;
1182 iotlb
= memory_region_section_get_iotlb(cpu
, section
) + xlat
;
1184 * Writes to romd devices must go through MMIO to enable write.
1185 * Reads to romd devices go through the ram_ptr found above,
1186 * but of course reads to I/O must go through MMIO.
1188 write_flags
|= TLB_MMIO
;
1190 read_flags
= write_flags
;
1194 wp_flags
= cpu_watchpoint_address_matches(cpu
, addr_page
,
1197 index
= tlb_index(cpu
, mmu_idx
, addr_page
);
1198 te
= tlb_entry(cpu
, mmu_idx
, addr_page
);
1201 * Hold the TLB lock for the rest of the function. We could acquire/release
1202 * the lock several times in the function, but it is faster to amortize the
1203 * acquisition cost by acquiring it just once. Note that this leads to
1204 * a longer critical section, but this is not a concern since the TLB lock
1205 * is unlikely to be contended.
1207 qemu_spin_lock(&tlb
->c
.lock
);
1209 /* Note that the tlb is no longer clean. */
1210 tlb
->c
.dirty
|= 1 << mmu_idx
;
1212 /* Make sure there's no cached translation for the new page. */
1213 tlb_flush_vtlb_page_locked(cpu
, mmu_idx
, addr_page
);
1216 * Only evict the old entry to the victim tlb if it's for a
1217 * different page; otherwise just overwrite the stale data.
1219 if (!tlb_hit_page_anyprot(te
, addr_page
) && !tlb_entry_is_empty(te
)) {
1220 unsigned vidx
= desc
->vindex
++ % CPU_VTLB_SIZE
;
1221 CPUTLBEntry
*tv
= &desc
->vtable
[vidx
];
1223 /* Evict the old entry into the victim tlb. */
1224 copy_tlb_helper_locked(tv
, te
);
1225 desc
->vfulltlb
[vidx
] = desc
->fulltlb
[index
];
1226 tlb_n_used_entries_dec(cpu
, mmu_idx
);
1229 /* refill the tlb */
1231 * When memory region is ram, iotlb contains a TARGET_PAGE_BITS
1232 * aligned ram_addr_t of the page base of the target RAM.
1233 * Otherwise, iotlb contains
1234 * - a physical section number in the lower TARGET_PAGE_BITS
1235 * - the offset within section->mr of the page base (I/O, ROMD) with the
1236 * TARGET_PAGE_BITS masked off.
1237 * We subtract addr_page (which is page aligned and thus won't
1238 * disturb the low bits) to give an offset which can be added to the
1239 * (non-page-aligned) vaddr of the eventual memory access to get
1240 * the MemoryRegion offset for the access. Note that the vaddr we
1241 * subtract here is that of the page base, and not the same as the
1242 * vaddr we add back in io_prepare()/get_page_addr_code().
1244 desc
->fulltlb
[index
] = *full
;
1245 full
= &desc
->fulltlb
[index
];
1246 full
->xlat_section
= iotlb
- addr_page
;
1247 full
->phys_addr
= paddr_page
;
1249 /* Now calculate the new entry */
1250 tn
.addend
= addend
- addr_page
;
1252 tlb_set_compare(full
, &tn
, addr_page
, read_flags
,
1253 MMU_INST_FETCH
, prot
& PAGE_EXEC
);
1255 if (wp_flags
& BP_MEM_READ
) {
1256 read_flags
|= TLB_WATCHPOINT
;
1258 tlb_set_compare(full
, &tn
, addr_page
, read_flags
,
1259 MMU_DATA_LOAD
, prot
& PAGE_READ
);
1261 if (prot
& PAGE_WRITE_INV
) {
1262 write_flags
|= TLB_INVALID_MASK
;
1264 if (wp_flags
& BP_MEM_WRITE
) {
1265 write_flags
|= TLB_WATCHPOINT
;
1267 tlb_set_compare(full
, &tn
, addr_page
, write_flags
,
1268 MMU_DATA_STORE
, prot
& PAGE_WRITE
);
1270 copy_tlb_helper_locked(te
, &tn
);
1271 tlb_n_used_entries_inc(cpu
, mmu_idx
);
1272 qemu_spin_unlock(&tlb
->c
.lock
);
1275 void tlb_set_page_with_attrs(CPUState
*cpu
, vaddr addr
,
1276 hwaddr paddr
, MemTxAttrs attrs
, int prot
,
1277 int mmu_idx
, uint64_t size
)
1279 CPUTLBEntryFull full
= {
1283 .lg_page_size
= ctz64(size
)
1286 assert(is_power_of_2(size
));
1287 tlb_set_page_full(cpu
, mmu_idx
, addr
, &full
);
1290 void tlb_set_page(CPUState
*cpu
, vaddr addr
,
1291 hwaddr paddr
, int prot
,
1292 int mmu_idx
, uint64_t size
)
1294 tlb_set_page_with_attrs(cpu
, addr
, paddr
, MEMTXATTRS_UNSPECIFIED
,
1295 prot
, mmu_idx
, size
);
1299 * Note: tlb_fill() can trigger a resize of the TLB. This means that all of the
1300 * caller's prior references to the TLB table (e.g. CPUTLBEntry pointers) must
1301 * be discarded and looked up again (e.g. via tlb_entry()).
1303 static void tlb_fill(CPUState
*cpu
, vaddr addr
, int size
,
1304 MMUAccessType access_type
, int mmu_idx
, uintptr_t retaddr
)
1309 * This is not a probe, so only valid return is success; failure
1310 * should result in exception + longjmp to the cpu loop.
1312 ok
= cpu
->cc
->tcg_ops
->tlb_fill(cpu
, addr
, size
,
1313 access_type
, mmu_idx
, false, retaddr
);
1317 static inline void cpu_unaligned_access(CPUState
*cpu
, vaddr addr
,
1318 MMUAccessType access_type
,
1319 int mmu_idx
, uintptr_t retaddr
)
1321 cpu
->cc
->tcg_ops
->do_unaligned_access(cpu
, addr
, access_type
,
1325 static MemoryRegionSection
*
1326 io_prepare(hwaddr
*out_offset
, CPUState
*cpu
, hwaddr xlat
,
1327 MemTxAttrs attrs
, vaddr addr
, uintptr_t retaddr
)
1329 MemoryRegionSection
*section
;
1332 section
= iotlb_to_section(cpu
, xlat
, attrs
);
1333 mr_offset
= (xlat
& TARGET_PAGE_MASK
) + addr
;
1334 cpu
->mem_io_pc
= retaddr
;
1335 if (!cpu
->neg
.can_do_io
) {
1336 cpu_io_recompile(cpu
, retaddr
);
1339 *out_offset
= mr_offset
;
1343 static void io_failed(CPUState
*cpu
, CPUTLBEntryFull
*full
, vaddr addr
,
1344 unsigned size
, MMUAccessType access_type
, int mmu_idx
,
1345 MemTxResult response
, uintptr_t retaddr
)
1347 if (!cpu
->ignore_memory_transaction_failures
1348 && cpu
->cc
->tcg_ops
->do_transaction_failed
) {
1349 hwaddr physaddr
= full
->phys_addr
| (addr
& ~TARGET_PAGE_MASK
);
1351 cpu
->cc
->tcg_ops
->do_transaction_failed(cpu
, physaddr
, addr
, size
,
1352 access_type
, mmu_idx
,
1353 full
->attrs
, response
, retaddr
);
1357 /* Return true if ADDR is present in the victim tlb, and has been copied
1358 back to the main tlb. */
1359 static bool victim_tlb_hit(CPUState
*cpu
, size_t mmu_idx
, size_t index
,
1360 MMUAccessType access_type
, vaddr page
)
1364 assert_cpu_is_self(cpu
);
1365 for (vidx
= 0; vidx
< CPU_VTLB_SIZE
; ++vidx
) {
1366 CPUTLBEntry
*vtlb
= &cpu
->neg
.tlb
.d
[mmu_idx
].vtable
[vidx
];
1367 uint64_t cmp
= tlb_read_idx(vtlb
, access_type
);
1370 /* Found entry in victim tlb, swap tlb and iotlb. */
1371 CPUTLBEntry tmptlb
, *tlb
= &cpu
->neg
.tlb
.f
[mmu_idx
].table
[index
];
1373 qemu_spin_lock(&cpu
->neg
.tlb
.c
.lock
);
1374 copy_tlb_helper_locked(&tmptlb
, tlb
);
1375 copy_tlb_helper_locked(tlb
, vtlb
);
1376 copy_tlb_helper_locked(vtlb
, &tmptlb
);
1377 qemu_spin_unlock(&cpu
->neg
.tlb
.c
.lock
);
1379 CPUTLBEntryFull
*f1
= &cpu
->neg
.tlb
.d
[mmu_idx
].fulltlb
[index
];
1380 CPUTLBEntryFull
*f2
= &cpu
->neg
.tlb
.d
[mmu_idx
].vfulltlb
[vidx
];
1381 CPUTLBEntryFull tmpf
;
1382 tmpf
= *f1
; *f1
= *f2
; *f2
= tmpf
;
1389 static void notdirty_write(CPUState
*cpu
, vaddr mem_vaddr
, unsigned size
,
1390 CPUTLBEntryFull
*full
, uintptr_t retaddr
)
1392 ram_addr_t ram_addr
= mem_vaddr
+ full
->xlat_section
;
1394 trace_memory_notdirty_write_access(mem_vaddr
, ram_addr
, size
);
1396 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1397 tb_invalidate_phys_range_fast(ram_addr
, size
, retaddr
);
1401 * Set both VGA and migration bits for simplicity and to remove
1402 * the notdirty callback faster.
1404 cpu_physical_memory_set_dirty_range(ram_addr
, size
, DIRTY_CLIENTS_NOCODE
);
1406 /* We remove the notdirty callback only if the code has been flushed. */
1407 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1408 trace_memory_notdirty_set_dirty(mem_vaddr
);
1409 tlb_set_dirty(cpu
, mem_vaddr
);
1413 static int probe_access_internal(CPUState
*cpu
, vaddr addr
,
1414 int fault_size
, MMUAccessType access_type
,
1415 int mmu_idx
, bool nonfault
,
1416 void **phost
, CPUTLBEntryFull
**pfull
,
1417 uintptr_t retaddr
, bool check_mem_cbs
)
1419 uintptr_t index
= tlb_index(cpu
, mmu_idx
, addr
);
1420 CPUTLBEntry
*entry
= tlb_entry(cpu
, mmu_idx
, addr
);
1421 uint64_t tlb_addr
= tlb_read_idx(entry
, access_type
);
1422 vaddr page_addr
= addr
& TARGET_PAGE_MASK
;
1423 int flags
= TLB_FLAGS_MASK
& ~TLB_FORCE_SLOW
;
1424 bool force_mmio
= check_mem_cbs
&& cpu_plugin_mem_cbs_enabled(cpu
);
1425 CPUTLBEntryFull
*full
;
1427 if (!tlb_hit_page(tlb_addr
, page_addr
)) {
1428 if (!victim_tlb_hit(cpu
, mmu_idx
, index
, access_type
, page_addr
)) {
1429 if (!cpu
->cc
->tcg_ops
->tlb_fill(cpu
, addr
, fault_size
, access_type
,
1430 mmu_idx
, nonfault
, retaddr
)) {
1431 /* Non-faulting page table read failed. */
1434 return TLB_INVALID_MASK
;
1437 /* TLB resize via tlb_fill may have moved the entry. */
1438 index
= tlb_index(cpu
, mmu_idx
, addr
);
1439 entry
= tlb_entry(cpu
, mmu_idx
, addr
);
1442 * With PAGE_WRITE_INV, we set TLB_INVALID_MASK immediately,
1443 * to force the next access through tlb_fill. We've just
1444 * called tlb_fill, so we know that this entry *is* valid.
1446 flags
&= ~TLB_INVALID_MASK
;
1448 tlb_addr
= tlb_read_idx(entry
, access_type
);
1452 *pfull
= full
= &cpu
->neg
.tlb
.d
[mmu_idx
].fulltlb
[index
];
1453 flags
|= full
->slow_flags
[access_type
];
1455 /* Fold all "mmio-like" bits into TLB_MMIO. This is not RAM. */
1456 if (unlikely(flags
& ~(TLB_WATCHPOINT
| TLB_NOTDIRTY
| TLB_CHECK_ALIGNED
))
1457 || (access_type
!= MMU_INST_FETCH
&& force_mmio
)) {
1462 /* Everything else is RAM. */
1463 *phost
= (void *)((uintptr_t)addr
+ entry
->addend
);
1467 int probe_access_full(CPUArchState
*env
, vaddr addr
, int size
,
1468 MMUAccessType access_type
, int mmu_idx
,
1469 bool nonfault
, void **phost
, CPUTLBEntryFull
**pfull
,
1472 int flags
= probe_access_internal(env_cpu(env
), addr
, size
, access_type
,
1473 mmu_idx
, nonfault
, phost
, pfull
, retaddr
,
1476 /* Handle clean RAM pages. */
1477 if (unlikely(flags
& TLB_NOTDIRTY
)) {
1478 int dirtysize
= size
== 0 ? 1 : size
;
1479 notdirty_write(env_cpu(env
), addr
, dirtysize
, *pfull
, retaddr
);
1480 flags
&= ~TLB_NOTDIRTY
;
1486 int probe_access_full_mmu(CPUArchState
*env
, vaddr addr
, int size
,
1487 MMUAccessType access_type
, int mmu_idx
,
1488 void **phost
, CPUTLBEntryFull
**pfull
)
1490 void *discard_phost
;
1491 CPUTLBEntryFull
*discard_tlb
;
1493 /* privately handle users that don't need full results */
1494 phost
= phost
? phost
: &discard_phost
;
1495 pfull
= pfull
? pfull
: &discard_tlb
;
1497 int flags
= probe_access_internal(env_cpu(env
), addr
, size
, access_type
,
1498 mmu_idx
, true, phost
, pfull
, 0, false);
1500 /* Handle clean RAM pages. */
1501 if (unlikely(flags
& TLB_NOTDIRTY
)) {
1502 int dirtysize
= size
== 0 ? 1 : size
;
1503 notdirty_write(env_cpu(env
), addr
, dirtysize
, *pfull
, 0);
1504 flags
&= ~TLB_NOTDIRTY
;
1510 int probe_access_flags(CPUArchState
*env
, vaddr addr
, int size
,
1511 MMUAccessType access_type
, int mmu_idx
,
1512 bool nonfault
, void **phost
, uintptr_t retaddr
)
1514 CPUTLBEntryFull
*full
;
1517 g_assert(-(addr
| TARGET_PAGE_MASK
) >= size
);
1519 flags
= probe_access_internal(env_cpu(env
), addr
, size
, access_type
,
1520 mmu_idx
, nonfault
, phost
, &full
, retaddr
,
1523 /* Handle clean RAM pages. */
1524 if (unlikely(flags
& TLB_NOTDIRTY
)) {
1525 int dirtysize
= size
== 0 ? 1 : size
;
1526 notdirty_write(env_cpu(env
), addr
, dirtysize
, full
, retaddr
);
1527 flags
&= ~TLB_NOTDIRTY
;
1533 void *probe_access(CPUArchState
*env
, vaddr addr
, int size
,
1534 MMUAccessType access_type
, int mmu_idx
, uintptr_t retaddr
)
1536 CPUTLBEntryFull
*full
;
1540 g_assert(-(addr
| TARGET_PAGE_MASK
) >= size
);
1542 flags
= probe_access_internal(env_cpu(env
), addr
, size
, access_type
,
1543 mmu_idx
, false, &host
, &full
, retaddr
,
1546 /* Per the interface, size == 0 merely faults the access. */
1551 if (unlikely(flags
& (TLB_NOTDIRTY
| TLB_WATCHPOINT
))) {
1552 /* Handle watchpoints. */
1553 if (flags
& TLB_WATCHPOINT
) {
1554 int wp_access
= (access_type
== MMU_DATA_STORE
1555 ? BP_MEM_WRITE
: BP_MEM_READ
);
1556 cpu_check_watchpoint(env_cpu(env
), addr
, size
,
1557 full
->attrs
, wp_access
, retaddr
);
1560 /* Handle clean RAM pages. */
1561 if (flags
& TLB_NOTDIRTY
) {
1562 notdirty_write(env_cpu(env
), addr
, size
, full
, retaddr
);
1569 void *tlb_vaddr_to_host(CPUArchState
*env
, abi_ptr addr
,
1570 MMUAccessType access_type
, int mmu_idx
)
1572 CPUTLBEntryFull
*full
;
1576 flags
= probe_access_internal(env_cpu(env
), addr
, 0, access_type
,
1577 mmu_idx
, true, &host
, &full
, 0, false);
1579 /* No combination of flags are expected by the caller. */
1580 return flags
? NULL
: host
;
1584 * Return a ram_addr_t for the virtual address for execution.
1586 * Return -1 if we can't translate and execute from an entire page
1587 * of RAM. This will force us to execute by loading and translating
1588 * one insn at a time, without caching.
1590 * NOTE: This function will trigger an exception if the page is
1593 tb_page_addr_t
get_page_addr_code_hostp(CPUArchState
*env
, vaddr addr
,
1596 CPUTLBEntryFull
*full
;
1599 (void)probe_access_internal(env_cpu(env
), addr
, 1, MMU_INST_FETCH
,
1600 cpu_mmu_index(env_cpu(env
), true), false,
1601 &p
, &full
, 0, false);
1606 if (full
->lg_page_size
< TARGET_PAGE_BITS
) {
1613 return qemu_ram_addr_from_host_nofail(p
);
1616 /* Load/store with atomicity primitives. */
1617 #include "ldst_atomicity.c.inc"
1619 #ifdef CONFIG_PLUGIN
1621 * Perform a TLB lookup and populate the qemu_plugin_hwaddr structure.
1622 * This should be a hot path as we will have just looked this path up
1623 * in the softmmu lookup code (or helper). We don't handle re-fills or
1624 * checking the victim table. This is purely informational.
1626 * The one corner case is i/o write, which can cause changes to the
1627 * address space. Those changes, and the corresponding tlb flush,
1628 * should be delayed until the next TB, so even then this ought not fail.
1629 * But check, Just in Case.
1631 bool tlb_plugin_lookup(CPUState
*cpu
, vaddr addr
, int mmu_idx
,
1632 bool is_store
, struct qemu_plugin_hwaddr
*data
)
1634 CPUTLBEntry
*tlbe
= tlb_entry(cpu
, mmu_idx
, addr
);
1635 uintptr_t index
= tlb_index(cpu
, mmu_idx
, addr
);
1636 MMUAccessType access_type
= is_store
? MMU_DATA_STORE
: MMU_DATA_LOAD
;
1637 uint64_t tlb_addr
= tlb_read_idx(tlbe
, access_type
);
1638 CPUTLBEntryFull
*full
;
1640 if (unlikely(!tlb_hit(tlb_addr
, addr
))) {
1644 full
= &cpu
->neg
.tlb
.d
[mmu_idx
].fulltlb
[index
];
1645 data
->phys_addr
= full
->phys_addr
| (addr
& ~TARGET_PAGE_MASK
);
1647 /* We must have an iotlb entry for MMIO */
1648 if (tlb_addr
& TLB_MMIO
) {
1649 MemoryRegionSection
*section
=
1650 iotlb_to_section(cpu
, full
->xlat_section
& ~TARGET_PAGE_MASK
,
1653 data
->mr
= section
->mr
;
1655 data
->is_io
= false;
1663 * Probe for a load/store operation.
1664 * Return the host address and into @flags.
1667 typedef struct MMULookupPageData
{
1668 CPUTLBEntryFull
*full
;
1673 } MMULookupPageData
;
1675 typedef struct MMULookupLocals
{
1676 MMULookupPageData page
[2];
1682 * mmu_lookup1: translate one page
1683 * @cpu: generic cpu state
1684 * @data: lookup parameters
1685 * @mmu_idx: virtual address context
1686 * @access_type: load/store/code
1687 * @ra: return address into tcg generated code, or 0
1689 * Resolve the translation for the one page at @data.addr, filling in
1690 * the rest of @data with the results. If the translation fails,
1691 * tlb_fill will longjmp out. Return true if the softmmu tlb for
1692 * @mmu_idx may have resized.
1694 static bool mmu_lookup1(CPUState
*cpu
, MMULookupPageData
*data
,
1695 int mmu_idx
, MMUAccessType access_type
, uintptr_t ra
)
1697 vaddr addr
= data
->addr
;
1698 uintptr_t index
= tlb_index(cpu
, mmu_idx
, addr
);
1699 CPUTLBEntry
*entry
= tlb_entry(cpu
, mmu_idx
, addr
);
1700 uint64_t tlb_addr
= tlb_read_idx(entry
, access_type
);
1701 bool maybe_resized
= false;
1702 CPUTLBEntryFull
*full
;
1705 /* If the TLB entry is for a different page, reload and try again. */
1706 if (!tlb_hit(tlb_addr
, addr
)) {
1707 if (!victim_tlb_hit(cpu
, mmu_idx
, index
, access_type
,
1708 addr
& TARGET_PAGE_MASK
)) {
1709 tlb_fill(cpu
, addr
, data
->size
, access_type
, mmu_idx
, ra
);
1710 maybe_resized
= true;
1711 index
= tlb_index(cpu
, mmu_idx
, addr
);
1712 entry
= tlb_entry(cpu
, mmu_idx
, addr
);
1714 tlb_addr
= tlb_read_idx(entry
, access_type
) & ~TLB_INVALID_MASK
;
1717 full
= &cpu
->neg
.tlb
.d
[mmu_idx
].fulltlb
[index
];
1718 flags
= tlb_addr
& (TLB_FLAGS_MASK
& ~TLB_FORCE_SLOW
);
1719 flags
|= full
->slow_flags
[access_type
];
1722 data
->flags
= flags
;
1723 /* Compute haddr speculatively; depending on flags it might be invalid. */
1724 data
->haddr
= (void *)((uintptr_t)addr
+ entry
->addend
);
1726 return maybe_resized
;
1730 * mmu_watch_or_dirty
1731 * @cpu: generic cpu state
1732 * @data: lookup parameters
1733 * @access_type: load/store/code
1734 * @ra: return address into tcg generated code, or 0
1736 * Trigger watchpoints for @data.addr:@data.size;
1737 * record writes to protected clean pages.
1739 static void mmu_watch_or_dirty(CPUState
*cpu
, MMULookupPageData
*data
,
1740 MMUAccessType access_type
, uintptr_t ra
)
1742 CPUTLBEntryFull
*full
= data
->full
;
1743 vaddr addr
= data
->addr
;
1744 int flags
= data
->flags
;
1745 int size
= data
->size
;
1747 /* On watchpoint hit, this will longjmp out. */
1748 if (flags
& TLB_WATCHPOINT
) {
1749 int wp
= access_type
== MMU_DATA_STORE
? BP_MEM_WRITE
: BP_MEM_READ
;
1750 cpu_check_watchpoint(cpu
, addr
, size
, full
->attrs
, wp
, ra
);
1751 flags
&= ~TLB_WATCHPOINT
;
1754 /* Note that notdirty is only set for writes. */
1755 if (flags
& TLB_NOTDIRTY
) {
1756 notdirty_write(cpu
, addr
, size
, full
, ra
);
1757 flags
&= ~TLB_NOTDIRTY
;
1759 data
->flags
= flags
;
1763 * mmu_lookup: translate page(s)
1764 * @cpu: generic cpu state
1765 * @addr: virtual address
1766 * @oi: combined mmu_idx and MemOp
1767 * @ra: return address into tcg generated code, or 0
1768 * @access_type: load/store/code
1771 * Resolve the translation for the page(s) beginning at @addr, for MemOp.size
1772 * bytes. Return true if the lookup crosses a page boundary.
1774 static bool mmu_lookup(CPUState
*cpu
, vaddr addr
, MemOpIdx oi
,
1775 uintptr_t ra
, MMUAccessType type
, MMULookupLocals
*l
)
1781 l
->memop
= get_memop(oi
);
1782 l
->mmu_idx
= get_mmuidx(oi
);
1784 tcg_debug_assert(l
->mmu_idx
< NB_MMU_MODES
);
1786 /* Handle CPU specific unaligned behaviour */
1787 a_bits
= get_alignment_bits(l
->memop
);
1788 if (addr
& ((1 << a_bits
) - 1)) {
1789 cpu_unaligned_access(cpu
, addr
, type
, l
->mmu_idx
, ra
);
1792 l
->page
[0].addr
= addr
;
1793 l
->page
[0].size
= memop_size(l
->memop
);
1794 l
->page
[1].addr
= (addr
+ l
->page
[0].size
- 1) & TARGET_PAGE_MASK
;
1795 l
->page
[1].size
= 0;
1796 crosspage
= (addr
^ l
->page
[1].addr
) & TARGET_PAGE_MASK
;
1798 if (likely(!crosspage
)) {
1799 mmu_lookup1(cpu
, &l
->page
[0], l
->mmu_idx
, type
, ra
);
1801 flags
= l
->page
[0].flags
;
1802 if (unlikely(flags
& (TLB_WATCHPOINT
| TLB_NOTDIRTY
))) {
1803 mmu_watch_or_dirty(cpu
, &l
->page
[0], type
, ra
);
1805 if (unlikely(flags
& TLB_BSWAP
)) {
1806 l
->memop
^= MO_BSWAP
;
1809 /* Finish compute of page crossing. */
1810 int size0
= l
->page
[1].addr
- addr
;
1811 l
->page
[1].size
= l
->page
[0].size
- size0
;
1812 l
->page
[0].size
= size0
;
1815 * Lookup both pages, recognizing exceptions from either. If the
1816 * second lookup potentially resized, refresh first CPUTLBEntryFull.
1818 mmu_lookup1(cpu
, &l
->page
[0], l
->mmu_idx
, type
, ra
);
1819 if (mmu_lookup1(cpu
, &l
->page
[1], l
->mmu_idx
, type
, ra
)) {
1820 uintptr_t index
= tlb_index(cpu
, l
->mmu_idx
, addr
);
1821 l
->page
[0].full
= &cpu
->neg
.tlb
.d
[l
->mmu_idx
].fulltlb
[index
];
1824 flags
= l
->page
[0].flags
| l
->page
[1].flags
;
1825 if (unlikely(flags
& (TLB_WATCHPOINT
| TLB_NOTDIRTY
))) {
1826 mmu_watch_or_dirty(cpu
, &l
->page
[0], type
, ra
);
1827 mmu_watch_or_dirty(cpu
, &l
->page
[1], type
, ra
);
1831 * Since target/sparc is the only user of TLB_BSWAP, and all
1832 * Sparc accesses are aligned, any treatment across two pages
1833 * would be arbitrary. Refuse it until there's a use.
1835 tcg_debug_assert((flags
& TLB_BSWAP
) == 0);
1839 * This alignment check differs from the one above, in that this is
1840 * based on the atomicity of the operation. The intended use case is
1841 * the ARM memory type field of each PTE, where access to pages with
1842 * Device memory type require alignment.
1844 if (unlikely(flags
& TLB_CHECK_ALIGNED
)) {
1845 MemOp size
= l
->memop
& MO_SIZE
;
1847 switch (l
->memop
& MO_ATOM_MASK
) {
1851 case MO_ATOM_IFALIGN_PAIR
:
1852 case MO_ATOM_WITHIN16_PAIR
:
1853 size
= size
? size
- 1 : 0;
1858 if (addr
& ((1 << size
) - 1)) {
1859 cpu_unaligned_access(cpu
, addr
, type
, l
->mmu_idx
, ra
);
1867 * Probe for an atomic operation. Do not allow unaligned operations,
1868 * or io operations to proceed. Return the host address.
1870 static void *atomic_mmu_lookup(CPUState
*cpu
, vaddr addr
, MemOpIdx oi
,
1871 int size
, uintptr_t retaddr
)
1873 uintptr_t mmu_idx
= get_mmuidx(oi
);
1874 MemOp mop
= get_memop(oi
);
1875 int a_bits
= get_alignment_bits(mop
);
1880 CPUTLBEntryFull
*full
;
1882 tcg_debug_assert(mmu_idx
< NB_MMU_MODES
);
1884 /* Adjust the given return address. */
1885 retaddr
-= GETPC_ADJ
;
1887 /* Enforce guest required alignment. */
1888 if (unlikely(a_bits
> 0 && (addr
& ((1 << a_bits
) - 1)))) {
1889 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
1890 cpu_unaligned_access(cpu
, addr
, MMU_DATA_STORE
,
1894 /* Enforce qemu required alignment. */
1895 if (unlikely(addr
& (size
- 1))) {
1896 /* We get here if guest alignment was not requested,
1897 or was not enforced by cpu_unaligned_access above.
1898 We might widen the access and emulate, but for now
1899 mark an exception and exit the cpu loop. */
1900 goto stop_the_world
;
1903 index
= tlb_index(cpu
, mmu_idx
, addr
);
1904 tlbe
= tlb_entry(cpu
, mmu_idx
, addr
);
1906 /* Check TLB entry and enforce page permissions. */
1907 tlb_addr
= tlb_addr_write(tlbe
);
1908 if (!tlb_hit(tlb_addr
, addr
)) {
1909 if (!victim_tlb_hit(cpu
, mmu_idx
, index
, MMU_DATA_STORE
,
1910 addr
& TARGET_PAGE_MASK
)) {
1911 tlb_fill(cpu
, addr
, size
,
1912 MMU_DATA_STORE
, mmu_idx
, retaddr
);
1913 index
= tlb_index(cpu
, mmu_idx
, addr
);
1914 tlbe
= tlb_entry(cpu
, mmu_idx
, addr
);
1916 tlb_addr
= tlb_addr_write(tlbe
) & ~TLB_INVALID_MASK
;
1920 * Let the guest notice RMW on a write-only page.
1921 * We have just verified that the page is writable.
1922 * Subpage lookups may have left TLB_INVALID_MASK set,
1923 * but addr_read will only be -1 if PAGE_READ was unset.
1925 if (unlikely(tlbe
->addr_read
== -1)) {
1926 tlb_fill(cpu
, addr
, size
, MMU_DATA_LOAD
, mmu_idx
, retaddr
);
1928 * Since we don't support reads and writes to different
1929 * addresses, and we do have the proper page loaded for
1930 * write, this shouldn't ever return. But just in case,
1931 * handle via stop-the-world.
1933 goto stop_the_world
;
1935 /* Collect tlb flags for read. */
1936 tlb_addr
|= tlbe
->addr_read
;
1938 /* Notice an IO access or a needs-MMU-lookup access */
1939 if (unlikely(tlb_addr
& (TLB_MMIO
| TLB_DISCARD_WRITE
))) {
1940 /* There's really nothing that can be done to
1941 support this apart from stop-the-world. */
1942 goto stop_the_world
;
1945 hostaddr
= (void *)((uintptr_t)addr
+ tlbe
->addend
);
1946 full
= &cpu
->neg
.tlb
.d
[mmu_idx
].fulltlb
[index
];
1948 if (unlikely(tlb_addr
& TLB_NOTDIRTY
)) {
1949 notdirty_write(cpu
, addr
, size
, full
, retaddr
);
1952 if (unlikely(tlb_addr
& TLB_FORCE_SLOW
)) {
1955 if (full
->slow_flags
[MMU_DATA_STORE
] & TLB_WATCHPOINT
) {
1956 wp_flags
|= BP_MEM_WRITE
;
1958 if (full
->slow_flags
[MMU_DATA_LOAD
] & TLB_WATCHPOINT
) {
1959 wp_flags
|= BP_MEM_READ
;
1962 cpu_check_watchpoint(cpu
, addr
, size
,
1963 full
->attrs
, wp_flags
, retaddr
);
1970 cpu_loop_exit_atomic(cpu
, retaddr
);
1976 * We support two different access types. SOFTMMU_CODE_ACCESS is
1977 * specifically for reading instructions from system memory. It is
1978 * called by the translation loop and in some helpers where the code
1979 * is disassembled. It shouldn't be called directly by guest code.
1981 * For the benefit of TCG generated code, we want to avoid the
1982 * complication of ABI-specific return type promotion and always
1983 * return a value extended to the register size of the host. This is
1984 * tcg_target_long, except in the case of a 32-bit host and 64-bit
1985 * data, and for that we always have uint64_t.
1987 * We don't bother with this widened value for SOFTMMU_CODE_ACCESS.
1992 * @cpu: generic cpu state
1993 * @full: page parameters
1994 * @ret_be: accumulated data
1995 * @addr: virtual address
1996 * @size: number of bytes
1997 * @mmu_idx: virtual address context
1998 * @ra: return address into tcg generated code, or 0
2001 * Load @size bytes from @addr, which is memory-mapped i/o.
2002 * The bytes are concatenated in big-endian order with @ret_be.
2004 static uint64_t int_ld_mmio_beN(CPUState
*cpu
, CPUTLBEntryFull
*full
,
2005 uint64_t ret_be
, vaddr addr
, int size
,
2006 int mmu_idx
, MMUAccessType type
, uintptr_t ra
,
2007 MemoryRegion
*mr
, hwaddr mr_offset
)
2015 /* Read aligned pieces up to 8 bytes. */
2016 this_mop
= ctz32(size
| (int)addr
| 8);
2017 this_size
= 1 << this_mop
;
2020 r
= memory_region_dispatch_read(mr
, mr_offset
, &val
,
2021 this_mop
, full
->attrs
);
2022 if (unlikely(r
!= MEMTX_OK
)) {
2023 io_failed(cpu
, full
, addr
, this_size
, type
, mmu_idx
, r
, ra
);
2025 if (this_size
== 8) {
2029 ret_be
= (ret_be
<< (this_size
* 8)) | val
;
2031 mr_offset
+= this_size
;
2038 static uint64_t do_ld_mmio_beN(CPUState
*cpu
, CPUTLBEntryFull
*full
,
2039 uint64_t ret_be
, vaddr addr
, int size
,
2040 int mmu_idx
, MMUAccessType type
, uintptr_t ra
)
2042 MemoryRegionSection
*section
;
2047 tcg_debug_assert(size
> 0 && size
<= 8);
2049 attrs
= full
->attrs
;
2050 section
= io_prepare(&mr_offset
, cpu
, full
->xlat_section
, attrs
, addr
, ra
);
2054 return int_ld_mmio_beN(cpu
, full
, ret_be
, addr
, size
, mmu_idx
,
2055 type
, ra
, mr
, mr_offset
);
2058 static Int128
do_ld16_mmio_beN(CPUState
*cpu
, CPUTLBEntryFull
*full
,
2059 uint64_t ret_be
, vaddr addr
, int size
,
2060 int mmu_idx
, uintptr_t ra
)
2062 MemoryRegionSection
*section
;
2068 tcg_debug_assert(size
> 8 && size
<= 16);
2070 attrs
= full
->attrs
;
2071 section
= io_prepare(&mr_offset
, cpu
, full
->xlat_section
, attrs
, addr
, ra
);
2075 a
= int_ld_mmio_beN(cpu
, full
, ret_be
, addr
, size
- 8, mmu_idx
,
2076 MMU_DATA_LOAD
, ra
, mr
, mr_offset
);
2077 b
= int_ld_mmio_beN(cpu
, full
, ret_be
, addr
+ size
- 8, 8, mmu_idx
,
2078 MMU_DATA_LOAD
, ra
, mr
, mr_offset
+ size
- 8);
2079 return int128_make128(b
, a
);
2084 * @p: translation parameters
2085 * @ret_be: accumulated data
2087 * Load @p->size bytes from @p->haddr, which is RAM.
2088 * The bytes to concatenated in big-endian order with @ret_be.
2090 static uint64_t do_ld_bytes_beN(MMULookupPageData
*p
, uint64_t ret_be
)
2092 uint8_t *haddr
= p
->haddr
;
2093 int i
, size
= p
->size
;
2095 for (i
= 0; i
< size
; i
++) {
2096 ret_be
= (ret_be
<< 8) | haddr
[i
];
2103 * @p: translation parameters
2104 * @ret_be: accumulated data
2106 * As do_ld_bytes_beN, but atomically on each aligned part.
2108 static uint64_t do_ld_parts_beN(MMULookupPageData
*p
, uint64_t ret_be
)
2110 void *haddr
= p
->haddr
;
2118 * Find minimum of alignment and size.
2119 * This is slightly stronger than required by MO_ATOM_SUBALIGN, which
2120 * would have only checked the low bits of addr|size once at the start,
2121 * but is just as easy.
2123 switch (((uintptr_t)haddr
| size
) & 7) {
2125 x
= cpu_to_be32(load_atomic4(haddr
));
2126 ret_be
= (ret_be
<< 32) | x
;
2131 x
= cpu_to_be16(load_atomic2(haddr
));
2132 ret_be
= (ret_be
<< 16) | x
;
2136 x
= *(uint8_t *)haddr
;
2137 ret_be
= (ret_be
<< 8) | x
;
2141 g_assert_not_reached();
2145 } while (size
!= 0);
2151 * @p: translation parameters
2152 * @ret_be: accumulated data
2154 * As do_ld_bytes_beN, but with one atomic load.
2155 * Four aligned bytes are guaranteed to cover the load.
2157 static uint64_t do_ld_whole_be4(MMULookupPageData
*p
, uint64_t ret_be
)
2159 int o
= p
->addr
& 3;
2160 uint32_t x
= load_atomic4(p
->haddr
- o
);
2164 x
>>= (4 - p
->size
) * 8;
2165 return (ret_be
<< (p
->size
* 8)) | x
;
2170 * @p: translation parameters
2171 * @ret_be: accumulated data
2173 * As do_ld_bytes_beN, but with one atomic load.
2174 * Eight aligned bytes are guaranteed to cover the load.
2176 static uint64_t do_ld_whole_be8(CPUState
*cpu
, uintptr_t ra
,
2177 MMULookupPageData
*p
, uint64_t ret_be
)
2179 int o
= p
->addr
& 7;
2180 uint64_t x
= load_atomic8_or_exit(cpu
, ra
, p
->haddr
- o
);
2184 x
>>= (8 - p
->size
) * 8;
2185 return (ret_be
<< (p
->size
* 8)) | x
;
2190 * @p: translation parameters
2191 * @ret_be: accumulated data
2193 * As do_ld_bytes_beN, but with one atomic load.
2194 * 16 aligned bytes are guaranteed to cover the load.
2196 static Int128
do_ld_whole_be16(CPUState
*cpu
, uintptr_t ra
,
2197 MMULookupPageData
*p
, uint64_t ret_be
)
2199 int o
= p
->addr
& 15;
2200 Int128 x
, y
= load_atomic16_or_exit(cpu
, ra
, p
->haddr
- o
);
2203 if (!HOST_BIG_ENDIAN
) {
2206 y
= int128_lshift(y
, o
* 8);
2207 y
= int128_urshift(y
, (16 - size
) * 8);
2208 x
= int128_make64(ret_be
);
2209 x
= int128_lshift(x
, size
* 8);
2210 return int128_or(x
, y
);
2214 * Wrapper for the above.
2216 static uint64_t do_ld_beN(CPUState
*cpu
, MMULookupPageData
*p
,
2217 uint64_t ret_be
, int mmu_idx
, MMUAccessType type
,
2218 MemOp mop
, uintptr_t ra
)
2221 unsigned tmp
, half_size
;
2223 if (unlikely(p
->flags
& TLB_MMIO
)) {
2224 return do_ld_mmio_beN(cpu
, p
->full
, ret_be
, p
->addr
, p
->size
,
2229 * It is a given that we cross a page and therefore there is no
2230 * atomicity for the load as a whole, but subobjects may need attention.
2232 atom
= mop
& MO_ATOM_MASK
;
2234 case MO_ATOM_SUBALIGN
:
2235 return do_ld_parts_beN(p
, ret_be
);
2237 case MO_ATOM_IFALIGN_PAIR
:
2238 case MO_ATOM_WITHIN16_PAIR
:
2239 tmp
= mop
& MO_SIZE
;
2240 tmp
= tmp
? tmp
- 1 : 0;
2241 half_size
= 1 << tmp
;
2242 if (atom
== MO_ATOM_IFALIGN_PAIR
2243 ? p
->size
== half_size
2244 : p
->size
>= half_size
) {
2245 if (!HAVE_al8_fast
&& p
->size
< 4) {
2246 return do_ld_whole_be4(p
, ret_be
);
2248 return do_ld_whole_be8(cpu
, ra
, p
, ret_be
);
2253 case MO_ATOM_IFALIGN
:
2254 case MO_ATOM_WITHIN16
:
2256 return do_ld_bytes_beN(p
, ret_be
);
2259 g_assert_not_reached();
2264 * Wrapper for the above, for 8 < size < 16.
2266 static Int128
do_ld16_beN(CPUState
*cpu
, MMULookupPageData
*p
,
2267 uint64_t a
, int mmu_idx
, MemOp mop
, uintptr_t ra
)
2273 if (unlikely(p
->flags
& TLB_MMIO
)) {
2274 return do_ld16_mmio_beN(cpu
, p
->full
, a
, p
->addr
, size
, mmu_idx
, ra
);
2278 * It is a given that we cross a page and therefore there is no
2279 * atomicity for the load as a whole, but subobjects may need attention.
2281 atom
= mop
& MO_ATOM_MASK
;
2283 case MO_ATOM_SUBALIGN
:
2285 a
= do_ld_parts_beN(p
, a
);
2286 p
->haddr
+= size
- 8;
2288 b
= do_ld_parts_beN(p
, 0);
2291 case MO_ATOM_WITHIN16_PAIR
:
2292 /* Since size > 8, this is the half that must be atomic. */
2293 return do_ld_whole_be16(cpu
, ra
, p
, a
);
2295 case MO_ATOM_IFALIGN_PAIR
:
2297 * Since size > 8, both halves are misaligned,
2298 * and so neither is atomic.
2300 case MO_ATOM_IFALIGN
:
2301 case MO_ATOM_WITHIN16
:
2304 a
= do_ld_bytes_beN(p
, a
);
2305 b
= ldq_be_p(p
->haddr
+ size
- 8);
2309 g_assert_not_reached();
2312 return int128_make128(b
, a
);
2315 static uint8_t do_ld_1(CPUState
*cpu
, MMULookupPageData
*p
, int mmu_idx
,
2316 MMUAccessType type
, uintptr_t ra
)
2318 if (unlikely(p
->flags
& TLB_MMIO
)) {
2319 return do_ld_mmio_beN(cpu
, p
->full
, 0, p
->addr
, 1, mmu_idx
, type
, ra
);
2321 return *(uint8_t *)p
->haddr
;
2325 static uint16_t do_ld_2(CPUState
*cpu
, MMULookupPageData
*p
, int mmu_idx
,
2326 MMUAccessType type
, MemOp memop
, uintptr_t ra
)
2330 if (unlikely(p
->flags
& TLB_MMIO
)) {
2331 ret
= do_ld_mmio_beN(cpu
, p
->full
, 0, p
->addr
, 2, mmu_idx
, type
, ra
);
2332 if ((memop
& MO_BSWAP
) == MO_LE
) {
2336 /* Perform the load host endian, then swap if necessary. */
2337 ret
= load_atom_2(cpu
, ra
, p
->haddr
, memop
);
2338 if (memop
& MO_BSWAP
) {
2345 static uint32_t do_ld_4(CPUState
*cpu
, MMULookupPageData
*p
, int mmu_idx
,
2346 MMUAccessType type
, MemOp memop
, uintptr_t ra
)
2350 if (unlikely(p
->flags
& TLB_MMIO
)) {
2351 ret
= do_ld_mmio_beN(cpu
, p
->full
, 0, p
->addr
, 4, mmu_idx
, type
, ra
);
2352 if ((memop
& MO_BSWAP
) == MO_LE
) {
2356 /* Perform the load host endian. */
2357 ret
= load_atom_4(cpu
, ra
, p
->haddr
, memop
);
2358 if (memop
& MO_BSWAP
) {
2365 static uint64_t do_ld_8(CPUState
*cpu
, MMULookupPageData
*p
, int mmu_idx
,
2366 MMUAccessType type
, MemOp memop
, uintptr_t ra
)
2370 if (unlikely(p
->flags
& TLB_MMIO
)) {
2371 ret
= do_ld_mmio_beN(cpu
, p
->full
, 0, p
->addr
, 8, mmu_idx
, type
, ra
);
2372 if ((memop
& MO_BSWAP
) == MO_LE
) {
2376 /* Perform the load host endian. */
2377 ret
= load_atom_8(cpu
, ra
, p
->haddr
, memop
);
2378 if (memop
& MO_BSWAP
) {
2385 static uint8_t do_ld1_mmu(CPUState
*cpu
, vaddr addr
, MemOpIdx oi
,
2386 uintptr_t ra
, MMUAccessType access_type
)
2391 cpu_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2392 crosspage
= mmu_lookup(cpu
, addr
, oi
, ra
, access_type
, &l
);
2393 tcg_debug_assert(!crosspage
);
2395 return do_ld_1(cpu
, &l
.page
[0], l
.mmu_idx
, access_type
, ra
);
2398 static uint16_t do_ld2_mmu(CPUState
*cpu
, vaddr addr
, MemOpIdx oi
,
2399 uintptr_t ra
, MMUAccessType access_type
)
2406 cpu_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2407 crosspage
= mmu_lookup(cpu
, addr
, oi
, ra
, access_type
, &l
);
2408 if (likely(!crosspage
)) {
2409 return do_ld_2(cpu
, &l
.page
[0], l
.mmu_idx
, access_type
, l
.memop
, ra
);
2412 a
= do_ld_1(cpu
, &l
.page
[0], l
.mmu_idx
, access_type
, ra
);
2413 b
= do_ld_1(cpu
, &l
.page
[1], l
.mmu_idx
, access_type
, ra
);
2415 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2423 static uint32_t do_ld4_mmu(CPUState
*cpu
, vaddr addr
, MemOpIdx oi
,
2424 uintptr_t ra
, MMUAccessType access_type
)
2430 cpu_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2431 crosspage
= mmu_lookup(cpu
, addr
, oi
, ra
, access_type
, &l
);
2432 if (likely(!crosspage
)) {
2433 return do_ld_4(cpu
, &l
.page
[0], l
.mmu_idx
, access_type
, l
.memop
, ra
);
2436 ret
= do_ld_beN(cpu
, &l
.page
[0], 0, l
.mmu_idx
, access_type
, l
.memop
, ra
);
2437 ret
= do_ld_beN(cpu
, &l
.page
[1], ret
, l
.mmu_idx
, access_type
, l
.memop
, ra
);
2438 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2444 static uint64_t do_ld8_mmu(CPUState
*cpu
, vaddr addr
, MemOpIdx oi
,
2445 uintptr_t ra
, MMUAccessType access_type
)
2451 cpu_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2452 crosspage
= mmu_lookup(cpu
, addr
, oi
, ra
, access_type
, &l
);
2453 if (likely(!crosspage
)) {
2454 return do_ld_8(cpu
, &l
.page
[0], l
.mmu_idx
, access_type
, l
.memop
, ra
);
2457 ret
= do_ld_beN(cpu
, &l
.page
[0], 0, l
.mmu_idx
, access_type
, l
.memop
, ra
);
2458 ret
= do_ld_beN(cpu
, &l
.page
[1], ret
, l
.mmu_idx
, access_type
, l
.memop
, ra
);
2459 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2465 static Int128
do_ld16_mmu(CPUState
*cpu
, vaddr addr
,
2466 MemOpIdx oi
, uintptr_t ra
)
2474 cpu_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2475 crosspage
= mmu_lookup(cpu
, addr
, oi
, ra
, MMU_DATA_LOAD
, &l
);
2476 if (likely(!crosspage
)) {
2477 if (unlikely(l
.page
[0].flags
& TLB_MMIO
)) {
2478 ret
= do_ld16_mmio_beN(cpu
, l
.page
[0].full
, 0, addr
, 16,
2480 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2481 ret
= bswap128(ret
);
2484 /* Perform the load host endian. */
2485 ret
= load_atom_16(cpu
, ra
, l
.page
[0].haddr
, l
.memop
);
2486 if (l
.memop
& MO_BSWAP
) {
2487 ret
= bswap128(ret
);
2493 first
= l
.page
[0].size
;
2495 MemOp mop8
= (l
.memop
& ~MO_SIZE
) | MO_64
;
2497 a
= do_ld_8(cpu
, &l
.page
[0], l
.mmu_idx
, MMU_DATA_LOAD
, mop8
, ra
);
2498 b
= do_ld_8(cpu
, &l
.page
[1], l
.mmu_idx
, MMU_DATA_LOAD
, mop8
, ra
);
2499 if ((mop8
& MO_BSWAP
) == MO_LE
) {
2500 ret
= int128_make128(a
, b
);
2502 ret
= int128_make128(b
, a
);
2508 a
= do_ld_beN(cpu
, &l
.page
[0], 0, l
.mmu_idx
,
2509 MMU_DATA_LOAD
, l
.memop
, ra
);
2510 ret
= do_ld16_beN(cpu
, &l
.page
[1], a
, l
.mmu_idx
, l
.memop
, ra
);
2512 ret
= do_ld16_beN(cpu
, &l
.page
[0], 0, l
.mmu_idx
, l
.memop
, ra
);
2513 b
= int128_getlo(ret
);
2514 ret
= int128_lshift(ret
, l
.page
[1].size
* 8);
2515 a
= int128_gethi(ret
);
2516 b
= do_ld_beN(cpu
, &l
.page
[1], b
, l
.mmu_idx
,
2517 MMU_DATA_LOAD
, l
.memop
, ra
);
2518 ret
= int128_make128(b
, a
);
2520 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2521 ret
= bswap128(ret
);
2532 * @cpu: generic cpu state
2533 * @full: page parameters
2534 * @val_le: data to store
2535 * @addr: virtual address
2536 * @size: number of bytes
2537 * @mmu_idx: virtual address context
2538 * @ra: return address into tcg generated code, or 0
2541 * Store @size bytes at @addr, which is memory-mapped i/o.
2542 * The bytes to store are extracted in little-endian order from @val_le;
2543 * return the bytes of @val_le beyond @p->size that have not been stored.
2545 static uint64_t int_st_mmio_leN(CPUState
*cpu
, CPUTLBEntryFull
*full
,
2546 uint64_t val_le
, vaddr addr
, int size
,
2547 int mmu_idx
, uintptr_t ra
,
2548 MemoryRegion
*mr
, hwaddr mr_offset
)
2555 /* Store aligned pieces up to 8 bytes. */
2556 this_mop
= ctz32(size
| (int)addr
| 8);
2557 this_size
= 1 << this_mop
;
2560 r
= memory_region_dispatch_write(mr
, mr_offset
, val_le
,
2561 this_mop
, full
->attrs
);
2562 if (unlikely(r
!= MEMTX_OK
)) {
2563 io_failed(cpu
, full
, addr
, this_size
, MMU_DATA_STORE
,
2566 if (this_size
== 8) {
2570 val_le
>>= this_size
* 8;
2572 mr_offset
+= this_size
;
2579 static uint64_t do_st_mmio_leN(CPUState
*cpu
, CPUTLBEntryFull
*full
,
2580 uint64_t val_le
, vaddr addr
, int size
,
2581 int mmu_idx
, uintptr_t ra
)
2583 MemoryRegionSection
*section
;
2588 tcg_debug_assert(size
> 0 && size
<= 8);
2590 attrs
= full
->attrs
;
2591 section
= io_prepare(&mr_offset
, cpu
, full
->xlat_section
, attrs
, addr
, ra
);
2595 return int_st_mmio_leN(cpu
, full
, val_le
, addr
, size
, mmu_idx
,
2599 static uint64_t do_st16_mmio_leN(CPUState
*cpu
, CPUTLBEntryFull
*full
,
2600 Int128 val_le
, vaddr addr
, int size
,
2601 int mmu_idx
, uintptr_t ra
)
2603 MemoryRegionSection
*section
;
2608 tcg_debug_assert(size
> 8 && size
<= 16);
2610 attrs
= full
->attrs
;
2611 section
= io_prepare(&mr_offset
, cpu
, full
->xlat_section
, attrs
, addr
, ra
);
2615 int_st_mmio_leN(cpu
, full
, int128_getlo(val_le
), addr
, 8,
2616 mmu_idx
, ra
, mr
, mr_offset
);
2617 return int_st_mmio_leN(cpu
, full
, int128_gethi(val_le
), addr
+ 8,
2618 size
- 8, mmu_idx
, ra
, mr
, mr_offset
+ 8);
2622 * Wrapper for the above.
2624 static uint64_t do_st_leN(CPUState
*cpu
, MMULookupPageData
*p
,
2625 uint64_t val_le
, int mmu_idx
,
2626 MemOp mop
, uintptr_t ra
)
2629 unsigned tmp
, half_size
;
2631 if (unlikely(p
->flags
& TLB_MMIO
)) {
2632 return do_st_mmio_leN(cpu
, p
->full
, val_le
, p
->addr
,
2633 p
->size
, mmu_idx
, ra
);
2634 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2635 return val_le
>> (p
->size
* 8);
2639 * It is a given that we cross a page and therefore there is no atomicity
2640 * for the store as a whole, but subobjects may need attention.
2642 atom
= mop
& MO_ATOM_MASK
;
2644 case MO_ATOM_SUBALIGN
:
2645 return store_parts_leN(p
->haddr
, p
->size
, val_le
);
2647 case MO_ATOM_IFALIGN_PAIR
:
2648 case MO_ATOM_WITHIN16_PAIR
:
2649 tmp
= mop
& MO_SIZE
;
2650 tmp
= tmp
? tmp
- 1 : 0;
2651 half_size
= 1 << tmp
;
2652 if (atom
== MO_ATOM_IFALIGN_PAIR
2653 ? p
->size
== half_size
2654 : p
->size
>= half_size
) {
2655 if (!HAVE_al8_fast
&& p
->size
<= 4) {
2656 return store_whole_le4(p
->haddr
, p
->size
, val_le
);
2657 } else if (HAVE_al8
) {
2658 return store_whole_le8(p
->haddr
, p
->size
, val_le
);
2660 cpu_loop_exit_atomic(cpu
, ra
);
2665 case MO_ATOM_IFALIGN
:
2666 case MO_ATOM_WITHIN16
:
2668 return store_bytes_leN(p
->haddr
, p
->size
, val_le
);
2671 g_assert_not_reached();
2676 * Wrapper for the above, for 8 < size < 16.
2678 static uint64_t do_st16_leN(CPUState
*cpu
, MMULookupPageData
*p
,
2679 Int128 val_le
, int mmu_idx
,
2680 MemOp mop
, uintptr_t ra
)
2685 if (unlikely(p
->flags
& TLB_MMIO
)) {
2686 return do_st16_mmio_leN(cpu
, p
->full
, val_le
, p
->addr
,
2688 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2689 return int128_gethi(val_le
) >> ((size
- 8) * 8);
2693 * It is a given that we cross a page and therefore there is no atomicity
2694 * for the store as a whole, but subobjects may need attention.
2696 atom
= mop
& MO_ATOM_MASK
;
2698 case MO_ATOM_SUBALIGN
:
2699 store_parts_leN(p
->haddr
, 8, int128_getlo(val_le
));
2700 return store_parts_leN(p
->haddr
+ 8, p
->size
- 8,
2701 int128_gethi(val_le
));
2703 case MO_ATOM_WITHIN16_PAIR
:
2704 /* Since size > 8, this is the half that must be atomic. */
2705 if (!HAVE_CMPXCHG128
) {
2706 cpu_loop_exit_atomic(cpu
, ra
);
2708 return store_whole_le16(p
->haddr
, p
->size
, val_le
);
2710 case MO_ATOM_IFALIGN_PAIR
:
2712 * Since size > 8, both halves are misaligned,
2713 * and so neither is atomic.
2715 case MO_ATOM_IFALIGN
:
2716 case MO_ATOM_WITHIN16
:
2718 stq_le_p(p
->haddr
, int128_getlo(val_le
));
2719 return store_bytes_leN(p
->haddr
+ 8, p
->size
- 8,
2720 int128_gethi(val_le
));
2723 g_assert_not_reached();
2727 static void do_st_1(CPUState
*cpu
, MMULookupPageData
*p
, uint8_t val
,
2728 int mmu_idx
, uintptr_t ra
)
2730 if (unlikely(p
->flags
& TLB_MMIO
)) {
2731 do_st_mmio_leN(cpu
, p
->full
, val
, p
->addr
, 1, mmu_idx
, ra
);
2732 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2735 *(uint8_t *)p
->haddr
= val
;
2739 static void do_st_2(CPUState
*cpu
, MMULookupPageData
*p
, uint16_t val
,
2740 int mmu_idx
, MemOp memop
, uintptr_t ra
)
2742 if (unlikely(p
->flags
& TLB_MMIO
)) {
2743 if ((memop
& MO_BSWAP
) != MO_LE
) {
2746 do_st_mmio_leN(cpu
, p
->full
, val
, p
->addr
, 2, mmu_idx
, ra
);
2747 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2750 /* Swap to host endian if necessary, then store. */
2751 if (memop
& MO_BSWAP
) {
2754 store_atom_2(cpu
, ra
, p
->haddr
, memop
, val
);
2758 static void do_st_4(CPUState
*cpu
, MMULookupPageData
*p
, uint32_t val
,
2759 int mmu_idx
, MemOp memop
, uintptr_t ra
)
2761 if (unlikely(p
->flags
& TLB_MMIO
)) {
2762 if ((memop
& MO_BSWAP
) != MO_LE
) {
2765 do_st_mmio_leN(cpu
, p
->full
, val
, p
->addr
, 4, mmu_idx
, ra
);
2766 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2769 /* Swap to host endian if necessary, then store. */
2770 if (memop
& MO_BSWAP
) {
2773 store_atom_4(cpu
, ra
, p
->haddr
, memop
, val
);
2777 static void do_st_8(CPUState
*cpu
, MMULookupPageData
*p
, uint64_t val
,
2778 int mmu_idx
, MemOp memop
, uintptr_t ra
)
2780 if (unlikely(p
->flags
& TLB_MMIO
)) {
2781 if ((memop
& MO_BSWAP
) != MO_LE
) {
2784 do_st_mmio_leN(cpu
, p
->full
, val
, p
->addr
, 8, mmu_idx
, ra
);
2785 } else if (unlikely(p
->flags
& TLB_DISCARD_WRITE
)) {
2788 /* Swap to host endian if necessary, then store. */
2789 if (memop
& MO_BSWAP
) {
2792 store_atom_8(cpu
, ra
, p
->haddr
, memop
, val
);
2796 static void do_st1_mmu(CPUState
*cpu
, vaddr addr
, uint8_t val
,
2797 MemOpIdx oi
, uintptr_t ra
)
2802 cpu_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2803 crosspage
= mmu_lookup(cpu
, addr
, oi
, ra
, MMU_DATA_STORE
, &l
);
2804 tcg_debug_assert(!crosspage
);
2806 do_st_1(cpu
, &l
.page
[0], val
, l
.mmu_idx
, ra
);
2809 static void do_st2_mmu(CPUState
*cpu
, vaddr addr
, uint16_t val
,
2810 MemOpIdx oi
, uintptr_t ra
)
2816 cpu_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2817 crosspage
= mmu_lookup(cpu
, addr
, oi
, ra
, MMU_DATA_STORE
, &l
);
2818 if (likely(!crosspage
)) {
2819 do_st_2(cpu
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2823 if ((l
.memop
& MO_BSWAP
) == MO_LE
) {
2824 a
= val
, b
= val
>> 8;
2826 b
= val
, a
= val
>> 8;
2828 do_st_1(cpu
, &l
.page
[0], a
, l
.mmu_idx
, ra
);
2829 do_st_1(cpu
, &l
.page
[1], b
, l
.mmu_idx
, ra
);
2832 static void do_st4_mmu(CPUState
*cpu
, vaddr addr
, uint32_t val
,
2833 MemOpIdx oi
, uintptr_t ra
)
2838 cpu_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2839 crosspage
= mmu_lookup(cpu
, addr
, oi
, ra
, MMU_DATA_STORE
, &l
);
2840 if (likely(!crosspage
)) {
2841 do_st_4(cpu
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2845 /* Swap to little endian for simplicity, then store by bytes. */
2846 if ((l
.memop
& MO_BSWAP
) != MO_LE
) {
2849 val
= do_st_leN(cpu
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2850 (void) do_st_leN(cpu
, &l
.page
[1], val
, l
.mmu_idx
, l
.memop
, ra
);
2853 static void do_st8_mmu(CPUState
*cpu
, vaddr addr
, uint64_t val
,
2854 MemOpIdx oi
, uintptr_t ra
)
2859 cpu_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2860 crosspage
= mmu_lookup(cpu
, addr
, oi
, ra
, MMU_DATA_STORE
, &l
);
2861 if (likely(!crosspage
)) {
2862 do_st_8(cpu
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2866 /* Swap to little endian for simplicity, then store by bytes. */
2867 if ((l
.memop
& MO_BSWAP
) != MO_LE
) {
2870 val
= do_st_leN(cpu
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2871 (void) do_st_leN(cpu
, &l
.page
[1], val
, l
.mmu_idx
, l
.memop
, ra
);
2874 static void do_st16_mmu(CPUState
*cpu
, vaddr addr
, Int128 val
,
2875 MemOpIdx oi
, uintptr_t ra
)
2882 cpu_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2883 crosspage
= mmu_lookup(cpu
, addr
, oi
, ra
, MMU_DATA_STORE
, &l
);
2884 if (likely(!crosspage
)) {
2885 if (unlikely(l
.page
[0].flags
& TLB_MMIO
)) {
2886 if ((l
.memop
& MO_BSWAP
) != MO_LE
) {
2887 val
= bswap128(val
);
2889 do_st16_mmio_leN(cpu
, l
.page
[0].full
, val
, addr
, 16, l
.mmu_idx
, ra
);
2890 } else if (unlikely(l
.page
[0].flags
& TLB_DISCARD_WRITE
)) {
2893 /* Swap to host endian if necessary, then store. */
2894 if (l
.memop
& MO_BSWAP
) {
2895 val
= bswap128(val
);
2897 store_atom_16(cpu
, ra
, l
.page
[0].haddr
, l
.memop
, val
);
2902 first
= l
.page
[0].size
;
2904 MemOp mop8
= (l
.memop
& ~(MO_SIZE
| MO_BSWAP
)) | MO_64
;
2906 if (l
.memop
& MO_BSWAP
) {
2907 val
= bswap128(val
);
2909 if (HOST_BIG_ENDIAN
) {
2910 b
= int128_getlo(val
), a
= int128_gethi(val
);
2912 a
= int128_getlo(val
), b
= int128_gethi(val
);
2914 do_st_8(cpu
, &l
.page
[0], a
, l
.mmu_idx
, mop8
, ra
);
2915 do_st_8(cpu
, &l
.page
[1], b
, l
.mmu_idx
, mop8
, ra
);
2919 if ((l
.memop
& MO_BSWAP
) != MO_LE
) {
2920 val
= bswap128(val
);
2923 do_st_leN(cpu
, &l
.page
[0], int128_getlo(val
), l
.mmu_idx
, l
.memop
, ra
);
2924 val
= int128_urshift(val
, first
* 8);
2925 do_st16_leN(cpu
, &l
.page
[1], val
, l
.mmu_idx
, l
.memop
, ra
);
2927 b
= do_st16_leN(cpu
, &l
.page
[0], val
, l
.mmu_idx
, l
.memop
, ra
);
2928 do_st_leN(cpu
, &l
.page
[1], b
, l
.mmu_idx
, l
.memop
, ra
);
2932 #include "ldst_common.c.inc"
2935 * First set of functions passes in OI and RETADDR.
2936 * This makes them callable from other helpers.
2939 #define ATOMIC_NAME(X) \
2940 glue(glue(glue(cpu_atomic_ ## X, SUFFIX), END), _mmu)
2942 #define ATOMIC_MMU_CLEANUP
2944 #include "atomic_common.c.inc"
2947 #include "atomic_template.h"
2950 #include "atomic_template.h"
2953 #include "atomic_template.h"
2955 #ifdef CONFIG_ATOMIC64
2957 #include "atomic_template.h"
2960 #if defined(CONFIG_ATOMIC128) || HAVE_CMPXCHG128
2961 #define DATA_SIZE 16
2962 #include "atomic_template.h"
2965 /* Code access functions. */
2967 uint32_t cpu_ldub_code(CPUArchState
*env
, abi_ptr addr
)
2969 CPUState
*cs
= env_cpu(env
);
2970 MemOpIdx oi
= make_memop_idx(MO_UB
, cpu_mmu_index(cs
, true));
2971 return do_ld1_mmu(cs
, addr
, oi
, 0, MMU_INST_FETCH
);
2974 uint32_t cpu_lduw_code(CPUArchState
*env
, abi_ptr addr
)
2976 CPUState
*cs
= env_cpu(env
);
2977 MemOpIdx oi
= make_memop_idx(MO_TEUW
, cpu_mmu_index(cs
, true));
2978 return do_ld2_mmu(cs
, addr
, oi
, 0, MMU_INST_FETCH
);
2981 uint32_t cpu_ldl_code(CPUArchState
*env
, abi_ptr addr
)
2983 CPUState
*cs
= env_cpu(env
);
2984 MemOpIdx oi
= make_memop_idx(MO_TEUL
, cpu_mmu_index(cs
, true));
2985 return do_ld4_mmu(cs
, addr
, oi
, 0, MMU_INST_FETCH
);
2988 uint64_t cpu_ldq_code(CPUArchState
*env
, abi_ptr addr
)
2990 CPUState
*cs
= env_cpu(env
);
2991 MemOpIdx oi
= make_memop_idx(MO_TEUQ
, cpu_mmu_index(cs
, true));
2992 return do_ld8_mmu(cs
, addr
, oi
, 0, MMU_INST_FETCH
);
2995 uint8_t cpu_ldb_code_mmu(CPUArchState
*env
, abi_ptr addr
,
2996 MemOpIdx oi
, uintptr_t retaddr
)
2998 return do_ld1_mmu(env_cpu(env
), addr
, oi
, retaddr
, MMU_INST_FETCH
);
3001 uint16_t cpu_ldw_code_mmu(CPUArchState
*env
, abi_ptr addr
,
3002 MemOpIdx oi
, uintptr_t retaddr
)
3004 return do_ld2_mmu(env_cpu(env
), addr
, oi
, retaddr
, MMU_INST_FETCH
);
3007 uint32_t cpu_ldl_code_mmu(CPUArchState
*env
, abi_ptr addr
,
3008 MemOpIdx oi
, uintptr_t retaddr
)
3010 return do_ld4_mmu(env_cpu(env
), addr
, oi
, retaddr
, MMU_INST_FETCH
);
3013 uint64_t cpu_ldq_code_mmu(CPUArchState
*env
, abi_ptr addr
,
3014 MemOpIdx oi
, uintptr_t retaddr
)
3016 return do_ld8_mmu(env_cpu(env
), addr
, oi
, retaddr
, MMU_INST_FETCH
);