]> git.ipfire.org Git - thirdparty/qemu.git/blame_incremental - exec.c
projects / thirdparty / qemu.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
dma: eliminate DMAContext
[thirdparty/qemu.git] / exec.c
... / ...
CommitLineData
1/*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19#include "config.h"
20#ifdef _WIN32
21#include <windows.h>
22#else
23#include <sys/types.h>
24#include <sys/mman.h>
25#endif
26
27#include "qemu-common.h"
28#include "cpu.h"
29#include "tcg.h"
30#include "hw/hw.h"
31#include "hw/qdev.h"
32#include "qemu/osdep.h"
33#include "sysemu/kvm.h"
34#include "hw/xen/xen.h"
35#include "qemu/timer.h"
36#include "qemu/config-file.h"
37#include "exec/memory.h"
38#include "sysemu/dma.h"
39#include "exec/address-spaces.h"
40#if defined(CONFIG_USER_ONLY)
41#include <qemu.h>
42#else /* !CONFIG_USER_ONLY */
43#include "sysemu/xen-mapcache.h"
44#include "trace.h"
45#endif
46#include "exec/cpu-all.h"
47
48#include "exec/cputlb.h"
49#include "translate-all.h"
50
51#include "exec/memory-internal.h"
52
53//#define DEBUG_SUBPAGE
54
55#if !defined(CONFIG_USER_ONLY)
56int phys_ram_fd;
57static int in_migration;
58
59RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61static MemoryRegion *system_memory;
62static MemoryRegion *system_io;
63
64AddressSpace address_space_io;
65AddressSpace address_space_memory;
66
67MemoryRegion io_mem_rom, io_mem_notdirty;
68static MemoryRegion io_mem_unassigned;
69
70#endif
71
72CPUArchState *first_cpu;
73/* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75DEFINE_TLS(CPUArchState *,cpu_single_env);
76/* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79int use_icount;
80
81#if !defined(CONFIG_USER_ONLY)
82
83typedef struct PhysPageEntry PhysPageEntry;
84
85struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89};
90
91struct AddressSpaceDispatch {
92 /* This is a multi-level map on the physical address space.
93 * The bottom level has pointers to MemoryRegionSections.
94 */
95 PhysPageEntry phys_map;
96 MemoryListener listener;
97 AddressSpace *as;
98};
99
100#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
101typedef struct subpage_t {
102 MemoryRegion iomem;
103 AddressSpace *as;
104 hwaddr base;
105 uint16_t sub_section[TARGET_PAGE_SIZE];
106} subpage_t;
107
108static MemoryRegionSection *phys_sections;
109static unsigned phys_sections_nb, phys_sections_nb_alloc;
110static uint16_t phys_section_unassigned;
111static uint16_t phys_section_notdirty;
112static uint16_t phys_section_rom;
113static uint16_t phys_section_watch;
114
115/* Simple allocator for PhysPageEntry nodes */
116static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
117static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
118
119#define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
120
121static void io_mem_init(void);
122static void memory_map_init(void);
123static void *qemu_safe_ram_ptr(ram_addr_t addr);
124
125static MemoryRegion io_mem_watch;
126#endif
127
128#if !defined(CONFIG_USER_ONLY)
129
130static void phys_map_node_reserve(unsigned nodes)
131{
132 if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
133 typedef PhysPageEntry Node[L2_SIZE];
134 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
135 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
136 phys_map_nodes_nb + nodes);
137 phys_map_nodes = g_renew(Node, phys_map_nodes,
138 phys_map_nodes_nb_alloc);
139 }
140}
141
142static uint16_t phys_map_node_alloc(void)
143{
144 unsigned i;
145 uint16_t ret;
146
147 ret = phys_map_nodes_nb++;
148 assert(ret != PHYS_MAP_NODE_NIL);
149 assert(ret != phys_map_nodes_nb_alloc);
150 for (i = 0; i < L2_SIZE; ++i) {
151 phys_map_nodes[ret][i].is_leaf = 0;
152 phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
153 }
154 return ret;
155}
156
157static void phys_map_nodes_reset(void)
158{
159 phys_map_nodes_nb = 0;
160}
161
162
163static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
164 hwaddr *nb, uint16_t leaf,
165 int level)
166{
167 PhysPageEntry *p;
168 int i;
169 hwaddr step = (hwaddr)1 << (level * L2_BITS);
170
171 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
172 lp->ptr = phys_map_node_alloc();
173 p = phys_map_nodes[lp->ptr];
174 if (level == 0) {
175 for (i = 0; i < L2_SIZE; i++) {
176 p[i].is_leaf = 1;
177 p[i].ptr = phys_section_unassigned;
178 }
179 }
180 } else {
181 p = phys_map_nodes[lp->ptr];
182 }
183 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
184
185 while (*nb && lp < &p[L2_SIZE]) {
186 if ((*index & (step - 1)) == 0 && *nb >= step) {
187 lp->is_leaf = true;
188 lp->ptr = leaf;
189 *index += step;
190 *nb -= step;
191 } else {
192 phys_page_set_level(lp, index, nb, leaf, level - 1);
193 }
194 ++lp;
195 }
196}
197
198static void phys_page_set(AddressSpaceDispatch *d,
199 hwaddr index, hwaddr nb,
200 uint16_t leaf)
201{
202 /* Wildly overreserve - it doesn't matter much. */
203 phys_map_node_reserve(3 * P_L2_LEVELS);
204
205 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
206}
207
208static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
209{
210 PhysPageEntry lp = d->phys_map;
211 PhysPageEntry *p;
212 int i;
213
214 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
215 if (lp.ptr == PHYS_MAP_NODE_NIL) {
216 return &phys_sections[phys_section_unassigned];
217 }
218 p = phys_map_nodes[lp.ptr];
219 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
220 }
221 return &phys_sections[lp.ptr];
222}
223
224bool memory_region_is_unassigned(MemoryRegion *mr)
225{
226 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
227 && mr != &io_mem_watch;
228}
229
230static MemoryRegionSection *address_space_lookup_region(AddressSpace *as,
231 hwaddr addr,
232 bool resolve_subpage)
233{
234 MemoryRegionSection *section;
235 subpage_t *subpage;
236
237 section = phys_page_find(as->dispatch, addr >> TARGET_PAGE_BITS);
238 if (resolve_subpage && section->mr->subpage) {
239 subpage = container_of(section->mr, subpage_t, iomem);
240 section = &phys_sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
241 }
242 return section;
243}
244
245static MemoryRegionSection *
246address_space_translate_internal(AddressSpace *as, hwaddr addr, hwaddr *xlat,
247 hwaddr *plen, bool resolve_subpage)
248{
249 MemoryRegionSection *section;
250 Int128 diff;
251
252 section = address_space_lookup_region(as, addr, resolve_subpage);
253 /* Compute offset within MemoryRegionSection */
254 addr -= section->offset_within_address_space;
255
256 /* Compute offset within MemoryRegion */
257 *xlat = addr + section->offset_within_region;
258
259 diff = int128_sub(section->mr->size, int128_make64(addr));
260 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
261 return section;
262}
263
264MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
265 hwaddr *xlat, hwaddr *plen,
266 bool is_write)
267{
268 IOMMUTLBEntry iotlb;
269 MemoryRegionSection *section;
270 MemoryRegion *mr;
271 hwaddr len = *plen;
272
273 for (;;) {
274 section = address_space_translate_internal(as, addr, &addr, plen, true);
275 mr = section->mr;
276
277 if (!mr->iommu_ops) {
278 break;
279 }
280
281 iotlb = mr->iommu_ops->translate(mr, addr);
282 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
283 | (addr & iotlb.addr_mask));
284 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
285 if (!(iotlb.perm & (1 << is_write))) {
286 mr = &io_mem_unassigned;
287 break;
288 }
289
290 as = iotlb.target_as;
291 }
292
293 *plen = len;
294 *xlat = addr;
295 return mr;
296}
297
298MemoryRegionSection *
299address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
300 hwaddr *plen)
301{
302 MemoryRegionSection *section;
303 section = address_space_translate_internal(as, addr, xlat, plen, false);
304
305 assert(!section->mr->iommu_ops);
306 return section;
307}
308#endif
309
310void cpu_exec_init_all(void)
311{
312#if !defined(CONFIG_USER_ONLY)
313 qemu_mutex_init(&ram_list.mutex);
314 memory_map_init();
315 io_mem_init();
316#endif
317}
318
319#if !defined(CONFIG_USER_ONLY)
320
321static int cpu_common_post_load(void *opaque, int version_id)
322{
323 CPUState *cpu = opaque;
324
325 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
326 version_id is increased. */
327 cpu->interrupt_request &= ~0x01;
328 tlb_flush(cpu->env_ptr, 1);
329
330 return 0;
331}
332
333static const VMStateDescription vmstate_cpu_common = {
334 .name = "cpu_common",
335 .version_id = 1,
336 .minimum_version_id = 1,
337 .minimum_version_id_old = 1,
338 .post_load = cpu_common_post_load,
339 .fields = (VMStateField []) {
340 VMSTATE_UINT32(halted, CPUState),
341 VMSTATE_UINT32(interrupt_request, CPUState),
342 VMSTATE_END_OF_LIST()
343 }
344};
345#else
346#define vmstate_cpu_common vmstate_dummy
347#endif
348
349CPUState *qemu_get_cpu(int index)
350{
351 CPUArchState *env = first_cpu;
352 CPUState *cpu = NULL;
353
354 while (env) {
355 cpu = ENV_GET_CPU(env);
356 if (cpu->cpu_index == index) {
357 break;
358 }
359 env = env->next_cpu;
360 }
361
362 return env ? cpu : NULL;
363}
364
365void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
366{
367 CPUArchState *env = first_cpu;
368
369 while (env) {
370 func(ENV_GET_CPU(env), data);
371 env = env->next_cpu;
372 }
373}
374
375void cpu_exec_init(CPUArchState *env)
376{
377 CPUState *cpu = ENV_GET_CPU(env);
378 CPUClass *cc = CPU_GET_CLASS(cpu);
379 CPUArchState **penv;
380 int cpu_index;
381
382#if defined(CONFIG_USER_ONLY)
383 cpu_list_lock();
384#endif
385 env->next_cpu = NULL;
386 penv = &first_cpu;
387 cpu_index = 0;
388 while (*penv != NULL) {
389 penv = &(*penv)->next_cpu;
390 cpu_index++;
391 }
392 cpu->cpu_index = cpu_index;
393 cpu->numa_node = 0;
394 QTAILQ_INIT(&env->breakpoints);
395 QTAILQ_INIT(&env->watchpoints);
396#ifndef CONFIG_USER_ONLY
397 cpu->thread_id = qemu_get_thread_id();
398#endif
399 *penv = env;
400#if defined(CONFIG_USER_ONLY)
401 cpu_list_unlock();
402#endif
403 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
404#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
405 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
406 cpu_save, cpu_load, env);
407 assert(cc->vmsd == NULL);
408#endif
409 if (cc->vmsd != NULL) {
410 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
411 }
412}
413
414#if defined(TARGET_HAS_ICE)
415#if defined(CONFIG_USER_ONLY)
416static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
417{
418 tb_invalidate_phys_page_range(pc, pc + 1, 0);
419}
420#else
421static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
422{
423 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
424 (pc & ~TARGET_PAGE_MASK));
425}
426#endif
427#endif /* TARGET_HAS_ICE */
428
429#if defined(CONFIG_USER_ONLY)
430void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
431
432{
433}
434
435int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
436 int flags, CPUWatchpoint **watchpoint)
437{
438 return -ENOSYS;
439}
440#else
441/* Add a watchpoint. */
442int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
443 int flags, CPUWatchpoint **watchpoint)
444{
445 target_ulong len_mask = ~(len - 1);
446 CPUWatchpoint *wp;
447
448 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
449 if ((len & (len - 1)) || (addr & ~len_mask) ||
450 len == 0 || len > TARGET_PAGE_SIZE) {
451 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
452 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
453 return -EINVAL;
454 }
455 wp = g_malloc(sizeof(*wp));
456
457 wp->vaddr = addr;
458 wp->len_mask = len_mask;
459 wp->flags = flags;
460
461 /* keep all GDB-injected watchpoints in front */
462 if (flags & BP_GDB)
463 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
464 else
465 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
466
467 tlb_flush_page(env, addr);
468
469 if (watchpoint)
470 *watchpoint = wp;
471 return 0;
472}
473
474/* Remove a specific watchpoint. */
475int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
476 int flags)
477{
478 target_ulong len_mask = ~(len - 1);
479 CPUWatchpoint *wp;
480
481 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
482 if (addr == wp->vaddr && len_mask == wp->len_mask
483 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
484 cpu_watchpoint_remove_by_ref(env, wp);
485 return 0;
486 }
487 }
488 return -ENOENT;
489}
490
491/* Remove a specific watchpoint by reference. */
492void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
493{
494 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
495
496 tlb_flush_page(env, watchpoint->vaddr);
497
498 g_free(watchpoint);
499}
500
501/* Remove all matching watchpoints. */
502void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
503{
504 CPUWatchpoint *wp, *next;
505
506 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
507 if (wp->flags & mask)
508 cpu_watchpoint_remove_by_ref(env, wp);
509 }
510}
511#endif
512
513/* Add a breakpoint. */
514int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
515 CPUBreakpoint **breakpoint)
516{
517#if defined(TARGET_HAS_ICE)
518 CPUBreakpoint *bp;
519
520 bp = g_malloc(sizeof(*bp));
521
522 bp->pc = pc;
523 bp->flags = flags;
524
525 /* keep all GDB-injected breakpoints in front */
526 if (flags & BP_GDB)
527 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
528 else
529 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
530
531 breakpoint_invalidate(env, pc);
532
533 if (breakpoint)
534 *breakpoint = bp;
535 return 0;
536#else
537 return -ENOSYS;
538#endif
539}
540
541/* Remove a specific breakpoint. */
542int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
543{
544#if defined(TARGET_HAS_ICE)
545 CPUBreakpoint *bp;
546
547 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
548 if (bp->pc == pc && bp->flags == flags) {
549 cpu_breakpoint_remove_by_ref(env, bp);
550 return 0;
551 }
552 }
553 return -ENOENT;
554#else
555 return -ENOSYS;
556#endif
557}
558
559/* Remove a specific breakpoint by reference. */
560void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
561{
562#if defined(TARGET_HAS_ICE)
563 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
564
565 breakpoint_invalidate(env, breakpoint->pc);
566
567 g_free(breakpoint);
568#endif
569}
570
571/* Remove all matching breakpoints. */
572void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
573{
574#if defined(TARGET_HAS_ICE)
575 CPUBreakpoint *bp, *next;
576
577 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
578 if (bp->flags & mask)
579 cpu_breakpoint_remove_by_ref(env, bp);
580 }
581#endif
582}
583
584/* enable or disable single step mode. EXCP_DEBUG is returned by the
585 CPU loop after each instruction */
586void cpu_single_step(CPUArchState *env, int enabled)
587{
588#if defined(TARGET_HAS_ICE)
589 if (env->singlestep_enabled != enabled) {
590 env->singlestep_enabled = enabled;
591 if (kvm_enabled())
592 kvm_update_guest_debug(env, 0);
593 else {
594 /* must flush all the translated code to avoid inconsistencies */
595 /* XXX: only flush what is necessary */
596 tb_flush(env);
597 }
598 }
599#endif
600}
601
602void cpu_exit(CPUArchState *env)
603{
604 CPUState *cpu = ENV_GET_CPU(env);
605
606 cpu->exit_request = 1;
607 cpu->tcg_exit_req = 1;
608}
609
610void cpu_abort(CPUArchState *env, const char *fmt, ...)
611{
612 va_list ap;
613 va_list ap2;
614
615 va_start(ap, fmt);
616 va_copy(ap2, ap);
617 fprintf(stderr, "qemu: fatal: ");
618 vfprintf(stderr, fmt, ap);
619 fprintf(stderr, "\n");
620 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
621 if (qemu_log_enabled()) {
622 qemu_log("qemu: fatal: ");
623 qemu_log_vprintf(fmt, ap2);
624 qemu_log("\n");
625 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
626 qemu_log_flush();
627 qemu_log_close();
628 }
629 va_end(ap2);
630 va_end(ap);
631#if defined(CONFIG_USER_ONLY)
632 {
633 struct sigaction act;
634 sigfillset(&act.sa_mask);
635 act.sa_handler = SIG_DFL;
636 sigaction(SIGABRT, &act, NULL);
637 }
638#endif
639 abort();
640}
641
642CPUArchState *cpu_copy(CPUArchState *env)
643{
644 CPUArchState *new_env = cpu_init(env->cpu_model_str);
645 CPUArchState *next_cpu = new_env->next_cpu;
646#if defined(TARGET_HAS_ICE)
647 CPUBreakpoint *bp;
648 CPUWatchpoint *wp;
649#endif
650
651 memcpy(new_env, env, sizeof(CPUArchState));
652
653 /* Preserve chaining. */
654 new_env->next_cpu = next_cpu;
655
656 /* Clone all break/watchpoints.
657 Note: Once we support ptrace with hw-debug register access, make sure
658 BP_CPU break/watchpoints are handled correctly on clone. */
659 QTAILQ_INIT(&env->breakpoints);
660 QTAILQ_INIT(&env->watchpoints);
661#if defined(TARGET_HAS_ICE)
662 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
663 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
664 }
665 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
666 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
667 wp->flags, NULL);
668 }
669#endif
670
671 return new_env;
672}
673
674#if !defined(CONFIG_USER_ONLY)
675static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
676 uintptr_t length)
677{
678 uintptr_t start1;
679
680 /* we modify the TLB cache so that the dirty bit will be set again
681 when accessing the range */
682 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
683 /* Check that we don't span multiple blocks - this breaks the
684 address comparisons below. */
685 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
686 != (end - 1) - start) {
687 abort();
688 }
689 cpu_tlb_reset_dirty_all(start1, length);
690
691}
692
693/* Note: start and end must be within the same ram block. */
694void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
695 int dirty_flags)
696{
697 uintptr_t length;
698
699 start &= TARGET_PAGE_MASK;
700 end = TARGET_PAGE_ALIGN(end);
701
702 length = end - start;
703 if (length == 0)
704 return;
705 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
706
707 if (tcg_enabled()) {
708 tlb_reset_dirty_range_all(start, end, length);
709 }
710}
711
712static int cpu_physical_memory_set_dirty_tracking(int enable)
713{
714 int ret = 0;
715 in_migration = enable;
716 return ret;
717}
718
719hwaddr memory_region_section_get_iotlb(CPUArchState *env,
720 MemoryRegionSection *section,
721 target_ulong vaddr,
722 hwaddr paddr, hwaddr xlat,
723 int prot,
724 target_ulong *address)
725{
726 hwaddr iotlb;
727 CPUWatchpoint *wp;
728
729 if (memory_region_is_ram(section->mr)) {
730 /* Normal RAM. */
731 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
732 + xlat;
733 if (!section->readonly) {
734 iotlb |= phys_section_notdirty;
735 } else {
736 iotlb |= phys_section_rom;
737 }
738 } else {
739 iotlb = section - phys_sections;
740 iotlb += xlat;
741 }
742
743 /* Make accesses to pages with watchpoints go via the
744 watchpoint trap routines. */
745 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
746 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
747 /* Avoid trapping reads of pages with a write breakpoint. */
748 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
749 iotlb = phys_section_watch + paddr;
750 *address |= TLB_MMIO;
751 break;
752 }
753 }
754 }
755
756 return iotlb;
757}
758#endif /* defined(CONFIG_USER_ONLY) */
759
760#if !defined(CONFIG_USER_ONLY)
761
762static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
763 uint16_t section);
764static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
765static void destroy_page_desc(uint16_t section_index)
766{
767 MemoryRegionSection *section = &phys_sections[section_index];
768 MemoryRegion *mr = section->mr;
769
770 if (mr->subpage) {
771 subpage_t *subpage = container_of(mr, subpage_t, iomem);
772 memory_region_destroy(&subpage->iomem);
773 g_free(subpage);
774 }
775}
776
777static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
778{
779 unsigned i;
780 PhysPageEntry *p;
781
782 if (lp->ptr == PHYS_MAP_NODE_NIL) {
783 return;
784 }
785
786 p = phys_map_nodes[lp->ptr];
787 for (i = 0; i < L2_SIZE; ++i) {
788 if (!p[i].is_leaf) {
789 destroy_l2_mapping(&p[i], level - 1);
790 } else {
791 destroy_page_desc(p[i].ptr);
792 }
793 }
794 lp->is_leaf = 0;
795 lp->ptr = PHYS_MAP_NODE_NIL;
796}
797
798static void destroy_all_mappings(AddressSpaceDispatch *d)
799{
800 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
801 phys_map_nodes_reset();
802}
803
804static uint16_t phys_section_add(MemoryRegionSection *section)
805{
806 /* The physical section number is ORed with a page-aligned
807 * pointer to produce the iotlb entries. Thus it should
808 * never overflow into the page-aligned value.
809 */
810 assert(phys_sections_nb < TARGET_PAGE_SIZE);
811
812 if (phys_sections_nb == phys_sections_nb_alloc) {
813 phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
814 phys_sections = g_renew(MemoryRegionSection, phys_sections,
815 phys_sections_nb_alloc);
816 }
817 phys_sections[phys_sections_nb] = *section;
818 return phys_sections_nb++;
819}
820
821static void phys_sections_clear(void)
822{
823 phys_sections_nb = 0;
824}
825
826static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
827{
828 subpage_t *subpage;
829 hwaddr base = section->offset_within_address_space
830 & TARGET_PAGE_MASK;
831 MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
832 MemoryRegionSection subsection = {
833 .offset_within_address_space = base,
834 .size = int128_make64(TARGET_PAGE_SIZE),
835 };
836 hwaddr start, end;
837
838 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
839
840 if (!(existing->mr->subpage)) {
841 subpage = subpage_init(d->as, base);
842 subsection.mr = &subpage->iomem;
843 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
844 phys_section_add(&subsection));
845 } else {
846 subpage = container_of(existing->mr, subpage_t, iomem);
847 }
848 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
849 end = start + int128_get64(section->size) - 1;
850 subpage_register(subpage, start, end, phys_section_add(section));
851}
852
853
854static void register_multipage(AddressSpaceDispatch *d,
855 MemoryRegionSection *section)
856{
857 hwaddr start_addr = section->offset_within_address_space;
858 uint16_t section_index = phys_section_add(section);
859 uint64_t num_pages = int128_get64(int128_rshift(section->size,
860 TARGET_PAGE_BITS));
861
862 assert(num_pages);
863 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
864}
865
866static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
867{
868 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
869 MemoryRegionSection now = *section, remain = *section;
870 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
871
872 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
873 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
874 - now.offset_within_address_space;
875
876 now.size = int128_min(int128_make64(left), now.size);
877 register_subpage(d, &now);
878 } else {
879 now.size = int128_zero();
880 }
881 while (int128_ne(remain.size, now.size)) {
882 remain.size = int128_sub(remain.size, now.size);
883 remain.offset_within_address_space += int128_get64(now.size);
884 remain.offset_within_region += int128_get64(now.size);
885 now = remain;
886 if (int128_lt(remain.size, page_size)) {
887 register_subpage(d, &now);
888 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
889 now.size = page_size;
890 register_subpage(d, &now);
891 } else {
892 now.size = int128_and(now.size, int128_neg(page_size));
893 register_multipage(d, &now);
894 }
895 }
896}
897
898void qemu_flush_coalesced_mmio_buffer(void)
899{
900 if (kvm_enabled())
901 kvm_flush_coalesced_mmio_buffer();
902}
903
904void qemu_mutex_lock_ramlist(void)
905{
906 qemu_mutex_lock(&ram_list.mutex);
907}
908
909void qemu_mutex_unlock_ramlist(void)
910{
911 qemu_mutex_unlock(&ram_list.mutex);
912}
913
914#if defined(__linux__) && !defined(TARGET_S390X)
915
916#include <sys/vfs.h>
917
918#define HUGETLBFS_MAGIC 0x958458f6
919
920static long gethugepagesize(const char *path)
921{
922 struct statfs fs;
923 int ret;
924
925 do {
926 ret = statfs(path, &fs);
927 } while (ret != 0 && errno == EINTR);
928
929 if (ret != 0) {
930 perror(path);
931 return 0;
932 }
933
934 if (fs.f_type != HUGETLBFS_MAGIC)
935 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
936
937 return fs.f_bsize;
938}
939
940static void *file_ram_alloc(RAMBlock *block,
941 ram_addr_t memory,
942 const char *path)
943{
944 char *filename;
945 char *sanitized_name;
946 char *c;
947 void *area;
948 int fd;
949#ifdef MAP_POPULATE
950 int flags;
951#endif
952 unsigned long hpagesize;
953
954 hpagesize = gethugepagesize(path);
955 if (!hpagesize) {
956 return NULL;
957 }
958
959 if (memory < hpagesize) {
960 return NULL;
961 }
962
963 if (kvm_enabled() && !kvm_has_sync_mmu()) {
964 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
965 return NULL;
966 }
967
968 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
969 sanitized_name = g_strdup(block->mr->name);
970 for (c = sanitized_name; *c != '\0'; c++) {
971 if (*c == '/')
972 *c = '_';
973 }
974
975 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
976 sanitized_name);
977 g_free(sanitized_name);
978
979 fd = mkstemp(filename);
980 if (fd < 0) {
981 perror("unable to create backing store for hugepages");
982 g_free(filename);
983 return NULL;
984 }
985 unlink(filename);
986 g_free(filename);
987
988 memory = (memory+hpagesize-1) & ~(hpagesize-1);
989
990 /*
991 * ftruncate is not supported by hugetlbfs in older
992 * hosts, so don't bother bailing out on errors.
993 * If anything goes wrong with it under other filesystems,
994 * mmap will fail.
995 */
996 if (ftruncate(fd, memory))
997 perror("ftruncate");
998
999#ifdef MAP_POPULATE
1000 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
1001 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
1002 * to sidestep this quirk.
1003 */
1004 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
1005 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
1006#else
1007 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1008#endif
1009 if (area == MAP_FAILED) {
1010 perror("file_ram_alloc: can't mmap RAM pages");
1011 close(fd);
1012 return (NULL);
1013 }
1014 block->fd = fd;
1015 return area;
1016}
1017#endif
1018
1019static ram_addr_t find_ram_offset(ram_addr_t size)
1020{
1021 RAMBlock *block, *next_block;
1022 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1023
1024 assert(size != 0); /* it would hand out same offset multiple times */
1025
1026 if (QTAILQ_EMPTY(&ram_list.blocks))
1027 return 0;
1028
1029 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1030 ram_addr_t end, next = RAM_ADDR_MAX;
1031
1032 end = block->offset + block->length;
1033
1034 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1035 if (next_block->offset >= end) {
1036 next = MIN(next, next_block->offset);
1037 }
1038 }
1039 if (next - end >= size && next - end < mingap) {
1040 offset = end;
1041 mingap = next - end;
1042 }
1043 }
1044
1045 if (offset == RAM_ADDR_MAX) {
1046 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1047 (uint64_t)size);
1048 abort();
1049 }
1050
1051 return offset;
1052}
1053
1054ram_addr_t last_ram_offset(void)
1055{
1056 RAMBlock *block;
1057 ram_addr_t last = 0;
1058
1059 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1060 last = MAX(last, block->offset + block->length);
1061
1062 return last;
1063}
1064
1065static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1066{
1067 int ret;
1068 QemuOpts *machine_opts;
1069
1070 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1071 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1072 if (machine_opts &&
1073 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
1074 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1075 if (ret) {
1076 perror("qemu_madvise");
1077 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1078 "but dump_guest_core=off specified\n");
1079 }
1080 }
1081}
1082
1083void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1084{
1085 RAMBlock *new_block, *block;
1086
1087 new_block = NULL;
1088 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1089 if (block->offset == addr) {
1090 new_block = block;
1091 break;
1092 }
1093 }
1094 assert(new_block);
1095 assert(!new_block->idstr[0]);
1096
1097 if (dev) {
1098 char *id = qdev_get_dev_path(dev);
1099 if (id) {
1100 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1101 g_free(id);
1102 }
1103 }
1104 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1105
1106 /* This assumes the iothread lock is taken here too. */
1107 qemu_mutex_lock_ramlist();
1108 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1109 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1110 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1111 new_block->idstr);
1112 abort();
1113 }
1114 }
1115 qemu_mutex_unlock_ramlist();
1116}
1117
1118static int memory_try_enable_merging(void *addr, size_t len)
1119{
1120 QemuOpts *opts;
1121
1122 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1123 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1124 /* disabled by the user */
1125 return 0;
1126 }
1127
1128 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1129}
1130
1131ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1132 MemoryRegion *mr)
1133{
1134 RAMBlock *block, *new_block;
1135
1136 size = TARGET_PAGE_ALIGN(size);
1137 new_block = g_malloc0(sizeof(*new_block));
1138
1139 /* This assumes the iothread lock is taken here too. */
1140 qemu_mutex_lock_ramlist();
1141 new_block->mr = mr;
1142 new_block->offset = find_ram_offset(size);
1143 if (host) {
1144 new_block->host = host;
1145 new_block->flags |= RAM_PREALLOC_MASK;
1146 } else {
1147 if (mem_path) {
1148#if defined (__linux__) && !defined(TARGET_S390X)
1149 new_block->host = file_ram_alloc(new_block, size, mem_path);
1150 if (!new_block->host) {
1151 new_block->host = qemu_anon_ram_alloc(size);
1152 memory_try_enable_merging(new_block->host, size);
1153 }
1154#else
1155 fprintf(stderr, "-mem-path option unsupported\n");
1156 exit(1);
1157#endif
1158 } else {
1159 if (xen_enabled()) {
1160 xen_ram_alloc(new_block->offset, size, mr);
1161 } else if (kvm_enabled()) {
1162 /* some s390/kvm configurations have special constraints */
1163 new_block->host = kvm_ram_alloc(size);
1164 } else {
1165 new_block->host = qemu_anon_ram_alloc(size);
1166 }
1167 memory_try_enable_merging(new_block->host, size);
1168 }
1169 }
1170 new_block->length = size;
1171
1172 /* Keep the list sorted from biggest to smallest block. */
1173 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1174 if (block->length < new_block->length) {
1175 break;
1176 }
1177 }
1178 if (block) {
1179 QTAILQ_INSERT_BEFORE(block, new_block, next);
1180 } else {
1181 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1182 }
1183 ram_list.mru_block = NULL;
1184
1185 ram_list.version++;
1186 qemu_mutex_unlock_ramlist();
1187
1188 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1189 last_ram_offset() >> TARGET_PAGE_BITS);
1190 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1191 0, size >> TARGET_PAGE_BITS);
1192 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1193
1194 qemu_ram_setup_dump(new_block->host, size);
1195 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1196
1197 if (kvm_enabled())
1198 kvm_setup_guest_memory(new_block->host, size);
1199
1200 return new_block->offset;
1201}
1202
1203ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1204{
1205 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1206}
1207
1208void qemu_ram_free_from_ptr(ram_addr_t addr)
1209{
1210 RAMBlock *block;
1211
1212 /* This assumes the iothread lock is taken here too. */
1213 qemu_mutex_lock_ramlist();
1214 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1215 if (addr == block->offset) {
1216 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1217 ram_list.mru_block = NULL;
1218 ram_list.version++;
1219 g_free(block);
1220 break;
1221 }
1222 }
1223 qemu_mutex_unlock_ramlist();
1224}
1225
1226void qemu_ram_free(ram_addr_t addr)
1227{
1228 RAMBlock *block;
1229
1230 /* This assumes the iothread lock is taken here too. */
1231 qemu_mutex_lock_ramlist();
1232 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1233 if (addr == block->offset) {
1234 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1235 ram_list.mru_block = NULL;
1236 ram_list.version++;
1237 if (block->flags & RAM_PREALLOC_MASK) {
1238 ;
1239 } else if (mem_path) {
1240#if defined (__linux__) && !defined(TARGET_S390X)
1241 if (block->fd) {
1242 munmap(block->host, block->length);
1243 close(block->fd);
1244 } else {
1245 qemu_anon_ram_free(block->host, block->length);
1246 }
1247#else
1248 abort();
1249#endif
1250 } else {
1251 if (xen_enabled()) {
1252 xen_invalidate_map_cache_entry(block->host);
1253 } else {
1254 qemu_anon_ram_free(block->host, block->length);
1255 }
1256 }
1257 g_free(block);
1258 break;
1259 }
1260 }
1261 qemu_mutex_unlock_ramlist();
1262
1263}
1264
1265#ifndef _WIN32
1266void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1267{
1268 RAMBlock *block;
1269 ram_addr_t offset;
1270 int flags;
1271 void *area, *vaddr;
1272
1273 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1274 offset = addr - block->offset;
1275 if (offset < block->length) {
1276 vaddr = block->host + offset;
1277 if (block->flags & RAM_PREALLOC_MASK) {
1278 ;
1279 } else {
1280 flags = MAP_FIXED;
1281 munmap(vaddr, length);
1282 if (mem_path) {
1283#if defined(__linux__) && !defined(TARGET_S390X)
1284 if (block->fd) {
1285#ifdef MAP_POPULATE
1286 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1287 MAP_PRIVATE;
1288#else
1289 flags |= MAP_PRIVATE;
1290#endif
1291 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1292 flags, block->fd, offset);
1293 } else {
1294 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1295 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1296 flags, -1, 0);
1297 }
1298#else
1299 abort();
1300#endif
1301 } else {
1302#if defined(TARGET_S390X) && defined(CONFIG_KVM)
1303 flags |= MAP_SHARED | MAP_ANONYMOUS;
1304 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1305 flags, -1, 0);
1306#else
1307 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1308 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1309 flags, -1, 0);
1310#endif
1311 }
1312 if (area != vaddr) {
1313 fprintf(stderr, "Could not remap addr: "
1314 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1315 length, addr);
1316 exit(1);
1317 }
1318 memory_try_enable_merging(vaddr, length);
1319 qemu_ram_setup_dump(vaddr, length);
1320 }
1321 return;
1322 }
1323 }
1324}
1325#endif /* !_WIN32 */
1326
1327/* Return a host pointer to ram allocated with qemu_ram_alloc.
1328 With the exception of the softmmu code in this file, this should
1329 only be used for local memory (e.g. video ram) that the device owns,
1330 and knows it isn't going to access beyond the end of the block.
1331
1332 It should not be used for general purpose DMA.
1333 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1334 */
1335void *qemu_get_ram_ptr(ram_addr_t addr)
1336{
1337 RAMBlock *block;
1338
1339 /* The list is protected by the iothread lock here. */
1340 block = ram_list.mru_block;
1341 if (block && addr - block->offset < block->length) {
1342 goto found;
1343 }
1344 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1345 if (addr - block->offset < block->length) {
1346 goto found;
1347 }
1348 }
1349
1350 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1351 abort();
1352
1353found:
1354 ram_list.mru_block = block;
1355 if (xen_enabled()) {
1356 /* We need to check if the requested address is in the RAM
1357 * because we don't want to map the entire memory in QEMU.
1358 * In that case just map until the end of the page.
1359 */
1360 if (block->offset == 0) {
1361 return xen_map_cache(addr, 0, 0);
1362 } else if (block->host == NULL) {
1363 block->host =
1364 xen_map_cache(block->offset, block->length, 1);
1365 }
1366 }
1367 return block->host + (addr - block->offset);
1368}
1369
1370/* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1371 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1372 *
1373 * ??? Is this still necessary?
1374 */
1375static void *qemu_safe_ram_ptr(ram_addr_t addr)
1376{
1377 RAMBlock *block;
1378
1379 /* The list is protected by the iothread lock here. */
1380 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1381 if (addr - block->offset < block->length) {
1382 if (xen_enabled()) {
1383 /* We need to check if the requested address is in the RAM
1384 * because we don't want to map the entire memory in QEMU.
1385 * In that case just map until the end of the page.
1386 */
1387 if (block->offset == 0) {
1388 return xen_map_cache(addr, 0, 0);
1389 } else if (block->host == NULL) {
1390 block->host =
1391 xen_map_cache(block->offset, block->length, 1);
1392 }
1393 }
1394 return block->host + (addr - block->offset);
1395 }
1396 }
1397
1398 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1399 abort();
1400
1401 return NULL;
1402}
1403
1404/* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1405 * but takes a size argument */
1406static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1407{
1408 if (*size == 0) {
1409 return NULL;
1410 }
1411 if (xen_enabled()) {
1412 return xen_map_cache(addr, *size, 1);
1413 } else {
1414 RAMBlock *block;
1415
1416 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1417 if (addr - block->offset < block->length) {
1418 if (addr - block->offset + *size > block->length)
1419 *size = block->length - addr + block->offset;
1420 return block->host + (addr - block->offset);
1421 }
1422 }
1423
1424 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1425 abort();
1426 }
1427}
1428
1429int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1430{
1431 RAMBlock *block;
1432 uint8_t *host = ptr;
1433
1434 if (xen_enabled()) {
1435 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1436 return 0;
1437 }
1438
1439 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1440 /* This case append when the block is not mapped. */
1441 if (block->host == NULL) {
1442 continue;
1443 }
1444 if (host - block->host < block->length) {
1445 *ram_addr = block->offset + (host - block->host);
1446 return 0;
1447 }
1448 }
1449
1450 return -1;
1451}
1452
1453/* Some of the softmmu routines need to translate from a host pointer
1454 (typically a TLB entry) back to a ram offset. */
1455ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1456{
1457 ram_addr_t ram_addr;
1458
1459 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1460 fprintf(stderr, "Bad ram pointer %p\n", ptr);
1461 abort();
1462 }
1463 return ram_addr;
1464}
1465
1466static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1467 uint64_t val, unsigned size)
1468{
1469 int dirty_flags;
1470 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1471 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1472 tb_invalidate_phys_page_fast(ram_addr, size);
1473 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1474 }
1475 switch (size) {
1476 case 1:
1477 stb_p(qemu_get_ram_ptr(ram_addr), val);
1478 break;
1479 case 2:
1480 stw_p(qemu_get_ram_ptr(ram_addr), val);
1481 break;
1482 case 4:
1483 stl_p(qemu_get_ram_ptr(ram_addr), val);
1484 break;
1485 default:
1486 abort();
1487 }
1488 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1489 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1490 /* we remove the notdirty callback only if the code has been
1491 flushed */
1492 if (dirty_flags == 0xff)
1493 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1494}
1495
1496static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1497 unsigned size, bool is_write)
1498{
1499 return is_write;
1500}
1501
1502static const MemoryRegionOps notdirty_mem_ops = {
1503 .write = notdirty_mem_write,
1504 .valid.accepts = notdirty_mem_accepts,
1505 .endianness = DEVICE_NATIVE_ENDIAN,
1506};
1507
1508/* Generate a debug exception if a watchpoint has been hit. */
1509static void check_watchpoint(int offset, int len_mask, int flags)
1510{
1511 CPUArchState *env = cpu_single_env;
1512 target_ulong pc, cs_base;
1513 target_ulong vaddr;
1514 CPUWatchpoint *wp;
1515 int cpu_flags;
1516
1517 if (env->watchpoint_hit) {
1518 /* We re-entered the check after replacing the TB. Now raise
1519 * the debug interrupt so that is will trigger after the
1520 * current instruction. */
1521 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1522 return;
1523 }
1524 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1525 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1526 if ((vaddr == (wp->vaddr & len_mask) ||
1527 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1528 wp->flags |= BP_WATCHPOINT_HIT;
1529 if (!env->watchpoint_hit) {
1530 env->watchpoint_hit = wp;
1531 tb_check_watchpoint(env);
1532 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1533 env->exception_index = EXCP_DEBUG;
1534 cpu_loop_exit(env);
1535 } else {
1536 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1537 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1538 cpu_resume_from_signal(env, NULL);
1539 }
1540 }
1541 } else {
1542 wp->flags &= ~BP_WATCHPOINT_HIT;
1543 }
1544 }
1545}
1546
1547/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1548 so these check for a hit then pass through to the normal out-of-line
1549 phys routines. */
1550static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1551 unsigned size)
1552{
1553 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1554 switch (size) {
1555 case 1: return ldub_phys(addr);
1556 case 2: return lduw_phys(addr);
1557 case 4: return ldl_phys(addr);
1558 default: abort();
1559 }
1560}
1561
1562static void watch_mem_write(void *opaque, hwaddr addr,
1563 uint64_t val, unsigned size)
1564{
1565 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1566 switch (size) {
1567 case 1:
1568 stb_phys(addr, val);
1569 break;
1570 case 2:
1571 stw_phys(addr, val);
1572 break;
1573 case 4:
1574 stl_phys(addr, val);
1575 break;
1576 default: abort();
1577 }
1578}
1579
1580static const MemoryRegionOps watch_mem_ops = {
1581 .read = watch_mem_read,
1582 .write = watch_mem_write,
1583 .endianness = DEVICE_NATIVE_ENDIAN,
1584};
1585
1586static uint64_t subpage_read(void *opaque, hwaddr addr,
1587 unsigned len)
1588{
1589 subpage_t *subpage = opaque;
1590 uint8_t buf[4];
1591
1592#if defined(DEBUG_SUBPAGE)
1593 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1594 subpage, len, addr);
1595#endif
1596 address_space_read(subpage->as, addr + subpage->base, buf, len);
1597 switch (len) {
1598 case 1:
1599 return ldub_p(buf);
1600 case 2:
1601 return lduw_p(buf);
1602 case 4:
1603 return ldl_p(buf);
1604 default:
1605 abort();
1606 }
1607}
1608
1609static void subpage_write(void *opaque, hwaddr addr,
1610 uint64_t value, unsigned len)
1611{
1612 subpage_t *subpage = opaque;
1613 uint8_t buf[4];
1614
1615#if defined(DEBUG_SUBPAGE)
1616 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1617 " value %"PRIx64"\n",
1618 __func__, subpage, len, addr, value);
1619#endif
1620 switch (len) {
1621 case 1:
1622 stb_p(buf, value);
1623 break;
1624 case 2:
1625 stw_p(buf, value);
1626 break;
1627 case 4:
1628 stl_p(buf, value);
1629 break;
1630 default:
1631 abort();
1632 }
1633 address_space_write(subpage->as, addr + subpage->base, buf, len);
1634}
1635
1636static bool subpage_accepts(void *opaque, hwaddr addr,
1637 unsigned size, bool is_write)
1638{
1639 subpage_t *subpage = opaque;
1640#if defined(DEBUG_SUBPAGE)
1641 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1642 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1643#endif
1644
1645 return address_space_access_valid(subpage->as, addr + subpage->base,
1646 size, is_write);
1647}
1648
1649static const MemoryRegionOps subpage_ops = {
1650 .read = subpage_read,
1651 .write = subpage_write,
1652 .valid.accepts = subpage_accepts,
1653 .endianness = DEVICE_NATIVE_ENDIAN,
1654};
1655
1656static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1657 uint16_t section)
1658{
1659 int idx, eidx;
1660
1661 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1662 return -1;
1663 idx = SUBPAGE_IDX(start);
1664 eidx = SUBPAGE_IDX(end);
1665#if defined(DEBUG_SUBPAGE)
1666 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1667 mmio, start, end, idx, eidx, memory);
1668#endif
1669 for (; idx <= eidx; idx++) {
1670 mmio->sub_section[idx] = section;
1671 }
1672
1673 return 0;
1674}
1675
1676static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1677{
1678 subpage_t *mmio;
1679
1680 mmio = g_malloc0(sizeof(subpage_t));
1681
1682 mmio->as = as;
1683 mmio->base = base;
1684 memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1685 "subpage", TARGET_PAGE_SIZE);
1686 mmio->iomem.subpage = true;
1687#if defined(DEBUG_SUBPAGE)
1688 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1689 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1690#endif
1691 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1692
1693 return mmio;
1694}
1695
1696static uint16_t dummy_section(MemoryRegion *mr)
1697{
1698 MemoryRegionSection section = {
1699 .mr = mr,
1700 .offset_within_address_space = 0,
1701 .offset_within_region = 0,
1702 .size = int128_2_64(),
1703 };
1704
1705 return phys_section_add(&section);
1706}
1707
1708MemoryRegion *iotlb_to_region(hwaddr index)
1709{
1710 return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1711}
1712
1713static void io_mem_init(void)
1714{
1715 memory_region_init_io(&io_mem_rom, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1716 memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1717 "unassigned", UINT64_MAX);
1718 memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1719 "notdirty", UINT64_MAX);
1720 memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1721 "watch", UINT64_MAX);
1722}
1723
1724static void mem_begin(MemoryListener *listener)
1725{
1726 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1727
1728 destroy_all_mappings(d);
1729 d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1730}
1731
1732static void core_begin(MemoryListener *listener)
1733{
1734 phys_sections_clear();
1735 phys_section_unassigned = dummy_section(&io_mem_unassigned);
1736 phys_section_notdirty = dummy_section(&io_mem_notdirty);
1737 phys_section_rom = dummy_section(&io_mem_rom);
1738 phys_section_watch = dummy_section(&io_mem_watch);
1739}
1740
1741static void tcg_commit(MemoryListener *listener)
1742{
1743 CPUArchState *env;
1744
1745 /* since each CPU stores ram addresses in its TLB cache, we must
1746 reset the modified entries */
1747 /* XXX: slow ! */
1748 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1749 tlb_flush(env, 1);
1750 }
1751}
1752
1753static void core_log_global_start(MemoryListener *listener)
1754{
1755 cpu_physical_memory_set_dirty_tracking(1);
1756}
1757
1758static void core_log_global_stop(MemoryListener *listener)
1759{
1760 cpu_physical_memory_set_dirty_tracking(0);
1761}
1762
1763static void io_region_add(MemoryListener *listener,
1764 MemoryRegionSection *section)
1765{
1766 MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1767
1768 mrio->mr = section->mr;
1769 mrio->offset = section->offset_within_region;
1770 iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1771 section->offset_within_address_space,
1772 int128_get64(section->size));
1773 ioport_register(&mrio->iorange);
1774}
1775
1776static void io_region_del(MemoryListener *listener,
1777 MemoryRegionSection *section)
1778{
1779 isa_unassign_ioport(section->offset_within_address_space,
1780 int128_get64(section->size));
1781}
1782
1783static MemoryListener core_memory_listener = {
1784 .begin = core_begin,
1785 .log_global_start = core_log_global_start,
1786 .log_global_stop = core_log_global_stop,
1787 .priority = 1,
1788};
1789
1790static MemoryListener io_memory_listener = {
1791 .region_add = io_region_add,
1792 .region_del = io_region_del,
1793 .priority = 0,
1794};
1795
1796static MemoryListener tcg_memory_listener = {
1797 .commit = tcg_commit,
1798};
1799
1800void address_space_init_dispatch(AddressSpace *as)
1801{
1802 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1803
1804 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1805 d->listener = (MemoryListener) {
1806 .begin = mem_begin,
1807 .region_add = mem_add,
1808 .region_nop = mem_add,
1809 .priority = 0,
1810 };
1811 d->as = as;
1812 as->dispatch = d;
1813 memory_listener_register(&d->listener, as);
1814}
1815
1816void address_space_destroy_dispatch(AddressSpace *as)
1817{
1818 AddressSpaceDispatch *d = as->dispatch;
1819
1820 memory_listener_unregister(&d->listener);
1821 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1822 g_free(d);
1823 as->dispatch = NULL;
1824}
1825
1826static void memory_map_init(void)
1827{
1828 system_memory = g_malloc(sizeof(*system_memory));
1829 memory_region_init(system_memory, "system", INT64_MAX);
1830 address_space_init(&address_space_memory, system_memory);
1831 address_space_memory.name = "memory";
1832
1833 system_io = g_malloc(sizeof(*system_io));
1834 memory_region_init(system_io, "io", 65536);
1835 address_space_init(&address_space_io, system_io);
1836 address_space_io.name = "I/O";
1837
1838 memory_listener_register(&core_memory_listener, &address_space_memory);
1839 memory_listener_register(&io_memory_listener, &address_space_io);
1840 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1841}
1842
1843MemoryRegion *get_system_memory(void)
1844{
1845 return system_memory;
1846}
1847
1848MemoryRegion *get_system_io(void)
1849{
1850 return system_io;
1851}
1852
1853#endif /* !defined(CONFIG_USER_ONLY) */
1854
1855/* physical memory access (slow version, mainly for debug) */
1856#if defined(CONFIG_USER_ONLY)
1857int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1858 uint8_t *buf, int len, int is_write)
1859{
1860 int l, flags;
1861 target_ulong page;
1862 void * p;
1863
1864 while (len > 0) {
1865 page = addr & TARGET_PAGE_MASK;
1866 l = (page + TARGET_PAGE_SIZE) - addr;
1867 if (l > len)
1868 l = len;
1869 flags = page_get_flags(page);
1870 if (!(flags & PAGE_VALID))
1871 return -1;
1872 if (is_write) {
1873 if (!(flags & PAGE_WRITE))
1874 return -1;
1875 /* XXX: this code should not depend on lock_user */
1876 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1877 return -1;
1878 memcpy(p, buf, l);
1879 unlock_user(p, addr, l);
1880 } else {
1881 if (!(flags & PAGE_READ))
1882 return -1;
1883 /* XXX: this code should not depend on lock_user */
1884 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1885 return -1;
1886 memcpy(buf, p, l);
1887 unlock_user(p, addr, 0);
1888 }
1889 len -= l;
1890 buf += l;
1891 addr += l;
1892 }
1893 return 0;
1894}
1895
1896#else
1897
1898static void invalidate_and_set_dirty(hwaddr addr,
1899 hwaddr length)
1900{
1901 if (!cpu_physical_memory_is_dirty(addr)) {
1902 /* invalidate code */
1903 tb_invalidate_phys_page_range(addr, addr + length, 0);
1904 /* set dirty bit */
1905 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1906 }
1907 xen_modified_memory(addr, length);
1908}
1909
1910static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1911{
1912 if (memory_region_is_ram(mr)) {
1913 return !(is_write && mr->readonly);
1914 }
1915 if (memory_region_is_romd(mr)) {
1916 return !is_write;
1917 }
1918
1919 return false;
1920}
1921
1922static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1923{
1924 if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1925 return 4;
1926 }
1927 if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1928 return 2;
1929 }
1930 return 1;
1931}
1932
1933bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1934 int len, bool is_write)
1935{
1936 hwaddr l;
1937 uint8_t *ptr;
1938 uint64_t val;
1939 hwaddr addr1;
1940 MemoryRegion *mr;
1941 bool error = false;
1942
1943 while (len > 0) {
1944 l = len;
1945 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1946
1947 if (is_write) {
1948 if (!memory_access_is_direct(mr, is_write)) {
1949 l = memory_access_size(mr, l, addr1);
1950 /* XXX: could force cpu_single_env to NULL to avoid
1951 potential bugs */
1952 if (l == 4) {
1953 /* 32 bit write access */
1954 val = ldl_p(buf);
1955 error |= io_mem_write(mr, addr1, val, 4);
1956 } else if (l == 2) {
1957 /* 16 bit write access */
1958 val = lduw_p(buf);
1959 error |= io_mem_write(mr, addr1, val, 2);
1960 } else {
1961 /* 8 bit write access */
1962 val = ldub_p(buf);
1963 error |= io_mem_write(mr, addr1, val, 1);
1964 }
1965 } else {
1966 addr1 += memory_region_get_ram_addr(mr);
1967 /* RAM case */
1968 ptr = qemu_get_ram_ptr(addr1);
1969 memcpy(ptr, buf, l);
1970 invalidate_and_set_dirty(addr1, l);
1971 }
1972 } else {
1973 if (!memory_access_is_direct(mr, is_write)) {
1974 /* I/O case */
1975 l = memory_access_size(mr, l, addr1);
1976 if (l == 4) {
1977 /* 32 bit read access */
1978 error |= io_mem_read(mr, addr1, &val, 4);
1979 stl_p(buf, val);
1980 } else if (l == 2) {
1981 /* 16 bit read access */
1982 error |= io_mem_read(mr, addr1, &val, 2);
1983 stw_p(buf, val);
1984 } else {
1985 /* 8 bit read access */
1986 error |= io_mem_read(mr, addr1, &val, 1);
1987 stb_p(buf, val);
1988 }
1989 } else {
1990 /* RAM case */
1991 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1992 memcpy(buf, ptr, l);
1993 }
1994 }
1995 len -= l;
1996 buf += l;
1997 addr += l;
1998 }
1999
2000 return error;
2001}
2002
2003bool address_space_write(AddressSpace *as, hwaddr addr,
2004 const uint8_t *buf, int len)
2005{
2006 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2007}
2008
2009bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2010{
2011 return address_space_rw(as, addr, buf, len, false);
2012}
2013
2014
2015void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2016 int len, int is_write)
2017{
2018 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2019}
2020
2021/* used for ROM loading : can write in RAM and ROM */
2022void cpu_physical_memory_write_rom(hwaddr addr,
2023 const uint8_t *buf, int len)
2024{
2025 hwaddr l;
2026 uint8_t *ptr;
2027 hwaddr addr1;
2028 MemoryRegion *mr;
2029
2030 while (len > 0) {
2031 l = len;
2032 mr = address_space_translate(&address_space_memory,
2033 addr, &addr1, &l, true);
2034
2035 if (!(memory_region_is_ram(mr) ||
2036 memory_region_is_romd(mr))) {
2037 /* do nothing */
2038 } else {
2039 addr1 += memory_region_get_ram_addr(mr);
2040 /* ROM/RAM case */
2041 ptr = qemu_get_ram_ptr(addr1);
2042 memcpy(ptr, buf, l);
2043 invalidate_and_set_dirty(addr1, l);
2044 }
2045 len -= l;
2046 buf += l;
2047 addr += l;
2048 }
2049}
2050
2051typedef struct {
2052 void *buffer;
2053 hwaddr addr;
2054 hwaddr len;
2055} BounceBuffer;
2056
2057static BounceBuffer bounce;
2058
2059typedef struct MapClient {
2060 void *opaque;
2061 void (*callback)(void *opaque);
2062 QLIST_ENTRY(MapClient) link;
2063} MapClient;
2064
2065static QLIST_HEAD(map_client_list, MapClient) map_client_list
2066 = QLIST_HEAD_INITIALIZER(map_client_list);
2067
2068void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2069{
2070 MapClient *client = g_malloc(sizeof(*client));
2071
2072 client->opaque = opaque;
2073 client->callback = callback;
2074 QLIST_INSERT_HEAD(&map_client_list, client, link);
2075 return client;
2076}
2077
2078static void cpu_unregister_map_client(void *_client)
2079{
2080 MapClient *client = (MapClient *)_client;
2081
2082 QLIST_REMOVE(client, link);
2083 g_free(client);
2084}
2085
2086static void cpu_notify_map_clients(void)
2087{
2088 MapClient *client;
2089
2090 while (!QLIST_EMPTY(&map_client_list)) {
2091 client = QLIST_FIRST(&map_client_list);
2092 client->callback(client->opaque);
2093 cpu_unregister_map_client(client);
2094 }
2095}
2096
2097bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2098{
2099 MemoryRegion *mr;
2100 hwaddr l, xlat;
2101
2102 while (len > 0) {
2103 l = len;
2104 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2105 if (!memory_access_is_direct(mr, is_write)) {
2106 l = memory_access_size(mr, l, addr);
2107 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2108 return false;
2109 }
2110 }
2111
2112 len -= l;
2113 addr += l;
2114 }
2115 return true;
2116}
2117
2118/* Map a physical memory region into a host virtual address.
2119 * May map a subset of the requested range, given by and returned in *plen.
2120 * May return NULL if resources needed to perform the mapping are exhausted.
2121 * Use only for reads OR writes - not for read-modify-write operations.
2122 * Use cpu_register_map_client() to know when retrying the map operation is
2123 * likely to succeed.
2124 */
2125void *address_space_map(AddressSpace *as,
2126 hwaddr addr,
2127 hwaddr *plen,
2128 bool is_write)
2129{
2130 hwaddr len = *plen;
2131 hwaddr todo = 0;
2132 hwaddr l, xlat;
2133 MemoryRegion *mr;
2134 ram_addr_t raddr = RAM_ADDR_MAX;
2135 ram_addr_t rlen;
2136 void *ret;
2137
2138 while (len > 0) {
2139 l = len;
2140 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2141
2142 if (!memory_access_is_direct(mr, is_write)) {
2143 if (todo || bounce.buffer) {
2144 break;
2145 }
2146 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2147 bounce.addr = addr;
2148 bounce.len = l;
2149 if (!is_write) {
2150 address_space_read(as, addr, bounce.buffer, l);
2151 }
2152
2153 *plen = l;
2154 return bounce.buffer;
2155 }
2156 if (!todo) {
2157 raddr = memory_region_get_ram_addr(mr) + xlat;
2158 } else {
2159 if (memory_region_get_ram_addr(mr) + xlat != raddr + todo) {
2160 break;
2161 }
2162 }
2163
2164 len -= l;
2165 addr += l;
2166 todo += l;
2167 }
2168 rlen = todo;
2169 ret = qemu_ram_ptr_length(raddr, &rlen);
2170 *plen = rlen;
2171 return ret;
2172}
2173
2174/* Unmaps a memory region previously mapped by address_space_map().
2175 * Will also mark the memory as dirty if is_write == 1. access_len gives
2176 * the amount of memory that was actually read or written by the caller.
2177 */
2178void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2179 int is_write, hwaddr access_len)
2180{
2181 if (buffer != bounce.buffer) {
2182 if (is_write) {
2183 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2184 while (access_len) {
2185 unsigned l;
2186 l = TARGET_PAGE_SIZE;
2187 if (l > access_len)
2188 l = access_len;
2189 invalidate_and_set_dirty(addr1, l);
2190 addr1 += l;
2191 access_len -= l;
2192 }
2193 }
2194 if (xen_enabled()) {
2195 xen_invalidate_map_cache_entry(buffer);
2196 }
2197 return;
2198 }
2199 if (is_write) {
2200 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2201 }
2202 qemu_vfree(bounce.buffer);
2203 bounce.buffer = NULL;
2204 cpu_notify_map_clients();
2205}
2206
2207void *cpu_physical_memory_map(hwaddr addr,
2208 hwaddr *plen,
2209 int is_write)
2210{
2211 return address_space_map(&address_space_memory, addr, plen, is_write);
2212}
2213
2214void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2215 int is_write, hwaddr access_len)
2216{
2217 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2218}
2219
2220/* warning: addr must be aligned */
2221static inline uint32_t ldl_phys_internal(hwaddr addr,
2222 enum device_endian endian)
2223{
2224 uint8_t *ptr;
2225 uint64_t val;
2226 MemoryRegion *mr;
2227 hwaddr l = 4;
2228 hwaddr addr1;
2229
2230 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2231 false);
2232 if (l < 4 || !memory_access_is_direct(mr, false)) {
2233 /* I/O case */
2234 io_mem_read(mr, addr1, &val, 4);
2235#if defined(TARGET_WORDS_BIGENDIAN)
2236 if (endian == DEVICE_LITTLE_ENDIAN) {
2237 val = bswap32(val);
2238 }
2239#else
2240 if (endian == DEVICE_BIG_ENDIAN) {
2241 val = bswap32(val);
2242 }
2243#endif
2244 } else {
2245 /* RAM case */
2246 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2247 & TARGET_PAGE_MASK)
2248 + addr1);
2249 switch (endian) {
2250 case DEVICE_LITTLE_ENDIAN:
2251 val = ldl_le_p(ptr);
2252 break;
2253 case DEVICE_BIG_ENDIAN:
2254 val = ldl_be_p(ptr);
2255 break;
2256 default:
2257 val = ldl_p(ptr);
2258 break;
2259 }
2260 }
2261 return val;
2262}
2263
2264uint32_t ldl_phys(hwaddr addr)
2265{
2266 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2267}
2268
2269uint32_t ldl_le_phys(hwaddr addr)
2270{
2271 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2272}
2273
2274uint32_t ldl_be_phys(hwaddr addr)
2275{
2276 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2277}
2278
2279/* warning: addr must be aligned */
2280static inline uint64_t ldq_phys_internal(hwaddr addr,
2281 enum device_endian endian)
2282{
2283 uint8_t *ptr;
2284 uint64_t val;
2285 MemoryRegion *mr;
2286 hwaddr l = 8;
2287 hwaddr addr1;
2288
2289 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2290 false);
2291 if (l < 8 || !memory_access_is_direct(mr, false)) {
2292 /* I/O case */
2293 io_mem_read(mr, addr1, &val, 8);
2294#if defined(TARGET_WORDS_BIGENDIAN)
2295 if (endian == DEVICE_LITTLE_ENDIAN) {
2296 val = bswap64(val);
2297 }
2298#else
2299 if (endian == DEVICE_BIG_ENDIAN) {
2300 val = bswap64(val);
2301 }
2302#endif
2303 } else {
2304 /* RAM case */
2305 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2306 & TARGET_PAGE_MASK)
2307 + addr1);
2308 switch (endian) {
2309 case DEVICE_LITTLE_ENDIAN:
2310 val = ldq_le_p(ptr);
2311 break;
2312 case DEVICE_BIG_ENDIAN:
2313 val = ldq_be_p(ptr);
2314 break;
2315 default:
2316 val = ldq_p(ptr);
2317 break;
2318 }
2319 }
2320 return val;
2321}
2322
2323uint64_t ldq_phys(hwaddr addr)
2324{
2325 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2326}
2327
2328uint64_t ldq_le_phys(hwaddr addr)
2329{
2330 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2331}
2332
2333uint64_t ldq_be_phys(hwaddr addr)
2334{
2335 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2336}
2337
2338/* XXX: optimize */
2339uint32_t ldub_phys(hwaddr addr)
2340{
2341 uint8_t val;
2342 cpu_physical_memory_read(addr, &val, 1);
2343 return val;
2344}
2345
2346/* warning: addr must be aligned */
2347static inline uint32_t lduw_phys_internal(hwaddr addr,
2348 enum device_endian endian)
2349{
2350 uint8_t *ptr;
2351 uint64_t val;
2352 MemoryRegion *mr;
2353 hwaddr l = 2;
2354 hwaddr addr1;
2355
2356 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2357 false);
2358 if (l < 2 || !memory_access_is_direct(mr, false)) {
2359 /* I/O case */
2360 io_mem_read(mr, addr1, &val, 2);
2361#if defined(TARGET_WORDS_BIGENDIAN)
2362 if (endian == DEVICE_LITTLE_ENDIAN) {
2363 val = bswap16(val);
2364 }
2365#else
2366 if (endian == DEVICE_BIG_ENDIAN) {
2367 val = bswap16(val);
2368 }
2369#endif
2370 } else {
2371 /* RAM case */
2372 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2373 & TARGET_PAGE_MASK)
2374 + addr1);
2375 switch (endian) {
2376 case DEVICE_LITTLE_ENDIAN:
2377 val = lduw_le_p(ptr);
2378 break;
2379 case DEVICE_BIG_ENDIAN:
2380 val = lduw_be_p(ptr);
2381 break;
2382 default:
2383 val = lduw_p(ptr);
2384 break;
2385 }
2386 }
2387 return val;
2388}
2389
2390uint32_t lduw_phys(hwaddr addr)
2391{
2392 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2393}
2394
2395uint32_t lduw_le_phys(hwaddr addr)
2396{
2397 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2398}
2399
2400uint32_t lduw_be_phys(hwaddr addr)
2401{
2402 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2403}
2404
2405/* warning: addr must be aligned. The ram page is not masked as dirty
2406 and the code inside is not invalidated. It is useful if the dirty
2407 bits are used to track modified PTEs */
2408void stl_phys_notdirty(hwaddr addr, uint32_t val)
2409{
2410 uint8_t *ptr;
2411 MemoryRegion *mr;
2412 hwaddr l = 4;
2413 hwaddr addr1;
2414
2415 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2416 true);
2417 if (l < 4 || !memory_access_is_direct(mr, true)) {
2418 io_mem_write(mr, addr1, val, 4);
2419 } else {
2420 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2421 ptr = qemu_get_ram_ptr(addr1);
2422 stl_p(ptr, val);
2423
2424 if (unlikely(in_migration)) {
2425 if (!cpu_physical_memory_is_dirty(addr1)) {
2426 /* invalidate code */
2427 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2428 /* set dirty bit */
2429 cpu_physical_memory_set_dirty_flags(
2430 addr1, (0xff & ~CODE_DIRTY_FLAG));
2431 }
2432 }
2433 }
2434}
2435
2436/* warning: addr must be aligned */
2437static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2438 enum device_endian endian)
2439{
2440 uint8_t *ptr;
2441 MemoryRegion *mr;
2442 hwaddr l = 4;
2443 hwaddr addr1;
2444
2445 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2446 true);
2447 if (l < 4 || !memory_access_is_direct(mr, true)) {
2448#if defined(TARGET_WORDS_BIGENDIAN)
2449 if (endian == DEVICE_LITTLE_ENDIAN) {
2450 val = bswap32(val);
2451 }
2452#else
2453 if (endian == DEVICE_BIG_ENDIAN) {
2454 val = bswap32(val);
2455 }
2456#endif
2457 io_mem_write(mr, addr1, val, 4);
2458 } else {
2459 /* RAM case */
2460 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2461 ptr = qemu_get_ram_ptr(addr1);
2462 switch (endian) {
2463 case DEVICE_LITTLE_ENDIAN:
2464 stl_le_p(ptr, val);
2465 break;
2466 case DEVICE_BIG_ENDIAN:
2467 stl_be_p(ptr, val);
2468 break;
2469 default:
2470 stl_p(ptr, val);
2471 break;
2472 }
2473 invalidate_and_set_dirty(addr1, 4);
2474 }
2475}
2476
2477void stl_phys(hwaddr addr, uint32_t val)
2478{
2479 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2480}
2481
2482void stl_le_phys(hwaddr addr, uint32_t val)
2483{
2484 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2485}
2486
2487void stl_be_phys(hwaddr addr, uint32_t val)
2488{
2489 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2490}
2491
2492/* XXX: optimize */
2493void stb_phys(hwaddr addr, uint32_t val)
2494{
2495 uint8_t v = val;
2496 cpu_physical_memory_write(addr, &v, 1);
2497}
2498
2499/* warning: addr must be aligned */
2500static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2501 enum device_endian endian)
2502{
2503 uint8_t *ptr;
2504 MemoryRegion *mr;
2505 hwaddr l = 2;
2506 hwaddr addr1;
2507
2508 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2509 true);
2510 if (l < 2 || !memory_access_is_direct(mr, true)) {
2511#if defined(TARGET_WORDS_BIGENDIAN)
2512 if (endian == DEVICE_LITTLE_ENDIAN) {
2513 val = bswap16(val);
2514 }
2515#else
2516 if (endian == DEVICE_BIG_ENDIAN) {
2517 val = bswap16(val);
2518 }
2519#endif
2520 io_mem_write(mr, addr1, val, 2);
2521 } else {
2522 /* RAM case */
2523 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2524 ptr = qemu_get_ram_ptr(addr1);
2525 switch (endian) {
2526 case DEVICE_LITTLE_ENDIAN:
2527 stw_le_p(ptr, val);
2528 break;
2529 case DEVICE_BIG_ENDIAN:
2530 stw_be_p(ptr, val);
2531 break;
2532 default:
2533 stw_p(ptr, val);
2534 break;
2535 }
2536 invalidate_and_set_dirty(addr1, 2);
2537 }
2538}
2539
2540void stw_phys(hwaddr addr, uint32_t val)
2541{
2542 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2543}
2544
2545void stw_le_phys(hwaddr addr, uint32_t val)
2546{
2547 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2548}
2549
2550void stw_be_phys(hwaddr addr, uint32_t val)
2551{
2552 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2553}
2554
2555/* XXX: optimize */
2556void stq_phys(hwaddr addr, uint64_t val)
2557{
2558 val = tswap64(val);
2559 cpu_physical_memory_write(addr, &val, 8);
2560}
2561
2562void stq_le_phys(hwaddr addr, uint64_t val)
2563{
2564 val = cpu_to_le64(val);
2565 cpu_physical_memory_write(addr, &val, 8);
2566}
2567
2568void stq_be_phys(hwaddr addr, uint64_t val)
2569{
2570 val = cpu_to_be64(val);
2571 cpu_physical_memory_write(addr, &val, 8);
2572}
2573
2574/* virtual memory access for debug (includes writing to ROM) */
2575int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2576 uint8_t *buf, int len, int is_write)
2577{
2578 int l;
2579 hwaddr phys_addr;
2580 target_ulong page;
2581
2582 while (len > 0) {
2583 page = addr & TARGET_PAGE_MASK;
2584 phys_addr = cpu_get_phys_page_debug(env, page);
2585 /* if no physical page mapped, return an error */
2586 if (phys_addr == -1)
2587 return -1;
2588 l = (page + TARGET_PAGE_SIZE) - addr;
2589 if (l > len)
2590 l = len;
2591 phys_addr += (addr & ~TARGET_PAGE_MASK);
2592 if (is_write)
2593 cpu_physical_memory_write_rom(phys_addr, buf, l);
2594 else
2595 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2596 len -= l;
2597 buf += l;
2598 addr += l;
2599 }
2600 return 0;
2601}
2602#endif
2603
2604#if !defined(CONFIG_USER_ONLY)
2605
2606/*
2607 * A helper function for the _utterly broken_ virtio device model to find out if
2608 * it's running on a big endian machine. Don't do this at home kids!
2609 */
2610bool virtio_is_big_endian(void);
2611bool virtio_is_big_endian(void)
2612{
2613#if defined(TARGET_WORDS_BIGENDIAN)
2614 return true;
2615#else
2616 return false;
2617#endif
2618}
2619
2620#endif
2621
2622#ifndef CONFIG_USER_ONLY
2623bool cpu_physical_memory_is_io(hwaddr phys_addr)
2624{
2625 MemoryRegion*mr;
2626 hwaddr l = 1;
2627
2628 mr = address_space_translate(&address_space_memory,
2629 phys_addr, &phys_addr, &l, false);
2630
2631 return !(memory_region_is_ram(mr) ||
2632 memory_region_is_romd(mr));
2633}
2634#endif
Mirror of https://git.qemu.org/git/qemu.git