]> git.ipfire.org Git - thirdparty/qemu.git/blob - exec.c
projects / thirdparty / qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
char: use common function to disable callbacks on chardev close
[thirdparty/qemu.git] / exec.c
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu = first_cpu;
355
356 while (cpu) {
357 if (cpu->cpu_index == index) {
358 break;
359 }
360 cpu = cpu->next_cpu;
361 }
362
363 return cpu;
364 }
365
366 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
367 {
368 CPUState *cpu;
369
370 cpu = first_cpu;
371 while (cpu) {
372 func(cpu, data);
373 cpu = cpu->next_cpu;
374 }
375 }
376
377 void cpu_exec_init(CPUArchState *env)
378 {
379 CPUState *cpu = ENV_GET_CPU(env);
380 CPUClass *cc = CPU_GET_CLASS(cpu);
381 CPUState **pcpu;
382 int cpu_index;
383
384 #if defined(CONFIG_USER_ONLY)
385 cpu_list_lock();
386 #endif
387 cpu->next_cpu = NULL;
388 pcpu = &first_cpu;
389 cpu_index = 0;
390 while (*pcpu != NULL) {
391 pcpu = &(*pcpu)->next_cpu;
392 cpu_index++;
393 }
394 cpu->cpu_index = cpu_index;
395 cpu->numa_node = 0;
396 QTAILQ_INIT(&env->breakpoints);
397 QTAILQ_INIT(&env->watchpoints);
398 #ifndef CONFIG_USER_ONLY
399 cpu->thread_id = qemu_get_thread_id();
400 #endif
401 *pcpu = cpu;
402 #if defined(CONFIG_USER_ONLY)
403 cpu_list_unlock();
404 #endif
405 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
406 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
407 }
408 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
409 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
410 cpu_save, cpu_load, env);
411 assert(cc->vmsd == NULL);
412 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
413 #endif
414 if (cc->vmsd != NULL) {
415 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
416 }
417 }
418
419 #if defined(TARGET_HAS_ICE)
420 #if defined(CONFIG_USER_ONLY)
421 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
422 {
423 tb_invalidate_phys_page_range(pc, pc + 1, 0);
424 }
425 #else
426 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
427 {
428 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
429 (pc & ~TARGET_PAGE_MASK));
430 }
431 #endif
432 #endif /* TARGET_HAS_ICE */
433
434 #if defined(CONFIG_USER_ONLY)
435 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
436
437 {
438 }
439
440 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
441 int flags, CPUWatchpoint **watchpoint)
442 {
443 return -ENOSYS;
444 }
445 #else
446 /* Add a watchpoint. */
447 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
448 int flags, CPUWatchpoint **watchpoint)
449 {
450 target_ulong len_mask = ~(len - 1);
451 CPUWatchpoint *wp;
452
453 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
454 if ((len & (len - 1)) || (addr & ~len_mask) ||
455 len == 0 || len > TARGET_PAGE_SIZE) {
456 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
457 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
458 return -EINVAL;
459 }
460 wp = g_malloc(sizeof(*wp));
461
462 wp->vaddr = addr;
463 wp->len_mask = len_mask;
464 wp->flags = flags;
465
466 /* keep all GDB-injected watchpoints in front */
467 if (flags & BP_GDB)
468 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
469 else
470 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
471
472 tlb_flush_page(env, addr);
473
474 if (watchpoint)
475 *watchpoint = wp;
476 return 0;
477 }
478
479 /* Remove a specific watchpoint. */
480 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
481 int flags)
482 {
483 target_ulong len_mask = ~(len - 1);
484 CPUWatchpoint *wp;
485
486 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
487 if (addr == wp->vaddr && len_mask == wp->len_mask
488 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
489 cpu_watchpoint_remove_by_ref(env, wp);
490 return 0;
491 }
492 }
493 return -ENOENT;
494 }
495
496 /* Remove a specific watchpoint by reference. */
497 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
498 {
499 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
500
501 tlb_flush_page(env, watchpoint->vaddr);
502
503 g_free(watchpoint);
504 }
505
506 /* Remove all matching watchpoints. */
507 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
508 {
509 CPUWatchpoint *wp, *next;
510
511 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
512 if (wp->flags & mask)
513 cpu_watchpoint_remove_by_ref(env, wp);
514 }
515 }
516 #endif
517
518 /* Add a breakpoint. */
519 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
520 CPUBreakpoint **breakpoint)
521 {
522 #if defined(TARGET_HAS_ICE)
523 CPUBreakpoint *bp;
524
525 bp = g_malloc(sizeof(*bp));
526
527 bp->pc = pc;
528 bp->flags = flags;
529
530 /* keep all GDB-injected breakpoints in front */
531 if (flags & BP_GDB) {
532 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
533 } else {
534 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
535 }
536
537 breakpoint_invalidate(ENV_GET_CPU(env), pc);
538
539 if (breakpoint) {
540 *breakpoint = bp;
541 }
542 return 0;
543 #else
544 return -ENOSYS;
545 #endif
546 }
547
548 /* Remove a specific breakpoint. */
549 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
550 {
551 #if defined(TARGET_HAS_ICE)
552 CPUBreakpoint *bp;
553
554 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
555 if (bp->pc == pc && bp->flags == flags) {
556 cpu_breakpoint_remove_by_ref(env, bp);
557 return 0;
558 }
559 }
560 return -ENOENT;
561 #else
562 return -ENOSYS;
563 #endif
564 }
565
566 /* Remove a specific breakpoint by reference. */
567 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
568 {
569 #if defined(TARGET_HAS_ICE)
570 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
571
572 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
573
574 g_free(breakpoint);
575 #endif
576 }
577
578 /* Remove all matching breakpoints. */
579 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
580 {
581 #if defined(TARGET_HAS_ICE)
582 CPUBreakpoint *bp, *next;
583
584 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
585 if (bp->flags & mask)
586 cpu_breakpoint_remove_by_ref(env, bp);
587 }
588 #endif
589 }
590
591 /* enable or disable single step mode. EXCP_DEBUG is returned by the
592 CPU loop after each instruction */
593 void cpu_single_step(CPUState *cpu, int enabled)
594 {
595 #if defined(TARGET_HAS_ICE)
596 if (cpu->singlestep_enabled != enabled) {
597 cpu->singlestep_enabled = enabled;
598 if (kvm_enabled()) {
599 kvm_update_guest_debug(cpu, 0);
600 } else {
601 /* must flush all the translated code to avoid inconsistencies */
602 /* XXX: only flush what is necessary */
603 CPUArchState *env = cpu->env_ptr;
604 tb_flush(env);
605 }
606 }
607 #endif
608 }
609
610 void cpu_abort(CPUArchState *env, const char *fmt, ...)
611 {
612 CPUState *cpu = ENV_GET_CPU(env);
613 va_list ap;
614 va_list ap2;
615
616 va_start(ap, fmt);
617 va_copy(ap2, ap);
618 fprintf(stderr, "qemu: fatal: ");
619 vfprintf(stderr, fmt, ap);
620 fprintf(stderr, "\n");
621 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
622 if (qemu_log_enabled()) {
623 qemu_log("qemu: fatal: ");
624 qemu_log_vprintf(fmt, ap2);
625 qemu_log("\n");
626 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
627 qemu_log_flush();
628 qemu_log_close();
629 }
630 va_end(ap2);
631 va_end(ap);
632 #if defined(CONFIG_USER_ONLY)
633 {
634 struct sigaction act;
635 sigfillset(&act.sa_mask);
636 act.sa_handler = SIG_DFL;
637 sigaction(SIGABRT, &act, NULL);
638 }
639 #endif
640 abort();
641 }
642
643 CPUArchState *cpu_copy(CPUArchState *env)
644 {
645 CPUArchState *new_env = cpu_init(env->cpu_model_str);
646 #if defined(TARGET_HAS_ICE)
647 CPUBreakpoint *bp;
648 CPUWatchpoint *wp;
649 #endif
650
651 /* Reset non arch specific state */
652 cpu_reset(ENV_GET_CPU(new_env));
653
654 /* Copy arch specific state into the new CPU */
655 memcpy(new_env, env, sizeof(CPUArchState));
656
657 /* Clone all break/watchpoints.
658 Note: Once we support ptrace with hw-debug register access, make sure
659 BP_CPU break/watchpoints are handled correctly on clone. */
660 QTAILQ_INIT(&env->breakpoints);
661 QTAILQ_INIT(&env->watchpoints);
662 #if defined(TARGET_HAS_ICE)
663 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
664 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
665 }
666 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
667 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
668 wp->flags, NULL);
669 }
670 #endif
671
672 return new_env;
673 }
674
675 #if !defined(CONFIG_USER_ONLY)
676 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
677 uintptr_t length)
678 {
679 uintptr_t start1;
680
681 /* we modify the TLB cache so that the dirty bit will be set again
682 when accessing the range */
683 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
684 /* Check that we don't span multiple blocks - this breaks the
685 address comparisons below. */
686 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
687 != (end - 1) - start) {
688 abort();
689 }
690 cpu_tlb_reset_dirty_all(start1, length);
691
692 }
693
694 /* Note: start and end must be within the same ram block. */
695 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
696 int dirty_flags)
697 {
698 uintptr_t length;
699
700 start &= TARGET_PAGE_MASK;
701 end = TARGET_PAGE_ALIGN(end);
702
703 length = end - start;
704 if (length == 0)
705 return;
706 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
707
708 if (tcg_enabled()) {
709 tlb_reset_dirty_range_all(start, end, length);
710 }
711 }
712
713 static int cpu_physical_memory_set_dirty_tracking(int enable)
714 {
715 int ret = 0;
716 in_migration = enable;
717 return ret;
718 }
719
720 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
721 MemoryRegionSection *section,
722 target_ulong vaddr,
723 hwaddr paddr, hwaddr xlat,
724 int prot,
725 target_ulong *address)
726 {
727 hwaddr iotlb;
728 CPUWatchpoint *wp;
729
730 if (memory_region_is_ram(section->mr)) {
731 /* Normal RAM. */
732 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
733 + xlat;
734 if (!section->readonly) {
735 iotlb |= PHYS_SECTION_NOTDIRTY;
736 } else {
737 iotlb |= PHYS_SECTION_ROM;
738 }
739 } else {
740 iotlb = section - address_space_memory.dispatch->sections;
741 iotlb += xlat;
742 }
743
744 /* Make accesses to pages with watchpoints go via the
745 watchpoint trap routines. */
746 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
747 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
748 /* Avoid trapping reads of pages with a write breakpoint. */
749 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
750 iotlb = PHYS_SECTION_WATCH + paddr;
751 *address |= TLB_MMIO;
752 break;
753 }
754 }
755 }
756
757 return iotlb;
758 }
759 #endif /* defined(CONFIG_USER_ONLY) */
760
761 #if !defined(CONFIG_USER_ONLY)
762
763 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
764 uint16_t section);
765 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
766
767 static uint16_t phys_section_add(MemoryRegionSection *section)
768 {
769 /* The physical section number is ORed with a page-aligned
770 * pointer to produce the iotlb entries. Thus it should
771 * never overflow into the page-aligned value.
772 */
773 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
774
775 if (next_map.sections_nb == next_map.sections_nb_alloc) {
776 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
777 16);
778 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
779 next_map.sections_nb_alloc);
780 }
781 next_map.sections[next_map.sections_nb] = *section;
782 memory_region_ref(section->mr);
783 return next_map.sections_nb++;
784 }
785
786 static void phys_section_destroy(MemoryRegion *mr)
787 {
788 memory_region_unref(mr);
789
790 if (mr->subpage) {
791 subpage_t *subpage = container_of(mr, subpage_t, iomem);
792 memory_region_destroy(&subpage->iomem);
793 g_free(subpage);
794 }
795 }
796
797 static void phys_sections_free(PhysPageMap *map)
798 {
799 while (map->sections_nb > 0) {
800 MemoryRegionSection *section = &map->sections[--map->sections_nb];
801 phys_section_destroy(section->mr);
802 }
803 g_free(map->sections);
804 g_free(map->nodes);
805 g_free(map);
806 }
807
808 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
809 {
810 subpage_t *subpage;
811 hwaddr base = section->offset_within_address_space
812 & TARGET_PAGE_MASK;
813 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
814 next_map.nodes, next_map.sections);
815 MemoryRegionSection subsection = {
816 .offset_within_address_space = base,
817 .size = int128_make64(TARGET_PAGE_SIZE),
818 };
819 hwaddr start, end;
820
821 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
822
823 if (!(existing->mr->subpage)) {
824 subpage = subpage_init(d->as, base);
825 subsection.mr = &subpage->iomem;
826 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
827 phys_section_add(&subsection));
828 } else {
829 subpage = container_of(existing->mr, subpage_t, iomem);
830 }
831 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
832 end = start + int128_get64(section->size) - 1;
833 subpage_register(subpage, start, end, phys_section_add(section));
834 }
835
836
837 static void register_multipage(AddressSpaceDispatch *d,
838 MemoryRegionSection *section)
839 {
840 hwaddr start_addr = section->offset_within_address_space;
841 uint16_t section_index = phys_section_add(section);
842 uint64_t num_pages = int128_get64(int128_rshift(section->size,
843 TARGET_PAGE_BITS));
844
845 assert(num_pages);
846 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
847 }
848
849 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
850 {
851 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
852 AddressSpaceDispatch *d = as->next_dispatch;
853 MemoryRegionSection now = *section, remain = *section;
854 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
855
856 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
857 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
858 - now.offset_within_address_space;
859
860 now.size = int128_min(int128_make64(left), now.size);
861 register_subpage(d, &now);
862 } else {
863 now.size = int128_zero();
864 }
865 while (int128_ne(remain.size, now.size)) {
866 remain.size = int128_sub(remain.size, now.size);
867 remain.offset_within_address_space += int128_get64(now.size);
868 remain.offset_within_region += int128_get64(now.size);
869 now = remain;
870 if (int128_lt(remain.size, page_size)) {
871 register_subpage(d, &now);
872 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
873 now.size = page_size;
874 register_subpage(d, &now);
875 } else {
876 now.size = int128_and(now.size, int128_neg(page_size));
877 register_multipage(d, &now);
878 }
879 }
880 }
881
882 void qemu_flush_coalesced_mmio_buffer(void)
883 {
884 if (kvm_enabled())
885 kvm_flush_coalesced_mmio_buffer();
886 }
887
888 void qemu_mutex_lock_ramlist(void)
889 {
890 qemu_mutex_lock(&ram_list.mutex);
891 }
892
893 void qemu_mutex_unlock_ramlist(void)
894 {
895 qemu_mutex_unlock(&ram_list.mutex);
896 }
897
898 #if defined(__linux__) && !defined(TARGET_S390X)
899
900 #include <sys/vfs.h>
901
902 #define HUGETLBFS_MAGIC 0x958458f6
903
904 static long gethugepagesize(const char *path)
905 {
906 struct statfs fs;
907 int ret;
908
909 do {
910 ret = statfs(path, &fs);
911 } while (ret != 0 && errno == EINTR);
912
913 if (ret != 0) {
914 perror(path);
915 return 0;
916 }
917
918 if (fs.f_type != HUGETLBFS_MAGIC)
919 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
920
921 return fs.f_bsize;
922 }
923
924 static void *file_ram_alloc(RAMBlock *block,
925 ram_addr_t memory,
926 const char *path)
927 {
928 char *filename;
929 char *sanitized_name;
930 char *c;
931 void *area;
932 int fd;
933 #ifdef MAP_POPULATE
934 int flags;
935 #endif
936 unsigned long hpagesize;
937
938 hpagesize = gethugepagesize(path);
939 if (!hpagesize) {
940 return NULL;
941 }
942
943 if (memory < hpagesize) {
944 return NULL;
945 }
946
947 if (kvm_enabled() && !kvm_has_sync_mmu()) {
948 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
949 return NULL;
950 }
951
952 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
953 sanitized_name = g_strdup(block->mr->name);
954 for (c = sanitized_name; *c != '\0'; c++) {
955 if (*c == '/')
956 *c = '_';
957 }
958
959 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
960 sanitized_name);
961 g_free(sanitized_name);
962
963 fd = mkstemp(filename);
964 if (fd < 0) {
965 perror("unable to create backing store for hugepages");
966 g_free(filename);
967 return NULL;
968 }
969 unlink(filename);
970 g_free(filename);
971
972 memory = (memory+hpagesize-1) & ~(hpagesize-1);
973
974 /*
975 * ftruncate is not supported by hugetlbfs in older
976 * hosts, so don't bother bailing out on errors.
977 * If anything goes wrong with it under other filesystems,
978 * mmap will fail.
979 */
980 if (ftruncate(fd, memory))
981 perror("ftruncate");
982
983 #ifdef MAP_POPULATE
984 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
985 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
986 * to sidestep this quirk.
987 */
988 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
989 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
990 #else
991 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
992 #endif
993 if (area == MAP_FAILED) {
994 perror("file_ram_alloc: can't mmap RAM pages");
995 close(fd);
996 return (NULL);
997 }
998 block->fd = fd;
999 return area;
1000 }
1001 #endif
1002
1003 static ram_addr_t find_ram_offset(ram_addr_t size)
1004 {
1005 RAMBlock *block, *next_block;
1006 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1007
1008 assert(size != 0); /* it would hand out same offset multiple times */
1009
1010 if (QTAILQ_EMPTY(&ram_list.blocks))
1011 return 0;
1012
1013 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1014 ram_addr_t end, next = RAM_ADDR_MAX;
1015
1016 end = block->offset + block->length;
1017
1018 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1019 if (next_block->offset >= end) {
1020 next = MIN(next, next_block->offset);
1021 }
1022 }
1023 if (next - end >= size && next - end < mingap) {
1024 offset = end;
1025 mingap = next - end;
1026 }
1027 }
1028
1029 if (offset == RAM_ADDR_MAX) {
1030 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1031 (uint64_t)size);
1032 abort();
1033 }
1034
1035 return offset;
1036 }
1037
1038 ram_addr_t last_ram_offset(void)
1039 {
1040 RAMBlock *block;
1041 ram_addr_t last = 0;
1042
1043 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1044 last = MAX(last, block->offset + block->length);
1045
1046 return last;
1047 }
1048
1049 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1050 {
1051 int ret;
1052
1053 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1054 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1055 "dump-guest-core", true)) {
1056 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1057 if (ret) {
1058 perror("qemu_madvise");
1059 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1060 "but dump_guest_core=off specified\n");
1061 }
1062 }
1063 }
1064
1065 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1066 {
1067 RAMBlock *new_block, *block;
1068
1069 new_block = NULL;
1070 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1071 if (block->offset == addr) {
1072 new_block = block;
1073 break;
1074 }
1075 }
1076 assert(new_block);
1077 assert(!new_block->idstr[0]);
1078
1079 if (dev) {
1080 char *id = qdev_get_dev_path(dev);
1081 if (id) {
1082 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1083 g_free(id);
1084 }
1085 }
1086 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1087
1088 /* This assumes the iothread lock is taken here too. */
1089 qemu_mutex_lock_ramlist();
1090 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1091 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1092 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1093 new_block->idstr);
1094 abort();
1095 }
1096 }
1097 qemu_mutex_unlock_ramlist();
1098 }
1099
1100 static int memory_try_enable_merging(void *addr, size_t len)
1101 {
1102 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1103 /* disabled by the user */
1104 return 0;
1105 }
1106
1107 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1108 }
1109
1110 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1111 MemoryRegion *mr)
1112 {
1113 RAMBlock *block, *new_block;
1114
1115 size = TARGET_PAGE_ALIGN(size);
1116 new_block = g_malloc0(sizeof(*new_block));
1117
1118 /* This assumes the iothread lock is taken here too. */
1119 qemu_mutex_lock_ramlist();
1120 new_block->mr = mr;
1121 new_block->offset = find_ram_offset(size);
1122 if (host) {
1123 new_block->host = host;
1124 new_block->flags |= RAM_PREALLOC_MASK;
1125 } else {
1126 if (mem_path) {
1127 #if defined (__linux__) && !defined(TARGET_S390X)
1128 new_block->host = file_ram_alloc(new_block, size, mem_path);
1129 if (!new_block->host) {
1130 new_block->host = qemu_anon_ram_alloc(size);
1131 memory_try_enable_merging(new_block->host, size);
1132 }
1133 #else
1134 fprintf(stderr, "-mem-path option unsupported\n");
1135 exit(1);
1136 #endif
1137 } else {
1138 if (xen_enabled()) {
1139 xen_ram_alloc(new_block->offset, size, mr);
1140 } else if (kvm_enabled()) {
1141 /* some s390/kvm configurations have special constraints */
1142 new_block->host = kvm_ram_alloc(size);
1143 } else {
1144 new_block->host = qemu_anon_ram_alloc(size);
1145 }
1146 memory_try_enable_merging(new_block->host, size);
1147 }
1148 }
1149 new_block->length = size;
1150
1151 /* Keep the list sorted from biggest to smallest block. */
1152 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1153 if (block->length < new_block->length) {
1154 break;
1155 }
1156 }
1157 if (block) {
1158 QTAILQ_INSERT_BEFORE(block, new_block, next);
1159 } else {
1160 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1161 }
1162 ram_list.mru_block = NULL;
1163
1164 ram_list.version++;
1165 qemu_mutex_unlock_ramlist();
1166
1167 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1168 last_ram_offset() >> TARGET_PAGE_BITS);
1169 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1170 0, size >> TARGET_PAGE_BITS);
1171 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1172
1173 qemu_ram_setup_dump(new_block->host, size);
1174 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1175 qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1176
1177 if (kvm_enabled())
1178 kvm_setup_guest_memory(new_block->host, size);
1179
1180 return new_block->offset;
1181 }
1182
1183 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1184 {
1185 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1186 }
1187
1188 void qemu_ram_free_from_ptr(ram_addr_t addr)
1189 {
1190 RAMBlock *block;
1191
1192 /* This assumes the iothread lock is taken here too. */
1193 qemu_mutex_lock_ramlist();
1194 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1195 if (addr == block->offset) {
1196 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1197 ram_list.mru_block = NULL;
1198 ram_list.version++;
1199 g_free(block);
1200 break;
1201 }
1202 }
1203 qemu_mutex_unlock_ramlist();
1204 }
1205
1206 void qemu_ram_free(ram_addr_t addr)
1207 {
1208 RAMBlock *block;
1209
1210 /* This assumes the iothread lock is taken here too. */
1211 qemu_mutex_lock_ramlist();
1212 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1213 if (addr == block->offset) {
1214 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1215 ram_list.mru_block = NULL;
1216 ram_list.version++;
1217 if (block->flags & RAM_PREALLOC_MASK) {
1218 ;
1219 } else if (mem_path) {
1220 #if defined (__linux__) && !defined(TARGET_S390X)
1221 if (block->fd) {
1222 munmap(block->host, block->length);
1223 close(block->fd);
1224 } else {
1225 qemu_anon_ram_free(block->host, block->length);
1226 }
1227 #else
1228 abort();
1229 #endif
1230 } else {
1231 if (xen_enabled()) {
1232 xen_invalidate_map_cache_entry(block->host);
1233 } else {
1234 qemu_anon_ram_free(block->host, block->length);
1235 }
1236 }
1237 g_free(block);
1238 break;
1239 }
1240 }
1241 qemu_mutex_unlock_ramlist();
1242
1243 }
1244
1245 #ifndef _WIN32
1246 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1247 {
1248 RAMBlock *block;
1249 ram_addr_t offset;
1250 int flags;
1251 void *area, *vaddr;
1252
1253 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1254 offset = addr - block->offset;
1255 if (offset < block->length) {
1256 vaddr = block->host + offset;
1257 if (block->flags & RAM_PREALLOC_MASK) {
1258 ;
1259 } else {
1260 flags = MAP_FIXED;
1261 munmap(vaddr, length);
1262 if (mem_path) {
1263 #if defined(__linux__) && !defined(TARGET_S390X)
1264 if (block->fd) {
1265 #ifdef MAP_POPULATE
1266 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1267 MAP_PRIVATE;
1268 #else
1269 flags |= MAP_PRIVATE;
1270 #endif
1271 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1272 flags, block->fd, offset);
1273 } else {
1274 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1275 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1276 flags, -1, 0);
1277 }
1278 #else
1279 abort();
1280 #endif
1281 } else {
1282 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1283 flags |= MAP_SHARED | MAP_ANONYMOUS;
1284 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1285 flags, -1, 0);
1286 #else
1287 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1288 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1289 flags, -1, 0);
1290 #endif
1291 }
1292 if (area != vaddr) {
1293 fprintf(stderr, "Could not remap addr: "
1294 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1295 length, addr);
1296 exit(1);
1297 }
1298 memory_try_enable_merging(vaddr, length);
1299 qemu_ram_setup_dump(vaddr, length);
1300 }
1301 return;
1302 }
1303 }
1304 }
1305 #endif /* !_WIN32 */
1306
1307 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1308 {
1309 RAMBlock *block;
1310
1311 /* The list is protected by the iothread lock here. */
1312 block = ram_list.mru_block;
1313 if (block && addr - block->offset < block->length) {
1314 goto found;
1315 }
1316 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1317 if (addr - block->offset < block->length) {
1318 goto found;
1319 }
1320 }
1321
1322 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1323 abort();
1324
1325 found:
1326 ram_list.mru_block = block;
1327 return block;
1328 }
1329
1330 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1331 With the exception of the softmmu code in this file, this should
1332 only be used for local memory (e.g. video ram) that the device owns,
1333 and knows it isn't going to access beyond the end of the block.
1334
1335 It should not be used for general purpose DMA.
1336 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1337 */
1338 void *qemu_get_ram_ptr(ram_addr_t addr)
1339 {
1340 RAMBlock *block = qemu_get_ram_block(addr);
1341
1342 if (xen_enabled()) {
1343 /* We need to check if the requested address is in the RAM
1344 * because we don't want to map the entire memory in QEMU.
1345 * In that case just map until the end of the page.
1346 */
1347 if (block->offset == 0) {
1348 return xen_map_cache(addr, 0, 0);
1349 } else if (block->host == NULL) {
1350 block->host =
1351 xen_map_cache(block->offset, block->length, 1);
1352 }
1353 }
1354 return block->host + (addr - block->offset);
1355 }
1356
1357 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1358 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1359 *
1360 * ??? Is this still necessary?
1361 */
1362 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1363 {
1364 RAMBlock *block;
1365
1366 /* The list is protected by the iothread lock here. */
1367 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1368 if (addr - block->offset < block->length) {
1369 if (xen_enabled()) {
1370 /* We need to check if the requested address is in the RAM
1371 * because we don't want to map the entire memory in QEMU.
1372 * In that case just map until the end of the page.
1373 */
1374 if (block->offset == 0) {
1375 return xen_map_cache(addr, 0, 0);
1376 } else if (block->host == NULL) {
1377 block->host =
1378 xen_map_cache(block->offset, block->length, 1);
1379 }
1380 }
1381 return block->host + (addr - block->offset);
1382 }
1383 }
1384
1385 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1386 abort();
1387
1388 return NULL;
1389 }
1390
1391 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1392 * but takes a size argument */
1393 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1394 {
1395 if (*size == 0) {
1396 return NULL;
1397 }
1398 if (xen_enabled()) {
1399 return xen_map_cache(addr, *size, 1);
1400 } else {
1401 RAMBlock *block;
1402
1403 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1404 if (addr - block->offset < block->length) {
1405 if (addr - block->offset + *size > block->length)
1406 *size = block->length - addr + block->offset;
1407 return block->host + (addr - block->offset);
1408 }
1409 }
1410
1411 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1412 abort();
1413 }
1414 }
1415
1416 /* Some of the softmmu routines need to translate from a host pointer
1417 (typically a TLB entry) back to a ram offset. */
1418 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1419 {
1420 RAMBlock *block;
1421 uint8_t *host = ptr;
1422
1423 if (xen_enabled()) {
1424 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1425 return qemu_get_ram_block(*ram_addr)->mr;
1426 }
1427
1428 block = ram_list.mru_block;
1429 if (block && block->host && host - block->host < block->length) {
1430 goto found;
1431 }
1432
1433 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1434 /* This case append when the block is not mapped. */
1435 if (block->host == NULL) {
1436 continue;
1437 }
1438 if (host - block->host < block->length) {
1439 goto found;
1440 }
1441 }
1442
1443 return NULL;
1444
1445 found:
1446 *ram_addr = block->offset + (host - block->host);
1447 return block->mr;
1448 }
1449
1450 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1451 uint64_t val, unsigned size)
1452 {
1453 int dirty_flags;
1454 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1455 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1456 tb_invalidate_phys_page_fast(ram_addr, size);
1457 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1458 }
1459 switch (size) {
1460 case 1:
1461 stb_p(qemu_get_ram_ptr(ram_addr), val);
1462 break;
1463 case 2:
1464 stw_p(qemu_get_ram_ptr(ram_addr), val);
1465 break;
1466 case 4:
1467 stl_p(qemu_get_ram_ptr(ram_addr), val);
1468 break;
1469 default:
1470 abort();
1471 }
1472 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1473 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1474 /* we remove the notdirty callback only if the code has been
1475 flushed */
1476 if (dirty_flags == 0xff) {
1477 CPUArchState *env = current_cpu->env_ptr;
1478 tlb_set_dirty(env, env->mem_io_vaddr);
1479 }
1480 }
1481
1482 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1483 unsigned size, bool is_write)
1484 {
1485 return is_write;
1486 }
1487
1488 static const MemoryRegionOps notdirty_mem_ops = {
1489 .write = notdirty_mem_write,
1490 .valid.accepts = notdirty_mem_accepts,
1491 .endianness = DEVICE_NATIVE_ENDIAN,
1492 };
1493
1494 /* Generate a debug exception if a watchpoint has been hit. */
1495 static void check_watchpoint(int offset, int len_mask, int flags)
1496 {
1497 CPUArchState *env = current_cpu->env_ptr;
1498 target_ulong pc, cs_base;
1499 target_ulong vaddr;
1500 CPUWatchpoint *wp;
1501 int cpu_flags;
1502
1503 if (env->watchpoint_hit) {
1504 /* We re-entered the check after replacing the TB. Now raise
1505 * the debug interrupt so that is will trigger after the
1506 * current instruction. */
1507 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1508 return;
1509 }
1510 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1511 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1512 if ((vaddr == (wp->vaddr & len_mask) ||
1513 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1514 wp->flags |= BP_WATCHPOINT_HIT;
1515 if (!env->watchpoint_hit) {
1516 env->watchpoint_hit = wp;
1517 tb_check_watchpoint(env);
1518 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1519 env->exception_index = EXCP_DEBUG;
1520 cpu_loop_exit(env);
1521 } else {
1522 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1523 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1524 cpu_resume_from_signal(env, NULL);
1525 }
1526 }
1527 } else {
1528 wp->flags &= ~BP_WATCHPOINT_HIT;
1529 }
1530 }
1531 }
1532
1533 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1534 so these check for a hit then pass through to the normal out-of-line
1535 phys routines. */
1536 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1537 unsigned size)
1538 {
1539 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1540 switch (size) {
1541 case 1: return ldub_phys(addr);
1542 case 2: return lduw_phys(addr);
1543 case 4: return ldl_phys(addr);
1544 default: abort();
1545 }
1546 }
1547
1548 static void watch_mem_write(void *opaque, hwaddr addr,
1549 uint64_t val, unsigned size)
1550 {
1551 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1552 switch (size) {
1553 case 1:
1554 stb_phys(addr, val);
1555 break;
1556 case 2:
1557 stw_phys(addr, val);
1558 break;
1559 case 4:
1560 stl_phys(addr, val);
1561 break;
1562 default: abort();
1563 }
1564 }
1565
1566 static const MemoryRegionOps watch_mem_ops = {
1567 .read = watch_mem_read,
1568 .write = watch_mem_write,
1569 .endianness = DEVICE_NATIVE_ENDIAN,
1570 };
1571
1572 static uint64_t subpage_read(void *opaque, hwaddr addr,
1573 unsigned len)
1574 {
1575 subpage_t *subpage = opaque;
1576 uint8_t buf[4];
1577
1578 #if defined(DEBUG_SUBPAGE)
1579 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1580 subpage, len, addr);
1581 #endif
1582 address_space_read(subpage->as, addr + subpage->base, buf, len);
1583 switch (len) {
1584 case 1:
1585 return ldub_p(buf);
1586 case 2:
1587 return lduw_p(buf);
1588 case 4:
1589 return ldl_p(buf);
1590 default:
1591 abort();
1592 }
1593 }
1594
1595 static void subpage_write(void *opaque, hwaddr addr,
1596 uint64_t value, unsigned len)
1597 {
1598 subpage_t *subpage = opaque;
1599 uint8_t buf[4];
1600
1601 #if defined(DEBUG_SUBPAGE)
1602 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1603 " value %"PRIx64"\n",
1604 __func__, subpage, len, addr, value);
1605 #endif
1606 switch (len) {
1607 case 1:
1608 stb_p(buf, value);
1609 break;
1610 case 2:
1611 stw_p(buf, value);
1612 break;
1613 case 4:
1614 stl_p(buf, value);
1615 break;
1616 default:
1617 abort();
1618 }
1619 address_space_write(subpage->as, addr + subpage->base, buf, len);
1620 }
1621
1622 static bool subpage_accepts(void *opaque, hwaddr addr,
1623 unsigned size, bool is_write)
1624 {
1625 subpage_t *subpage = opaque;
1626 #if defined(DEBUG_SUBPAGE)
1627 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1628 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1629 #endif
1630
1631 return address_space_access_valid(subpage->as, addr + subpage->base,
1632 size, is_write);
1633 }
1634
1635 static const MemoryRegionOps subpage_ops = {
1636 .read = subpage_read,
1637 .write = subpage_write,
1638 .valid.accepts = subpage_accepts,
1639 .endianness = DEVICE_NATIVE_ENDIAN,
1640 };
1641
1642 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1643 uint16_t section)
1644 {
1645 int idx, eidx;
1646
1647 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1648 return -1;
1649 idx = SUBPAGE_IDX(start);
1650 eidx = SUBPAGE_IDX(end);
1651 #if defined(DEBUG_SUBPAGE)
1652 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1653 mmio, start, end, idx, eidx, memory);
1654 #endif
1655 for (; idx <= eidx; idx++) {
1656 mmio->sub_section[idx] = section;
1657 }
1658
1659 return 0;
1660 }
1661
1662 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1663 {
1664 subpage_t *mmio;
1665
1666 mmio = g_malloc0(sizeof(subpage_t));
1667
1668 mmio->as = as;
1669 mmio->base = base;
1670 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1671 "subpage", TARGET_PAGE_SIZE);
1672 mmio->iomem.subpage = true;
1673 #if defined(DEBUG_SUBPAGE)
1674 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1675 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1676 #endif
1677 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1678
1679 return mmio;
1680 }
1681
1682 static uint16_t dummy_section(MemoryRegion *mr)
1683 {
1684 MemoryRegionSection section = {
1685 .mr = mr,
1686 .offset_within_address_space = 0,
1687 .offset_within_region = 0,
1688 .size = int128_2_64(),
1689 };
1690
1691 return phys_section_add(&section);
1692 }
1693
1694 MemoryRegion *iotlb_to_region(hwaddr index)
1695 {
1696 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1697 }
1698
1699 static void io_mem_init(void)
1700 {
1701 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1702 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1703 "unassigned", UINT64_MAX);
1704 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1705 "notdirty", UINT64_MAX);
1706 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1707 "watch", UINT64_MAX);
1708 }
1709
1710 static void mem_begin(MemoryListener *listener)
1711 {
1712 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1713 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1714
1715 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1716 d->as = as;
1717 as->next_dispatch = d;
1718 }
1719
1720 static void mem_commit(MemoryListener *listener)
1721 {
1722 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1723 AddressSpaceDispatch *cur = as->dispatch;
1724 AddressSpaceDispatch *next = as->next_dispatch;
1725
1726 next->nodes = next_map.nodes;
1727 next->sections = next_map.sections;
1728
1729 as->dispatch = next;
1730 g_free(cur);
1731 }
1732
1733 static void core_begin(MemoryListener *listener)
1734 {
1735 uint16_t n;
1736
1737 prev_map = g_new(PhysPageMap, 1);
1738 *prev_map = next_map;
1739
1740 memset(&next_map, 0, sizeof(next_map));
1741 n = dummy_section(&io_mem_unassigned);
1742 assert(n == PHYS_SECTION_UNASSIGNED);
1743 n = dummy_section(&io_mem_notdirty);
1744 assert(n == PHYS_SECTION_NOTDIRTY);
1745 n = dummy_section(&io_mem_rom);
1746 assert(n == PHYS_SECTION_ROM);
1747 n = dummy_section(&io_mem_watch);
1748 assert(n == PHYS_SECTION_WATCH);
1749 }
1750
1751 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1752 * All AddressSpaceDispatch instances have switched to the next map.
1753 */
1754 static void core_commit(MemoryListener *listener)
1755 {
1756 phys_sections_free(prev_map);
1757 }
1758
1759 static void tcg_commit(MemoryListener *listener)
1760 {
1761 CPUState *cpu;
1762
1763 /* since each CPU stores ram addresses in its TLB cache, we must
1764 reset the modified entries */
1765 /* XXX: slow ! */
1766 for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1767 CPUArchState *env = cpu->env_ptr;
1768
1769 tlb_flush(env, 1);
1770 }
1771 }
1772
1773 static void core_log_global_start(MemoryListener *listener)
1774 {
1775 cpu_physical_memory_set_dirty_tracking(1);
1776 }
1777
1778 static void core_log_global_stop(MemoryListener *listener)
1779 {
1780 cpu_physical_memory_set_dirty_tracking(0);
1781 }
1782
1783 static MemoryListener core_memory_listener = {
1784 .begin = core_begin,
1785 .commit = core_commit,
1786 .log_global_start = core_log_global_start,
1787 .log_global_stop = core_log_global_stop,
1788 .priority = 1,
1789 };
1790
1791 static MemoryListener tcg_memory_listener = {
1792 .commit = tcg_commit,
1793 };
1794
1795 void address_space_init_dispatch(AddressSpace *as)
1796 {
1797 as->dispatch = NULL;
1798 as->dispatch_listener = (MemoryListener) {
1799 .begin = mem_begin,
1800 .commit = mem_commit,
1801 .region_add = mem_add,
1802 .region_nop = mem_add,
1803 .priority = 0,
1804 };
1805 memory_listener_register(&as->dispatch_listener, as);
1806 }
1807
1808 void address_space_destroy_dispatch(AddressSpace *as)
1809 {
1810 AddressSpaceDispatch *d = as->dispatch;
1811
1812 memory_listener_unregister(&as->dispatch_listener);
1813 g_free(d);
1814 as->dispatch = NULL;
1815 }
1816
1817 static void memory_map_init(void)
1818 {
1819 system_memory = g_malloc(sizeof(*system_memory));
1820 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1821 address_space_init(&address_space_memory, system_memory, "memory");
1822
1823 system_io = g_malloc(sizeof(*system_io));
1824 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1825 65536);
1826 address_space_init(&address_space_io, system_io, "I/O");
1827
1828 memory_listener_register(&core_memory_listener, &address_space_memory);
1829 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1830 }
1831
1832 MemoryRegion *get_system_memory(void)
1833 {
1834 return system_memory;
1835 }
1836
1837 MemoryRegion *get_system_io(void)
1838 {
1839 return system_io;
1840 }
1841
1842 #endif /* !defined(CONFIG_USER_ONLY) */
1843
1844 /* physical memory access (slow version, mainly for debug) */
1845 #if defined(CONFIG_USER_ONLY)
1846 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1847 uint8_t *buf, int len, int is_write)
1848 {
1849 int l, flags;
1850 target_ulong page;
1851 void * p;
1852
1853 while (len > 0) {
1854 page = addr & TARGET_PAGE_MASK;
1855 l = (page + TARGET_PAGE_SIZE) - addr;
1856 if (l > len)
1857 l = len;
1858 flags = page_get_flags(page);
1859 if (!(flags & PAGE_VALID))
1860 return -1;
1861 if (is_write) {
1862 if (!(flags & PAGE_WRITE))
1863 return -1;
1864 /* XXX: this code should not depend on lock_user */
1865 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1866 return -1;
1867 memcpy(p, buf, l);
1868 unlock_user(p, addr, l);
1869 } else {
1870 if (!(flags & PAGE_READ))
1871 return -1;
1872 /* XXX: this code should not depend on lock_user */
1873 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1874 return -1;
1875 memcpy(buf, p, l);
1876 unlock_user(p, addr, 0);
1877 }
1878 len -= l;
1879 buf += l;
1880 addr += l;
1881 }
1882 return 0;
1883 }
1884
1885 #else
1886
1887 static void invalidate_and_set_dirty(hwaddr addr,
1888 hwaddr length)
1889 {
1890 if (!cpu_physical_memory_is_dirty(addr)) {
1891 /* invalidate code */
1892 tb_invalidate_phys_page_range(addr, addr + length, 0);
1893 /* set dirty bit */
1894 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1895 }
1896 xen_modified_memory(addr, length);
1897 }
1898
1899 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1900 {
1901 if (memory_region_is_ram(mr)) {
1902 return !(is_write && mr->readonly);
1903 }
1904 if (memory_region_is_romd(mr)) {
1905 return !is_write;
1906 }
1907
1908 return false;
1909 }
1910
1911 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1912 {
1913 unsigned access_size_max = mr->ops->valid.max_access_size;
1914
1915 /* Regions are assumed to support 1-4 byte accesses unless
1916 otherwise specified. */
1917 if (access_size_max == 0) {
1918 access_size_max = 4;
1919 }
1920
1921 /* Bound the maximum access by the alignment of the address. */
1922 if (!mr->ops->impl.unaligned) {
1923 unsigned align_size_max = addr & -addr;
1924 if (align_size_max != 0 && align_size_max < access_size_max) {
1925 access_size_max = align_size_max;
1926 }
1927 }
1928
1929 /* Don't attempt accesses larger than the maximum. */
1930 if (l > access_size_max) {
1931 l = access_size_max;
1932 }
1933 if (l & (l - 1)) {
1934 l = 1 << (qemu_fls(l) - 1);
1935 }
1936
1937 return l;
1938 }
1939
1940 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1941 int len, bool is_write)
1942 {
1943 hwaddr l;
1944 uint8_t *ptr;
1945 uint64_t val;
1946 hwaddr addr1;
1947 MemoryRegion *mr;
1948 bool error = false;
1949
1950 while (len > 0) {
1951 l = len;
1952 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1953
1954 if (is_write) {
1955 if (!memory_access_is_direct(mr, is_write)) {
1956 l = memory_access_size(mr, l, addr1);
1957 /* XXX: could force current_cpu to NULL to avoid
1958 potential bugs */
1959 switch (l) {
1960 case 8:
1961 /* 64 bit write access */
1962 val = ldq_p(buf);
1963 error |= io_mem_write(mr, addr1, val, 8);
1964 break;
1965 case 4:
1966 /* 32 bit write access */
1967 val = ldl_p(buf);
1968 error |= io_mem_write(mr, addr1, val, 4);
1969 break;
1970 case 2:
1971 /* 16 bit write access */
1972 val = lduw_p(buf);
1973 error |= io_mem_write(mr, addr1, val, 2);
1974 break;
1975 case 1:
1976 /* 8 bit write access */
1977 val = ldub_p(buf);
1978 error |= io_mem_write(mr, addr1, val, 1);
1979 break;
1980 default:
1981 abort();
1982 }
1983 } else {
1984 addr1 += memory_region_get_ram_addr(mr);
1985 /* RAM case */
1986 ptr = qemu_get_ram_ptr(addr1);
1987 memcpy(ptr, buf, l);
1988 invalidate_and_set_dirty(addr1, l);
1989 }
1990 } else {
1991 if (!memory_access_is_direct(mr, is_write)) {
1992 /* I/O case */
1993 l = memory_access_size(mr, l, addr1);
1994 switch (l) {
1995 case 8:
1996 /* 64 bit read access */
1997 error |= io_mem_read(mr, addr1, &val, 8);
1998 stq_p(buf, val);
1999 break;
2000 case 4:
2001 /* 32 bit read access */
2002 error |= io_mem_read(mr, addr1, &val, 4);
2003 stl_p(buf, val);
2004 break;
2005 case 2:
2006 /* 16 bit read access */
2007 error |= io_mem_read(mr, addr1, &val, 2);
2008 stw_p(buf, val);
2009 break;
2010 case 1:
2011 /* 8 bit read access */
2012 error |= io_mem_read(mr, addr1, &val, 1);
2013 stb_p(buf, val);
2014 break;
2015 default:
2016 abort();
2017 }
2018 } else {
2019 /* RAM case */
2020 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2021 memcpy(buf, ptr, l);
2022 }
2023 }
2024 len -= l;
2025 buf += l;
2026 addr += l;
2027 }
2028
2029 return error;
2030 }
2031
2032 bool address_space_write(AddressSpace *as, hwaddr addr,
2033 const uint8_t *buf, int len)
2034 {
2035 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2036 }
2037
2038 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2039 {
2040 return address_space_rw(as, addr, buf, len, false);
2041 }
2042
2043
2044 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2045 int len, int is_write)
2046 {
2047 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2048 }
2049
2050 /* used for ROM loading : can write in RAM and ROM */
2051 void cpu_physical_memory_write_rom(hwaddr addr,
2052 const uint8_t *buf, int len)
2053 {
2054 hwaddr l;
2055 uint8_t *ptr;
2056 hwaddr addr1;
2057 MemoryRegion *mr;
2058
2059 while (len > 0) {
2060 l = len;
2061 mr = address_space_translate(&address_space_memory,
2062 addr, &addr1, &l, true);
2063
2064 if (!(memory_region_is_ram(mr) ||
2065 memory_region_is_romd(mr))) {
2066 /* do nothing */
2067 } else {
2068 addr1 += memory_region_get_ram_addr(mr);
2069 /* ROM/RAM case */
2070 ptr = qemu_get_ram_ptr(addr1);
2071 memcpy(ptr, buf, l);
2072 invalidate_and_set_dirty(addr1, l);
2073 }
2074 len -= l;
2075 buf += l;
2076 addr += l;
2077 }
2078 }
2079
2080 typedef struct {
2081 MemoryRegion *mr;
2082 void *buffer;
2083 hwaddr addr;
2084 hwaddr len;
2085 } BounceBuffer;
2086
2087 static BounceBuffer bounce;
2088
2089 typedef struct MapClient {
2090 void *opaque;
2091 void (*callback)(void *opaque);
2092 QLIST_ENTRY(MapClient) link;
2093 } MapClient;
2094
2095 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2096 = QLIST_HEAD_INITIALIZER(map_client_list);
2097
2098 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2099 {
2100 MapClient *client = g_malloc(sizeof(*client));
2101
2102 client->opaque = opaque;
2103 client->callback = callback;
2104 QLIST_INSERT_HEAD(&map_client_list, client, link);
2105 return client;
2106 }
2107
2108 static void cpu_unregister_map_client(void *_client)
2109 {
2110 MapClient *client = (MapClient *)_client;
2111
2112 QLIST_REMOVE(client, link);
2113 g_free(client);
2114 }
2115
2116 static void cpu_notify_map_clients(void)
2117 {
2118 MapClient *client;
2119
2120 while (!QLIST_EMPTY(&map_client_list)) {
2121 client = QLIST_FIRST(&map_client_list);
2122 client->callback(client->opaque);
2123 cpu_unregister_map_client(client);
2124 }
2125 }
2126
2127 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2128 {
2129 MemoryRegion *mr;
2130 hwaddr l, xlat;
2131
2132 while (len > 0) {
2133 l = len;
2134 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2135 if (!memory_access_is_direct(mr, is_write)) {
2136 l = memory_access_size(mr, l, addr);
2137 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2138 return false;
2139 }
2140 }
2141
2142 len -= l;
2143 addr += l;
2144 }
2145 return true;
2146 }
2147
2148 /* Map a physical memory region into a host virtual address.
2149 * May map a subset of the requested range, given by and returned in *plen.
2150 * May return NULL if resources needed to perform the mapping are exhausted.
2151 * Use only for reads OR writes - not for read-modify-write operations.
2152 * Use cpu_register_map_client() to know when retrying the map operation is
2153 * likely to succeed.
2154 */
2155 void *address_space_map(AddressSpace *as,
2156 hwaddr addr,
2157 hwaddr *plen,
2158 bool is_write)
2159 {
2160 hwaddr len = *plen;
2161 hwaddr done = 0;
2162 hwaddr l, xlat, base;
2163 MemoryRegion *mr, *this_mr;
2164 ram_addr_t raddr;
2165
2166 if (len == 0) {
2167 return NULL;
2168 }
2169
2170 l = len;
2171 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2172 if (!memory_access_is_direct(mr, is_write)) {
2173 if (bounce.buffer) {
2174 return NULL;
2175 }
2176 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2177 bounce.addr = addr;
2178 bounce.len = l;
2179
2180 memory_region_ref(mr);
2181 bounce.mr = mr;
2182 if (!is_write) {
2183 address_space_read(as, addr, bounce.buffer, l);
2184 }
2185
2186 *plen = l;
2187 return bounce.buffer;
2188 }
2189
2190 base = xlat;
2191 raddr = memory_region_get_ram_addr(mr);
2192
2193 for (;;) {
2194 len -= l;
2195 addr += l;
2196 done += l;
2197 if (len == 0) {
2198 break;
2199 }
2200
2201 l = len;
2202 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2203 if (this_mr != mr || xlat != base + done) {
2204 break;
2205 }
2206 }
2207
2208 memory_region_ref(mr);
2209 *plen = done;
2210 return qemu_ram_ptr_length(raddr + base, plen);
2211 }
2212
2213 /* Unmaps a memory region previously mapped by address_space_map().
2214 * Will also mark the memory as dirty if is_write == 1. access_len gives
2215 * the amount of memory that was actually read or written by the caller.
2216 */
2217 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2218 int is_write, hwaddr access_len)
2219 {
2220 if (buffer != bounce.buffer) {
2221 MemoryRegion *mr;
2222 ram_addr_t addr1;
2223
2224 mr = qemu_ram_addr_from_host(buffer, &addr1);
2225 assert(mr != NULL);
2226 if (is_write) {
2227 while (access_len) {
2228 unsigned l;
2229 l = TARGET_PAGE_SIZE;
2230 if (l > access_len)
2231 l = access_len;
2232 invalidate_and_set_dirty(addr1, l);
2233 addr1 += l;
2234 access_len -= l;
2235 }
2236 }
2237 if (xen_enabled()) {
2238 xen_invalidate_map_cache_entry(buffer);
2239 }
2240 memory_region_unref(mr);
2241 return;
2242 }
2243 if (is_write) {
2244 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2245 }
2246 qemu_vfree(bounce.buffer);
2247 bounce.buffer = NULL;
2248 memory_region_unref(bounce.mr);
2249 cpu_notify_map_clients();
2250 }
2251
2252 void *cpu_physical_memory_map(hwaddr addr,
2253 hwaddr *plen,
2254 int is_write)
2255 {
2256 return address_space_map(&address_space_memory, addr, plen, is_write);
2257 }
2258
2259 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2260 int is_write, hwaddr access_len)
2261 {
2262 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2263 }
2264
2265 /* warning: addr must be aligned */
2266 static inline uint32_t ldl_phys_internal(hwaddr addr,
2267 enum device_endian endian)
2268 {
2269 uint8_t *ptr;
2270 uint64_t val;
2271 MemoryRegion *mr;
2272 hwaddr l = 4;
2273 hwaddr addr1;
2274
2275 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2276 false);
2277 if (l < 4 || !memory_access_is_direct(mr, false)) {
2278 /* I/O case */
2279 io_mem_read(mr, addr1, &val, 4);
2280 #if defined(TARGET_WORDS_BIGENDIAN)
2281 if (endian == DEVICE_LITTLE_ENDIAN) {
2282 val = bswap32(val);
2283 }
2284 #else
2285 if (endian == DEVICE_BIG_ENDIAN) {
2286 val = bswap32(val);
2287 }
2288 #endif
2289 } else {
2290 /* RAM case */
2291 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2292 & TARGET_PAGE_MASK)
2293 + addr1);
2294 switch (endian) {
2295 case DEVICE_LITTLE_ENDIAN:
2296 val = ldl_le_p(ptr);
2297 break;
2298 case DEVICE_BIG_ENDIAN:
2299 val = ldl_be_p(ptr);
2300 break;
2301 default:
2302 val = ldl_p(ptr);
2303 break;
2304 }
2305 }
2306 return val;
2307 }
2308
2309 uint32_t ldl_phys(hwaddr addr)
2310 {
2311 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2312 }
2313
2314 uint32_t ldl_le_phys(hwaddr addr)
2315 {
2316 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2317 }
2318
2319 uint32_t ldl_be_phys(hwaddr addr)
2320 {
2321 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2322 }
2323
2324 /* warning: addr must be aligned */
2325 static inline uint64_t ldq_phys_internal(hwaddr addr,
2326 enum device_endian endian)
2327 {
2328 uint8_t *ptr;
2329 uint64_t val;
2330 MemoryRegion *mr;
2331 hwaddr l = 8;
2332 hwaddr addr1;
2333
2334 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2335 false);
2336 if (l < 8 || !memory_access_is_direct(mr, false)) {
2337 /* I/O case */
2338 io_mem_read(mr, addr1, &val, 8);
2339 #if defined(TARGET_WORDS_BIGENDIAN)
2340 if (endian == DEVICE_LITTLE_ENDIAN) {
2341 val = bswap64(val);
2342 }
2343 #else
2344 if (endian == DEVICE_BIG_ENDIAN) {
2345 val = bswap64(val);
2346 }
2347 #endif
2348 } else {
2349 /* RAM case */
2350 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2351 & TARGET_PAGE_MASK)
2352 + addr1);
2353 switch (endian) {
2354 case DEVICE_LITTLE_ENDIAN:
2355 val = ldq_le_p(ptr);
2356 break;
2357 case DEVICE_BIG_ENDIAN:
2358 val = ldq_be_p(ptr);
2359 break;
2360 default:
2361 val = ldq_p(ptr);
2362 break;
2363 }
2364 }
2365 return val;
2366 }
2367
2368 uint64_t ldq_phys(hwaddr addr)
2369 {
2370 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2371 }
2372
2373 uint64_t ldq_le_phys(hwaddr addr)
2374 {
2375 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2376 }
2377
2378 uint64_t ldq_be_phys(hwaddr addr)
2379 {
2380 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2381 }
2382
2383 /* XXX: optimize */
2384 uint32_t ldub_phys(hwaddr addr)
2385 {
2386 uint8_t val;
2387 cpu_physical_memory_read(addr, &val, 1);
2388 return val;
2389 }
2390
2391 /* warning: addr must be aligned */
2392 static inline uint32_t lduw_phys_internal(hwaddr addr,
2393 enum device_endian endian)
2394 {
2395 uint8_t *ptr;
2396 uint64_t val;
2397 MemoryRegion *mr;
2398 hwaddr l = 2;
2399 hwaddr addr1;
2400
2401 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2402 false);
2403 if (l < 2 || !memory_access_is_direct(mr, false)) {
2404 /* I/O case */
2405 io_mem_read(mr, addr1, &val, 2);
2406 #if defined(TARGET_WORDS_BIGENDIAN)
2407 if (endian == DEVICE_LITTLE_ENDIAN) {
2408 val = bswap16(val);
2409 }
2410 #else
2411 if (endian == DEVICE_BIG_ENDIAN) {
2412 val = bswap16(val);
2413 }
2414 #endif
2415 } else {
2416 /* RAM case */
2417 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2418 & TARGET_PAGE_MASK)
2419 + addr1);
2420 switch (endian) {
2421 case DEVICE_LITTLE_ENDIAN:
2422 val = lduw_le_p(ptr);
2423 break;
2424 case DEVICE_BIG_ENDIAN:
2425 val = lduw_be_p(ptr);
2426 break;
2427 default:
2428 val = lduw_p(ptr);
2429 break;
2430 }
2431 }
2432 return val;
2433 }
2434
2435 uint32_t lduw_phys(hwaddr addr)
2436 {
2437 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2438 }
2439
2440 uint32_t lduw_le_phys(hwaddr addr)
2441 {
2442 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2443 }
2444
2445 uint32_t lduw_be_phys(hwaddr addr)
2446 {
2447 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2448 }
2449
2450 /* warning: addr must be aligned. The ram page is not masked as dirty
2451 and the code inside is not invalidated. It is useful if the dirty
2452 bits are used to track modified PTEs */
2453 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2454 {
2455 uint8_t *ptr;
2456 MemoryRegion *mr;
2457 hwaddr l = 4;
2458 hwaddr addr1;
2459
2460 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2461 true);
2462 if (l < 4 || !memory_access_is_direct(mr, true)) {
2463 io_mem_write(mr, addr1, val, 4);
2464 } else {
2465 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2466 ptr = qemu_get_ram_ptr(addr1);
2467 stl_p(ptr, val);
2468
2469 if (unlikely(in_migration)) {
2470 if (!cpu_physical_memory_is_dirty(addr1)) {
2471 /* invalidate code */
2472 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2473 /* set dirty bit */
2474 cpu_physical_memory_set_dirty_flags(
2475 addr1, (0xff & ~CODE_DIRTY_FLAG));
2476 }
2477 }
2478 }
2479 }
2480
2481 /* warning: addr must be aligned */
2482 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2483 enum device_endian endian)
2484 {
2485 uint8_t *ptr;
2486 MemoryRegion *mr;
2487 hwaddr l = 4;
2488 hwaddr addr1;
2489
2490 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2491 true);
2492 if (l < 4 || !memory_access_is_direct(mr, true)) {
2493 #if defined(TARGET_WORDS_BIGENDIAN)
2494 if (endian == DEVICE_LITTLE_ENDIAN) {
2495 val = bswap32(val);
2496 }
2497 #else
2498 if (endian == DEVICE_BIG_ENDIAN) {
2499 val = bswap32(val);
2500 }
2501 #endif
2502 io_mem_write(mr, addr1, val, 4);
2503 } else {
2504 /* RAM case */
2505 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2506 ptr = qemu_get_ram_ptr(addr1);
2507 switch (endian) {
2508 case DEVICE_LITTLE_ENDIAN:
2509 stl_le_p(ptr, val);
2510 break;
2511 case DEVICE_BIG_ENDIAN:
2512 stl_be_p(ptr, val);
2513 break;
2514 default:
2515 stl_p(ptr, val);
2516 break;
2517 }
2518 invalidate_and_set_dirty(addr1, 4);
2519 }
2520 }
2521
2522 void stl_phys(hwaddr addr, uint32_t val)
2523 {
2524 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2525 }
2526
2527 void stl_le_phys(hwaddr addr, uint32_t val)
2528 {
2529 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2530 }
2531
2532 void stl_be_phys(hwaddr addr, uint32_t val)
2533 {
2534 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2535 }
2536
2537 /* XXX: optimize */
2538 void stb_phys(hwaddr addr, uint32_t val)
2539 {
2540 uint8_t v = val;
2541 cpu_physical_memory_write(addr, &v, 1);
2542 }
2543
2544 /* warning: addr must be aligned */
2545 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2546 enum device_endian endian)
2547 {
2548 uint8_t *ptr;
2549 MemoryRegion *mr;
2550 hwaddr l = 2;
2551 hwaddr addr1;
2552
2553 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2554 true);
2555 if (l < 2 || !memory_access_is_direct(mr, true)) {
2556 #if defined(TARGET_WORDS_BIGENDIAN)
2557 if (endian == DEVICE_LITTLE_ENDIAN) {
2558 val = bswap16(val);
2559 }
2560 #else
2561 if (endian == DEVICE_BIG_ENDIAN) {
2562 val = bswap16(val);
2563 }
2564 #endif
2565 io_mem_write(mr, addr1, val, 2);
2566 } else {
2567 /* RAM case */
2568 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2569 ptr = qemu_get_ram_ptr(addr1);
2570 switch (endian) {
2571 case DEVICE_LITTLE_ENDIAN:
2572 stw_le_p(ptr, val);
2573 break;
2574 case DEVICE_BIG_ENDIAN:
2575 stw_be_p(ptr, val);
2576 break;
2577 default:
2578 stw_p(ptr, val);
2579 break;
2580 }
2581 invalidate_and_set_dirty(addr1, 2);
2582 }
2583 }
2584
2585 void stw_phys(hwaddr addr, uint32_t val)
2586 {
2587 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2588 }
2589
2590 void stw_le_phys(hwaddr addr, uint32_t val)
2591 {
2592 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2593 }
2594
2595 void stw_be_phys(hwaddr addr, uint32_t val)
2596 {
2597 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2598 }
2599
2600 /* XXX: optimize */
2601 void stq_phys(hwaddr addr, uint64_t val)
2602 {
2603 val = tswap64(val);
2604 cpu_physical_memory_write(addr, &val, 8);
2605 }
2606
2607 void stq_le_phys(hwaddr addr, uint64_t val)
2608 {
2609 val = cpu_to_le64(val);
2610 cpu_physical_memory_write(addr, &val, 8);
2611 }
2612
2613 void stq_be_phys(hwaddr addr, uint64_t val)
2614 {
2615 val = cpu_to_be64(val);
2616 cpu_physical_memory_write(addr, &val, 8);
2617 }
2618
2619 /* virtual memory access for debug (includes writing to ROM) */
2620 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2621 uint8_t *buf, int len, int is_write)
2622 {
2623 int l;
2624 hwaddr phys_addr;
2625 target_ulong page;
2626
2627 while (len > 0) {
2628 page = addr & TARGET_PAGE_MASK;
2629 phys_addr = cpu_get_phys_page_debug(cpu, page);
2630 /* if no physical page mapped, return an error */
2631 if (phys_addr == -1)
2632 return -1;
2633 l = (page + TARGET_PAGE_SIZE) - addr;
2634 if (l > len)
2635 l = len;
2636 phys_addr += (addr & ~TARGET_PAGE_MASK);
2637 if (is_write)
2638 cpu_physical_memory_write_rom(phys_addr, buf, l);
2639 else
2640 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2641 len -= l;
2642 buf += l;
2643 addr += l;
2644 }
2645 return 0;
2646 }
2647 #endif
2648
2649 #if !defined(CONFIG_USER_ONLY)
2650
2651 /*
2652 * A helper function for the _utterly broken_ virtio device model to find out if
2653 * it's running on a big endian machine. Don't do this at home kids!
2654 */
2655 bool virtio_is_big_endian(void);
2656 bool virtio_is_big_endian(void)
2657 {
2658 #if defined(TARGET_WORDS_BIGENDIAN)
2659 return true;
2660 #else
2661 return false;
2662 #endif
2663 }
2664
2665 #endif
2666
2667 #ifndef CONFIG_USER_ONLY
2668 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2669 {
2670 MemoryRegion*mr;
2671 hwaddr l = 1;
2672
2673 mr = address_space_translate(&address_space_memory,
2674 phys_addr, &phys_addr, &l, false);
2675
2676 return !(memory_region_is_ram(mr) ||
2677 memory_region_is_romd(mr));
2678 }
2679
2680 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2681 {
2682 RAMBlock *block;
2683
2684 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2685 func(block->host, block->offset, block->length, opaque);
2686 }
2687 }
2688 #endif
Mirror of https://git.qemu.org/git/qemu.git