]>
Commit | Line | Data |
---|---|---|
220c3ebd JQ |
1 | /* |
2 | * Declarations for cpu physical memory functions | |
3 | * | |
4 | * Copyright 2011 Red Hat, Inc. and/or its affiliates | |
5 | * | |
6 | * Authors: | |
7 | * Avi Kivity <avi@redhat.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2 or | |
10 | * later. See the COPYING file in the top-level directory. | |
11 | * | |
12 | */ | |
13 | ||
14 | /* | |
15 | * This header is for use by exec.c and memory.c ONLY. Do not include it. | |
16 | * The functions declared here will be removed soon. | |
17 | */ | |
18 | ||
19 | #ifndef RAM_ADDR_H | |
20 | #define RAM_ADDR_H | |
21 | ||
22 | #ifndef CONFIG_USER_ONLY | |
23 | #include "hw/xen/xen.h" | |
0987d735 | 24 | #include "exec/ramlist.h" |
220c3ebd | 25 | |
3c9589e1 DDAG |
26 | struct RAMBlock { |
27 | struct rcu_head rcu; | |
28 | struct MemoryRegion *mr; | |
29 | uint8_t *host; | |
30 | ram_addr_t offset; | |
31 | ram_addr_t used_length; | |
32 | ram_addr_t max_length; | |
33 | void (*resized)(const char*, uint64_t length, void *host); | |
34 | uint32_t flags; | |
35 | /* Protected by iothread lock. */ | |
36 | char idstr[256]; | |
37 | /* RCU-enabled, writes protected by the ramlist lock */ | |
38 | QLIST_ENTRY(RAMBlock) next; | |
0987d735 | 39 | QLIST_HEAD(, RAMBlockNotifier) ramblock_notifiers; |
3c9589e1 | 40 | int fd; |
863e9621 | 41 | size_t page_size; |
6b6712ef JQ |
42 | /* dirty bitmap used during migration */ |
43 | unsigned long *bmap; | |
44 | /* bitmap of pages that haven't been sent even once | |
45 | * only maintained and used in postcopy at the moment | |
46 | * where it's used to send the dirtymap at the start | |
47 | * of the postcopy phase | |
48 | */ | |
49 | unsigned long *unsentmap; | |
f9494614 AP |
50 | /* bitmap of already received pages in postcopy */ |
51 | unsigned long *receivedmap; | |
3c9589e1 DDAG |
52 | }; |
53 | ||
4c4bad48 HZ |
54 | static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) |
55 | { | |
56 | return (b && b->host && offset < b->used_length) ? true : false; | |
57 | } | |
58 | ||
3c9589e1 DDAG |
59 | static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset) |
60 | { | |
4c4bad48 | 61 | assert(offset_in_ramblock(block, offset)); |
3c9589e1 DDAG |
62 | return (char *)block->host + offset; |
63 | } | |
64 | ||
f9494614 AP |
65 | static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr, |
66 | RAMBlock *rb) | |
67 | { | |
68 | uint64_t host_addr_offset = | |
69 | (uint64_t)(uintptr_t)(host_addr - (void *)rb->host); | |
70 | return host_addr_offset >> TARGET_PAGE_BITS; | |
71 | } | |
72 | ||
9c607668 | 73 | long qemu_getrampagesize(void); |
528f46af FZ |
74 | RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, |
75 | bool share, const char *mem_path, | |
76 | Error **errp); | |
38b3362d MAL |
77 | RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, |
78 | bool share, int fd, | |
79 | Error **errp); | |
528f46af FZ |
80 | RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, |
81 | MemoryRegion *mr, Error **errp); | |
06329cce MA |
82 | RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share, MemoryRegion *mr, |
83 | Error **errp); | |
528f46af FZ |
84 | RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size, |
85 | void (*resized)(const char*, | |
86 | uint64_t length, | |
87 | void *host), | |
88 | MemoryRegion *mr, Error **errp); | |
f1060c55 | 89 | void qemu_ram_free(RAMBlock *block); |
220c3ebd | 90 | |
fa53a0e5 | 91 | int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp); |
62be4e3a | 92 | |
58d2707e PB |
93 | #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) |
94 | #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) | |
95 | ||
8bca9a03 PB |
96 | void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end); |
97 | ||
220c3ebd JQ |
98 | static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, |
99 | ram_addr_t length, | |
100 | unsigned client) | |
101 | { | |
5b82b703 SH |
102 | DirtyMemoryBlocks *blocks; |
103 | unsigned long end, page; | |
88c73d16 | 104 | unsigned long idx, offset, base; |
5b82b703 | 105 | bool dirty = false; |
220c3ebd JQ |
106 | |
107 | assert(client < DIRTY_MEMORY_NUM); | |
108 | ||
109 | end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; | |
110 | page = start >> TARGET_PAGE_BITS; | |
220c3ebd | 111 | |
5b82b703 SH |
112 | rcu_read_lock(); |
113 | ||
114 | blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); | |
115 | ||
88c73d16 PB |
116 | idx = page / DIRTY_MEMORY_BLOCK_SIZE; |
117 | offset = page % DIRTY_MEMORY_BLOCK_SIZE; | |
118 | base = page - offset; | |
5b82b703 | 119 | while (page < end) { |
88c73d16 PB |
120 | unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); |
121 | unsigned long num = next - base; | |
122 | unsigned long found = find_next_bit(blocks->blocks[idx], num, offset); | |
123 | if (found < num) { | |
5b82b703 SH |
124 | dirty = true; |
125 | break; | |
126 | } | |
127 | ||
88c73d16 PB |
128 | page = next; |
129 | idx++; | |
130 | offset = 0; | |
131 | base += DIRTY_MEMORY_BLOCK_SIZE; | |
5b82b703 SH |
132 | } |
133 | ||
134 | rcu_read_unlock(); | |
135 | ||
136 | return dirty; | |
220c3ebd JQ |
137 | } |
138 | ||
72b47e79 | 139 | static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, |
f874bf90 PM |
140 | ram_addr_t length, |
141 | unsigned client) | |
142 | { | |
5b82b703 SH |
143 | DirtyMemoryBlocks *blocks; |
144 | unsigned long end, page; | |
88c73d16 | 145 | unsigned long idx, offset, base; |
5b82b703 | 146 | bool dirty = true; |
f874bf90 PM |
147 | |
148 | assert(client < DIRTY_MEMORY_NUM); | |
149 | ||
150 | end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; | |
151 | page = start >> TARGET_PAGE_BITS; | |
f874bf90 | 152 | |
5b82b703 SH |
153 | rcu_read_lock(); |
154 | ||
155 | blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); | |
156 | ||
88c73d16 PB |
157 | idx = page / DIRTY_MEMORY_BLOCK_SIZE; |
158 | offset = page % DIRTY_MEMORY_BLOCK_SIZE; | |
159 | base = page - offset; | |
5b82b703 | 160 | while (page < end) { |
88c73d16 PB |
161 | unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); |
162 | unsigned long num = next - base; | |
163 | unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset); | |
164 | if (found < num) { | |
5b82b703 SH |
165 | dirty = false; |
166 | break; | |
167 | } | |
168 | ||
88c73d16 PB |
169 | page = next; |
170 | idx++; | |
171 | offset = 0; | |
172 | base += DIRTY_MEMORY_BLOCK_SIZE; | |
5b82b703 SH |
173 | } |
174 | ||
175 | rcu_read_unlock(); | |
176 | ||
177 | return dirty; | |
f874bf90 PM |
178 | } |
179 | ||
220c3ebd JQ |
180 | static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, |
181 | unsigned client) | |
182 | { | |
183 | return cpu_physical_memory_get_dirty(addr, 1, client); | |
184 | } | |
185 | ||
186 | static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) | |
187 | { | |
188 | bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); | |
189 | bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); | |
190 | bool migration = | |
191 | cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); | |
192 | return !(vga && code && migration); | |
193 | } | |
194 | ||
e87f7778 PB |
195 | static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, |
196 | ram_addr_t length, | |
197 | uint8_t mask) | |
f874bf90 | 198 | { |
e87f7778 PB |
199 | uint8_t ret = 0; |
200 | ||
201 | if (mask & (1 << DIRTY_MEMORY_VGA) && | |
202 | !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { | |
203 | ret |= (1 << DIRTY_MEMORY_VGA); | |
204 | } | |
205 | if (mask & (1 << DIRTY_MEMORY_CODE) && | |
206 | !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { | |
207 | ret |= (1 << DIRTY_MEMORY_CODE); | |
208 | } | |
209 | if (mask & (1 << DIRTY_MEMORY_MIGRATION) && | |
210 | !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { | |
211 | ret |= (1 << DIRTY_MEMORY_MIGRATION); | |
212 | } | |
213 | return ret; | |
f874bf90 PM |
214 | } |
215 | ||
220c3ebd JQ |
216 | static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, |
217 | unsigned client) | |
218 | { | |
5b82b703 SH |
219 | unsigned long page, idx, offset; |
220 | DirtyMemoryBlocks *blocks; | |
221 | ||
220c3ebd | 222 | assert(client < DIRTY_MEMORY_NUM); |
5b82b703 SH |
223 | |
224 | page = addr >> TARGET_PAGE_BITS; | |
225 | idx = page / DIRTY_MEMORY_BLOCK_SIZE; | |
226 | offset = page % DIRTY_MEMORY_BLOCK_SIZE; | |
227 | ||
228 | rcu_read_lock(); | |
229 | ||
230 | blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); | |
231 | ||
232 | set_bit_atomic(offset, blocks->blocks[idx]); | |
233 | ||
234 | rcu_read_unlock(); | |
220c3ebd JQ |
235 | } |
236 | ||
237 | static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, | |
58d2707e PB |
238 | ram_addr_t length, |
239 | uint8_t mask) | |
220c3ebd | 240 | { |
5b82b703 | 241 | DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; |
220c3ebd | 242 | unsigned long end, page; |
88c73d16 | 243 | unsigned long idx, offset, base; |
5b82b703 | 244 | int i; |
220c3ebd | 245 | |
8bafcb21 PB |
246 | if (!mask && !xen_enabled()) { |
247 | return; | |
248 | } | |
249 | ||
220c3ebd JQ |
250 | end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; |
251 | page = start >> TARGET_PAGE_BITS; | |
5b82b703 SH |
252 | |
253 | rcu_read_lock(); | |
254 | ||
255 | for (i = 0; i < DIRTY_MEMORY_NUM; i++) { | |
256 | blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]); | |
58d2707e | 257 | } |
5b82b703 | 258 | |
88c73d16 PB |
259 | idx = page / DIRTY_MEMORY_BLOCK_SIZE; |
260 | offset = page % DIRTY_MEMORY_BLOCK_SIZE; | |
261 | base = page - offset; | |
5b82b703 | 262 | while (page < end) { |
88c73d16 | 263 | unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); |
5b82b703 SH |
264 | |
265 | if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { | |
266 | bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], | |
88c73d16 | 267 | offset, next - page); |
5b82b703 SH |
268 | } |
269 | if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { | |
270 | bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], | |
88c73d16 | 271 | offset, next - page); |
5b82b703 SH |
272 | } |
273 | if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { | |
274 | bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], | |
88c73d16 | 275 | offset, next - page); |
5b82b703 SH |
276 | } |
277 | ||
88c73d16 PB |
278 | page = next; |
279 | idx++; | |
280 | offset = 0; | |
281 | base += DIRTY_MEMORY_BLOCK_SIZE; | |
58d2707e | 282 | } |
5b82b703 SH |
283 | |
284 | rcu_read_unlock(); | |
285 | ||
5100afb5 | 286 | xen_hvm_modified_memory(start, length); |
220c3ebd JQ |
287 | } |
288 | ||
fb3ecb7e | 289 | #if !defined(_WIN32) |
5ff7fb77 JQ |
290 | static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, |
291 | ram_addr_t start, | |
292 | ram_addr_t pages) | |
293 | { | |
ae2810c4 | 294 | unsigned long i, j; |
5ff7fb77 JQ |
295 | unsigned long page_number, c; |
296 | hwaddr addr; | |
297 | ram_addr_t ram_addr; | |
ae2810c4 | 298 | unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; |
5ff7fb77 | 299 | unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE; |
ae2810c4 | 300 | unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); |
5ff7fb77 | 301 | |
ae2810c4 | 302 | /* start address is aligned at the start of a word? */ |
f9ee9f9a AK |
303 | if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && |
304 | (hpratio == 1)) { | |
5b82b703 SH |
305 | unsigned long **blocks[DIRTY_MEMORY_NUM]; |
306 | unsigned long idx; | |
307 | unsigned long offset; | |
ae2810c4 JQ |
308 | long k; |
309 | long nr = BITS_TO_LONGS(pages); | |
310 | ||
5b82b703 SH |
311 | idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; |
312 | offset = BIT_WORD((start >> TARGET_PAGE_BITS) % | |
313 | DIRTY_MEMORY_BLOCK_SIZE); | |
314 | ||
315 | rcu_read_lock(); | |
316 | ||
317 | for (i = 0; i < DIRTY_MEMORY_NUM; i++) { | |
318 | blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks; | |
319 | } | |
320 | ||
ae2810c4 JQ |
321 | for (k = 0; k < nr; k++) { |
322 | if (bitmap[k]) { | |
323 | unsigned long temp = leul_to_cpu(bitmap[k]); | |
324 | ||
5b82b703 SH |
325 | atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp); |
326 | atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); | |
9460dee4 | 327 | if (tcg_enabled()) { |
5b82b703 | 328 | atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp); |
9460dee4 | 329 | } |
ae2810c4 | 330 | } |
5b82b703 SH |
331 | |
332 | if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { | |
333 | offset = 0; | |
334 | idx++; | |
335 | } | |
ae2810c4 | 336 | } |
5b82b703 SH |
337 | |
338 | rcu_read_unlock(); | |
339 | ||
5100afb5 | 340 | xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); |
ae2810c4 | 341 | } else { |
9460dee4 | 342 | uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; |
ae2810c4 JQ |
343 | /* |
344 | * bitmap-traveling is faster than memory-traveling (for addr...) | |
345 | * especially when most of the memory is not dirty. | |
346 | */ | |
347 | for (i = 0; i < len; i++) { | |
348 | if (bitmap[i] != 0) { | |
349 | c = leul_to_cpu(bitmap[i]); | |
350 | do { | |
7224f66e | 351 | j = ctzl(c); |
ae2810c4 JQ |
352 | c &= ~(1ul << j); |
353 | page_number = (i * HOST_LONG_BITS + j) * hpratio; | |
354 | addr = page_number * TARGET_PAGE_SIZE; | |
355 | ram_addr = start + addr; | |
356 | cpu_physical_memory_set_dirty_range(ram_addr, | |
9460dee4 | 357 | TARGET_PAGE_SIZE * hpratio, clients); |
ae2810c4 JQ |
358 | } while (c != 0); |
359 | } | |
5ff7fb77 JQ |
360 | } |
361 | } | |
362 | } | |
fb3ecb7e | 363 | #endif /* not _WIN32 */ |
5ff7fb77 | 364 | |
03eebc9e SH |
365 | bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, |
366 | ram_addr_t length, | |
367 | unsigned client); | |
220c3ebd | 368 | |
8deaf12c GH |
369 | DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty |
370 | (ram_addr_t start, ram_addr_t length, unsigned client); | |
371 | ||
372 | bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, | |
373 | ram_addr_t start, | |
374 | ram_addr_t length); | |
375 | ||
c8d6f66a MT |
376 | static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, |
377 | ram_addr_t length) | |
378 | { | |
03eebc9e SH |
379 | cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); |
380 | cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); | |
381 | cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); | |
c8d6f66a MT |
382 | } |
383 | ||
384 | ||
20015f72 | 385 | static inline |
6b6712ef | 386 | uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, |
20015f72 | 387 | ram_addr_t start, |
1ffb5dfd | 388 | ram_addr_t length, |
68908ed6 | 389 | uint64_t *real_dirty_pages) |
20015f72 SH |
390 | { |
391 | ram_addr_t addr; | |
f70d3451 | 392 | unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS); |
20015f72 | 393 | uint64_t num_dirty = 0; |
6b6712ef | 394 | unsigned long *dest = rb->bmap; |
20015f72 | 395 | |
aa777e29 | 396 | /* start address and length is aligned at the start of a word? */ |
f70d3451 | 397 | if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) == |
aa777e29 DDAG |
398 | (start + rb->offset) && |
399 | !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) { | |
20015f72 SH |
400 | int k; |
401 | int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); | |
5b82b703 | 402 | unsigned long * const *src; |
084140bd HZ |
403 | unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE; |
404 | unsigned long offset = BIT_WORD((word * BITS_PER_LONG) % | |
5b82b703 | 405 | DIRTY_MEMORY_BLOCK_SIZE); |
f70d3451 | 406 | unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); |
5b82b703 SH |
407 | |
408 | rcu_read_lock(); | |
409 | ||
410 | src = atomic_rcu_read( | |
411 | &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; | |
20015f72 SH |
412 | |
413 | for (k = page; k < page + nr; k++) { | |
5b82b703 SH |
414 | if (src[idx][offset]) { |
415 | unsigned long bits = atomic_xchg(&src[idx][offset], 0); | |
20015f72 | 416 | unsigned long new_dirty; |
1ffb5dfd | 417 | *real_dirty_pages += ctpopl(bits); |
20015f72 | 418 | new_dirty = ~dest[k]; |
5f2cb946 SH |
419 | dest[k] |= bits; |
420 | new_dirty &= bits; | |
20015f72 | 421 | num_dirty += ctpopl(new_dirty); |
20015f72 | 422 | } |
5b82b703 SH |
423 | |
424 | if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { | |
425 | offset = 0; | |
426 | idx++; | |
427 | } | |
20015f72 | 428 | } |
5b82b703 SH |
429 | |
430 | rcu_read_unlock(); | |
20015f72 | 431 | } else { |
084140bd HZ |
432 | ram_addr_t offset = rb->offset; |
433 | ||
20015f72 | 434 | for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { |
03eebc9e | 435 | if (cpu_physical_memory_test_and_clear_dirty( |
084140bd | 436 | start + addr + offset, |
03eebc9e SH |
437 | TARGET_PAGE_SIZE, |
438 | DIRTY_MEMORY_MIGRATION)) { | |
1ffb5dfd | 439 | *real_dirty_pages += 1; |
20015f72 SH |
440 | long k = (start + addr) >> TARGET_PAGE_BITS; |
441 | if (!test_and_set_bit(k, dest)) { | |
442 | num_dirty++; | |
443 | } | |
20015f72 SH |
444 | } |
445 | } | |
446 | } | |
447 | ||
448 | return num_dirty; | |
449 | } | |
220c3ebd JQ |
450 | #endif |
451 | #endif |