]>
Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
b79cd8f1 | 2 | /* |
640e1b38 | 3 | * Low level x86 E820 memory map handling functions. |
b79cd8f1 | 4 | * |
640e1b38 IM |
5 | * The firmware and bootloader passes us the "E820 table", which is the primary |
6 | * physical memory layout description available about x86 systems. | |
b79cd8f1 | 7 | * |
640e1b38 IM |
8 | * The kernel takes the E820 memory layout and optionally modifies it with |
9 | * quirks and other tweaks, and feeds that into the generic Linux memory | |
10 | * allocation code routines via a platform independent interface (memblock, etc.). | |
b79cd8f1 | 11 | */ |
93a72052 | 12 | #include <linux/crash_dump.h> |
57c8a661 | 13 | #include <linux/memblock.h> |
bf62f398 | 14 | #include <linux/suspend.h> |
976513db | 15 | #include <linux/acpi.h> |
5dfcf14d | 16 | #include <linux/firmware-map.h> |
d1bbdd66 | 17 | #include <linux/sort.h> |
357b4da5 | 18 | #include <linux/memory_hotplug.h> |
b79cd8f1 | 19 | |
66441bd3 | 20 | #include <asm/e820/api.h> |
b79cd8f1 YL |
21 | #include <asm/setup.h> |
22 | ||
5dfcf14d | 23 | /* |
12df216c | 24 | * We organize the E820 table into three main data structures: |
544a0f47 | 25 | * |
12df216c CY |
26 | * - 'e820_table_firmware': the original firmware version passed to us by the |
27 | * bootloader - not modified by the kernel. It is composed of two parts: | |
28 | * the first 128 E820 memory entries in boot_params.e820_table and the remaining | |
29 | * (if any) entries of the SETUP_E820_EXT nodes. We use this to: | |
544a0f47 IM |
30 | * |
31 | * - inform the user about the firmware's notion of memory layout | |
32 | * via /sys/firmware/memmap | |
33 | * | |
34 | * - the hibernation code uses it to generate a kernel-independent MD5 | |
35 | * fingerprint of the physical memory layout of a system. | |
36 | * | |
12df216c CY |
37 | * - 'e820_table_kexec': a slightly modified (by the kernel) firmware version |
38 | * passed to us by the bootloader - the major difference between | |
39 | * e820_table_firmware[] and this one is that, the latter marks the setup_data | |
40 | * list created by the EFI boot stub as reserved, so that kexec can reuse the | |
41 | * setup_data information in the second kernel. Besides, e820_table_kexec[] | |
42 | * might also be modified by the kexec itself to fake a mptable. | |
43 | * We use this to: | |
44 | * | |
640e1b38 | 45 | * - kexec, which is a bootloader in disguise, uses the original E820 |
544a0f47 | 46 | * layout to pass to the kexec-ed kernel. This way the original kernel |
640e1b38 | 47 | * can have a restricted E820 map while the kexec()-ed kexec-kernel |
544a0f47 IM |
48 | * can have access to full memory - etc. |
49 | * | |
640e1b38 | 50 | * - 'e820_table': this is the main E820 table that is massaged by the |
544a0f47 IM |
51 | * low level x86 platform code, or modified by boot parameters, before |
52 | * passed on to higher level MM layers. | |
53 | * | |
640e1b38 | 54 | * Once the E820 map has been converted to the standard Linux memory layout |
544a0f47 IM |
55 | * information its role stops - modifying it has no effect and does not get |
56 | * re-propagated. So itsmain role is a temporary bootstrap storage of firmware | |
57 | * specific memory layout data during early bootup. | |
5dfcf14d | 58 | */ |
544a0f47 | 59 | static struct e820_table e820_table_init __initdata; |
a09bae0f | 60 | static struct e820_table e820_table_kexec_init __initdata; |
12df216c | 61 | static struct e820_table e820_table_firmware_init __initdata; |
544a0f47 IM |
62 | |
63 | struct e820_table *e820_table __refdata = &e820_table_init; | |
a09bae0f | 64 | struct e820_table *e820_table_kexec __refdata = &e820_table_kexec_init; |
12df216c | 65 | struct e820_table *e820_table_firmware __refdata = &e820_table_firmware_init; |
b79cd8f1 YL |
66 | |
67 | /* For PCI or other memory-mapped resources */ | |
68 | unsigned long pci_mem_start = 0xaeedbabe; | |
69 | #ifdef CONFIG_PCI | |
70 | EXPORT_SYMBOL(pci_mem_start); | |
71 | #endif | |
72 | ||
73 | /* | |
74 | * This function checks if any part of the range <start,end> is mapped | |
75 | * with type. | |
76 | */ | |
0c55671f KA |
77 | static bool _e820__mapped_any(struct e820_table *table, |
78 | u64 start, u64 end, enum e820_type type) | |
b79cd8f1 YL |
79 | { |
80 | int i; | |
81 | ||
0c55671f KA |
82 | for (i = 0; i < table->nr_entries; i++) { |
83 | struct e820_entry *entry = &table->entries[i]; | |
b79cd8f1 | 84 | |
e5540f87 | 85 | if (type && entry->type != type) |
b79cd8f1 | 86 | continue; |
e5540f87 | 87 | if (entry->addr >= end || entry->addr + entry->size <= start) |
b79cd8f1 | 88 | continue; |
f709f814 | 89 | return true; |
b79cd8f1 | 90 | } |
f709f814 | 91 | return false; |
b79cd8f1 | 92 | } |
0c55671f KA |
93 | |
94 | bool e820__mapped_raw_any(u64 start, u64 end, enum e820_type type) | |
95 | { | |
96 | return _e820__mapped_any(e820_table_firmware, start, end, type); | |
97 | } | |
98 | EXPORT_SYMBOL_GPL(e820__mapped_raw_any); | |
99 | ||
100 | bool e820__mapped_any(u64 start, u64 end, enum e820_type type) | |
101 | { | |
102 | return _e820__mapped_any(e820_table, start, end, type); | |
103 | } | |
3bce64f0 | 104 | EXPORT_SYMBOL_GPL(e820__mapped_any); |
b79cd8f1 YL |
105 | |
106 | /* | |
640e1b38 | 107 | * This function checks if the entire <start,end> range is mapped with 'type'. |
b79cd8f1 | 108 | * |
640e1b38 IM |
109 | * Note: this function only works correctly once the E820 table is sorted and |
110 | * not-overlapping (at least for the range specified), which is the case normally. | |
b79cd8f1 | 111 | */ |
d68baa3f TL |
112 | static struct e820_entry *__e820__mapped_all(u64 start, u64 end, |
113 | enum e820_type type) | |
b79cd8f1 YL |
114 | { |
115 | int i; | |
116 | ||
bf495573 | 117 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 118 | struct e820_entry *entry = &e820_table->entries[i]; |
b79cd8f1 | 119 | |
e5540f87 | 120 | if (type && entry->type != type) |
b79cd8f1 | 121 | continue; |
640e1b38 IM |
122 | |
123 | /* Is the region (part) in overlap with the current region? */ | |
e5540f87 | 124 | if (entry->addr >= end || entry->addr + entry->size <= start) |
b79cd8f1 YL |
125 | continue; |
126 | ||
640e1b38 IM |
127 | /* |
128 | * If the region is at the beginning of <start,end> we move | |
129 | * 'start' to the end of the region since it's ok until there | |
b79cd8f1 | 130 | */ |
e5540f87 IM |
131 | if (entry->addr <= start) |
132 | start = entry->addr + entry->size; | |
640e1b38 | 133 | |
b79cd8f1 | 134 | /* |
640e1b38 IM |
135 | * If 'start' is now at or beyond 'end', we're done, full |
136 | * coverage of the desired range exists: | |
b79cd8f1 YL |
137 | */ |
138 | if (start >= end) | |
d68baa3f | 139 | return entry; |
b79cd8f1 | 140 | } |
d68baa3f TL |
141 | |
142 | return NULL; | |
143 | } | |
144 | ||
145 | /* | |
146 | * This function checks if the entire range <start,end> is mapped with type. | |
147 | */ | |
148 | bool __init e820__mapped_all(u64 start, u64 end, enum e820_type type) | |
149 | { | |
150 | return __e820__mapped_all(start, end, type); | |
151 | } | |
152 | ||
153 | /* | |
154 | * This function returns the type associated with the range <start,end>. | |
155 | */ | |
156 | int e820__get_entry_type(u64 start, u64 end) | |
157 | { | |
158 | struct e820_entry *entry = __e820__mapped_all(start, end, 0); | |
159 | ||
160 | return entry ? entry->type : -EINVAL; | |
b79cd8f1 YL |
161 | } |
162 | ||
163 | /* | |
640e1b38 | 164 | * Add a memory region to the kernel E820 map. |
b79cd8f1 | 165 | */ |
6afc03b8 | 166 | static void __init __e820__range_add(struct e820_table *table, u64 start, u64 size, enum e820_type type) |
b79cd8f1 | 167 | { |
bf495573 | 168 | int x = table->nr_entries; |
b79cd8f1 | 169 | |
bf495573 | 170 | if (x >= ARRAY_SIZE(table->entries)) { |
1de392f5 JP |
171 | pr_err("too many entries; ignoring [mem %#010llx-%#010llx]\n", |
172 | start, start + size - 1); | |
b79cd8f1 YL |
173 | return; |
174 | } | |
175 | ||
bf495573 IM |
176 | table->entries[x].addr = start; |
177 | table->entries[x].size = size; | |
178 | table->entries[x].type = type; | |
179 | table->nr_entries++; | |
773e673d YL |
180 | } |
181 | ||
6afc03b8 | 182 | void __init e820__range_add(u64 start, u64 size, enum e820_type type) |
773e673d | 183 | { |
ab6bc04c | 184 | __e820__range_add(e820_table, start, size, type); |
b79cd8f1 YL |
185 | } |
186 | ||
6afc03b8 | 187 | static void __init e820_print_type(enum e820_type type) |
c61cf4cf YL |
188 | { |
189 | switch (type) { | |
09821ff1 IM |
190 | case E820_TYPE_RAM: /* Fall through: */ |
191 | case E820_TYPE_RESERVED_KERN: pr_cont("usable"); break; | |
192 | case E820_TYPE_RESERVED: pr_cont("reserved"); break; | |
262b45ae | 193 | case E820_TYPE_SOFT_RESERVED: pr_cont("soft reserved"); break; |
09821ff1 IM |
194 | case E820_TYPE_ACPI: pr_cont("ACPI data"); break; |
195 | case E820_TYPE_NVS: pr_cont("ACPI NVS"); break; | |
196 | case E820_TYPE_UNUSABLE: pr_cont("unusable"); break; | |
197 | case E820_TYPE_PMEM: /* Fall through: */ | |
198 | case E820_TYPE_PRAM: pr_cont("persistent (type %u)", type); break; | |
01259ef1 | 199 | default: pr_cont("type %u", type); break; |
c61cf4cf YL |
200 | } |
201 | } | |
202 | ||
be0c3f0f | 203 | void __init e820__print_table(char *who) |
b79cd8f1 YL |
204 | { |
205 | int i; | |
206 | ||
bf495573 | 207 | for (i = 0; i < e820_table->nr_entries; i++) { |
1de392f5 JP |
208 | pr_info("%s: [mem %#018Lx-%#018Lx] ", |
209 | who, | |
210 | e820_table->entries[i].addr, | |
211 | e820_table->entries[i].addr + e820_table->entries[i].size - 1); | |
640e1b38 | 212 | |
bf495573 | 213 | e820_print_type(e820_table->entries[i].type); |
01259ef1 | 214 | pr_cont("\n"); |
b79cd8f1 YL |
215 | } |
216 | } | |
217 | ||
218 | /* | |
9a02fd0f | 219 | * Sanitize an E820 map. |
b79cd8f1 | 220 | * |
9a02fd0f | 221 | * Some E820 layouts include overlapping entries. The following |
640e1b38 | 222 | * replaces the original E820 map with a new one, removing overlaps, |
5b7eb2e9 PJ |
223 | * and resolving conflicting memory types in favor of highest |
224 | * numbered type. | |
b79cd8f1 | 225 | * |
9a02fd0f IM |
226 | * The input parameter 'entries' points to an array of 'struct |
227 | * e820_entry' which on entry has elements in the range [0, *nr_entries) | |
228 | * valid, and which has space for up to max_nr_entries entries. | |
640e1b38 | 229 | * On return, the resulting sanitized E820 map entries will be in |
9a02fd0f | 230 | * overwritten in the same location, starting at 'entries'. |
5b7eb2e9 | 231 | * |
9a02fd0f IM |
232 | * The integer pointed to by nr_entries must be valid on entry (the |
233 | * current number of valid entries located at 'entries'). If the | |
234 | * sanitizing succeeds the *nr_entries will be updated with the new | |
235 | * number of valid entries (something no more than max_nr_entries). | |
5b7eb2e9 | 236 | * |
f52355a9 | 237 | * The return value from e820__update_table() is zero if it |
5b7eb2e9 PJ |
238 | * successfully 'sanitized' the map entries passed in, and is -1 |
239 | * if it did nothing, which can happen if either of (1) it was | |
240 | * only passed one map entry, or (2) any of the input map entries | |
241 | * were invalid (start + size < start, meaning that the size was | |
242 | * so big the described memory range wrapped around through zero.) | |
243 | * | |
244 | * Visually we're performing the following | |
245 | * (1,2,3,4 = memory types)... | |
246 | * | |
247 | * Sample memory map (w/overlaps): | |
248 | * ____22__________________ | |
249 | * ______________________4_ | |
250 | * ____1111________________ | |
251 | * _44_____________________ | |
252 | * 11111111________________ | |
253 | * ____________________33__ | |
254 | * ___________44___________ | |
255 | * __________33333_________ | |
256 | * ______________22________ | |
257 | * ___________________2222_ | |
258 | * _________111111111______ | |
259 | * _____________________11_ | |
260 | * _________________4______ | |
261 | * | |
262 | * Sanitized equivalent (no overlap): | |
263 | * 1_______________________ | |
264 | * _44_____________________ | |
265 | * ___1____________________ | |
266 | * ____22__________________ | |
267 | * ______11________________ | |
268 | * _________1______________ | |
269 | * __________3_____________ | |
270 | * ___________44___________ | |
271 | * _____________33_________ | |
272 | * _______________2________ | |
273 | * ________________1_______ | |
274 | * _________________4______ | |
275 | * ___________________2____ | |
276 | * ____________________33__ | |
277 | * ______________________4_ | |
b79cd8f1 | 278 | */ |
d1bbdd66 | 279 | struct change_member { |
9a02fd0f IM |
280 | /* Pointer to the original entry: */ |
281 | struct e820_entry *entry; | |
640e1b38 IM |
282 | /* Address for this change point: */ |
283 | unsigned long long addr; | |
d1bbdd66 MD |
284 | }; |
285 | ||
441ac2f3 IM |
286 | static struct change_member change_point_list[2*E820_MAX_ENTRIES] __initdata; |
287 | static struct change_member *change_point[2*E820_MAX_ENTRIES] __initdata; | |
288 | static struct e820_entry *overlap_list[E820_MAX_ENTRIES] __initdata; | |
289 | static struct e820_entry new_entries[E820_MAX_ENTRIES] __initdata; | |
290 | ||
d1bbdd66 MD |
291 | static int __init cpcompare(const void *a, const void *b) |
292 | { | |
293 | struct change_member * const *app = a, * const *bpp = b; | |
294 | const struct change_member *ap = *app, *bp = *bpp; | |
295 | ||
296 | /* | |
297 | * Inputs are pointers to two elements of change_point[]. If their | |
640e1b38 | 298 | * addresses are not equal, their difference dominates. If the addresses |
d1bbdd66 MD |
299 | * are equal, then consider one that represents the end of its region |
300 | * to be greater than one that does not. | |
301 | */ | |
302 | if (ap->addr != bp->addr) | |
303 | return ap->addr > bp->addr ? 1 : -1; | |
304 | ||
9a02fd0f | 305 | return (ap->addr != ap->entry->addr) - (bp->addr != bp->entry->addr); |
d1bbdd66 | 306 | } |
5b7eb2e9 | 307 | |
441ac2f3 | 308 | int __init e820__update_table(struct e820_table *table) |
b79cd8f1 | 309 | { |
441ac2f3 IM |
310 | struct e820_entry *entries = table->entries; |
311 | u32 max_nr_entries = ARRAY_SIZE(table->entries); | |
6afc03b8 | 312 | enum e820_type current_type, last_type; |
b79cd8f1 | 313 | unsigned long long last_addr; |
441ac2f3 IM |
314 | u32 new_nr_entries, overlap_entries; |
315 | u32 i, chg_idx, chg_nr; | |
b79cd8f1 | 316 | |
640e1b38 | 317 | /* If there's only one memory region, don't bother: */ |
441ac2f3 | 318 | if (table->nr_entries < 2) |
b79cd8f1 YL |
319 | return -1; |
320 | ||
441ac2f3 | 321 | BUG_ON(table->nr_entries > max_nr_entries); |
b79cd8f1 | 322 | |
9a02fd0f | 323 | /* Bail out if we find any unreasonable addresses in the map: */ |
441ac2f3 | 324 | for (i = 0; i < table->nr_entries; i++) { |
9a02fd0f | 325 | if (entries[i].addr + entries[i].size < entries[i].addr) |
b79cd8f1 | 326 | return -1; |
640e1b38 | 327 | } |
b79cd8f1 | 328 | |
640e1b38 | 329 | /* Create pointers for initial change-point information (for sorting): */ |
441ac2f3 | 330 | for (i = 0; i < 2 * table->nr_entries; i++) |
b79cd8f1 YL |
331 | change_point[i] = &change_point_list[i]; |
332 | ||
640e1b38 IM |
333 | /* |
334 | * Record all known change-points (starting and ending addresses), | |
335 | * omitting empty memory regions: | |
336 | */ | |
441ac2f3 IM |
337 | chg_idx = 0; |
338 | for (i = 0; i < table->nr_entries; i++) { | |
9a02fd0f | 339 | if (entries[i].size != 0) { |
441ac2f3 IM |
340 | change_point[chg_idx]->addr = entries[i].addr; |
341 | change_point[chg_idx++]->entry = &entries[i]; | |
342 | change_point[chg_idx]->addr = entries[i].addr + entries[i].size; | |
343 | change_point[chg_idx++]->entry = &entries[i]; | |
b79cd8f1 YL |
344 | } |
345 | } | |
441ac2f3 | 346 | chg_nr = chg_idx; |
b79cd8f1 | 347 | |
640e1b38 | 348 | /* Sort change-point list by memory addresses (low -> high): */ |
d88961b5 | 349 | sort(change_point, chg_nr, sizeof(*change_point), cpcompare, NULL); |
b79cd8f1 | 350 | |
9a02fd0f | 351 | /* Create a new memory map, removing overlaps: */ |
640e1b38 | 352 | overlap_entries = 0; /* Number of entries in the overlap table */ |
9a02fd0f | 353 | new_nr_entries = 0; /* Index for creating new map entries */ |
640e1b38 IM |
354 | last_type = 0; /* Start with undefined memory type */ |
355 | last_addr = 0; /* Start with 0 as last starting address */ | |
b79cd8f1 | 356 | |
9a02fd0f | 357 | /* Loop through change-points, determining effect on the new map: */ |
441ac2f3 | 358 | for (chg_idx = 0; chg_idx < chg_nr; chg_idx++) { |
9a02fd0f | 359 | /* Keep track of all overlapping entries */ |
441ac2f3 | 360 | if (change_point[chg_idx]->addr == change_point[chg_idx]->entry->addr) { |
640e1b38 | 361 | /* Add map entry to overlap list (> 1 entry implies an overlap) */ |
441ac2f3 | 362 | overlap_list[overlap_entries++] = change_point[chg_idx]->entry; |
b79cd8f1 | 363 | } else { |
640e1b38 | 364 | /* Remove entry from list (order independent, so swap with last): */ |
b79cd8f1 | 365 | for (i = 0; i < overlap_entries; i++) { |
441ac2f3 | 366 | if (overlap_list[i] == change_point[chg_idx]->entry) |
640e1b38 | 367 | overlap_list[i] = overlap_list[overlap_entries-1]; |
b79cd8f1 YL |
368 | } |
369 | overlap_entries--; | |
370 | } | |
371 | /* | |
640e1b38 | 372 | * If there are overlapping entries, decide which |
b79cd8f1 YL |
373 | * "type" to use (larger value takes precedence -- |
374 | * 1=usable, 2,3,4,4+=unusable) | |
375 | */ | |
376 | current_type = 0; | |
640e1b38 | 377 | for (i = 0; i < overlap_entries; i++) { |
b79cd8f1 YL |
378 | if (overlap_list[i]->type > current_type) |
379 | current_type = overlap_list[i]->type; | |
640e1b38 IM |
380 | } |
381 | ||
9a02fd0f | 382 | /* Continue building up new map based on this information: */ |
09821ff1 | 383 | if (current_type != last_type || current_type == E820_TYPE_PRAM) { |
b79cd8f1 | 384 | if (last_type != 0) { |
441ac2f3 | 385 | new_entries[new_nr_entries].size = change_point[chg_idx]->addr - last_addr; |
640e1b38 | 386 | /* Move forward only if the new size was non-zero: */ |
9a02fd0f IM |
387 | if (new_entries[new_nr_entries].size != 0) |
388 | /* No more space left for new entries? */ | |
389 | if (++new_nr_entries >= max_nr_entries) | |
b79cd8f1 YL |
390 | break; |
391 | } | |
392 | if (current_type != 0) { | |
441ac2f3 | 393 | new_entries[new_nr_entries].addr = change_point[chg_idx]->addr; |
9a02fd0f | 394 | new_entries[new_nr_entries].type = current_type; |
441ac2f3 | 395 | last_addr = change_point[chg_idx]->addr; |
b79cd8f1 YL |
396 | } |
397 | last_type = current_type; | |
398 | } | |
399 | } | |
640e1b38 | 400 | |
9a02fd0f | 401 | /* Copy the new entries into the original location: */ |
441ac2f3 IM |
402 | memcpy(entries, new_entries, new_nr_entries*sizeof(*entries)); |
403 | table->nr_entries = new_nr_entries; | |
b79cd8f1 YL |
404 | |
405 | return 0; | |
406 | } | |
407 | ||
7410aa1c | 408 | static int __init __append_e820_table(struct boot_e820_entry *entries, u32 nr_entries) |
8c5beb50 | 409 | { |
7410aa1c | 410 | struct boot_e820_entry *entry = entries; |
9a02fd0f IM |
411 | |
412 | while (nr_entries) { | |
413 | u64 start = entry->addr; | |
414 | u64 size = entry->size; | |
3ec97965 | 415 | u64 end = start + size - 1; |
9a02fd0f | 416 | u32 type = entry->type; |
8c5beb50 | 417 | |
640e1b38 | 418 | /* Ignore the entry on 64-bit overflow: */ |
3ec97965 | 419 | if (start > end && likely(size)) |
8c5beb50 HY |
420 | return -1; |
421 | ||
ab6bc04c | 422 | e820__range_add(start, size, type); |
8c5beb50 | 423 | |
9a02fd0f IM |
424 | entry++; |
425 | nr_entries--; | |
8c5beb50 HY |
426 | } |
427 | return 0; | |
428 | } | |
429 | ||
b79cd8f1 | 430 | /* |
640e1b38 | 431 | * Copy the BIOS E820 map into a safe place. |
b79cd8f1 YL |
432 | * |
433 | * Sanity-check it while we're at it.. | |
434 | * | |
435 | * If we're lucky and live on a modern system, the setup code | |
436 | * will have given us a memory map that we can use to properly | |
437 | * set up memory. If we aren't, we'll fake a memory map. | |
438 | */ | |
7410aa1c | 439 | static int __init append_e820_table(struct boot_e820_entry *entries, u32 nr_entries) |
b79cd8f1 YL |
440 | { |
441 | /* Only one memory region (or negative)? Ignore it */ | |
9a02fd0f | 442 | if (nr_entries < 2) |
b79cd8f1 YL |
443 | return -1; |
444 | ||
9a02fd0f | 445 | return __append_e820_table(entries, nr_entries); |
b79cd8f1 YL |
446 | } |
447 | ||
640e1b38 | 448 | static u64 __init |
6afc03b8 | 449 | __e820__range_update(struct e820_table *table, u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) |
b79cd8f1 | 450 | { |
78a8b35b | 451 | u64 end; |
773e673d | 452 | unsigned int i; |
b79cd8f1 YL |
453 | u64 real_updated_size = 0; |
454 | ||
455 | BUG_ON(old_type == new_type); | |
456 | ||
232b957a YL |
457 | if (size > (ULLONG_MAX - start)) |
458 | size = ULLONG_MAX - start; | |
459 | ||
78a8b35b | 460 | end = start + size; |
e22af0be | 461 | printk(KERN_DEBUG "e820: update [mem %#010Lx-%#010Lx] ", start, end - 1); |
c61cf4cf | 462 | e820_print_type(old_type); |
01259ef1 | 463 | pr_cont(" ==> "); |
c61cf4cf | 464 | e820_print_type(new_type); |
01259ef1 | 465 | pr_cont("\n"); |
c61cf4cf | 466 | |
bf495573 | 467 | for (i = 0; i < table->nr_entries; i++) { |
e5540f87 | 468 | struct e820_entry *entry = &table->entries[i]; |
b79cd8f1 | 469 | u64 final_start, final_end; |
e5540f87 | 470 | u64 entry_end; |
78a8b35b | 471 | |
e5540f87 | 472 | if (entry->type != old_type) |
b79cd8f1 | 473 | continue; |
78a8b35b | 474 | |
e5540f87 | 475 | entry_end = entry->addr + entry->size; |
640e1b38 IM |
476 | |
477 | /* Completely covered by new range? */ | |
e5540f87 IM |
478 | if (entry->addr >= start && entry_end <= end) { |
479 | entry->type = new_type; | |
480 | real_updated_size += entry->size; | |
b79cd8f1 YL |
481 | continue; |
482 | } | |
78a8b35b | 483 | |
640e1b38 | 484 | /* New range is completely covered? */ |
e5540f87 | 485 | if (entry->addr < start && entry_end > end) { |
ab6bc04c IM |
486 | __e820__range_add(table, start, size, new_type); |
487 | __e820__range_add(table, end, entry_end - end, entry->type); | |
e5540f87 | 488 | entry->size = start - entry->addr; |
78a8b35b YL |
489 | real_updated_size += size; |
490 | continue; | |
491 | } | |
492 | ||
640e1b38 | 493 | /* Partially covered: */ |
e5540f87 IM |
494 | final_start = max(start, entry->addr); |
495 | final_end = min(end, entry_end); | |
b79cd8f1 YL |
496 | if (final_start >= final_end) |
497 | continue; | |
5c0e6f03 | 498 | |
ab6bc04c | 499 | __e820__range_add(table, final_start, final_end - final_start, new_type); |
5c0e6f03 | 500 | |
b79cd8f1 | 501 | real_updated_size += final_end - final_start; |
976dd4dc | 502 | |
773e673d | 503 | /* |
640e1b38 IM |
504 | * Left range could be head or tail, so need to update |
505 | * its size first: | |
773e673d | 506 | */ |
e5540f87 IM |
507 | entry->size -= final_end - final_start; |
508 | if (entry->addr < final_start) | |
976dd4dc | 509 | continue; |
640e1b38 | 510 | |
e5540f87 | 511 | entry->addr = final_end; |
b79cd8f1 YL |
512 | } |
513 | return real_updated_size; | |
514 | } | |
515 | ||
6afc03b8 | 516 | u64 __init e820__range_update(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) |
fc9036ea | 517 | { |
ab6bc04c | 518 | return __e820__range_update(e820_table, start, size, old_type, new_type); |
fc9036ea YL |
519 | } |
520 | ||
a09bae0f | 521 | static u64 __init e820__range_update_kexec(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type) |
fc9036ea | 522 | { |
a09bae0f | 523 | return __e820__range_update(e820_table_kexec, start, size, old_type, new_type); |
fc9036ea YL |
524 | } |
525 | ||
640e1b38 | 526 | /* Remove a range of memory from the E820 table: */ |
81b3e090 | 527 | u64 __init e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool check_type) |
7a1fd986 YL |
528 | { |
529 | int i; | |
1b5576e6 | 530 | u64 end; |
7a1fd986 YL |
531 | u64 real_removed_size = 0; |
532 | ||
232b957a YL |
533 | if (size > (ULLONG_MAX - start)) |
534 | size = ULLONG_MAX - start; | |
535 | ||
1b5576e6 | 536 | end = start + size; |
e22af0be | 537 | printk(KERN_DEBUG "e820: remove [mem %#010Lx-%#010Lx] ", start, end - 1); |
81b3e090 | 538 | if (check_type) |
9f3a5f52 | 539 | e820_print_type(old_type); |
01259ef1 | 540 | pr_cont("\n"); |
1b5576e6 | 541 | |
bf495573 | 542 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 543 | struct e820_entry *entry = &e820_table->entries[i]; |
7a1fd986 | 544 | u64 final_start, final_end; |
e5540f87 | 545 | u64 entry_end; |
7a1fd986 | 546 | |
81b3e090 | 547 | if (check_type && entry->type != old_type) |
7a1fd986 | 548 | continue; |
9f3a5f52 | 549 | |
e5540f87 | 550 | entry_end = entry->addr + entry->size; |
640e1b38 IM |
551 | |
552 | /* Completely covered? */ | |
e5540f87 IM |
553 | if (entry->addr >= start && entry_end <= end) { |
554 | real_removed_size += entry->size; | |
d88961b5 | 555 | memset(entry, 0, sizeof(*entry)); |
7a1fd986 YL |
556 | continue; |
557 | } | |
9f3a5f52 | 558 | |
640e1b38 | 559 | /* Is the new range completely covered? */ |
e5540f87 | 560 | if (entry->addr < start && entry_end > end) { |
ab6bc04c | 561 | e820__range_add(end, entry_end - end, entry->type); |
e5540f87 | 562 | entry->size = start - entry->addr; |
9f3a5f52 YL |
563 | real_removed_size += size; |
564 | continue; | |
565 | } | |
566 | ||
640e1b38 | 567 | /* Partially covered: */ |
e5540f87 IM |
568 | final_start = max(start, entry->addr); |
569 | final_end = min(end, entry_end); | |
7a1fd986 YL |
570 | if (final_start >= final_end) |
571 | continue; | |
640e1b38 | 572 | |
7a1fd986 YL |
573 | real_removed_size += final_end - final_start; |
574 | ||
9f3a5f52 | 575 | /* |
640e1b38 IM |
576 | * Left range could be head or tail, so need to update |
577 | * the size first: | |
9f3a5f52 | 578 | */ |
e5540f87 IM |
579 | entry->size -= final_end - final_start; |
580 | if (entry->addr < final_start) | |
7a1fd986 | 581 | continue; |
640e1b38 | 582 | |
e5540f87 | 583 | entry->addr = final_end; |
7a1fd986 YL |
584 | } |
585 | return real_removed_size; | |
586 | } | |
587 | ||
6464d294 | 588 | void __init e820__update_table_print(void) |
b79cd8f1 | 589 | { |
f9748fa0 | 590 | if (e820__update_table(e820_table)) |
b79cd8f1 | 591 | return; |
640e1b38 | 592 | |
1de392f5 | 593 | pr_info("modified physical RAM map:\n"); |
be0c3f0f | 594 | e820__print_table("modified"); |
b79cd8f1 | 595 | } |
640e1b38 | 596 | |
a09bae0f | 597 | static void __init e820__update_table_kexec(void) |
fc9036ea | 598 | { |
a09bae0f | 599 | e820__update_table(e820_table_kexec); |
fc9036ea | 600 | } |
640e1b38 | 601 | |
fd6493e1 | 602 | #define MAX_GAP_END 0x100000000ull |
640e1b38 | 603 | |
b79cd8f1 | 604 | /* |
640e1b38 | 605 | * Search for a gap in the E820 memory space from 0 to MAX_GAP_END (4GB). |
b79cd8f1 | 606 | */ |
640e1b38 | 607 | static int __init e820_search_gap(unsigned long *gapstart, unsigned long *gapsize) |
b79cd8f1 | 608 | { |
b4ed1d15 | 609 | unsigned long long last = MAX_GAP_END; |
bf495573 | 610 | int i = e820_table->nr_entries; |
b79cd8f1 YL |
611 | int found = 0; |
612 | ||
b79cd8f1 | 613 | while (--i >= 0) { |
bf495573 IM |
614 | unsigned long long start = e820_table->entries[i].addr; |
615 | unsigned long long end = start + e820_table->entries[i].size; | |
b79cd8f1 YL |
616 | |
617 | /* | |
618 | * Since "last" is at most 4GB, we know we'll | |
640e1b38 | 619 | * fit in 32 bits if this condition is true: |
b79cd8f1 YL |
620 | */ |
621 | if (last > end) { | |
622 | unsigned long gap = last - end; | |
623 | ||
3381959d AK |
624 | if (gap >= *gapsize) { |
625 | *gapsize = gap; | |
626 | *gapstart = end; | |
b79cd8f1 YL |
627 | found = 1; |
628 | } | |
629 | } | |
630 | if (start < last) | |
631 | last = start; | |
632 | } | |
3381959d AK |
633 | return found; |
634 | } | |
635 | ||
636 | /* | |
640e1b38 IM |
637 | * Search for the biggest gap in the low 32 bits of the E820 |
638 | * memory space. We pass this space to the PCI subsystem, so | |
639 | * that it can assign MMIO resources for hotplug or | |
640 | * unconfigured devices in. | |
641 | * | |
3381959d AK |
642 | * Hopefully the BIOS let enough space left. |
643 | */ | |
2df908ba | 644 | __init void e820__setup_pci_gap(void) |
3381959d | 645 | { |
5d423ccd | 646 | unsigned long gapstart, gapsize; |
3381959d AK |
647 | int found; |
648 | ||
3381959d | 649 | gapsize = 0x400000; |
b4ed1d15 | 650 | found = e820_search_gap(&gapstart, &gapsize); |
b79cd8f1 | 651 | |
b79cd8f1 | 652 | if (!found) { |
c19a5f35 | 653 | #ifdef CONFIG_X86_64 |
c987d12f | 654 | gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024; |
1de392f5 JP |
655 | pr_err("Cannot find an available gap in the 32-bit address range\n"); |
656 | pr_err("PCI devices with unassigned 32-bit BARs may not work!\n"); | |
c19a5f35 AB |
657 | #else |
658 | gapstart = 0x10000000; | |
b79cd8f1 | 659 | #endif |
c19a5f35 | 660 | } |
b79cd8f1 YL |
661 | |
662 | /* | |
1506c8dc | 663 | * e820__reserve_resources_late() protects stolen RAM already: |
b79cd8f1 | 664 | */ |
5d423ccd | 665 | pci_mem_start = gapstart; |
b79cd8f1 | 666 | |
1de392f5 JP |
667 | pr_info("[mem %#010lx-%#010lx] available for PCI devices\n", |
668 | gapstart, gapstart + gapsize - 1); | |
b79cd8f1 YL |
669 | } |
670 | ||
47533968 DV |
671 | /* |
672 | * Called late during init, in free_initmem(). | |
673 | * | |
a09bae0f | 674 | * Initial e820_table and e820_table_kexec are largish __initdata arrays. |
640e1b38 IM |
675 | * |
676 | * Copy them to a (usually much smaller) dynamically allocated area that is | |
677 | * sized precisely after the number of e820 entries. | |
678 | * | |
679 | * This is done after we've performed all the fixes and tweaks to the tables. | |
680 | * All functions which modify them are __init functions, which won't exist | |
681 | * after free_initmem(). | |
47533968 | 682 | */ |
0c6fc11a | 683 | __init void e820__reallocate_tables(void) |
47533968 | 684 | { |
61a50101 | 685 | struct e820_table *n; |
47533968 DV |
686 | int size; |
687 | ||
640e1b38 | 688 | size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table->nr_entries; |
345dca4c | 689 | n = kmemdup(e820_table, size, GFP_KERNEL); |
47533968 | 690 | BUG_ON(!n); |
61a50101 | 691 | e820_table = n; |
47533968 | 692 | |
a09bae0f | 693 | size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_kexec->nr_entries; |
345dca4c | 694 | n = kmemdup(e820_table_kexec, size, GFP_KERNEL); |
47533968 | 695 | BUG_ON(!n); |
a09bae0f | 696 | e820_table_kexec = n; |
12df216c CY |
697 | |
698 | size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry)*e820_table_firmware->nr_entries; | |
345dca4c | 699 | n = kmemdup(e820_table_firmware, size, GFP_KERNEL); |
12df216c | 700 | BUG_ON(!n); |
12df216c | 701 | e820_table_firmware = n; |
47533968 DV |
702 | } |
703 | ||
640e1b38 IM |
704 | /* |
705 | * Because of the small fixed size of struct boot_params, only the first | |
706 | * 128 E820 memory entries are passed to the kernel via boot_params.e820_table, | |
707 | * the remaining (if any) entries are passed via the SETUP_E820_EXT node of | |
708 | * struct setup_data, which is parsed here. | |
8c5beb50 | 709 | */ |
914053c0 | 710 | void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len) |
8c5beb50 | 711 | { |
8c5beb50 | 712 | int entries; |
7410aa1c | 713 | struct boot_e820_entry *extmap; |
30e46b57 | 714 | struct setup_data *sdata; |
8c5beb50 | 715 | |
30e46b57 | 716 | sdata = early_memremap(phys_addr, data_len); |
d88961b5 | 717 | entries = sdata->len / sizeof(*extmap); |
7410aa1c | 718 | extmap = (struct boot_e820_entry *)(sdata->data); |
640e1b38 | 719 | |
61a50101 | 720 | __append_e820_table(extmap, entries); |
f9748fa0 | 721 | e820__update_table(e820_table); |
640e1b38 | 722 | |
a09bae0f | 723 | memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); |
12df216c | 724 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); |
b7a67e02 | 725 | |
8d4a40bc | 726 | early_memunmap(sdata, data_len); |
1de392f5 | 727 | pr_info("extended physical RAM map:\n"); |
be0c3f0f | 728 | e820__print_table("extended"); |
8c5beb50 HY |
729 | } |
730 | ||
090d7171 | 731 | /* |
bf62f398 | 732 | * Find the ranges of physical addresses that do not correspond to |
090d7171 | 733 | * E820 RAM areas and register the corresponding pages as 'nosave' for |
640e1b38 | 734 | * hibernation (32-bit) or software suspend and suspend to RAM (64-bit). |
bf62f398 | 735 | * |
640e1b38 | 736 | * This function requires the E820 map to be sorted and without any |
84779575 | 737 | * overlapping entries. |
bf62f398 | 738 | */ |
090d7171 | 739 | void __init e820__register_nosave_regions(unsigned long limit_pfn) |
bf62f398 YL |
740 | { |
741 | int i; | |
84779575 | 742 | unsigned long pfn = 0; |
bf62f398 | 743 | |
bf495573 | 744 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 745 | struct e820_entry *entry = &e820_table->entries[i]; |
bf62f398 | 746 | |
e5540f87 IM |
747 | if (pfn < PFN_UP(entry->addr)) |
748 | register_nosave_region(pfn, PFN_UP(entry->addr)); | |
bf62f398 | 749 | |
e5540f87 | 750 | pfn = PFN_DOWN(entry->addr + entry->size); |
ec776ef6 | 751 | |
09821ff1 | 752 | if (entry->type != E820_TYPE_RAM && entry->type != E820_TYPE_RESERVED_KERN) |
e5540f87 | 753 | register_nosave_region(PFN_UP(entry->addr), pfn); |
bf62f398 YL |
754 | |
755 | if (pfn >= limit_pfn) | |
756 | break; | |
757 | } | |
758 | } | |
a4c81cf6 | 759 | |
b54ac6d2 | 760 | #ifdef CONFIG_ACPI |
640e1b38 IM |
761 | /* |
762 | * Register ACPI NVS memory regions, so that we can save/restore them during | |
763 | * hibernation and the subsequent resume: | |
b69edc76 | 764 | */ |
090d7171 | 765 | static int __init e820__register_nvs_regions(void) |
b69edc76 RW |
766 | { |
767 | int i; | |
768 | ||
bf495573 | 769 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 770 | struct e820_entry *entry = &e820_table->entries[i]; |
b69edc76 | 771 | |
09821ff1 | 772 | if (entry->type == E820_TYPE_NVS) |
e5540f87 | 773 | acpi_nvs_register(entry->addr, entry->size); |
b69edc76 RW |
774 | } |
775 | ||
776 | return 0; | |
777 | } | |
090d7171 | 778 | core_initcall(e820__register_nvs_regions); |
b69edc76 RW |
779 | #endif |
780 | ||
2944e16b | 781 | /* |
5da217ca IM |
782 | * Allocate the requested number of bytes with the requsted alignment |
783 | * and return (the physical address) to the caller. Also register this | |
a09bae0f | 784 | * range in the 'kexec' E820 table as a reserved range. |
5da217ca IM |
785 | * |
786 | * This allows kexec to fake a new mptable, as if it came from the real | |
787 | * system. | |
2944e16b | 788 | */ |
5da217ca | 789 | u64 __init e820__memblock_alloc_reserved(u64 size, u64 align) |
2944e16b | 790 | { |
2944e16b | 791 | u64 addr; |
2944e16b | 792 | |
42b46aef | 793 | addr = memblock_phys_alloc(size, align); |
ab5d140b | 794 | if (addr) { |
a09bae0f | 795 | e820__range_update_kexec(addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED); |
1de392f5 | 796 | pr_info("update e820_table_kexec for e820__memblock_alloc_reserved()\n"); |
a09bae0f | 797 | e820__update_table_kexec(); |
61438766 | 798 | } |
2944e16b | 799 | |
2944e16b YL |
800 | return addr; |
801 | } | |
802 | ||
ee0c80fa YL |
803 | #ifdef CONFIG_X86_32 |
804 | # ifdef CONFIG_X86_PAE | |
805 | # define MAX_ARCH_PFN (1ULL<<(36-PAGE_SHIFT)) | |
806 | # else | |
807 | # define MAX_ARCH_PFN (1ULL<<(32-PAGE_SHIFT)) | |
808 | # endif | |
809 | #else /* CONFIG_X86_32 */ | |
bd70e522 | 810 | # define MAX_ARCH_PFN MAXMEM>>PAGE_SHIFT |
ee0c80fa YL |
811 | #endif |
812 | ||
ee0c80fa YL |
813 | /* |
814 | * Find the highest page frame number we have available | |
815 | */ | |
6afc03b8 | 816 | static unsigned long __init e820_end_pfn(unsigned long limit_pfn, enum e820_type type) |
ee0c80fa | 817 | { |
2dc807b3 YL |
818 | int i; |
819 | unsigned long last_pfn = 0; | |
ee0c80fa YL |
820 | unsigned long max_arch_pfn = MAX_ARCH_PFN; |
821 | ||
bf495573 | 822 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 823 | struct e820_entry *entry = &e820_table->entries[i]; |
f361a450 | 824 | unsigned long start_pfn; |
2dc807b3 YL |
825 | unsigned long end_pfn; |
826 | ||
e5540f87 | 827 | if (entry->type != type) |
c22d4c18 | 828 | continue; |
c22d4c18 | 829 | |
e5540f87 IM |
830 | start_pfn = entry->addr >> PAGE_SHIFT; |
831 | end_pfn = (entry->addr + entry->size) >> PAGE_SHIFT; | |
f361a450 YL |
832 | |
833 | if (start_pfn >= limit_pfn) | |
834 | continue; | |
835 | if (end_pfn > limit_pfn) { | |
836 | last_pfn = limit_pfn; | |
837 | break; | |
838 | } | |
2dc807b3 YL |
839 | if (end_pfn > last_pfn) |
840 | last_pfn = end_pfn; | |
841 | } | |
ee0c80fa YL |
842 | |
843 | if (last_pfn > max_arch_pfn) | |
844 | last_pfn = max_arch_pfn; | |
ee0c80fa | 845 | |
1de392f5 JP |
846 | pr_info("last_pfn = %#lx max_arch_pfn = %#lx\n", |
847 | last_pfn, max_arch_pfn); | |
ee0c80fa YL |
848 | return last_pfn; |
849 | } | |
640e1b38 | 850 | |
0c6fc11a | 851 | unsigned long __init e820__end_of_ram_pfn(void) |
f361a450 | 852 | { |
09821ff1 | 853 | return e820_end_pfn(MAX_ARCH_PFN, E820_TYPE_RAM); |
f361a450 | 854 | } |
ee0c80fa | 855 | |
0c6fc11a | 856 | unsigned long __init e820__end_of_low_ram_pfn(void) |
f361a450 | 857 | { |
09821ff1 | 858 | return e820_end_pfn(1UL << (32 - PAGE_SHIFT), E820_TYPE_RAM); |
f361a450 | 859 | } |
ee0c80fa | 860 | |
8c2103f2 | 861 | static void __init early_panic(char *msg) |
ab4a465e YL |
862 | { |
863 | early_printk(msg); | |
864 | panic(msg); | |
865 | } | |
866 | ||
69a7704d YL |
867 | static int userdef __initdata; |
868 | ||
640e1b38 | 869 | /* The "mem=nopentium" boot option disables 4MB page tables on 32-bit kernels: */ |
ab4a465e YL |
870 | static int __init parse_memopt(char *p) |
871 | { | |
872 | u64 mem_size; | |
873 | ||
874 | if (!p) | |
875 | return -EINVAL; | |
876 | ||
ab4a465e | 877 | if (!strcmp(p, "nopentium")) { |
9a6d44b9 | 878 | #ifdef CONFIG_X86_32 |
ab4a465e YL |
879 | setup_clear_cpu_cap(X86_FEATURE_PSE); |
880 | return 0; | |
9a6d44b9 | 881 | #else |
01259ef1 | 882 | pr_warn("mem=nopentium ignored! (only supported on x86_32)\n"); |
9a6d44b9 | 883 | return -EINVAL; |
ab4a465e | 884 | #endif |
9a6d44b9 | 885 | } |
ab4a465e | 886 | |
69a7704d | 887 | userdef = 1; |
ab4a465e | 888 | mem_size = memparse(p, &p); |
640e1b38 IM |
889 | |
890 | /* Don't remove all memory when getting "mem={invalid}" parameter: */ | |
77eed821 KM |
891 | if (mem_size == 0) |
892 | return -EINVAL; | |
640e1b38 | 893 | |
09821ff1 | 894 | e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1); |
611dfd78 | 895 | |
357b4da5 JG |
896 | #ifdef CONFIG_MEMORY_HOTPLUG |
897 | max_mem_size = mem_size; | |
898 | #endif | |
899 | ||
ab4a465e YL |
900 | return 0; |
901 | } | |
902 | early_param("mem", parse_memopt); | |
903 | ||
9710f581 | 904 | static int __init parse_memmap_one(char *p) |
ab4a465e YL |
905 | { |
906 | char *oldp; | |
907 | u64 start_at, mem_size; | |
908 | ||
a737abd1 CG |
909 | if (!p) |
910 | return -EINVAL; | |
911 | ||
d6be118a | 912 | if (!strncmp(p, "exactmap", 8)) { |
bf495573 | 913 | e820_table->nr_entries = 0; |
ab4a465e YL |
914 | userdef = 1; |
915 | return 0; | |
916 | } | |
917 | ||
918 | oldp = p; | |
919 | mem_size = memparse(p, &p); | |
920 | if (p == oldp) | |
921 | return -EINVAL; | |
922 | ||
923 | userdef = 1; | |
924 | if (*p == '@') { | |
925 | start_at = memparse(p+1, &p); | |
09821ff1 | 926 | e820__range_add(start_at, mem_size, E820_TYPE_RAM); |
ab4a465e YL |
927 | } else if (*p == '#') { |
928 | start_at = memparse(p+1, &p); | |
09821ff1 | 929 | e820__range_add(start_at, mem_size, E820_TYPE_ACPI); |
ab4a465e YL |
930 | } else if (*p == '$') { |
931 | start_at = memparse(p+1, &p); | |
09821ff1 | 932 | e820__range_add(start_at, mem_size, E820_TYPE_RESERVED); |
ec776ef6 CH |
933 | } else if (*p == '!') { |
934 | start_at = memparse(p+1, &p); | |
09821ff1 | 935 | e820__range_add(start_at, mem_size, E820_TYPE_PRAM); |
ef61f8a3 JS |
936 | } else if (*p == '%') { |
937 | enum e820_type from = 0, to = 0; | |
938 | ||
939 | start_at = memparse(p + 1, &p); | |
940 | if (*p == '-') | |
941 | from = simple_strtoull(p + 1, &p, 0); | |
942 | if (*p == '+') | |
943 | to = simple_strtoull(p + 1, &p, 0); | |
944 | if (*p != '\0') | |
945 | return -EINVAL; | |
946 | if (from && to) | |
947 | e820__range_update(start_at, mem_size, from, to); | |
948 | else if (to) | |
949 | e820__range_add(start_at, mem_size, to); | |
950 | else if (from) | |
951 | e820__range_remove(start_at, mem_size, from, 1); | |
952 | else | |
953 | e820__range_remove(start_at, mem_size, 0, 0); | |
640e1b38 | 954 | } else { |
09821ff1 | 955 | e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1); |
640e1b38 | 956 | } |
7b479bec | 957 | |
ab4a465e YL |
958 | return *p == '\0' ? 0 : -EINVAL; |
959 | } | |
640e1b38 | 960 | |
9710f581 YL |
961 | static int __init parse_memmap_opt(char *str) |
962 | { | |
963 | while (str) { | |
964 | char *k = strchr(str, ','); | |
965 | ||
966 | if (k) | |
967 | *k++ = 0; | |
968 | ||
969 | parse_memmap_one(str); | |
970 | str = k; | |
971 | } | |
972 | ||
973 | return 0; | |
974 | } | |
ab4a465e YL |
975 | early_param("memmap", parse_memmap_opt); |
976 | ||
1a127034 IM |
977 | /* |
978 | * Reserve all entries from the bootloader's extensible data nodes list, | |
979 | * because if present we are going to use it later on to fetch e820 | |
980 | * entries from it: | |
981 | */ | |
982 | void __init e820__reserve_setup_data(void) | |
da92139b IM |
983 | { |
984 | struct setup_data *data; | |
985 | u64 pa_data; | |
986 | ||
987 | pa_data = boot_params.hdr.setup_data; | |
988 | if (!pa_data) | |
989 | return; | |
990 | ||
991 | while (pa_data) { | |
992 | data = early_memremap(pa_data, sizeof(*data)); | |
09821ff1 | 993 | e820__range_update(pa_data, sizeof(*data)+data->len, E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); |
a09bae0f | 994 | e820__range_update_kexec(pa_data, sizeof(*data)+data->len, E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); |
b3c72fc9 DK |
995 | |
996 | if (data->type == SETUP_INDIRECT && | |
997 | ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) { | |
998 | e820__range_update(((struct setup_indirect *)data->data)->addr, | |
999 | ((struct setup_indirect *)data->data)->len, | |
1000 | E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); | |
1001 | e820__range_update_kexec(((struct setup_indirect *)data->data)->addr, | |
1002 | ((struct setup_indirect *)data->data)->len, | |
1003 | E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); | |
1004 | } | |
1005 | ||
da92139b IM |
1006 | pa_data = data->next; |
1007 | early_memunmap(data, sizeof(*data)); | |
1008 | } | |
1009 | ||
f9748fa0 | 1010 | e820__update_table(e820_table); |
a09bae0f | 1011 | e820__update_table(e820_table_kexec); |
1a127034 IM |
1012 | |
1013 | pr_info("extended physical RAM map:\n"); | |
be0c3f0f | 1014 | e820__print_table("reserve setup_data"); |
da92139b IM |
1015 | } |
1016 | ||
9641bdaf IM |
1017 | /* |
1018 | * Called after parse_early_param(), after early parameters (such as mem=) | |
1019 | * have been processed, in which case we already have an E820 table filled in | |
1020 | * via the parameter callback function(s), but it's not sorted and printed yet: | |
1021 | */ | |
1022 | void __init e820__finish_early_params(void) | |
ab4a465e YL |
1023 | { |
1024 | if (userdef) { | |
f9748fa0 | 1025 | if (e820__update_table(e820_table) < 0) |
ab4a465e | 1026 | early_panic("Invalid user supplied memory map"); |
ab4a465e | 1027 | |
1de392f5 | 1028 | pr_info("user-defined physical RAM map:\n"); |
be0c3f0f | 1029 | e820__print_table("user"); |
ab4a465e YL |
1030 | } |
1031 | } | |
41c094fd | 1032 | |
c594761d | 1033 | static const char *__init e820_type_to_string(struct e820_entry *entry) |
5dfcf14d | 1034 | { |
c594761d | 1035 | switch (entry->type) { |
09821ff1 IM |
1036 | case E820_TYPE_RESERVED_KERN: /* Fall-through: */ |
1037 | case E820_TYPE_RAM: return "System RAM"; | |
1038 | case E820_TYPE_ACPI: return "ACPI Tables"; | |
1039 | case E820_TYPE_NVS: return "ACPI Non-volatile Storage"; | |
1040 | case E820_TYPE_UNUSABLE: return "Unusable memory"; | |
1041 | case E820_TYPE_PRAM: return "Persistent Memory (legacy)"; | |
1042 | case E820_TYPE_PMEM: return "Persistent Memory"; | |
c5231a57 | 1043 | case E820_TYPE_RESERVED: return "Reserved"; |
262b45ae | 1044 | case E820_TYPE_SOFT_RESERVED: return "Soft Reserved"; |
c5231a57 | 1045 | default: return "Unknown E820 type"; |
5dfcf14d BW |
1046 | } |
1047 | } | |
1048 | ||
c594761d | 1049 | static unsigned long __init e820_type_to_iomem_type(struct e820_entry *entry) |
f33b14a4 | 1050 | { |
c594761d | 1051 | switch (entry->type) { |
09821ff1 IM |
1052 | case E820_TYPE_RESERVED_KERN: /* Fall-through: */ |
1053 | case E820_TYPE_RAM: return IORESOURCE_SYSTEM_RAM; | |
1054 | case E820_TYPE_ACPI: /* Fall-through: */ | |
1055 | case E820_TYPE_NVS: /* Fall-through: */ | |
1056 | case E820_TYPE_UNUSABLE: /* Fall-through: */ | |
1057 | case E820_TYPE_PRAM: /* Fall-through: */ | |
1058 | case E820_TYPE_PMEM: /* Fall-through: */ | |
c5231a57 | 1059 | case E820_TYPE_RESERVED: /* Fall-through: */ |
262b45ae | 1060 | case E820_TYPE_SOFT_RESERVED: /* Fall-through: */ |
09821ff1 | 1061 | default: return IORESOURCE_MEM; |
f33b14a4 TK |
1062 | } |
1063 | } | |
1064 | ||
c594761d | 1065 | static unsigned long __init e820_type_to_iores_desc(struct e820_entry *entry) |
f33b14a4 | 1066 | { |
c594761d | 1067 | switch (entry->type) { |
09821ff1 IM |
1068 | case E820_TYPE_ACPI: return IORES_DESC_ACPI_TABLES; |
1069 | case E820_TYPE_NVS: return IORES_DESC_ACPI_NV_STORAGE; | |
1070 | case E820_TYPE_PMEM: return IORES_DESC_PERSISTENT_MEMORY; | |
1071 | case E820_TYPE_PRAM: return IORES_DESC_PERSISTENT_MEMORY_LEGACY; | |
ae9e13d6 | 1072 | case E820_TYPE_RESERVED: return IORES_DESC_RESERVED; |
262b45ae | 1073 | case E820_TYPE_SOFT_RESERVED: return IORES_DESC_SOFT_RESERVED; |
09821ff1 IM |
1074 | case E820_TYPE_RESERVED_KERN: /* Fall-through: */ |
1075 | case E820_TYPE_RAM: /* Fall-through: */ | |
1076 | case E820_TYPE_UNUSABLE: /* Fall-through: */ | |
1077 | default: return IORES_DESC_NONE; | |
f33b14a4 TK |
1078 | } |
1079 | } | |
1080 | ||
c5231a57 | 1081 | static bool __init do_mark_busy(enum e820_type type, struct resource *res) |
ad5fb870 DW |
1082 | { |
1083 | /* this is the legacy bios/dos rom-shadow + mmio region */ | |
1084 | if (res->start < (1ULL<<20)) | |
1085 | return true; | |
1086 | ||
1087 | /* | |
262b45ae DW |
1088 | * Treat persistent memory and other special memory ranges like |
1089 | * device memory, i.e. reserve it for exclusive use of a driver | |
ad5fb870 DW |
1090 | */ |
1091 | switch (type) { | |
09821ff1 | 1092 | case E820_TYPE_RESERVED: |
262b45ae | 1093 | case E820_TYPE_SOFT_RESERVED: |
09821ff1 IM |
1094 | case E820_TYPE_PRAM: |
1095 | case E820_TYPE_PMEM: | |
ad5fb870 | 1096 | return false; |
c5231a57 IM |
1097 | case E820_TYPE_RESERVED_KERN: |
1098 | case E820_TYPE_RAM: | |
1099 | case E820_TYPE_ACPI: | |
1100 | case E820_TYPE_NVS: | |
1101 | case E820_TYPE_UNUSABLE: | |
ad5fb870 DW |
1102 | default: |
1103 | return true; | |
1104 | } | |
1105 | } | |
1106 | ||
41c094fd | 1107 | /* |
640e1b38 | 1108 | * Mark E820 reserved areas as busy for the resource manager: |
41c094fd | 1109 | */ |
640e1b38 | 1110 | |
a5444d15 | 1111 | static struct resource __initdata *e820_res; |
640e1b38 | 1112 | |
1506c8dc | 1113 | void __init e820__reserve_resources(void) |
41c094fd YL |
1114 | { |
1115 | int i; | |
58f7c988 | 1116 | struct resource *res; |
a5444d15 | 1117 | u64 end; |
41c094fd | 1118 | |
7e1c4e27 MR |
1119 | res = memblock_alloc(sizeof(*res) * e820_table->nr_entries, |
1120 | SMP_CACHE_BYTES); | |
8a7f97b9 MR |
1121 | if (!res) |
1122 | panic("%s: Failed to allocate %zu bytes\n", __func__, | |
1123 | sizeof(*res) * e820_table->nr_entries); | |
58f7c988 | 1124 | e820_res = res; |
c594761d | 1125 | |
bf495573 | 1126 | for (i = 0; i < e820_table->nr_entries; i++) { |
c594761d IM |
1127 | struct e820_entry *entry = e820_table->entries + i; |
1128 | ||
1129 | end = entry->addr + entry->size - 1; | |
8308c54d | 1130 | if (end != (resource_size_t)end) { |
41c094fd YL |
1131 | res++; |
1132 | continue; | |
1133 | } | |
c594761d IM |
1134 | res->start = entry->addr; |
1135 | res->end = end; | |
1136 | res->name = e820_type_to_string(entry); | |
1137 | res->flags = e820_type_to_iomem_type(entry); | |
1138 | res->desc = e820_type_to_iores_desc(entry); | |
a5444d15 IM |
1139 | |
1140 | /* | |
1506c8dc IM |
1141 | * Don't register the region that could be conflicted with |
1142 | * PCI device BAR resources and insert them later in | |
1143 | * pcibios_resource_survey(): | |
a5444d15 | 1144 | */ |
c594761d | 1145 | if (do_mark_busy(entry->type, res)) { |
1f987577 | 1146 | res->flags |= IORESOURCE_BUSY; |
58f7c988 | 1147 | insert_resource(&iomem_resource, res); |
1f987577 | 1148 | } |
41c094fd YL |
1149 | res++; |
1150 | } | |
5dfcf14d | 1151 | |
12df216c CY |
1152 | /* Expose the bootloader-provided memory layout to the sysfs. */ |
1153 | for (i = 0; i < e820_table_firmware->nr_entries; i++) { | |
1154 | struct e820_entry *entry = e820_table_firmware->entries + i; | |
640e1b38 | 1155 | |
c594761d | 1156 | firmware_map_add_early(entry->addr, entry->addr + entry->size, e820_type_to_string(entry)); |
5dfcf14d | 1157 | } |
41c094fd YL |
1158 | } |
1159 | ||
1506c8dc IM |
1160 | /* |
1161 | * How much should we pad the end of RAM, depending on where it is? | |
1162 | */ | |
8c2103f2 | 1163 | static unsigned long __init ram_alignment(resource_size_t pos) |
45fbe3ee LT |
1164 | { |
1165 | unsigned long mb = pos >> 20; | |
1166 | ||
1167 | /* To 64kB in the first megabyte */ | |
1168 | if (!mb) | |
1169 | return 64*1024; | |
1170 | ||
1171 | /* To 1MB in the first 16MB */ | |
1172 | if (mb < 16) | |
1173 | return 1024*1024; | |
1174 | ||
15b812f1 YL |
1175 | /* To 64MB for anything above that */ |
1176 | return 64*1024*1024; | |
45fbe3ee LT |
1177 | } |
1178 | ||
7c5371c4 YL |
1179 | #define MAX_RESOURCE_SIZE ((resource_size_t)-1) |
1180 | ||
1506c8dc | 1181 | void __init e820__reserve_resources_late(void) |
58f7c988 YL |
1182 | { |
1183 | int i; | |
1184 | struct resource *res; | |
1185 | ||
1186 | res = e820_res; | |
bf495573 | 1187 | for (i = 0; i < e820_table->nr_entries; i++) { |
a5444d15 | 1188 | if (!res->parent && res->end) |
1f987577 | 1189 | insert_resource_expand_to_fit(&iomem_resource, res); |
58f7c988 YL |
1190 | res++; |
1191 | } | |
45fbe3ee LT |
1192 | |
1193 | /* | |
640e1b38 | 1194 | * Try to bump up RAM regions to reasonable boundaries, to |
45fbe3ee LT |
1195 | * avoid stolen RAM: |
1196 | */ | |
bf495573 IM |
1197 | for (i = 0; i < e820_table->nr_entries; i++) { |
1198 | struct e820_entry *entry = &e820_table->entries[i]; | |
7c5371c4 | 1199 | u64 start, end; |
45fbe3ee | 1200 | |
09821ff1 | 1201 | if (entry->type != E820_TYPE_RAM) |
45fbe3ee | 1202 | continue; |
640e1b38 | 1203 | |
45fbe3ee | 1204 | start = entry->addr + entry->size; |
7c5371c4 YL |
1205 | end = round_up(start, ram_alignment(start)) - 1; |
1206 | if (end > MAX_RESOURCE_SIZE) | |
1207 | end = MAX_RESOURCE_SIZE; | |
1208 | if (start >= end) | |
45fbe3ee | 1209 | continue; |
640e1b38 | 1210 | |
e22af0be | 1211 | printk(KERN_DEBUG "e820: reserve RAM buffer [mem %#010llx-%#010llx]\n", start, end); |
640e1b38 | 1212 | reserve_region_with_split(&iomem_resource, start, end, "RAM buffer"); |
45fbe3ee | 1213 | } |
58f7c988 YL |
1214 | } |
1215 | ||
640e1b38 IM |
1216 | /* |
1217 | * Pass the firmware (bootloader) E820 map to the kernel and process it: | |
1218 | */ | |
103e2063 | 1219 | char *__init e820__memory_setup_default(void) |
064d25f1 YL |
1220 | { |
1221 | char *who = "BIOS-e820"; | |
640e1b38 | 1222 | |
064d25f1 YL |
1223 | /* |
1224 | * Try to copy the BIOS-supplied E820-map. | |
1225 | * | |
1226 | * Otherwise fake a memory map; one section from 0k->640k, | |
1227 | * the next section from 1mb->appropriate_mem_k | |
1228 | */ | |
640e1b38 | 1229 | if (append_e820_table(boot_params.e820_table, boot_params.e820_entries) < 0) { |
95a71a45 | 1230 | u64 mem_size; |
064d25f1 | 1231 | |
640e1b38 IM |
1232 | /* Compare results from other methods and take the one that gives more RAM: */ |
1233 | if (boot_params.alt_mem_k < boot_params.screen_info.ext_mem_k) { | |
064d25f1 YL |
1234 | mem_size = boot_params.screen_info.ext_mem_k; |
1235 | who = "BIOS-88"; | |
1236 | } else { | |
1237 | mem_size = boot_params.alt_mem_k; | |
1238 | who = "BIOS-e801"; | |
1239 | } | |
1240 | ||
bf495573 | 1241 | e820_table->nr_entries = 0; |
09821ff1 IM |
1242 | e820__range_add(0, LOWMEMSIZE(), E820_TYPE_RAM); |
1243 | e820__range_add(HIGH_MEMORY, mem_size << 10, E820_TYPE_RAM); | |
064d25f1 YL |
1244 | } |
1245 | ||
7410aa1c IM |
1246 | /* We just appended a lot of ranges, sanitize the table: */ |
1247 | e820__update_table(e820_table); | |
1248 | ||
064d25f1 YL |
1249 | return who; |
1250 | } | |
1251 | ||
103e2063 IM |
1252 | /* |
1253 | * Calls e820__memory_setup_default() in essence to pick up the firmware/bootloader | |
1254 | * E820 map - with an optional platform quirk available for virtual platforms | |
1255 | * to override this method of boot environment processing: | |
1256 | */ | |
1257 | void __init e820__memory_setup(void) | |
064d25f1 | 1258 | { |
0be15526 YL |
1259 | char *who; |
1260 | ||
09c51513 | 1261 | /* This is a firmware interface ABI - make sure we don't break it: */ |
7410aa1c | 1262 | BUILD_BUG_ON(sizeof(struct boot_e820_entry) != 20); |
09c51513 | 1263 | |
6b18ae3e | 1264 | who = x86_init.resources.memory_setup(); |
640e1b38 | 1265 | |
a09bae0f | 1266 | memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); |
12df216c | 1267 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); |
640e1b38 | 1268 | |
1de392f5 | 1269 | pr_info("BIOS-provided physical RAM map:\n"); |
be0c3f0f | 1270 | e820__print_table(who); |
064d25f1 | 1271 | } |
72d7c3b3 | 1272 | |
4918e228 | 1273 | void __init e820__memblock_setup(void) |
72d7c3b3 YL |
1274 | { |
1275 | int i; | |
1276 | u64 end; | |
1277 | ||
1278 | /* | |
4918e228 IM |
1279 | * The bootstrap memblock region count maximum is 128 entries |
1280 | * (INIT_MEMBLOCK_REGIONS), but EFI might pass us more E820 entries | |
1281 | * than that - so allow memblock resizing. | |
1282 | * | |
1283 | * This is safe, because this call happens pretty late during x86 setup, | |
1284 | * so we know about reserved memory regions already. (This is important | |
1285 | * so that memblock resizing does no stomp over reserved areas.) | |
72d7c3b3 | 1286 | */ |
1aadc056 | 1287 | memblock_allow_resize(); |
72d7c3b3 | 1288 | |
bf495573 | 1289 | for (i = 0; i < e820_table->nr_entries; i++) { |
e5540f87 | 1290 | struct e820_entry *entry = &e820_table->entries[i]; |
72d7c3b3 | 1291 | |
e5540f87 | 1292 | end = entry->addr + entry->size; |
72d7c3b3 YL |
1293 | if (end != (resource_size_t)end) |
1294 | continue; | |
1295 | ||
262b45ae DW |
1296 | if (entry->type == E820_TYPE_SOFT_RESERVED) |
1297 | memblock_reserve(entry->addr, entry->size); | |
1298 | ||
09821ff1 | 1299 | if (entry->type != E820_TYPE_RAM && entry->type != E820_TYPE_RESERVED_KERN) |
9fd61bc9 MM |
1300 | continue; |
1301 | ||
1302 | memblock_add(entry->addr, entry->size); | |
72d7c3b3 YL |
1303 | } |
1304 | ||
4918e228 | 1305 | /* Throw away partial pages: */ |
6ede1fd3 YL |
1306 | memblock_trim_memory(PAGE_SIZE); |
1307 | ||
72d7c3b3 YL |
1308 | memblock_dump_all(); |
1309 | } |