1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright (c) 2021, Google LLC.
5 * Pasha Tatashin <pasha.tatashin@soleen.com>
8 #include <linux/page_table_check.h>
11 #define pr_fmt(fmt) "page_table_check: " fmt
13 struct page_table_check
{
14 atomic_t anon_map_count
;
15 atomic_t file_map_count
;
18 static bool __page_table_check_enabled __initdata
=
19 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED
);
21 DEFINE_STATIC_KEY_TRUE(page_table_check_disabled
);
22 EXPORT_SYMBOL(page_table_check_disabled
);
24 static int __init
early_page_table_check_param(char *buf
)
26 return strtobool(buf
, &__page_table_check_enabled
);
29 early_param("page_table_check", early_page_table_check_param
);
31 static bool __init
need_page_table_check(void)
33 return __page_table_check_enabled
;
36 static void __init
init_page_table_check(void)
38 if (!__page_table_check_enabled
)
40 static_branch_disable(&page_table_check_disabled
);
43 struct page_ext_operations page_table_check_ops
= {
44 .size
= sizeof(struct page_table_check
),
45 .need
= need_page_table_check
,
46 .init
= init_page_table_check
,
49 static struct page_table_check
*get_page_table_check(struct page_ext
*page_ext
)
52 return (void *)(page_ext
) + page_table_check_ops
.offset
;
56 * An entry is removed from the page table, decrement the counters for that page
57 * verify that it is of correct type and counters do not become negative.
59 static void page_table_check_clear(struct mm_struct
*mm
, unsigned long addr
,
60 unsigned long pfn
, unsigned long pgcnt
)
62 struct page_ext
*page_ext
;
70 page
= pfn_to_page(pfn
);
71 page_ext
= page_ext_get(page
);
72 anon
= PageAnon(page
);
74 for (i
= 0; i
< pgcnt
; i
++) {
75 struct page_table_check
*ptc
= get_page_table_check(page_ext
);
78 BUG_ON(atomic_read(&ptc
->file_map_count
));
79 BUG_ON(atomic_dec_return(&ptc
->anon_map_count
) < 0);
81 BUG_ON(atomic_read(&ptc
->anon_map_count
));
82 BUG_ON(atomic_dec_return(&ptc
->file_map_count
) < 0);
84 page_ext
= page_ext_next(page_ext
);
86 page_ext_put(page_ext
);
90 * A new entry is added to the page table, increment the counters for that page
91 * verify that it is of correct type and is not being mapped with a different
92 * type to a different process.
94 static void page_table_check_set(struct mm_struct
*mm
, unsigned long addr
,
95 unsigned long pfn
, unsigned long pgcnt
,
98 struct page_ext
*page_ext
;
106 page
= pfn_to_page(pfn
);
107 page_ext
= page_ext_get(page
);
108 anon
= PageAnon(page
);
110 for (i
= 0; i
< pgcnt
; i
++) {
111 struct page_table_check
*ptc
= get_page_table_check(page_ext
);
114 BUG_ON(atomic_read(&ptc
->file_map_count
));
115 BUG_ON(atomic_inc_return(&ptc
->anon_map_count
) > 1 && rw
);
117 BUG_ON(atomic_read(&ptc
->anon_map_count
));
118 BUG_ON(atomic_inc_return(&ptc
->file_map_count
) < 0);
120 page_ext
= page_ext_next(page_ext
);
122 page_ext_put(page_ext
);
126 * page is on free list, or is being allocated, verify that counters are zeroes
127 * crash if they are not.
129 void __page_table_check_zero(struct page
*page
, unsigned int order
)
131 struct page_ext
*page_ext
;
134 page_ext
= page_ext_get(page
);
136 for (i
= 0; i
< (1ul << order
); i
++) {
137 struct page_table_check
*ptc
= get_page_table_check(page_ext
);
139 BUG_ON(atomic_read(&ptc
->anon_map_count
));
140 BUG_ON(atomic_read(&ptc
->file_map_count
));
141 page_ext
= page_ext_next(page_ext
);
143 page_ext_put(page_ext
);
146 void __page_table_check_pte_clear(struct mm_struct
*mm
, unsigned long addr
,
152 if (pte_user_accessible_page(pte
)) {
153 page_table_check_clear(mm
, addr
, pte_pfn(pte
),
154 PAGE_SIZE
>> PAGE_SHIFT
);
157 EXPORT_SYMBOL(__page_table_check_pte_clear
);
159 void __page_table_check_pmd_clear(struct mm_struct
*mm
, unsigned long addr
,
165 if (pmd_user_accessible_page(pmd
)) {
166 page_table_check_clear(mm
, addr
, pmd_pfn(pmd
),
167 PMD_SIZE
>> PAGE_SHIFT
);
170 EXPORT_SYMBOL(__page_table_check_pmd_clear
);
172 void __page_table_check_pud_clear(struct mm_struct
*mm
, unsigned long addr
,
178 if (pud_user_accessible_page(pud
)) {
179 page_table_check_clear(mm
, addr
, pud_pfn(pud
),
180 PUD_SIZE
>> PAGE_SHIFT
);
183 EXPORT_SYMBOL(__page_table_check_pud_clear
);
185 void __page_table_check_pte_set(struct mm_struct
*mm
, unsigned long addr
,
186 pte_t
*ptep
, pte_t pte
)
191 __page_table_check_pte_clear(mm
, addr
, *ptep
);
192 if (pte_user_accessible_page(pte
)) {
193 page_table_check_set(mm
, addr
, pte_pfn(pte
),
194 PAGE_SIZE
>> PAGE_SHIFT
,
198 EXPORT_SYMBOL(__page_table_check_pte_set
);
200 void __page_table_check_pmd_set(struct mm_struct
*mm
, unsigned long addr
,
201 pmd_t
*pmdp
, pmd_t pmd
)
206 __page_table_check_pmd_clear(mm
, addr
, *pmdp
);
207 if (pmd_user_accessible_page(pmd
)) {
208 page_table_check_set(mm
, addr
, pmd_pfn(pmd
),
209 PMD_SIZE
>> PAGE_SHIFT
,
213 EXPORT_SYMBOL(__page_table_check_pmd_set
);
215 void __page_table_check_pud_set(struct mm_struct
*mm
, unsigned long addr
,
216 pud_t
*pudp
, pud_t pud
)
221 __page_table_check_pud_clear(mm
, addr
, *pudp
);
222 if (pud_user_accessible_page(pud
)) {
223 page_table_check_set(mm
, addr
, pud_pfn(pud
),
224 PUD_SIZE
>> PAGE_SHIFT
,
228 EXPORT_SYMBOL(__page_table_check_pud_set
);
230 void __page_table_check_pte_clear_range(struct mm_struct
*mm
,
237 if (!pmd_bad(pmd
) && !pmd_leaf(pmd
)) {
238 pte_t
*ptep
= pte_offset_map(&pmd
, addr
);
241 for (i
= 0; i
< PTRS_PER_PTE
; i
++) {
242 __page_table_check_pte_clear(mm
, addr
, *ptep
);
246 pte_unmap(ptep
- PTRS_PER_PTE
);