2 * memfd_create system call and file sealing support
4 * Code was originally included in shmem.c, and broken out to facilitate
5 * use by hugetlbfs as well as tmpfs.
7 * This file is released under the GPL.
11 #include <linux/vfs.h>
12 #include <linux/pagemap.h>
13 #include <linux/file.h>
15 #include <linux/sched/signal.h>
16 #include <linux/khugepaged.h>
17 #include <linux/syscalls.h>
18 #include <linux/hugetlb.h>
19 #include <linux/shmem_fs.h>
20 #include <linux/memfd.h>
21 #include <linux/pid_namespace.h>
22 #include <uapi/linux/memfd.h>
25 * We need a tag: a new tag would expand every xa_node by 8 bytes,
26 * so reuse a tag which we firmly believe is never set or cleared on tmpfs
27 * or hugetlbfs because they are memory only filesystems.
29 #define MEMFD_TAG_PINNED PAGECACHE_TAG_TOWRITE
30 #define LAST_SCAN 4 /* about 150ms max */
32 static void memfd_tag_pins(struct xa_state
*xas
)
41 xas_for_each(xas
, page
, ULONG_MAX
) {
43 if (!xa_is_value(page
) &&
44 PageTransHuge(page
) && !PageHuge(page
))
45 cache_count
= HPAGE_PMD_NR
;
47 if (!xa_is_value(page
) &&
48 page_count(page
) - total_mapcount(page
) != cache_count
)
49 xas_set_mark(xas
, MEMFD_TAG_PINNED
);
51 xas_set(xas
, page
->index
+ cache_count
);
53 latency
+= cache_count
;
54 if (latency
< XA_CHECK_SCHED
)
67 * Setting SEAL_WRITE requires us to verify there's no pending writer. However,
68 * via get_user_pages(), drivers might have some pending I/O without any active
69 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages
70 * and see whether it has an elevated ref-count. If so, we tag them and wait for
72 * The caller must guarantee that no new user will acquire writable references
73 * to those pages to avoid races.
75 static int memfd_wait_for_pins(struct address_space
*mapping
)
77 XA_STATE(xas
, &mapping
->i_pages
, 0);
84 for (scan
= 0; scan
<= LAST_SCAN
; scan
++) {
88 if (!xas_marked(&xas
, MEMFD_TAG_PINNED
))
93 else if (schedule_timeout_killable((HZ
<< scan
) / 200))
98 xas_for_each_marked(&xas
, page
, ULONG_MAX
, MEMFD_TAG_PINNED
) {
102 if (!xa_is_value(page
) &&
103 PageTransHuge(page
) && !PageHuge(page
))
104 cache_count
= HPAGE_PMD_NR
;
106 if (!xa_is_value(page
) && cache_count
!=
107 page_count(page
) - total_mapcount(page
)) {
109 * On the last scan, we clean up all those tags
110 * we inserted; but make a note that we still
111 * found pages pinned.
113 if (scan
== LAST_SCAN
)
119 xas_clear_mark(&xas
, MEMFD_TAG_PINNED
);
121 latency
+= cache_count
;
122 if (latency
< XA_CHECK_SCHED
)
127 xas_unlock_irq(&xas
);
131 xas_unlock_irq(&xas
);
137 static unsigned int *memfd_file_seals_ptr(struct file
*file
)
139 if (shmem_file(file
))
140 return &SHMEM_I(file_inode(file
))->seals
;
142 #ifdef CONFIG_HUGETLBFS
143 if (is_file_hugepages(file
))
144 return &HUGETLBFS_I(file_inode(file
))->seals
;
150 #define F_ALL_SEALS (F_SEAL_SEAL | \
157 static int memfd_add_seals(struct file
*file
, unsigned int seals
)
159 struct inode
*inode
= file_inode(file
);
160 unsigned int *file_seals
;
165 * Sealing allows multiple parties to share a tmpfs or hugetlbfs file
166 * but restrict access to a specific subset of file operations. Seals
167 * can only be added, but never removed. This way, mutually untrusted
168 * parties can share common memory regions with a well-defined policy.
169 * A malicious peer can thus never perform unwanted operations on a
172 * Seals are only supported on special tmpfs or hugetlbfs files and
173 * always affect the whole underlying inode. Once a seal is set, it
174 * may prevent some kinds of access to the file. Currently, the
175 * following seals are defined:
176 * SEAL_SEAL: Prevent further seals from being set on this file
177 * SEAL_SHRINK: Prevent the file from shrinking
178 * SEAL_GROW: Prevent the file from growing
179 * SEAL_WRITE: Prevent write access to the file
180 * SEAL_EXEC: Prevent modification of the exec bits in the file mode
182 * As we don't require any trust relationship between two parties, we
183 * must prevent seals from being removed. Therefore, sealing a file
184 * only adds a given set of seals to the file, it never touches
185 * existing seals. Furthermore, the "setting seals"-operation can be
186 * sealed itself, which basically prevents any further seal from being
189 * Semantics of sealing are only defined on volatile files. Only
190 * anonymous tmpfs and hugetlbfs files support sealing. More
191 * importantly, seals are never written to disk. Therefore, there's
192 * no plan to support it on other file types.
195 if (!(file
->f_mode
& FMODE_WRITE
))
197 if (seals
& ~(unsigned int)F_ALL_SEALS
)
202 file_seals
= memfd_file_seals_ptr(file
);
208 if (*file_seals
& F_SEAL_SEAL
) {
213 if ((seals
& F_SEAL_WRITE
) && !(*file_seals
& F_SEAL_WRITE
)) {
214 error
= mapping_deny_writable(file
->f_mapping
);
218 error
= memfd_wait_for_pins(file
->f_mapping
);
220 mapping_allow_writable(file
->f_mapping
);
226 * SEAL_EXEC implys SEAL_WRITE, making W^X from the start.
228 if (seals
& F_SEAL_EXEC
&& inode
->i_mode
& 0111)
229 seals
|= F_SEAL_SHRINK
|F_SEAL_GROW
|F_SEAL_WRITE
|F_SEAL_FUTURE_WRITE
;
231 *file_seals
|= seals
;
239 static int memfd_get_seals(struct file
*file
)
241 unsigned int *seals
= memfd_file_seals_ptr(file
);
243 return seals
? *seals
: -EINVAL
;
246 long memfd_fcntl(struct file
*file
, unsigned int cmd
, unsigned int arg
)
252 error
= memfd_add_seals(file
, arg
);
255 error
= memfd_get_seals(file
);
265 #define MFD_NAME_PREFIX "memfd:"
266 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
267 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
269 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB | MFD_NOEXEC_SEAL | MFD_EXEC)
271 SYSCALL_DEFINE2(memfd_create
,
272 const char __user
*, uname
,
275 char comm
[TASK_COMM_LEN
];
276 unsigned int *file_seals
;
282 if (!(flags
& MFD_HUGETLB
)) {
283 if (flags
& ~(unsigned int)MFD_ALL_FLAGS
)
286 /* Allow huge page size encoding in flags. */
287 if (flags
& ~(unsigned int)(MFD_ALL_FLAGS
|
288 (MFD_HUGE_MASK
<< MFD_HUGE_SHIFT
)))
292 /* Invalid if both EXEC and NOEXEC_SEAL are set.*/
293 if ((flags
& MFD_EXEC
) && (flags
& MFD_NOEXEC_SEAL
))
296 if (!(flags
& (MFD_EXEC
| MFD_NOEXEC_SEAL
))) {
298 int sysctl
= MEMFD_NOEXEC_SCOPE_EXEC
;
299 struct pid_namespace
*ns
;
301 ns
= task_active_pid_ns(current
);
303 sysctl
= ns
->memfd_noexec_scope
;
306 case MEMFD_NOEXEC_SCOPE_EXEC
:
309 case MEMFD_NOEXEC_SCOPE_NOEXEC_SEAL
:
310 flags
|= MFD_NOEXEC_SEAL
;
314 "memfd_create(): MFD_NOEXEC_SEAL is enforced, pid=%d '%s'\n",
315 task_pid_nr(current
), get_task_comm(comm
, current
));
322 "memfd_create() without MFD_EXEC nor MFD_NOEXEC_SEAL, pid=%d '%s'\n",
323 task_pid_nr(current
), get_task_comm(comm
, current
));
326 /* length includes terminating zero */
327 len
= strnlen_user(uname
, MFD_NAME_MAX_LEN
+ 1);
330 if (len
> MFD_NAME_MAX_LEN
+ 1)
333 name
= kmalloc(len
+ MFD_NAME_PREFIX_LEN
, GFP_KERNEL
);
337 strcpy(name
, MFD_NAME_PREFIX
);
338 if (copy_from_user(&name
[MFD_NAME_PREFIX_LEN
], uname
, len
)) {
343 /* terminating-zero may have changed after strnlen_user() returned */
344 if (name
[len
+ MFD_NAME_PREFIX_LEN
- 1]) {
349 fd
= get_unused_fd_flags((flags
& MFD_CLOEXEC
) ? O_CLOEXEC
: 0);
355 if (flags
& MFD_HUGETLB
) {
356 file
= hugetlb_file_setup(name
, 0, VM_NORESERVE
,
357 HUGETLB_ANONHUGE_INODE
,
358 (flags
>> MFD_HUGE_SHIFT
) &
361 file
= shmem_file_setup(name
, 0, VM_NORESERVE
);
363 error
= PTR_ERR(file
);
366 file
->f_mode
|= FMODE_LSEEK
| FMODE_PREAD
| FMODE_PWRITE
;
367 file
->f_flags
|= O_LARGEFILE
;
369 if (flags
& MFD_NOEXEC_SEAL
) {
370 struct inode
*inode
= file_inode(file
);
372 inode
->i_mode
&= ~0111;
373 file_seals
= memfd_file_seals_ptr(file
);
374 *file_seals
&= ~F_SEAL_SEAL
;
375 *file_seals
|= F_SEAL_EXEC
;
376 } else if (flags
& MFD_ALLOW_SEALING
) {
377 /* MFD_EXEC and MFD_ALLOW_SEALING are set */
378 file_seals
= memfd_file_seals_ptr(file
);
379 *file_seals
&= ~F_SEAL_SEAL
;
382 fd_install(fd
, file
);