1 // SPDX-License-Identifier: GPL-2.0-only
3 * Generic pidhash and scalable, time-bounded PID allocator
5 * (C) 2002-2003 Nadia Yvette Chambers, IBM
6 * (C) 2004 Nadia Yvette Chambers, Oracle
7 * (C) 2002-2004 Ingo Molnar, Red Hat
9 * pid-structures are backing objects for tasks sharing a given ID to chain
10 * against. There is very little to them aside from hashing them and
11 * parking tasks using given ID's on a list.
13 * The hash is always changed with the tasklist_lock write-acquired,
14 * and the hash is only accessed with the tasklist_lock at least
15 * read-acquired, so there's no additional SMP locking needed here.
17 * We have a list of bitmap pages, which bitmaps represent the PID space.
18 * Allocating and freeing PIDs is completely lockless. The worst-case
19 * allocation scenario when all but one out of 1 million PIDs possible are
20 * allocated already: the scanning of 32 list entries and at most PAGE_SIZE
21 * bytes. The typical fastpath is a single successful setbit. Freeing is O(1).
24 * (C) 2007 Pavel Emelyanov <xemul@openvz.org>, OpenVZ, SWsoft Inc.
25 * (C) 2007 Sukadev Bhattiprolu <sukadev@us.ibm.com>, IBM
26 * Many thanks to Oleg Nesterov for comments and help
31 #include <linux/export.h>
32 #include <linux/slab.h>
33 #include <linux/init.h>
34 #include <linux/rculist.h>
35 #include <linux/memblock.h>
36 #include <linux/pid_namespace.h>
37 #include <linux/init_task.h>
38 #include <linux/syscalls.h>
39 #include <linux/proc_ns.h>
40 #include <linux/proc_fs.h>
41 #include <linux/anon_inodes.h>
42 #include <linux/sched/signal.h>
43 #include <linux/sched/task.h>
44 #include <linux/idr.h>
46 struct pid init_struct_pid
= {
47 .count
= ATOMIC_INIT(1),
60 int pid_max
= PID_MAX_DEFAULT
;
62 #define RESERVED_PIDS 300
64 int pid_max_min
= RESERVED_PIDS
+ 1;
65 int pid_max_max
= PID_MAX_LIMIT
;
68 * PID-map pages start out as NULL, they get allocated upon
69 * first use and are never deallocated. This way a low pid_max
70 * value does not cause lots of bitmaps to be allocated, but
71 * the scheme scales to up to 4 million PIDs, runtime.
73 struct pid_namespace init_pid_ns
= {
75 .idr
= IDR_INIT(init_pid_ns
.idr
),
76 .pid_allocated
= PIDNS_ADDING
,
78 .child_reaper
= &init_task
,
79 .user_ns
= &init_user_ns
,
80 .ns
.inum
= PROC_PID_INIT_INO
,
82 .ns
.ops
= &pidns_operations
,
85 EXPORT_SYMBOL_GPL(init_pid_ns
);
88 * Note: disable interrupts while the pidmap_lock is held as an
89 * interrupt might come in and do read_lock(&tasklist_lock).
91 * If we don't disable interrupts there is a nasty deadlock between
92 * detach_pid()->free_pid() and another cpu that does
93 * spin_lock(&pidmap_lock) followed by an interrupt routine that does
94 * read_lock(&tasklist_lock);
96 * After we clean up the tasklist_lock and know there are no
97 * irq handlers that take it we can leave the interrupts enabled.
98 * For now it is easier to be safe than to prove it can't happen.
101 static __cacheline_aligned_in_smp
DEFINE_SPINLOCK(pidmap_lock
);
103 void put_pid(struct pid
*pid
)
105 struct pid_namespace
*ns
;
110 ns
= pid
->numbers
[pid
->level
].ns
;
111 if ((atomic_read(&pid
->count
) == 1) ||
112 atomic_dec_and_test(&pid
->count
)) {
113 kmem_cache_free(ns
->pid_cachep
, pid
);
117 EXPORT_SYMBOL_GPL(put_pid
);
119 static void delayed_put_pid(struct rcu_head
*rhp
)
121 struct pid
*pid
= container_of(rhp
, struct pid
, rcu
);
125 void free_pid(struct pid
*pid
)
127 /* We can be called with write_lock_irq(&tasklist_lock) held */
131 spin_lock_irqsave(&pidmap_lock
, flags
);
132 for (i
= 0; i
<= pid
->level
; i
++) {
133 struct upid
*upid
= pid
->numbers
+ i
;
134 struct pid_namespace
*ns
= upid
->ns
;
135 switch (--ns
->pid_allocated
) {
138 /* When all that is left in the pid namespace
139 * is the reaper wake up the reaper. The reaper
140 * may be sleeping in zap_pid_ns_processes().
142 wake_up_process(ns
->child_reaper
);
145 /* Handle a fork failure of the first process */
146 WARN_ON(ns
->child_reaper
);
147 ns
->pid_allocated
= 0;
150 schedule_work(&ns
->proc_work
);
154 idr_remove(&ns
->idr
, upid
->nr
);
156 spin_unlock_irqrestore(&pidmap_lock
, flags
);
158 call_rcu(&pid
->rcu
, delayed_put_pid
);
161 struct pid
*alloc_pid(struct pid_namespace
*ns
)
166 struct pid_namespace
*tmp
;
168 int retval
= -ENOMEM
;
170 pid
= kmem_cache_alloc(ns
->pid_cachep
, GFP_KERNEL
);
172 return ERR_PTR(retval
);
175 pid
->level
= ns
->level
;
177 for (i
= ns
->level
; i
>= 0; i
--) {
180 idr_preload(GFP_KERNEL
);
181 spin_lock_irq(&pidmap_lock
);
184 * init really needs pid 1, but after reaching the maximum
185 * wrap back to RESERVED_PIDS
187 if (idr_get_cursor(&tmp
->idr
) > RESERVED_PIDS
)
188 pid_min
= RESERVED_PIDS
;
191 * Store a null pointer so find_pid_ns does not find
192 * a partially initialized PID (see below).
194 nr
= idr_alloc_cyclic(&tmp
->idr
, NULL
, pid_min
,
195 pid_max
, GFP_ATOMIC
);
196 spin_unlock_irq(&pidmap_lock
);
200 retval
= (nr
== -ENOSPC
) ? -EAGAIN
: nr
;
204 pid
->numbers
[i
].nr
= nr
;
205 pid
->numbers
[i
].ns
= tmp
;
209 if (unlikely(is_child_reaper(pid
))) {
210 if (pid_ns_prepare_proc(ns
))
215 atomic_set(&pid
->count
, 1);
216 for (type
= 0; type
< PIDTYPE_MAX
; ++type
)
217 INIT_HLIST_HEAD(&pid
->tasks
[type
]);
219 init_waitqueue_head(&pid
->wait_pidfd
);
221 upid
= pid
->numbers
+ ns
->level
;
222 spin_lock_irq(&pidmap_lock
);
223 if (!(ns
->pid_allocated
& PIDNS_ADDING
))
225 for ( ; upid
>= pid
->numbers
; --upid
) {
226 /* Make the PID visible to find_pid_ns. */
227 idr_replace(&upid
->ns
->idr
, pid
, upid
->nr
);
228 upid
->ns
->pid_allocated
++;
230 spin_unlock_irq(&pidmap_lock
);
235 spin_unlock_irq(&pidmap_lock
);
239 spin_lock_irq(&pidmap_lock
);
240 while (++i
<= ns
->level
) {
241 upid
= pid
->numbers
+ i
;
242 idr_remove(&upid
->ns
->idr
, upid
->nr
);
245 /* On failure to allocate the first pid, reset the state */
246 if (ns
->pid_allocated
== PIDNS_ADDING
)
247 idr_set_cursor(&ns
->idr
, 0);
249 spin_unlock_irq(&pidmap_lock
);
251 kmem_cache_free(ns
->pid_cachep
, pid
);
252 return ERR_PTR(retval
);
255 void disable_pid_allocation(struct pid_namespace
*ns
)
257 spin_lock_irq(&pidmap_lock
);
258 ns
->pid_allocated
&= ~PIDNS_ADDING
;
259 spin_unlock_irq(&pidmap_lock
);
262 struct pid
*find_pid_ns(int nr
, struct pid_namespace
*ns
)
264 return idr_find(&ns
->idr
, nr
);
266 EXPORT_SYMBOL_GPL(find_pid_ns
);
268 struct pid
*find_vpid(int nr
)
270 return find_pid_ns(nr
, task_active_pid_ns(current
));
272 EXPORT_SYMBOL_GPL(find_vpid
);
274 static struct pid
**task_pid_ptr(struct task_struct
*task
, enum pid_type type
)
276 return (type
== PIDTYPE_PID
) ?
278 &task
->signal
->pids
[type
];
282 * attach_pid() must be called with the tasklist_lock write-held.
284 void attach_pid(struct task_struct
*task
, enum pid_type type
)
286 struct pid
*pid
= *task_pid_ptr(task
, type
);
287 hlist_add_head_rcu(&task
->pid_links
[type
], &pid
->tasks
[type
]);
290 static void __change_pid(struct task_struct
*task
, enum pid_type type
,
293 struct pid
**pid_ptr
= task_pid_ptr(task
, type
);
299 hlist_del_rcu(&task
->pid_links
[type
]);
302 for (tmp
= PIDTYPE_MAX
; --tmp
>= 0; )
303 if (!hlist_empty(&pid
->tasks
[tmp
]))
309 void detach_pid(struct task_struct
*task
, enum pid_type type
)
311 __change_pid(task
, type
, NULL
);
314 void change_pid(struct task_struct
*task
, enum pid_type type
,
317 __change_pid(task
, type
, pid
);
318 attach_pid(task
, type
);
321 /* transfer_pid is an optimization of attach_pid(new), detach_pid(old) */
322 void transfer_pid(struct task_struct
*old
, struct task_struct
*new,
325 if (type
== PIDTYPE_PID
)
326 new->thread_pid
= old
->thread_pid
;
327 hlist_replace_rcu(&old
->pid_links
[type
], &new->pid_links
[type
]);
330 struct task_struct
*pid_task(struct pid
*pid
, enum pid_type type
)
332 struct task_struct
*result
= NULL
;
334 struct hlist_node
*first
;
335 first
= rcu_dereference_check(hlist_first_rcu(&pid
->tasks
[type
]),
336 lockdep_tasklist_lock_is_held());
338 result
= hlist_entry(first
, struct task_struct
, pid_links
[(type
)]);
342 EXPORT_SYMBOL(pid_task
);
345 * Must be called under rcu_read_lock().
347 struct task_struct
*find_task_by_pid_ns(pid_t nr
, struct pid_namespace
*ns
)
349 RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
350 "find_task_by_pid_ns() needs rcu_read_lock() protection");
351 return pid_task(find_pid_ns(nr
, ns
), PIDTYPE_PID
);
354 struct task_struct
*find_task_by_vpid(pid_t vnr
)
356 return find_task_by_pid_ns(vnr
, task_active_pid_ns(current
));
359 struct task_struct
*find_get_task_by_vpid(pid_t nr
)
361 struct task_struct
*task
;
364 task
= find_task_by_vpid(nr
);
366 get_task_struct(task
);
372 struct pid
*get_task_pid(struct task_struct
*task
, enum pid_type type
)
376 pid
= get_pid(rcu_dereference(*task_pid_ptr(task
, type
)));
380 EXPORT_SYMBOL_GPL(get_task_pid
);
382 struct task_struct
*get_pid_task(struct pid
*pid
, enum pid_type type
)
384 struct task_struct
*result
;
386 result
= pid_task(pid
, type
);
388 get_task_struct(result
);
392 EXPORT_SYMBOL_GPL(get_pid_task
);
394 struct pid
*find_get_pid(pid_t nr
)
399 pid
= get_pid(find_vpid(nr
));
404 EXPORT_SYMBOL_GPL(find_get_pid
);
406 pid_t
pid_nr_ns(struct pid
*pid
, struct pid_namespace
*ns
)
411 if (pid
&& ns
->level
<= pid
->level
) {
412 upid
= &pid
->numbers
[ns
->level
];
418 EXPORT_SYMBOL_GPL(pid_nr_ns
);
420 pid_t
pid_vnr(struct pid
*pid
)
422 return pid_nr_ns(pid
, task_active_pid_ns(current
));
424 EXPORT_SYMBOL_GPL(pid_vnr
);
426 pid_t
__task_pid_nr_ns(struct task_struct
*task
, enum pid_type type
,
427 struct pid_namespace
*ns
)
433 ns
= task_active_pid_ns(current
);
434 if (likely(pid_alive(task
)))
435 nr
= pid_nr_ns(rcu_dereference(*task_pid_ptr(task
, type
)), ns
);
440 EXPORT_SYMBOL(__task_pid_nr_ns
);
442 struct pid_namespace
*task_active_pid_ns(struct task_struct
*tsk
)
444 return ns_of_pid(task_pid(tsk
));
446 EXPORT_SYMBOL_GPL(task_active_pid_ns
);
449 * Used by proc to find the first pid that is greater than or equal to nr.
451 * If there is a pid at nr this function is exactly the same as find_pid_ns.
453 struct pid
*find_ge_pid(int nr
, struct pid_namespace
*ns
)
455 return idr_get_next(&ns
->idr
, &nr
);
459 * pidfd_create() - Create a new pid file descriptor.
461 * @pid: struct pid that the pidfd will reference
463 * This creates a new pid file descriptor with the O_CLOEXEC flag set.
465 * Note, that this function can only be called after the fd table has
466 * been unshared to avoid leaking the pidfd to the new process.
468 * Return: On success, a cloexec pidfd is returned.
469 * On error, a negative errno number will be returned.
471 static int pidfd_create(struct pid
*pid
)
475 fd
= anon_inode_getfd("[pidfd]", &pidfd_fops
, get_pid(pid
),
484 * pidfd_open() - Open new pid file descriptor.
486 * @pid: pid for which to retrieve a pidfd
487 * @flags: flags to pass
489 * This creates a new pid file descriptor with the O_CLOEXEC flag set for
490 * the process identified by @pid. Currently, the process identified by
491 * @pid must be a thread-group leader. This restriction currently exists
492 * for all aspects of pidfds including pidfd creation (CLONE_PIDFD cannot
493 * be used with CLONE_THREAD) and pidfd polling (only supports thread group
496 * Return: On success, a cloexec pidfd is returned.
497 * On error, a negative errno number will be returned.
499 SYSCALL_DEFINE2(pidfd_open
, pid_t
, pid
, unsigned int, flags
)
510 p
= find_get_pid(pid
);
516 if (!pid_task(p
, PIDTYPE_TGID
))
520 fd
= ret
?: pidfd_create(p
);
525 void __init
pid_idr_init(void)
527 /* Verify no one has done anything silly: */
528 BUILD_BUG_ON(PID_MAX_LIMIT
>= PIDNS_ADDING
);
530 /* bump default and minimum pid_max based on number of cpus */
531 pid_max
= min(pid_max_max
, max_t(int, pid_max
,
532 PIDS_PER_CPU_DEFAULT
* num_possible_cpus()));
533 pid_max_min
= max_t(int, pid_max_min
,
534 PIDS_PER_CPU_MIN
* num_possible_cpus());
535 pr_info("pid_max: default: %u minimum: %u\n", pid_max
, pid_max_min
);
537 idr_init(&init_pid_ns
.idr
);
539 init_pid_ns
.pid_cachep
= KMEM_CACHE(pid
,
540 SLAB_HWCACHE_ALIGN
| SLAB_PANIC
| SLAB_ACCOUNT
);