1 // SPDX-License-Identifier: GPL-2.0
3 * Manage cache of swap slots to be used for and returned from
6 * Copyright(c) 2016 Intel Corporation.
8 * Author: Tim Chen <tim.c.chen@linux.intel.com>
10 * We allocate the swap slots from the global pool and put
11 * it into local per cpu caches. This has the advantage
12 * of no needing to acquire the swap_info lock every time
15 * There is also opportunity to simply return the slot
16 * to local caches without needing to acquire swap_info
17 * lock. We do not reuse the returned slots directly but
18 * move them back to the global pool in a batch. This
19 * allows the slots to coaellesce and reduce fragmentation.
21 * The swap entry allocated is marked with SWAP_HAS_CACHE
22 * flag in map_count that prevents it from being allocated
23 * again from the global pool.
25 * The swap slots cache is protected by a mutex instead of
26 * a spin lock as when we search for slots with scan_swap_map,
27 * we can possibly sleep.
30 #include <linux/swap_slots.h>
31 #include <linux/cpu.h>
32 #include <linux/cpumask.h>
33 #include <linux/vmalloc.h>
34 #include <linux/mutex.h>
39 static DEFINE_PER_CPU(struct swap_slots_cache
, swp_slots
);
40 static bool swap_slot_cache_active
;
41 bool swap_slot_cache_enabled
;
42 static bool swap_slot_cache_initialized
;
43 DEFINE_MUTEX(swap_slots_cache_mutex
);
44 /* Serialize swap slots cache enable/disable operations */
45 DEFINE_MUTEX(swap_slots_cache_enable_mutex
);
47 static void __drain_swap_slots_cache(unsigned int type
);
48 static void deactivate_swap_slots_cache(void);
49 static void reactivate_swap_slots_cache(void);
51 #define use_swap_slot_cache (swap_slot_cache_active && \
52 swap_slot_cache_enabled && swap_slot_cache_initialized)
53 #define SLOTS_CACHE 0x1
54 #define SLOTS_CACHE_RET 0x2
56 static void deactivate_swap_slots_cache(void)
58 mutex_lock(&swap_slots_cache_mutex
);
59 swap_slot_cache_active
= false;
60 __drain_swap_slots_cache(SLOTS_CACHE
|SLOTS_CACHE_RET
);
61 mutex_unlock(&swap_slots_cache_mutex
);
64 static void reactivate_swap_slots_cache(void)
66 mutex_lock(&swap_slots_cache_mutex
);
67 swap_slot_cache_active
= true;
68 mutex_unlock(&swap_slots_cache_mutex
);
71 /* Must not be called with cpu hot plug lock */
72 void disable_swap_slots_cache_lock(void)
74 mutex_lock(&swap_slots_cache_enable_mutex
);
75 swap_slot_cache_enabled
= false;
76 if (swap_slot_cache_initialized
) {
77 /* serialize with cpu hotplug operations */
79 __drain_swap_slots_cache(SLOTS_CACHE
|SLOTS_CACHE_RET
);
84 static void __reenable_swap_slots_cache(void)
86 swap_slot_cache_enabled
= has_usable_swap();
89 void reenable_swap_slots_cache_unlock(void)
91 __reenable_swap_slots_cache();
92 mutex_unlock(&swap_slots_cache_enable_mutex
);
95 static bool check_cache_active(void)
99 if (!swap_slot_cache_enabled
|| !swap_slot_cache_initialized
)
102 pages
= get_nr_swap_pages();
103 if (!swap_slot_cache_active
) {
104 if (pages
> num_online_cpus() *
105 THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE
)
106 reactivate_swap_slots_cache();
110 /* if global pool of slot caches too low, deactivate cache */
111 if (pages
< num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE
)
112 deactivate_swap_slots_cache();
114 return swap_slot_cache_active
;
117 static int alloc_swap_slot_cache(unsigned int cpu
)
119 struct swap_slots_cache
*cache
;
120 swp_entry_t
*slots
, *slots_ret
;
123 * Do allocation outside swap_slots_cache_mutex
124 * as kvzalloc could trigger reclaim and get_swap_page,
125 * which can lock swap_slots_cache_mutex.
127 slots
= kvzalloc(sizeof(swp_entry_t
) * SWAP_SLOTS_CACHE_SIZE
,
132 slots_ret
= kvzalloc(sizeof(swp_entry_t
) * SWAP_SLOTS_CACHE_SIZE
,
139 mutex_lock(&swap_slots_cache_mutex
);
140 cache
= &per_cpu(swp_slots
, cpu
);
141 if (cache
->slots
|| cache
->slots_ret
)
142 /* cache already allocated */
144 if (!cache
->lock_initialized
) {
145 mutex_init(&cache
->alloc_lock
);
146 spin_lock_init(&cache
->free_lock
);
147 cache
->lock_initialized
= true;
152 cache
->slots
= slots
;
154 cache
->slots_ret
= slots_ret
;
157 mutex_unlock(&swap_slots_cache_mutex
);
165 static void drain_slots_cache_cpu(unsigned int cpu
, unsigned int type
,
168 struct swap_slots_cache
*cache
;
169 swp_entry_t
*slots
= NULL
;
171 cache
= &per_cpu(swp_slots
, cpu
);
172 if ((type
& SLOTS_CACHE
) && cache
->slots
) {
173 mutex_lock(&cache
->alloc_lock
);
174 swapcache_free_entries(cache
->slots
+ cache
->cur
, cache
->nr
);
177 if (free_slots
&& cache
->slots
) {
178 kvfree(cache
->slots
);
181 mutex_unlock(&cache
->alloc_lock
);
183 if ((type
& SLOTS_CACHE_RET
) && cache
->slots_ret
) {
184 spin_lock_irq(&cache
->free_lock
);
185 swapcache_free_entries(cache
->slots_ret
, cache
->n_ret
);
187 if (free_slots
&& cache
->slots_ret
) {
188 slots
= cache
->slots_ret
;
189 cache
->slots_ret
= NULL
;
191 spin_unlock_irq(&cache
->free_lock
);
197 static void __drain_swap_slots_cache(unsigned int type
)
202 * This function is called during
203 * 1) swapoff, when we have to make sure no
204 * left over slots are in cache when we remove
206 * 2) disabling of swap slot cache, when we run low
207 * on swap slots when allocating memory and need
208 * to return swap slots to global pool.
210 * We cannot acquire cpu hot plug lock here as
211 * this function can be invoked in the cpu
213 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
214 * -> memory allocation -> direct reclaim -> get_swap_page
215 * -> drain_swap_slots_cache
217 * Hence the loop over current online cpu below could miss cpu that
218 * is being brought online but not yet marked as online.
219 * That is okay as we do not schedule and run anything on a
220 * cpu before it has been marked online. Hence, we will not
221 * fill any swap slots in slots cache of such cpu.
222 * There are no slots on such cpu that need to be drained.
224 for_each_online_cpu(cpu
)
225 drain_slots_cache_cpu(cpu
, type
, false);
228 static int free_slot_cache(unsigned int cpu
)
230 mutex_lock(&swap_slots_cache_mutex
);
231 drain_slots_cache_cpu(cpu
, SLOTS_CACHE
| SLOTS_CACHE_RET
, true);
232 mutex_unlock(&swap_slots_cache_mutex
);
236 int enable_swap_slots_cache(void)
240 mutex_lock(&swap_slots_cache_enable_mutex
);
241 if (swap_slot_cache_initialized
) {
242 __reenable_swap_slots_cache();
246 ret
= cpuhp_setup_state(CPUHP_AP_ONLINE_DYN
, "swap_slots_cache",
247 alloc_swap_slot_cache
, free_slot_cache
);
248 if (WARN_ONCE(ret
< 0, "Cache allocation failed (%s), operating "
249 "without swap slots cache.\n", __func__
))
252 swap_slot_cache_initialized
= true;
253 __reenable_swap_slots_cache();
255 mutex_unlock(&swap_slots_cache_enable_mutex
);
259 /* called with swap slot cache's alloc lock held */
260 static int refill_swap_slots_cache(struct swap_slots_cache
*cache
)
262 if (!use_swap_slot_cache
|| cache
->nr
)
266 if (swap_slot_cache_active
)
267 cache
->nr
= get_swap_pages(SWAP_SLOTS_CACHE_SIZE
, false,
273 int free_swap_slot(swp_entry_t entry
)
275 struct swap_slots_cache
*cache
;
277 cache
= raw_cpu_ptr(&swp_slots
);
278 if (use_swap_slot_cache
&& cache
->slots_ret
) {
279 spin_lock_irq(&cache
->free_lock
);
280 /* Swap slots cache may be deactivated before acquiring lock */
281 if (!use_swap_slot_cache
|| !cache
->slots_ret
) {
282 spin_unlock_irq(&cache
->free_lock
);
285 if (cache
->n_ret
>= SWAP_SLOTS_CACHE_SIZE
) {
287 * Return slots to global pool.
288 * The current swap_map value is SWAP_HAS_CACHE.
289 * Set it to 0 to indicate it is available for
290 * allocation in global pool
292 swapcache_free_entries(cache
->slots_ret
, cache
->n_ret
);
295 cache
->slots_ret
[cache
->n_ret
++] = entry
;
296 spin_unlock_irq(&cache
->free_lock
);
299 swapcache_free_entries(&entry
, 1);
305 swp_entry_t
get_swap_page(struct page
*page
)
307 swp_entry_t entry
, *pentry
;
308 struct swap_slots_cache
*cache
;
312 if (PageTransHuge(page
)) {
313 if (IS_ENABLED(CONFIG_THP_SWAP
))
314 get_swap_pages(1, true, &entry
);
319 * Preemption is allowed here, because we may sleep
320 * in refill_swap_slots_cache(). But it is safe, because
321 * accesses to the per-CPU data structure are protected by the
322 * mutex cache->alloc_lock.
324 * The alloc path here does not touch cache->slots_ret
325 * so cache->free_lock is not taken.
327 cache
= raw_cpu_ptr(&swp_slots
);
329 if (check_cache_active()) {
330 mutex_lock(&cache
->alloc_lock
);
334 pentry
= &cache
->slots
[cache
->cur
++];
339 if (refill_swap_slots_cache(cache
))
343 mutex_unlock(&cache
->alloc_lock
);
348 get_swap_pages(1, false, &entry
);
353 #endif /* CONFIG_SWAP */