struct plist_node list; /* entry in swap_active_head */
signed char type; /* strange name for an index */
unsigned int max; /* size of this swap device */
- unsigned long *zeromap; /* kvmalloc'ed bitmap to track zero pages */
struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
struct list_head free_clusters; /* free clusters list */
struct list_head full_clusters; /* full clusters list */
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
- * Check if the PTEs within a range are contiguous swap entries
- * and have consistent swapcache, zeromap.
+ * Check if the PTEs within a range are contiguous swap entries.
*/
static bool can_swapin_thp(struct vm_fault *vmf, pte_t *ptep, int nr_pages)
{
unsigned long addr;
- softleaf_t entry;
int idx;
pte_t pte;
if (!pte_same(pte, pte_move_swp_offset(vmf->orig_pte, -idx)))
return false;
- entry = softleaf_from_pte(pte);
- if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages)
- return false;
-
/*
* swap_read_folio() can't handle the case a large folio is hybridly
* from different backends. And they are likely corner cases. Similar
* things might be added once zswap support large folios.
*/
- if (unlikely(swap_zeromap_batch(entry, nr_pages, NULL) != nr_pages))
+ if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages)
return false;
-
return true;
}
#include <linux/delayacct.h>
#include <linux/zswap.h>
#include "swap.h"
+#include "swap_table.h"
static void __end_swap_bio_write(struct bio *bio)
{
static void swap_zeromap_folio_set(struct folio *folio)
{
struct obj_cgroup *objcg = get_obj_cgroup_from_folio(folio);
- struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
int nr_pages = folio_nr_pages(folio);
+ struct swap_cluster_info *ci;
swp_entry_t entry;
unsigned int i;
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
+
+ ci = swap_cluster_get_and_lock(folio);
for (i = 0; i < folio_nr_pages(folio); i++) {
entry = page_swap_entry(folio_page(folio, i));
- set_bit(swp_offset(entry), sis->zeromap);
+ __swap_table_set_zero(ci, swp_cluster_offset(entry));
}
+ swap_cluster_unlock(ci);
count_vm_events(SWPOUT_ZERO, nr_pages);
if (objcg) {
static void swap_zeromap_folio_clear(struct folio *folio)
{
- struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
+ struct swap_cluster_info *ci;
swp_entry_t entry;
unsigned int i;
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
+
+ ci = swap_cluster_get_and_lock(folio);
for (i = 0; i < folio_nr_pages(folio); i++) {
entry = page_swap_entry(folio_page(folio, i));
- clear_bit(swp_offset(entry), sis->zeromap);
+ __swap_table_clear_zero(ci, swp_cluster_offset(entry));
}
+ swap_cluster_unlock(ci);
}
/*
}
/*
- * Use a bitmap (zeromap) to avoid doing IO for zero-filled pages.
- * The bits in zeromap are protected by the locked swapcache folio
- * and atomic updates are used to protect against read-modify-write
- * corruption due to other zero swap entries seeing concurrent updates.
+ * Use the swap table zero mark to avoid doing IO for zero-filled
+ * pages. The zero mark is protected by the cluster lock, which is
+ * acquired internally by swap_zeromap_folio_set/clear.
*/
if (is_folio_zero_filled(folio)) {
swap_zeromap_folio_set(folio);
mempool_free(sio, sio_pool);
}
+/*
+ * Return the count of contiguous swap entries that share the same
+ * zeromap status as the starting entry. If is_zerop is not NULL,
+ * it will return the zeromap status of the starting entry.
+ *
+ * Context: Caller must ensure the cluster containing the entries
+ * that will be checked won't be freed.
+ */
+static int swap_zeromap_batch(swp_entry_t entry, int max_nr,
+ bool *is_zerop)
+{
+ int i;
+ bool is_zero;
+ unsigned int ci_start = swp_cluster_offset(entry);
+ struct swap_cluster_info *ci = __swap_entry_to_cluster(entry);
+
+ VM_WARN_ON_ONCE(ci_start + max_nr > SWAPFILE_CLUSTER);
+
+ rcu_read_lock();
+ is_zero = __swap_table_test_zero(ci, ci_start);
+ for (i = 1; i < max_nr; i++)
+ if (is_zero != __swap_table_test_zero(ci, ci_start + i))
+ break;
+ rcu_read_unlock();
+ if (is_zerop)
+ *is_zerop = is_zero;
+
+ return i;
+}
+
static bool swap_read_folio_zeromap(struct folio *folio)
{
int nr_pages = folio_nr_pages(folio);
struct obj_cgroup *objcg;
bool is_zeromap;
+ VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
+
/*
* Swapping in a large folio that is partially in the zeromap is not
* currently handled. Return true without marking the folio uptodate so
* that an IO error is emitted (e.g. do_swap_page() will sigbus).
+ * Folio lock stabilizes the cluster and map, so the check is safe.
*/
if (WARN_ON_ONCE(swap_zeromap_batch(folio->swap, nr_pages,
- &is_zeromap) != nr_pages))
+ &is_zeromap) != nr_pages))
return true;
if (!is_zeromap)
#define _MM_SWAP_H
#include <linux/atomic.h> /* for atomic_long_t */
+#include <linux/mm.h> /* for PAGE_SHIFT */
struct mempolicy;
struct swap_iocb;
struct swap_memcg_table;
extern int page_cluster;
+#if defined(MAX_POSSIBLE_PHYSMEM_BITS)
+#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
+#elif defined(MAX_PHYSMEM_BITS)
+#define SWAP_CACHE_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
+#else
+#define SWAP_CACHE_PFN_BITS (BITS_PER_LONG - PAGE_SHIFT)
+#endif
+
+/* Swap table marker, 0x1 means shadow, 0x2 means PFN (SWP_TB_PFN_MARK) */
+#define SWAP_CACHE_PFN_MARK_BITS 2
+/* At least 2 bits are needed to distinguish SWP_TB_COUNT_MAX, 1 and 0 */
+#define SWAP_COUNT_MIN_BITS 2
+/* If there are enough bits besides PFN and marker, store zero flag inline */
+#define SWAP_TABLE_HAS_ZEROFLAG ((BITS_PER_LONG - SWAP_CACHE_PFN_MARK_BITS - \
+ SWAP_CACHE_PFN_BITS) > SWAP_COUNT_MIN_BITS)
+
#ifdef CONFIG_THP_SWAP
#define SWAPFILE_CLUSTER HPAGE_PMD_NR
#define swap_entry_order(order) (order)
unsigned int *extend_table; /* For large swap count, protected by ci->lock */
#ifdef CONFIG_MEMCG
struct swap_memcg_table *memcg_table; /* Swap table entries' cgroup record */
+#endif
+#if !SWAP_TABLE_HAS_ZEROFLAG
+ unsigned long *zero_bitmap;
#endif
struct list_head list;
};
return __swap_entry_to_info(folio->swap)->flags;
}
-/*
- * Return the count of contiguous swap entries that share the same
- * zeromap status as the starting entry. If is_zeromap is not NULL,
- * it will return the zeromap status of the starting entry.
- */
-static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
- bool *is_zeromap)
-{
- struct swap_info_struct *sis = __swap_entry_to_info(entry);
- unsigned long start = swp_offset(entry);
- unsigned long end = start + max_nr;
- bool first_bit;
-
- first_bit = test_bit(start, sis->zeromap);
- if (is_zeromap)
- *is_zeromap = first_bit;
-
- if (max_nr <= 1)
- return max_nr;
- if (first_bit)
- return find_next_zero_bit(sis->zeromap, end, start) - start;
- else
- return find_next_bit(sis->zeromap, end, start) - start;
-}
-
#else /* CONFIG_SWAP */
struct swap_iocb;
static inline struct swap_cluster_info *swap_cluster_lock(
return 0;
}
-static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
- bool *has_zeromap)
-{
- return 0;
-}
#endif /* CONFIG_SWAP */
#endif /* _MM_SWAP_H */
{
unsigned int ci_off, ci_end;
unsigned long old_tb;
+ bool is_zero;
lockdep_assert_held(&ci->lock);
if (nr == 1)
return 0;
+ is_zero = __swap_table_test_zero(ci, ci_off);
ci_off = round_down(ci_off, nr);
ci_end = ci_off + nr;
do {
old_tb = __swap_table_get(ci, ci_off);
if (unlikely(swp_tb_is_folio(old_tb) ||
!__swp_tb_get_count(old_tb) ||
+ is_zero != __swap_table_test_zero(ci, ci_off) ||
(memcg_id && *memcg_id != __swap_cgroup_get(ci, ci_off))))
return -EBUSY;
} while (++ci_off < ci_end);
do {
old_tb = __swap_table_get(ci, ci_off);
VM_WARN_ON_ONCE(swp_tb_is_folio(old_tb));
- __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_count(old_tb)));
+ __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_flags(old_tb)));
} while (++ci_off < ci_end);
folio_ref_add(folio, nr_pages);
struct folio *folio,
swp_entry_t entry, void *shadow)
{
- int count;
unsigned long old_tb;
struct swap_info_struct *si;
unsigned int ci_start, ci_off, ci_end;
old_tb = __swap_table_get(ci, ci_off);
WARN_ON_ONCE(!swp_tb_is_folio(old_tb) ||
swp_tb_to_folio(old_tb) != folio);
- count = __swp_tb_get_count(old_tb);
- if (count)
+ if (__swp_tb_get_count(old_tb))
folio_swapped = true;
else
need_free = true;
/* If shadow is NULL, we set an empty shadow. */
- __swap_table_set(ci, ci_off, shadow_to_swp_tb(shadow, count));
+ __swap_table_set(ci, ci_off, shadow_to_swp_tb(shadow,
+ __swp_tb_get_flags(old_tb)));
} while (++ci_off < ci_end);
folio->swap.val = 0;
do {
old_tb = __swap_table_get(ci, ci_off);
WARN_ON_ONCE(!swp_tb_is_folio(old_tb) || swp_tb_to_folio(old_tb) != old);
- __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_count(old_tb)));
+ __swap_table_set(ci, ci_off, pfn_to_swp_tb(pfn, __swp_tb_get_flags(old_tb)));
} while (++ci_off < ci_end);
/*
* Swap table entry type and bits layouts:
*
* NULL: |---------------- 0 ---------------| - Free slot
- * Shadow: | SWAP_COUNT |---- SHADOW_VAL ---|1| - Swapped out slot
- * PFN: | SWAP_COUNT |------ PFN -------|10| - Cached slot
+ * Shadow: |SWAP_COUNT|Z|---- SHADOW_VAL ---|1| - Swapped out slot
+ * PFN: |SWAP_COUNT|Z|------ PFN -------|10| - Cached slot
* Pointer: |----------- Pointer ----------|100| - (Unused)
* Bad: |------------- 1 -------------|1000| - Bad slot
*
- * SWAP_COUNT is `SWP_TB_COUNT_BITS` long, each entry is an atomic long.
+ * COUNT is `SWP_TB_COUNT_BITS` long, Z is the `SWP_TB_ZERO_FLAG` bit,
+ * and together they form the `SWP_TB_FLAGS_BITS` wide flags field.
+ * Each entry is an atomic long.
*
* Usages:
*
* - Bad: Swap slot is reserved, protects swap header or holes on swap devices.
*/
-#if defined(MAX_POSSIBLE_PHYSMEM_BITS)
-#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
-#elif defined(MAX_PHYSMEM_BITS)
-#define SWAP_CACHE_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
-#else
-#define SWAP_CACHE_PFN_BITS (BITS_PER_LONG - PAGE_SHIFT)
-#endif
-
/* NULL Entry, all 0 */
#define SWP_TB_NULL 0UL
#define SWP_TB_SHADOW_MARK 0b1UL
/* Cached: PFN */
-#define SWP_TB_PFN_BITS (SWAP_CACHE_PFN_BITS + SWP_TB_PFN_MARK_BITS)
+#define SWP_TB_PFN_BITS (SWAP_CACHE_PFN_BITS + SWAP_CACHE_PFN_MARK_BITS)
#define SWP_TB_PFN_MARK 0b10UL
-#define SWP_TB_PFN_MARK_BITS 2
-#define SWP_TB_PFN_MARK_MASK (BIT(SWP_TB_PFN_MARK_BITS) - 1)
+#define SWP_TB_PFN_MARK_MASK (BIT(SWAP_CACHE_PFN_MARK_BITS) - 1)
-/* SWAP_COUNT part for PFN or shadow, the width can be shrunk or extended */
-#define SWP_TB_COUNT_BITS min(4, BITS_PER_LONG - SWP_TB_PFN_BITS)
+/* Flags: For PFN or shadow, contains SWAP_COUNT, width changes */
+#define SWP_TB_FLAGS_BITS min(5, BITS_PER_LONG - SWP_TB_PFN_BITS)
+#define SWP_TB_COUNT_BITS (SWP_TB_FLAGS_BITS - SWAP_TABLE_HAS_ZEROFLAG)
+#define SWP_TB_FLAGS_MASK (~((~0UL) >> SWP_TB_FLAGS_BITS))
#define SWP_TB_COUNT_MASK (~((~0UL) >> SWP_TB_COUNT_BITS))
+#define SWP_TB_FLAGS_SHIFT (BITS_PER_LONG - SWP_TB_FLAGS_BITS)
#define SWP_TB_COUNT_SHIFT (BITS_PER_LONG - SWP_TB_COUNT_BITS)
#define SWP_TB_COUNT_MAX ((1 << SWP_TB_COUNT_BITS) - 1)
+/* The first flag is zero bit (SWAP_TABLE_HAS_ZEROFLAG) */
+#define SWP_TB_ZERO_FLAG BIT(BITS_PER_LONG - SWP_TB_FLAGS_BITS)
/* Bad slot: ends with 0b1000 and rests of bits are all 1 */
#define SWP_TB_BAD ((~0UL) << 3)
/* Macro for shadow offset calculation */
-#define SWAP_COUNT_SHIFT SWP_TB_COUNT_BITS
+#define SWAP_COUNT_SHIFT SWP_TB_FLAGS_BITS
/*
* Helpers for casting one type of info into a swap table entry.
* used (count > 0 && count < SWP_TB_COUNT_MAX), and
* overflow (count == SWP_TB_COUNT_MAX).
*/
- BUILD_BUG_ON(SWP_TB_COUNT_MAX < 2 || SWP_TB_COUNT_BITS < 2);
+ BUILD_BUG_ON(SWP_TB_COUNT_BITS < SWAP_COUNT_MIN_BITS);
VM_WARN_ON(count > SWP_TB_COUNT_MAX);
return ((unsigned long)count) << SWP_TB_COUNT_SHIFT;
}
-static inline unsigned long pfn_to_swp_tb(unsigned long pfn, unsigned int count)
+static inline unsigned long __flags_to_swp_tb(unsigned char flags)
+{
+ BUILD_BUG_ON(SWP_TB_FLAGS_BITS > BITS_PER_BYTE);
+ VM_WARN_ON(flags >> SWP_TB_FLAGS_BITS);
+ return ((unsigned long)flags) << SWP_TB_FLAGS_SHIFT;
+}
+
+static inline unsigned long pfn_to_swp_tb(unsigned long pfn, unsigned char flags)
{
unsigned long swp_tb;
BUILD_BUG_ON(sizeof(unsigned long) != sizeof(void *));
BUILD_BUG_ON(SWAP_CACHE_PFN_BITS >
- (BITS_PER_LONG - SWP_TB_PFN_MARK_BITS - SWP_TB_COUNT_BITS));
+ (BITS_PER_LONG - SWAP_CACHE_PFN_MARK_BITS - SWP_TB_FLAGS_BITS));
- swp_tb = (pfn << SWP_TB_PFN_MARK_BITS) | SWP_TB_PFN_MARK;
- VM_WARN_ON_ONCE(swp_tb & SWP_TB_COUNT_MASK);
+ swp_tb = (pfn << SWAP_CACHE_PFN_MARK_BITS) | SWP_TB_PFN_MARK;
+ VM_WARN_ON_ONCE(swp_tb & SWP_TB_FLAGS_MASK);
- return swp_tb | __count_to_swp_tb(count);
+ return swp_tb | __flags_to_swp_tb(flags);
}
-static inline unsigned long folio_to_swp_tb(struct folio *folio, unsigned int count)
+static inline unsigned long folio_to_swp_tb(struct folio *folio, unsigned char flags)
{
- return pfn_to_swp_tb(folio_pfn(folio), count);
+ return pfn_to_swp_tb(folio_pfn(folio), flags);
}
-static inline unsigned long shadow_to_swp_tb(void *shadow, unsigned int count)
+static inline unsigned long shadow_to_swp_tb(void *shadow, unsigned char flags)
{
BUILD_BUG_ON((BITS_PER_XA_VALUE + 1) !=
BITS_PER_BYTE * sizeof(unsigned long));
BUILD_BUG_ON((unsigned long)xa_mk_value(0) != SWP_TB_SHADOW_MARK);
VM_WARN_ON_ONCE(shadow && !xa_is_value(shadow));
- VM_WARN_ON_ONCE(shadow && ((unsigned long)shadow & SWP_TB_COUNT_MASK));
+ VM_WARN_ON_ONCE(shadow && ((unsigned long)shadow & SWP_TB_FLAGS_MASK));
- return (unsigned long)shadow | __count_to_swp_tb(count) | SWP_TB_SHADOW_MARK;
+ return (unsigned long)shadow | SWP_TB_SHADOW_MARK | __flags_to_swp_tb(flags);
}
/*
static inline struct folio *swp_tb_to_folio(unsigned long swp_tb)
{
VM_WARN_ON(!swp_tb_is_folio(swp_tb));
- return pfn_folio((swp_tb & ~SWP_TB_COUNT_MASK) >> SWP_TB_PFN_MARK_BITS);
+ return pfn_folio((swp_tb & ~SWP_TB_FLAGS_MASK) >> SWAP_CACHE_PFN_MARK_BITS);
}
static inline void *swp_tb_to_shadow(unsigned long swp_tb)
{
VM_WARN_ON(!swp_tb_is_shadow(swp_tb));
/* No shift needed, xa_value is stored as it is in the lower bits. */
- return (void *)(swp_tb & ~SWP_TB_COUNT_MASK);
+ return (void *)(swp_tb & ~SWP_TB_FLAGS_MASK);
}
static inline unsigned char __swp_tb_get_count(unsigned long swp_tb)
return ((swp_tb & SWP_TB_COUNT_MASK) >> SWP_TB_COUNT_SHIFT);
}
+static inline unsigned char __swp_tb_get_flags(unsigned long swp_tb)
+{
+ VM_WARN_ON(!swp_tb_is_countable(swp_tb));
+ return ((swp_tb & SWP_TB_FLAGS_MASK) >> SWP_TB_FLAGS_SHIFT);
+}
+
static inline int swp_tb_get_count(unsigned long swp_tb)
{
if (swp_tb_is_countable(swp_tb))
return swp_tb;
}
+static inline void __swap_table_set_zero(struct swap_cluster_info *ci,
+ unsigned int ci_off)
+{
+#if SWAP_TABLE_HAS_ZEROFLAG
+ unsigned long swp_tb = __swap_table_get(ci, ci_off);
+
+ BUILD_BUG_ON(SWP_TB_ZERO_FLAG & ~SWP_TB_FLAGS_MASK);
+ VM_WARN_ON(!swp_tb_is_countable(swp_tb));
+ swp_tb |= SWP_TB_ZERO_FLAG;
+ __swap_table_set(ci, ci_off, swp_tb);
+#else
+ lockdep_assert_held(&ci->lock);
+ __set_bit(ci_off, ci->zero_bitmap);
+#endif
+}
+
+static inline bool __swap_table_test_zero(struct swap_cluster_info *ci,
+ unsigned int ci_off)
+{
+#if SWAP_TABLE_HAS_ZEROFLAG
+ unsigned long swp_tb = __swap_table_get(ci, ci_off);
+
+ VM_WARN_ON(!swp_tb_is_countable(swp_tb));
+ return !!(swp_tb & SWP_TB_ZERO_FLAG);
+#else
+ return test_bit(ci_off, ci->zero_bitmap);
+#endif
+}
+
+static inline void __swap_table_clear_zero(struct swap_cluster_info *ci,
+ unsigned int ci_off)
+{
+#if SWAP_TABLE_HAS_ZEROFLAG
+ unsigned long swp_tb = __swap_table_get(ci, ci_off);
+
+ VM_WARN_ON(!swp_tb_is_countable(swp_tb));
+ swp_tb &= ~SWP_TB_ZERO_FLAG;
+ __swap_table_set(ci, ci_off, swp_tb);
+#else
+ lockdep_assert_held(&ci->lock);
+ __clear_bit(ci_off, ci->zero_bitmap);
+#endif
+}
+
#ifdef CONFIG_MEMCG
static inline void __swap_cgroup_set(struct swap_cluster_info *ci,
unsigned int ci_off, unsigned long nr, unsigned short id)
ci->memcg_table = NULL;
#endif
+#if !SWAP_TABLE_HAS_ZEROFLAG
+ kfree(ci->zero_bitmap);
+ ci->zero_bitmap = NULL;
+#endif
+
table = (struct swap_table *)rcu_access_pointer(ci->table);
if (!table)
return;
{
struct swap_table *table = NULL;
struct folio *folio;
- int ret = 0;
/* The cluster must be empty and not on any list during allocation. */
VM_WARN_ON_ONCE(ci->flags || !cluster_is_empty(ci));
if (!mem_cgroup_disabled()) {
VM_WARN_ON_ONCE(ci->memcg_table);
ci->memcg_table = kzalloc_obj(*ci->memcg_table, gfp);
- if (!ci->memcg_table)
- ret = -ENOMEM;
+ if (!ci->memcg_table) {
+ swap_cluster_free_table(ci);
+ return -ENOMEM;
+ }
}
#endif
- if (ret)
- swap_cluster_free_table(ci);
- return ret;
+#if !SWAP_TABLE_HAS_ZEROFLAG
+ VM_WARN_ON_ONCE(ci->zero_bitmap);
+ ci->zero_bitmap = bitmap_zalloc(SWAPFILE_CLUSTER, gfp);
+ if (!ci->zero_bitmap) {
+ swap_cluster_free_table(ci);
+ return -ENOMEM;
+ }
+#endif
+ return 0;
}
/*
order = 0;
nr_pages = 1;
swap_cluster_assert_empty(ci, ci_off, 1, false);
- /* Sets a fake shadow as placeholder */
- __swap_table_set(ci, ci_off, shadow_to_swp_tb(NULL, 1));
+ /* Fake shadow placeholder with no flag, hibernation does not use the zeromap */
+ __swap_table_set(ci, ci_off, __swp_tb_mk_count(shadow_to_swp_tb(NULL, 0), 1));
} else {
/* Allocation without folio is only possible with hibernation */
WARN_ON_ONCE(1);
void (*swap_slot_free_notify)(struct block_device *, unsigned long);
unsigned int i;
- /*
- * Use atomic clear_bit operations only on zeromap instead of non-atomic
- * bitmap_clear to prevent adjacent bits corruption due to simultaneous writes.
- */
- for (i = 0; i < nr_entries; i++) {
- clear_bit(offset + i, si->zeromap);
+ for (i = 0; i < nr_entries; i++)
zswap_invalidate(swp_entry(si->type, offset + i));
- }
if (si->flags & SWP_BLKDEV)
swap_slot_free_notify =
* ref, or after swap cache is dropped
*/
VM_WARN_ON(!swp_tb_is_shadow(old_tb) || __swp_tb_get_count(old_tb) > 1);
+
+ /* Resetting the slot to NULL also clears the inline flags. */
__swap_table_set(ci, ci_off, null_to_swp_tb());
+ if (!SWAP_TABLE_HAS_ZEROFLAG)
+ __swap_table_clear_zero(ci, ci_off);
/*
* Uncharge swap slots by memcg in batches. Consecutive
SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
{
struct swap_info_struct *p = NULL;
- unsigned long *zeromap;
struct swap_cluster_info *cluster_info;
struct file *swap_file, *victim;
struct address_space *mapping;
swap_file = p->swap_file;
p->swap_file = NULL;
- zeromap = p->zeromap;
- p->zeromap = NULL;
maxpages = p->max;
cluster_info = p->cluster_info;
p->max = 0;
mutex_unlock(&swapon_mutex);
kfree(p->global_cluster);
p->global_cluster = NULL;
- kvfree(zeromap);
free_swap_cluster_info(cluster_info, maxpages);
inode = mapping->host;
if (error)
goto bad_swap_unlock_inode;
- /*
- * Use kvmalloc_array instead of bitmap_zalloc as the allocation order might
- * be above MAX_PAGE_ORDER incase of a large swap file.
- */
- si->zeromap = kvmalloc_array(BITS_TO_LONGS(maxpages), sizeof(long),
- GFP_KERNEL | __GFP_ZERO);
- if (!si->zeromap) {
- error = -ENOMEM;
- goto bad_swap_unlock_inode;
- }
-
if (si->bdev && bdev_stable_writes(si->bdev))
si->flags |= SWP_STABLE_WRITES;
destroy_swap_extents(si, swap_file);
free_swap_cluster_info(si->cluster_info, si->max);
si->cluster_info = NULL;
- kvfree(si->zeromap);
- si->zeromap = NULL;
/*
* Clear the SWP_USED flag after all resources are freed so
* alloc_swap_info can reuse this si safely.
projects / thirdparty / linux.git / commitdiff
