mm/swapfile.c: __swap_entry_free() always free 1 entry

[thirdparty/linux.git] / mm / swapfile.c

diff --git a/mm/swapfile.c b/mm/swapfile.c
index 106ff9c1a6b9299bf3c3a0b118b6ca7f9700943a..2aa272376cae18652ad7e3712cc6d78062820e7f 100644 (file)
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -601,7 +601,6 @@ static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
 {
        struct percpu_cluster *cluster;
        struct swap_cluster_info *ci;
-       bool found_free;
        unsigned long tmp, max;
 
 new_cluster:
@@ -623,8 +622,6 @@ new_cluster:
                        return false;
        }
 
-       found_free = false;
-
        /*
         * Other CPUs can use our cluster if they can't find a free cluster,
         * check if there is still free entry in the cluster
@@ -632,27 +629,23 @@ new_cluster:
        tmp = cluster->next;
        max = min_t(unsigned long, si->max,
                    (cluster_next(&cluster->index) + 1) * SWAPFILE_CLUSTER);
-       if (tmp >= max) {
-               cluster_set_null(&cluster->index);
-               goto new_cluster;
-       }
-       ci = lock_cluster(si, tmp);
-       while (tmp < max) {
-               if (!si->swap_map[tmp]) {
-                       found_free = true;
-                       break;
+       if (tmp < max) {
+               ci = lock_cluster(si, tmp);
+               while (tmp < max) {
+                       if (!si->swap_map[tmp])
+                               break;
+                       tmp++;
                }
-               tmp++;
+               unlock_cluster(ci);
        }
-       unlock_cluster(ci);
-       if (!found_free) {
+       if (tmp >= max) {
                cluster_set_null(&cluster->index);
                goto new_cluster;
        }
        cluster->next = tmp + 1;
        *offset = tmp;
        *scan_base = tmp;
-       return found_free;
+       return true;
 }
 
 static void __del_from_avail_list(struct swap_info_struct *p)
@@ -739,9 +732,7 @@ static int scan_swap_map_slots(struct swap_info_struct *si,
        unsigned long last_in_cluster = 0;
        int latency_ration = LATENCY_LIMIT;
        int n_ret = 0;
-
-       if (nr > SWAP_BATCH)
-               nr = SWAP_BATCH;
+       bool scanned_many = false;
 
        /*
         * We try to cluster swap pages by allocating them sequentially
@@ -759,13 +750,9 @@ static int scan_swap_map_slots(struct swap_info_struct *si,
 
        /* SSD algorithm */
        if (si->cluster_info) {
-               if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
-                       goto checks;
-               else
+               if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
                        goto scan;
-       }
-
-       if (unlikely(!si->cluster_nr--)) {
+       } else if (unlikely(!si->cluster_nr--)) {
                if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
                        si->cluster_nr = SWAPFILE_CLUSTER - 1;
                        goto checks;
@@ -871,18 +858,31 @@ checks:
        if (si->cluster_info) {
                if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
                        goto checks;
-               else
-                       goto done;
-       }
-       /* non-ssd case */
-       ++offset;
-
-       /* non-ssd case, still more slots in cluster? */
-       if (si->cluster_nr && !si->swap_map[offset]) {
+       } else if (si->cluster_nr && !si->swap_map[++offset]) {
+               /* non-ssd case, still more slots in cluster? */
                --si->cluster_nr;
                goto checks;
        }
 
+       /*
+        * Even if there's no free clusters available (fragmented),
+        * try to scan a little more quickly with lock held unless we
+        * have scanned too many slots already.
+        */
+       if (!scanned_many) {
+               unsigned long scan_limit;
+
+               if (offset < scan_base)
+                       scan_limit = scan_base;
+               else
+                       scan_limit = si->highest_bit;
+               for (; offset <= scan_limit && --latency_ration > 0;
+                    offset++) {
+                       if (!si->swap_map[offset])
+                               goto checks;
+               }
+       }
+
 done:
        si->flags -= SWP_SCANNING;
        return n_ret;
@@ -901,6 +901,7 @@ scan:
                if (unlikely(--latency_ration < 0)) {
                        cond_resched();
                        latency_ration = LATENCY_LIMIT;
+                       scanned_many = true;
                }
        }
        offset = si->lowest_bit;
@@ -916,6 +917,7 @@ scan:
                if (unlikely(--latency_ration < 0)) {
                        cond_resched();
                        latency_ration = LATENCY_LIMIT;
+                       scanned_many = true;
                }
                offset++;
        }
@@ -1004,11 +1006,7 @@ int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_size)
        if (avail_pgs <= 0)
                goto noswap;
 
-       if (n_goal > SWAP_BATCH)
-               n_goal = SWAP_BATCH;
-
-       if (n_goal > avail_pgs)
-               n_goal = avail_pgs;
+       n_goal = min3((long)n_goal, (long)SWAP_BATCH, avail_pgs);
 
        atomic_long_sub(n_goal * size, &nr_swap_pages);
 
@@ -1275,13 +1273,14 @@ unlock_out:
 }
 
 static unsigned char __swap_entry_free(struct swap_info_struct *p,
-                                      swp_entry_t entry, unsigned char usage)
+                                      swp_entry_t entry)
 {
        struct swap_cluster_info *ci;
        unsigned long offset = swp_offset(entry);
+       unsigned char usage;
 
        ci = lock_cluster_or_swap_info(p, offset);
-       usage = __swap_entry_free_locked(p, offset, usage);
+       usage = __swap_entry_free_locked(p, offset, 1);
        unlock_cluster_or_swap_info(p, ci);
        if (!usage)
                free_swap_slot(entry);
@@ -1316,7 +1315,7 @@ void swap_free(swp_entry_t entry)
 
        p = _swap_info_get(entry);
        if (p)
-               __swap_entry_free(p, entry, 1);
+               __swap_entry_free(p, entry);
 }
 
 /*
@@ -1739,7 +1738,7 @@ int free_swap_and_cache(swp_entry_t entry)
 
        p = _swap_info_get(entry);
        if (p) {
-               count = __swap_entry_free(p, entry, 1);
+               count = __swap_entry_free(p, entry);
                if (count == SWAP_HAS_CACHE &&
                    !swap_page_trans_huge_swapped(p, entry))
                        __try_to_reclaim_swap(p, swp_offset(entry),