mm: vmscan: fix IO/refault regression in cache workingset transition

[thirdparty/linux.git] / include / linux / memcontrol.h

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index c5ebb32fef49cd1521720e5f71dcfe5a3f40ee50..cfa91a3ca0ca76b0fcddf2b05d25bd861235d2c6 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -57,6 +57,9 @@ enum mem_cgroup_stat_index {
        MEMCG_SLAB_RECLAIMABLE,
        MEMCG_SLAB_UNRECLAIMABLE,
        MEMCG_SOCK,
+       MEMCG_WORKINGSET_REFAULT,
+       MEMCG_WORKINGSET_ACTIVATE,
+       MEMCG_WORKINGSET_NODERECLAIM,
        MEMCG_NR_STAT,
 };
 
@@ -495,6 +498,40 @@ extern int do_swap_account;
 void lock_page_memcg(struct page *page);
 void unlock_page_memcg(struct page *page);
 
+static inline unsigned long mem_cgroup_read_stat(struct mem_cgroup *memcg,
+                                                enum mem_cgroup_stat_index idx)
+{
+       long val = 0;
+       int cpu;
+
+       for_each_possible_cpu(cpu)
+               val += per_cpu(memcg->stat->count[idx], cpu);
+
+       if (val < 0)
+               val = 0;
+
+       return val;
+}
+
+static inline void mem_cgroup_update_stat(struct mem_cgroup *memcg,
+                                  enum mem_cgroup_stat_index idx, int val)
+{
+       if (!mem_cgroup_disabled())
+               this_cpu_add(memcg->stat->count[idx], val);
+}
+
+static inline void mem_cgroup_inc_stat(struct mem_cgroup *memcg,
+                                  enum mem_cgroup_stat_index idx)
+{
+       mem_cgroup_update_stat(memcg, idx, 1);
+}
+
+static inline void mem_cgroup_dec_stat(struct mem_cgroup *memcg,
+                                  enum mem_cgroup_stat_index idx)
+{
+       mem_cgroup_update_stat(memcg, idx, -1);
+}
+
 /**
  * mem_cgroup_update_page_stat - update page state statistics
  * @page: the page
@@ -509,14 +546,14 @@ void unlock_page_memcg(struct page *page);
  *   if (TestClearPageState(page))
  *     mem_cgroup_update_page_stat(page, state, -1);
  *   unlock_page(page) or unlock_page_memcg(page)
+ *
+ * Kernel pages are an exception to this, since they'll never move.
  */
 static inline void mem_cgroup_update_page_stat(struct page *page,
                                 enum mem_cgroup_stat_index idx, int val)
 {
-       VM_BUG_ON(!(rcu_read_lock_held() || PageLocked(page)));
-
        if (page->mem_cgroup)
-               this_cpu_add(page->mem_cgroup->stat->count[idx], val);
+               mem_cgroup_update_stat(page->mem_cgroup, idx, val);
 }
 
 static inline void mem_cgroup_inc_page_stat(struct page *page,
@@ -741,6 +778,27 @@ static inline bool mem_cgroup_oom_synchronize(bool wait)
        return false;
 }
 
+static inline unsigned long mem_cgroup_read_stat(struct mem_cgroup *memcg,
+                                                enum mem_cgroup_stat_index idx)
+{
+       return 0;
+}
+
+static inline void mem_cgroup_update_stat(struct mem_cgroup *memcg,
+                                  enum mem_cgroup_stat_index idx, int val)
+{
+}
+
+static inline void mem_cgroup_inc_stat(struct mem_cgroup *memcg,
+                                  enum mem_cgroup_stat_index idx)
+{
+}
+
+static inline void mem_cgroup_dec_stat(struct mem_cgroup *memcg,
+                                  enum mem_cgroup_stat_index idx)
+{
+}
+
 static inline void mem_cgroup_update_page_stat(struct page *page,
                                               enum mem_cgroup_stat_index idx,
                                               int nr)