1 // SPDX-License-Identifier: GPL-2.0-only
3 * Xen implementation for transcendent memory (tmem)
5 * Copyright (C) 2009-2011 Oracle Corp. All rights reserved.
6 * Author: Dan Magenheimer
9 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/types.h>
14 #include <linux/init.h>
15 #include <linux/pagemap.h>
16 #include <linux/cleancache.h>
17 #include <linux/frontswap.h>
20 #include <xen/interface/xen.h>
22 #include <asm/xen/hypercall.h>
23 #include <asm/xen/hypervisor.h>
26 #ifndef CONFIG_XEN_TMEM_MODULE
27 bool __read_mostly tmem_enabled
= false;
29 static int __init
enable_tmem(char *s
)
34 __setup("tmem", enable_tmem
);
37 #ifdef CONFIG_CLEANCACHE
38 static bool cleancache __read_mostly
= true;
39 module_param(cleancache
, bool, S_IRUGO
);
40 static bool selfballooning __read_mostly
= true;
41 module_param(selfballooning
, bool, S_IRUGO
);
42 #endif /* CONFIG_CLEANCACHE */
44 #ifdef CONFIG_FRONTSWAP
45 static bool frontswap __read_mostly
= true;
46 module_param(frontswap
, bool, S_IRUGO
);
47 #else /* CONFIG_FRONTSWAP */
49 #endif /* CONFIG_FRONTSWAP */
51 #ifdef CONFIG_XEN_SELFBALLOONING
52 static bool selfshrinking __read_mostly
= true;
53 module_param(selfshrinking
, bool, S_IRUGO
);
54 #endif /* CONFIG_XEN_SELFBALLOONING */
56 #define TMEM_CONTROL 0
57 #define TMEM_NEW_POOL 1
58 #define TMEM_DESTROY_POOL 2
59 #define TMEM_NEW_PAGE 3
60 #define TMEM_PUT_PAGE 4
61 #define TMEM_GET_PAGE 5
62 #define TMEM_FLUSH_PAGE 6
63 #define TMEM_FLUSH_OBJECT 7
68 /* Bits for HYPERVISOR_tmem_op(TMEM_NEW_POOL) */
69 #define TMEM_POOL_PERSIST 1
70 #define TMEM_POOL_SHARED 2
71 #define TMEM_POOL_PAGESIZE_SHIFT 4
72 #define TMEM_VERSION_SHIFT 24
75 struct tmem_pool_uuid
{
84 #define TMEM_POOL_PRIVATE_UUID { 0, 0 }
86 /* flags for tmem_ops.new_pool */
87 #define TMEM_POOL_PERSIST 1
88 #define TMEM_POOL_SHARED 2
90 /* xen tmem foundation ops/hypercalls */
92 static inline int xen_tmem_op(u32 tmem_cmd
, u32 tmem_pool
, struct tmem_oid oid
,
93 u32 index
, unsigned long gmfn
, u32 tmem_offset
, u32 pfn_offset
, u32 len
)
99 op
.pool_id
= tmem_pool
;
100 op
.u
.gen
.oid
[0] = oid
.oid
[0];
101 op
.u
.gen
.oid
[1] = oid
.oid
[1];
102 op
.u
.gen
.oid
[2] = oid
.oid
[2];
103 op
.u
.gen
.index
= index
;
104 op
.u
.gen
.tmem_offset
= tmem_offset
;
105 op
.u
.gen
.pfn_offset
= pfn_offset
;
107 set_xen_guest_handle(op
.u
.gen
.gmfn
, (void *)gmfn
);
108 rc
= HYPERVISOR_tmem_op(&op
);
112 static int xen_tmem_new_pool(struct tmem_pool_uuid uuid
,
113 u32 flags
, unsigned long pagesize
)
116 int rc
= 0, pageshift
;
118 for (pageshift
= 0; pagesize
!= 1; pageshift
++)
120 flags
|= (pageshift
- 12) << TMEM_POOL_PAGESIZE_SHIFT
;
121 flags
|= TMEM_SPEC_VERSION
<< TMEM_VERSION_SHIFT
;
122 op
.cmd
= TMEM_NEW_POOL
;
123 op
.u
.new.uuid
[0] = uuid
.uuid_lo
;
124 op
.u
.new.uuid
[1] = uuid
.uuid_hi
;
125 op
.u
.new.flags
= flags
;
126 rc
= HYPERVISOR_tmem_op(&op
);
130 /* xen generic tmem ops */
132 static int xen_tmem_put_page(u32 pool_id
, struct tmem_oid oid
,
133 u32 index
, struct page
*page
)
135 return xen_tmem_op(TMEM_PUT_PAGE
, pool_id
, oid
, index
,
136 xen_page_to_gfn(page
), 0, 0, 0);
139 static int xen_tmem_get_page(u32 pool_id
, struct tmem_oid oid
,
140 u32 index
, struct page
*page
)
142 return xen_tmem_op(TMEM_GET_PAGE
, pool_id
, oid
, index
,
143 xen_page_to_gfn(page
), 0, 0, 0);
146 static int xen_tmem_flush_page(u32 pool_id
, struct tmem_oid oid
, u32 index
)
148 return xen_tmem_op(TMEM_FLUSH_PAGE
, pool_id
, oid
, index
,
152 static int xen_tmem_flush_object(u32 pool_id
, struct tmem_oid oid
)
154 return xen_tmem_op(TMEM_FLUSH_OBJECT
, pool_id
, oid
, 0, 0, 0, 0, 0);
158 #ifdef CONFIG_CLEANCACHE
159 static int xen_tmem_destroy_pool(u32 pool_id
)
161 struct tmem_oid oid
= { { 0 } };
163 return xen_tmem_op(TMEM_DESTROY_POOL
, pool_id
, oid
, 0, 0, 0, 0, 0);
168 static void tmem_cleancache_put_page(int pool
, struct cleancache_filekey key
,
169 pgoff_t index
, struct page
*page
)
171 u32 ind
= (u32
) index
;
172 struct tmem_oid oid
= *(struct tmem_oid
*)&key
;
178 mb(); /* ensure page is quiescent; tmem may address it with an alias */
179 (void)xen_tmem_put_page((u32
)pool
, oid
, ind
, page
);
182 static int tmem_cleancache_get_page(int pool
, struct cleancache_filekey key
,
183 pgoff_t index
, struct page
*page
)
185 u32 ind
= (u32
) index
;
186 struct tmem_oid oid
= *(struct tmem_oid
*)&key
;
189 /* translate return values to linux semantics */
194 ret
= xen_tmem_get_page((u32
)pool
, oid
, ind
, page
);
201 static void tmem_cleancache_flush_page(int pool
, struct cleancache_filekey key
,
204 u32 ind
= (u32
) index
;
205 struct tmem_oid oid
= *(struct tmem_oid
*)&key
;
211 (void)xen_tmem_flush_page((u32
)pool
, oid
, ind
);
214 static void tmem_cleancache_flush_inode(int pool
, struct cleancache_filekey key
)
216 struct tmem_oid oid
= *(struct tmem_oid
*)&key
;
220 (void)xen_tmem_flush_object((u32
)pool
, oid
);
223 static void tmem_cleancache_flush_fs(int pool
)
227 (void)xen_tmem_destroy_pool((u32
)pool
);
230 static int tmem_cleancache_init_fs(size_t pagesize
)
232 struct tmem_pool_uuid uuid_private
= TMEM_POOL_PRIVATE_UUID
;
234 return xen_tmem_new_pool(uuid_private
, 0, pagesize
);
237 static int tmem_cleancache_init_shared_fs(uuid_t
*uuid
, size_t pagesize
)
239 struct tmem_pool_uuid shared_uuid
;
241 shared_uuid
.uuid_lo
= *(u64
*)&uuid
->b
[0];
242 shared_uuid
.uuid_hi
= *(u64
*)&uuid
->b
[8];
243 return xen_tmem_new_pool(shared_uuid
, TMEM_POOL_SHARED
, pagesize
);
246 static const struct cleancache_ops tmem_cleancache_ops
= {
247 .put_page
= tmem_cleancache_put_page
,
248 .get_page
= tmem_cleancache_get_page
,
249 .invalidate_page
= tmem_cleancache_flush_page
,
250 .invalidate_inode
= tmem_cleancache_flush_inode
,
251 .invalidate_fs
= tmem_cleancache_flush_fs
,
252 .init_shared_fs
= tmem_cleancache_init_shared_fs
,
253 .init_fs
= tmem_cleancache_init_fs
257 #ifdef CONFIG_FRONTSWAP
258 /* frontswap tmem operations */
260 /* a single tmem poolid is used for all frontswap "types" (swapfiles) */
261 static int tmem_frontswap_poolid
;
264 * Swizzling increases objects per swaptype, increasing tmem concurrency
265 * for heavy swaploads. Later, larger nr_cpus -> larger SWIZ_BITS
268 #define SWIZ_MASK ((1 << SWIZ_BITS) - 1)
269 #define _oswiz(_type, _ind) ((_type << SWIZ_BITS) | (_ind & SWIZ_MASK))
270 #define iswiz(_ind) (_ind >> SWIZ_BITS)
272 static inline struct tmem_oid
oswiz(unsigned type
, u32 ind
)
274 struct tmem_oid oid
= { .oid
= { 0 } };
275 oid
.oid
[0] = _oswiz(type
, ind
);
279 /* returns 0 if the page was successfully put into frontswap, -1 if not */
280 static int tmem_frontswap_store(unsigned type
, pgoff_t offset
,
283 u64 ind64
= (u64
)offset
;
284 u32 ind
= (u32
)offset
;
285 int pool
= tmem_frontswap_poolid
;
288 /* THP isn't supported */
289 if (PageTransHuge(page
))
296 mb(); /* ensure page is quiescent; tmem may address it with an alias */
297 ret
= xen_tmem_put_page(pool
, oswiz(type
, ind
), iswiz(ind
), page
);
298 /* translate Xen tmem return values to linux semantics */
306 * returns 0 if the page was successfully gotten from frontswap, -1 if
307 * was not present (should never happen!)
309 static int tmem_frontswap_load(unsigned type
, pgoff_t offset
,
312 u64 ind64
= (u64
)offset
;
313 u32 ind
= (u32
)offset
;
314 int pool
= tmem_frontswap_poolid
;
321 ret
= xen_tmem_get_page(pool
, oswiz(type
, ind
), iswiz(ind
), page
);
322 /* translate Xen tmem return values to linux semantics */
329 /* flush a single page from frontswap */
330 static void tmem_frontswap_flush_page(unsigned type
, pgoff_t offset
)
332 u64 ind64
= (u64
)offset
;
333 u32 ind
= (u32
)offset
;
334 int pool
= tmem_frontswap_poolid
;
340 (void) xen_tmem_flush_page(pool
, oswiz(type
, ind
), iswiz(ind
));
343 /* flush all pages from the passed swaptype */
344 static void tmem_frontswap_flush_area(unsigned type
)
346 int pool
= tmem_frontswap_poolid
;
351 for (ind
= SWIZ_MASK
; ind
>= 0; ind
--)
352 (void)xen_tmem_flush_object(pool
, oswiz(type
, ind
));
355 static void tmem_frontswap_init(unsigned ignored
)
357 struct tmem_pool_uuid
private = TMEM_POOL_PRIVATE_UUID
;
359 /* a single tmem poolid is used for all frontswap "types" (swapfiles) */
360 if (tmem_frontswap_poolid
< 0)
361 tmem_frontswap_poolid
=
362 xen_tmem_new_pool(private, TMEM_POOL_PERSIST
, PAGE_SIZE
);
365 static struct frontswap_ops tmem_frontswap_ops
= {
366 .store
= tmem_frontswap_store
,
367 .load
= tmem_frontswap_load
,
368 .invalidate_page
= tmem_frontswap_flush_page
,
369 .invalidate_area
= tmem_frontswap_flush_area
,
370 .init
= tmem_frontswap_init
374 static int __init
xen_tmem_init(void)
378 #ifdef CONFIG_FRONTSWAP
379 if (tmem_enabled
&& frontswap
) {
382 tmem_frontswap_poolid
= -1;
383 frontswap_register_ops(&tmem_frontswap_ops
);
384 pr_info("frontswap enabled, RAM provided by Xen Transcendent Memory%s\n",
388 #ifdef CONFIG_CLEANCACHE
389 BUILD_BUG_ON(sizeof(struct cleancache_filekey
) != sizeof(struct tmem_oid
));
390 if (tmem_enabled
&& cleancache
) {
393 err
= cleancache_register_ops(&tmem_cleancache_ops
);
395 pr_warn("xen-tmem: failed to enable cleancache: %d\n",
398 pr_info("cleancache enabled, RAM provided by "
399 "Xen Transcendent Memory\n");
402 #ifdef CONFIG_XEN_SELFBALLOONING
404 * There is no point of driving pages to the swap system if they
405 * aren't going anywhere in tmem universe.
408 selfshrinking
= false;
409 selfballooning
= false;
411 xen_selfballoon_init(selfballooning
, selfshrinking
);
416 module_init(xen_tmem_init
)
417 MODULE_LICENSE("GPL");
418 MODULE_AUTHOR("Dan Magenheimer <dan.magenheimer@oracle.com>");
419 MODULE_DESCRIPTION("Shim to Xen transcendent memory");