]>
Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
9b1d6c89 ML |
2 | #include <linux/module.h> |
3 | #include <linux/scatterlist.h> | |
4 | #include <linux/mempool.h> | |
5 | #include <linux/slab.h> | |
6 | ||
7 | #define SG_MEMPOOL_NR ARRAY_SIZE(sg_pools) | |
8 | #define SG_MEMPOOL_SIZE 2 | |
9 | ||
10 | struct sg_pool { | |
11 | size_t size; | |
12 | char *name; | |
13 | struct kmem_cache *slab; | |
14 | mempool_t *pool; | |
15 | }; | |
16 | ||
17 | #define SP(x) { .size = x, "sgpool-" __stringify(x) } | |
18 | #if (SG_CHUNK_SIZE < 32) | |
19 | #error SG_CHUNK_SIZE is too small (must be 32 or greater) | |
20 | #endif | |
21 | static struct sg_pool sg_pools[] = { | |
22 | SP(8), | |
23 | SP(16), | |
24 | #if (SG_CHUNK_SIZE > 32) | |
25 | SP(32), | |
26 | #if (SG_CHUNK_SIZE > 64) | |
27 | SP(64), | |
28 | #if (SG_CHUNK_SIZE > 128) | |
29 | SP(128), | |
30 | #if (SG_CHUNK_SIZE > 256) | |
31 | #error SG_CHUNK_SIZE is too large (256 MAX) | |
32 | #endif | |
33 | #endif | |
34 | #endif | |
35 | #endif | |
36 | SP(SG_CHUNK_SIZE) | |
37 | }; | |
38 | #undef SP | |
39 | ||
40 | static inline unsigned int sg_pool_index(unsigned short nents) | |
41 | { | |
42 | unsigned int index; | |
43 | ||
44 | BUG_ON(nents > SG_CHUNK_SIZE); | |
45 | ||
46 | if (nents <= 8) | |
47 | index = 0; | |
48 | else | |
49 | index = get_count_order(nents) - 3; | |
50 | ||
51 | return index; | |
52 | } | |
53 | ||
54 | static void sg_pool_free(struct scatterlist *sgl, unsigned int nents) | |
55 | { | |
56 | struct sg_pool *sgp; | |
57 | ||
58 | sgp = sg_pools + sg_pool_index(nents); | |
59 | mempool_free(sgl, sgp->pool); | |
60 | } | |
61 | ||
62 | static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask) | |
63 | { | |
64 | struct sg_pool *sgp; | |
65 | ||
66 | sgp = sg_pools + sg_pool_index(nents); | |
67 | return mempool_alloc(sgp->pool, gfp_mask); | |
68 | } | |
69 | ||
70 | /** | |
71 | * sg_free_table_chained - Free a previously mapped sg table | |
72 | * @table: The sg table header to use | |
73 | * @first_chunk: was first_chunk not NULL in sg_alloc_table_chained? | |
74 | * | |
75 | * Description: | |
76 | * Free an sg table previously allocated and setup with | |
77 | * sg_alloc_table_chained(). | |
78 | * | |
79 | **/ | |
80 | void sg_free_table_chained(struct sg_table *table, bool first_chunk) | |
81 | { | |
82 | if (first_chunk && table->orig_nents <= SG_CHUNK_SIZE) | |
83 | return; | |
84 | __sg_free_table(table, SG_CHUNK_SIZE, first_chunk, sg_pool_free); | |
85 | } | |
86 | EXPORT_SYMBOL_GPL(sg_free_table_chained); | |
87 | ||
88 | /** | |
89 | * sg_alloc_table_chained - Allocate and chain SGLs in an sg table | |
90 | * @table: The sg table header to use | |
91 | * @nents: Number of entries in sg list | |
92 | * @first_chunk: first SGL | |
93 | * | |
94 | * Description: | |
95 | * Allocate and chain SGLs in an sg table. If @nents@ is larger than | |
96 | * SG_CHUNK_SIZE a chained sg table will be setup. | |
97 | * | |
98 | **/ | |
99 | int sg_alloc_table_chained(struct sg_table *table, int nents, | |
100 | struct scatterlist *first_chunk) | |
101 | { | |
102 | int ret; | |
103 | ||
104 | BUG_ON(!nents); | |
105 | ||
106 | if (first_chunk) { | |
107 | if (nents <= SG_CHUNK_SIZE) { | |
108 | table->nents = table->orig_nents = nents; | |
109 | sg_init_table(table->sgl, nents); | |
110 | return 0; | |
111 | } | |
112 | } | |
113 | ||
114 | ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE, | |
115 | first_chunk, GFP_ATOMIC, sg_pool_alloc); | |
116 | if (unlikely(ret)) | |
117 | sg_free_table_chained(table, (bool)first_chunk); | |
118 | return ret; | |
119 | } | |
120 | EXPORT_SYMBOL_GPL(sg_alloc_table_chained); | |
121 | ||
122 | static __init int sg_pool_init(void) | |
123 | { | |
124 | int i; | |
125 | ||
126 | for (i = 0; i < SG_MEMPOOL_NR; i++) { | |
127 | struct sg_pool *sgp = sg_pools + i; | |
128 | int size = sgp->size * sizeof(struct scatterlist); | |
129 | ||
130 | sgp->slab = kmem_cache_create(sgp->name, size, 0, | |
131 | SLAB_HWCACHE_ALIGN, NULL); | |
132 | if (!sgp->slab) { | |
133 | printk(KERN_ERR "SG_POOL: can't init sg slab %s\n", | |
134 | sgp->name); | |
135 | goto cleanup_sdb; | |
136 | } | |
137 | ||
138 | sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE, | |
139 | sgp->slab); | |
140 | if (!sgp->pool) { | |
141 | printk(KERN_ERR "SG_POOL: can't init sg mempool %s\n", | |
142 | sgp->name); | |
143 | goto cleanup_sdb; | |
144 | } | |
145 | } | |
146 | ||
147 | return 0; | |
148 | ||
149 | cleanup_sdb: | |
150 | for (i = 0; i < SG_MEMPOOL_NR; i++) { | |
151 | struct sg_pool *sgp = sg_pools + i; | |
7f476715 | 152 | |
153 | mempool_destroy(sgp->pool); | |
154 | kmem_cache_destroy(sgp->slab); | |
9b1d6c89 ML |
155 | } |
156 | ||
157 | return -ENOMEM; | |
158 | } | |
159 | ||
160 | static __exit void sg_pool_exit(void) | |
161 | { | |
162 | int i; | |
163 | ||
164 | for (i = 0; i < SG_MEMPOOL_NR; i++) { | |
165 | struct sg_pool *sgp = sg_pools + i; | |
166 | mempool_destroy(sgp->pool); | |
167 | kmem_cache_destroy(sgp->slab); | |
168 | } | |
169 | } | |
170 | ||
171 | module_init(sg_pool_init); | |
172 | module_exit(sg_pool_exit); |