1 // SPDX-License-Identifier: GPL-2.0-or-later
3 #include <linux/module.h>
7 struct z_erofs_lzma
*next
;
8 struct xz_dec_microlzma
*state
;
13 /* considering the LZMA performance, no need to use a lockless list for now */
14 static DEFINE_SPINLOCK(z_erofs_lzma_lock
);
15 static unsigned int z_erofs_lzma_max_dictsize
;
16 static unsigned int z_erofs_lzma_nstrms
, z_erofs_lzma_avail_strms
;
17 static struct z_erofs_lzma
*z_erofs_lzma_head
;
18 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq
);
20 module_param_named(lzma_streams
, z_erofs_lzma_nstrms
, uint
, 0444);
22 void z_erofs_lzma_exit(void)
24 /* there should be no running fs instance */
25 while (z_erofs_lzma_avail_strms
) {
26 struct z_erofs_lzma
*strm
;
28 spin_lock(&z_erofs_lzma_lock
);
29 strm
= z_erofs_lzma_head
;
31 spin_unlock(&z_erofs_lzma_lock
);
35 z_erofs_lzma_head
= NULL
;
36 spin_unlock(&z_erofs_lzma_lock
);
39 struct z_erofs_lzma
*n
= strm
->next
;
42 xz_dec_microlzma_end(strm
->state
);
44 --z_erofs_lzma_avail_strms
;
50 int __init
z_erofs_lzma_init(void)
54 /* by default, use # of possible CPUs instead */
55 if (!z_erofs_lzma_nstrms
)
56 z_erofs_lzma_nstrms
= num_possible_cpus();
58 for (i
= 0; i
< z_erofs_lzma_nstrms
; ++i
) {
59 struct z_erofs_lzma
*strm
= kzalloc(sizeof(*strm
), GFP_KERNEL
);
65 spin_lock(&z_erofs_lzma_lock
);
66 strm
->next
= z_erofs_lzma_head
;
67 z_erofs_lzma_head
= strm
;
68 spin_unlock(&z_erofs_lzma_lock
);
69 ++z_erofs_lzma_avail_strms
;
74 int z_erofs_load_lzma_config(struct super_block
*sb
,
75 struct erofs_super_block
*dsb
,
76 struct z_erofs_lzma_cfgs
*lzma
, int size
)
78 static DEFINE_MUTEX(lzma_resize_mutex
);
79 unsigned int dict_size
, i
;
80 struct z_erofs_lzma
*strm
, *head
= NULL
;
83 if (!lzma
|| size
< sizeof(struct z_erofs_lzma_cfgs
)) {
84 erofs_err(sb
, "invalid lzma cfgs, size=%u", size
);
88 erofs_err(sb
, "unidentified lzma format %x, please check kernel version",
89 le16_to_cpu(lzma
->format
));
92 dict_size
= le32_to_cpu(lzma
->dict_size
);
93 if (dict_size
> Z_EROFS_LZMA_MAX_DICT_SIZE
|| dict_size
< 4096) {
94 erofs_err(sb
, "unsupported lzma dictionary size %u",
99 erofs_info(sb
, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!");
101 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
102 mutex_lock(&lzma_resize_mutex
);
104 if (z_erofs_lzma_max_dictsize
>= dict_size
) {
105 mutex_unlock(&lzma_resize_mutex
);
109 /* 1. collect/isolate all streams for the following check */
110 for (i
= 0; i
< z_erofs_lzma_avail_strms
; ++i
) {
111 struct z_erofs_lzma
*last
;
114 spin_lock(&z_erofs_lzma_lock
);
115 strm
= z_erofs_lzma_head
;
117 spin_unlock(&z_erofs_lzma_lock
);
118 wait_event(z_erofs_lzma_wq
,
119 READ_ONCE(z_erofs_lzma_head
));
122 z_erofs_lzma_head
= NULL
;
123 spin_unlock(&z_erofs_lzma_lock
);
125 for (last
= strm
; last
->next
; last
= last
->next
)
132 /* 2. walk each isolated stream and grow max dict_size if needed */
133 for (strm
= head
; strm
; strm
= strm
->next
) {
135 xz_dec_microlzma_end(strm
->state
);
136 strm
->state
= xz_dec_microlzma_alloc(XZ_PREALLOC
, dict_size
);
141 /* 3. push back all to the global list and update max dict_size */
142 spin_lock(&z_erofs_lzma_lock
);
143 DBG_BUGON(z_erofs_lzma_head
);
144 z_erofs_lzma_head
= head
;
145 spin_unlock(&z_erofs_lzma_lock
);
146 wake_up_all(&z_erofs_lzma_wq
);
148 z_erofs_lzma_max_dictsize
= dict_size
;
149 mutex_unlock(&lzma_resize_mutex
);
153 int z_erofs_lzma_decompress(struct z_erofs_decompress_req
*rq
,
154 struct page
**pagepool
)
156 const unsigned int nrpages_out
=
157 PAGE_ALIGN(rq
->pageofs_out
+ rq
->outputsize
) >> PAGE_SHIFT
;
158 const unsigned int nrpages_in
=
159 PAGE_ALIGN(rq
->inputsize
) >> PAGE_SHIFT
;
160 unsigned int inlen
, outlen
, pageofs
;
161 struct z_erofs_lzma
*strm
;
163 bool bounced
= false;
164 int no
, ni
, j
, err
= 0;
166 /* 1. get the exact LZMA compressed size */
168 err
= z_erofs_fixup_insize(rq
, kin
+ rq
->pageofs_in
,
169 min_t(unsigned int, rq
->inputsize
,
170 rq
->sb
->s_blocksize
- rq
->pageofs_in
));
176 /* 2. get an available lzma context */
178 spin_lock(&z_erofs_lzma_lock
);
179 strm
= z_erofs_lzma_head
;
181 spin_unlock(&z_erofs_lzma_lock
);
182 wait_event(z_erofs_lzma_wq
, READ_ONCE(z_erofs_lzma_head
));
185 z_erofs_lzma_head
= strm
->next
;
186 spin_unlock(&z_erofs_lzma_lock
);
188 /* 3. multi-call decompress */
189 inlen
= rq
->inputsize
;
190 outlen
= rq
->outputsize
;
191 xz_dec_microlzma_reset(strm
->state
, inlen
, outlen
,
192 !rq
->partial_decoding
);
193 pageofs
= rq
->pageofs_out
;
194 strm
->buf
.in
= kin
+ rq
->pageofs_in
;
195 strm
->buf
.in_pos
= 0;
196 strm
->buf
.in_size
= min_t(u32
, inlen
, PAGE_SIZE
- rq
->pageofs_in
);
197 inlen
-= strm
->buf
.in_size
;
198 strm
->buf
.out
= NULL
;
199 strm
->buf
.out_pos
= 0;
200 strm
->buf
.out_size
= 0;
202 for (ni
= 0, no
= -1;;) {
205 if (strm
->buf
.out_pos
== strm
->buf
.out_size
) {
208 strm
->buf
.out
= NULL
;
211 if (++no
>= nrpages_out
|| !outlen
) {
212 erofs_err(rq
->sb
, "decompressed buf out of bound");
216 strm
->buf
.out_pos
= 0;
217 strm
->buf
.out_size
= min_t(u32
, outlen
,
218 PAGE_SIZE
- pageofs
);
219 outlen
-= strm
->buf
.out_size
;
220 if (!rq
->out
[no
] && rq
->fillgaps
) /* deduped */
221 rq
->out
[no
] = erofs_allocpage(pagepool
,
222 GFP_KERNEL
| __GFP_NOFAIL
);
224 strm
->buf
.out
= kmap(rq
->out
[no
]) + pageofs
;
226 } else if (strm
->buf
.in_pos
== strm
->buf
.in_size
) {
229 if (++ni
>= nrpages_in
|| !inlen
) {
230 erofs_err(rq
->sb
, "compressed buf out of bound");
234 strm
->buf
.in_pos
= 0;
235 strm
->buf
.in_size
= min_t(u32
, inlen
, PAGE_SIZE
);
236 inlen
-= strm
->buf
.in_size
;
237 kin
= kmap(rq
->in
[ni
]);
243 * Handle overlapping: Use bounced buffer if the compressed
244 * data is under processing; Otherwise, Use short-lived pages
245 * from the on-stack pagepool where pages share with the same
248 if (!bounced
&& rq
->out
[no
] == rq
->in
[ni
]) {
249 memcpy(strm
->bounce
, strm
->buf
.in
, strm
->buf
.in_size
);
250 strm
->buf
.in
= strm
->bounce
;
253 for (j
= ni
+ 1; j
< nrpages_in
; ++j
) {
254 struct page
*tmppage
;
256 if (rq
->out
[no
] != rq
->in
[j
])
259 DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq
->sb
),
261 tmppage
= erofs_allocpage(pagepool
,
262 GFP_KERNEL
| __GFP_NOFAIL
);
263 set_page_private(tmppage
, Z_EROFS_SHORTLIVED_PAGE
);
264 copy_highpage(tmppage
, rq
->in
[j
]);
267 xz_err
= xz_dec_microlzma_run(strm
->state
, &strm
->buf
);
268 DBG_BUGON(strm
->buf
.out_pos
> strm
->buf
.out_size
);
269 DBG_BUGON(strm
->buf
.in_pos
> strm
->buf
.in_size
);
271 if (xz_err
!= XZ_OK
) {
272 if (xz_err
== XZ_STREAM_END
&& !outlen
)
274 erofs_err(rq
->sb
, "failed to decompress %d in[%u] out[%u]",
275 xz_err
, rq
->inputsize
, rq
->outputsize
);
280 if (no
< nrpages_out
&& strm
->buf
.out
)
284 /* 4. push back LZMA stream context to the global list */
285 spin_lock(&z_erofs_lzma_lock
);
286 strm
->next
= z_erofs_lzma_head
;
287 z_erofs_lzma_head
= strm
;
288 spin_unlock(&z_erofs_lzma_lock
);
289 wake_up(&z_erofs_lzma_wq
);